^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * rseq-x86.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <stdint.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * RSEQ_SIG is used with the following reserved undefined instructions, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * trap in user-space:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define RSEQ_SIG 0x53053053
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * address through a "r" input operand.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) /* Offset of cpu_id and rseq_cs fields in struct rseq. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define RSEQ_CPU_ID_OFFSET 4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define RSEQ_CS_OFFSET 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #ifdef __x86_64__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define rseq_smp_mb() \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define rseq_smp_rmb() rseq_barrier()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define rseq_smp_wmb() rseq_barrier()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #define rseq_smp_load_acquire(p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) __extension__ ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) rseq_barrier(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) ____p1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #define rseq_smp_store_release(p, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) rseq_barrier(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) RSEQ_WRITE_ONCE(*p, v); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #ifdef RSEQ_SKIP_FASTPATH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include "rseq-skip.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #else /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) start_ip, post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ".pushsection __rseq_cs, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) ".balign 32\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) ".popsection\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) ".quad " __rseq_str(label) "b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) (post_commit_ip - start_ip), abort_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * Exit points of a rseq critical section consist of all instructions outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * of the critical section where a critical section can either branch to or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * reach through the normal course of its execution. The abort IP and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) * post-commit IP are already part of the __rseq_cs section and should not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * explicitly defined as additional exit points. Knowing all exit points is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * useful to assist debuggers stepping over the critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) RSEQ_INJECT_ASM(1) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) __rseq_str(label) ":\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) RSEQ_INJECT_ASM(2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) "jnz " __rseq_str(label) "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) ".pushsection __rseq_failure, \"ax\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) ".byte 0x0f, 0xb9, 0x3d\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) ".long " __rseq_str(RSEQ_SIG) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) "jmp %l[" __rseq_str(abort_label) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ".pushsection __rseq_failure, \"ax\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) "movq %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) : "memory", "cc", "rax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * Compare @v against @expectnot. When it does _not_ match, load @v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) * into @load, and store the content of *@v + voffp into @v.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) off_t voffp, intptr_t *load, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) "movq %[v], %%rbx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) "cmpq %%rbx, %[expectnot]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) "je %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) "movq %[v], %%rbx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) "cmpq %%rbx, %[expectnot]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) "je %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) "movq %%rbx, %[load]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) "addq %[voffp], %%rbx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) "movq (%%rbx), %%rbx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) "movq %%rbx, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) [expectnot] "r" (expectnot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) [voffp] "er" (voffp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) [load] "m" (*load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) : "memory", "cc", "rax", "rbx"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) int rseq_addv(intptr_t *v, intptr_t count, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) "addq %[count], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) [count] "er" (count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) : "memory", "cc", "rax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) : abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) , error1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) * pval = *(ptr+off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) * *pval += inc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) /* get p+v */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) "movq %[ptr], %%rbx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) "addq %[off], %%rbx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* get pv */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) "movq (%%rbx), %%rcx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) /* *pv += inc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) "addq %[inc], (%%rcx)\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) [ptr] "m" (*ptr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) [off] "er" (off),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) [inc] "er" (inc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) : "memory", "cc", "rax", "rbx", "rcx"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) : abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) , error1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) /* try store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) "movq %[newv2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) "movq %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) /* try store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) [newv2] "r" (newv2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) : "memory", "cc", "rax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /* x86-64 is TSO. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) intptr_t *v2, intptr_t expect2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) "cmpq %[v2], %[expect2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) "cmpq %[v2], %[expect2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) "jnz %l[error3]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) "movq %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) /* cmp2 input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) [expect2] "r" (expect2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) : "memory", "cc", "rax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) , error1, error2, error3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) rseq_bug("1st expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) error3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) rseq_bug("2nd expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) uint64_t rseq_scratch[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) "movq %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) "movq %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) "movq %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) "jnz 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) "cmpq %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) "jnz 7f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) /* try memcpy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) "test %[len], %[len]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) "jz 333f\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) "222:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) "movb (%[src]), %%al\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) "movb %%al, (%[dst])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) "inc %[src]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) "inc %[dst]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) "dec %[len]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) "jnz 222b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) "333:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) "movq %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) "movq %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) "movq %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) "movq %[rseq_scratch0], %[src]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) RSEQ_ASM_DEFINE_ABORT(4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) "movq %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) "movq %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) "movq %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) RSEQ_ASM_DEFINE_CMPFAIL(5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) "movq %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) "movq %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) "movq %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) cmpfail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) RSEQ_ASM_DEFINE_CMPFAIL(6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) "movq %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) "movq %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) "movq %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) error1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) RSEQ_ASM_DEFINE_CMPFAIL(7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) "movq %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) "movq %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) "movq %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) error2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) /* try memcpy input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) [len] "r" (len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) [rseq_scratch0] "m" (rseq_scratch[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) [rseq_scratch1] "m" (rseq_scratch[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) [rseq_scratch2] "m" (rseq_scratch[2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) : "memory", "cc", "rax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) /* x86-64 is TSO. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) newv, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) #endif /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) #elif __i386__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) #define rseq_smp_mb() \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) #define rseq_smp_rmb() \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) #define rseq_smp_wmb() \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) #define rseq_smp_load_acquire(p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) __extension__ ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) rseq_smp_mb(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) ____p1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) #define rseq_smp_store_release(p, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) rseq_smp_mb(); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) RSEQ_WRITE_ONCE(*p, v); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) #ifdef RSEQ_SKIP_FASTPATH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) #include "rseq-skip.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) #else /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * Use eax as scratch register and take memory operands as input to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * lessen register pressure. Especially needed when compiling in O0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) start_ip, post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) ".pushsection __rseq_cs, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ".balign 32\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) ".popsection\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) ".long " __rseq_str(label) "b, 0x0\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) (post_commit_ip - start_ip), abort_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) * Exit points of a rseq critical section consist of all instructions outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) * of the critical section where a critical section can either branch to or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) * reach through the normal course of its execution. The abort IP and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * post-commit IP are already part of the __rseq_cs section and should not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * explicitly defined as additional exit points. Knowing all exit points is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * useful to assist debuggers stepping over the critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) RSEQ_INJECT_ASM(1) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) __rseq_str(label) ":\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) RSEQ_INJECT_ASM(2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) "jnz " __rseq_str(label) "\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) ".pushsection __rseq_failure, \"ax\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) ".byte 0x0f, 0xb9, 0x3d\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) ".long " __rseq_str(RSEQ_SIG) "\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) "jmp %l[" __rseq_str(abort_label) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) ".pushsection __rseq_failure, \"ax\"\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) __rseq_str(label) ":\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) teardown \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) ".popsection\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) "cmpl %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) "cmpl %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) "movl %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * Compare @v against @expectnot. When it does _not_ match, load @v
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * into @load, and store the content of *@v + voffp into @v.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) off_t voffp, intptr_t *load, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) "movl %[v], %%ebx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) "cmpl %%ebx, %[expectnot]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) "je %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) "movl %[v], %%ebx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) "cmpl %%ebx, %[expectnot]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) "je %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) "movl %%ebx, %[load]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) "addl %[voffp], %%ebx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) "movl (%%ebx), %%ebx\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) "movl %%ebx, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) [expectnot] "r" (expectnot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) [voffp] "ir" (voffp),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) [load] "m" (*load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) : "memory", "cc", "eax", "ebx"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) int rseq_addv(intptr_t *v, intptr_t count, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) "addl %[count], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) [count] "ir" (count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) : abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) , error1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) "cmpl %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) "cmpl %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) /* try store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) "movl %[newv2], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) "movl %%eax, %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) "movl %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) /* try store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) [newv2] "m" (newv2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) "movl %[expect], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) "cmpl %[v], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) "movl %[expect], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) "cmpl %[v], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) /* try store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) "movl %[newv2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) "lock; addl $0,-128(%%esp)\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) "movl %[newv], %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) /* try store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) [newv2] "r" (newv2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) [expect] "m" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) intptr_t *v2, intptr_t expect2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) "cmpl %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) "cmpl %[expect2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) "jnz %l[cmpfail]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) "cmpl %[v], %[expect]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) "jnz %l[error2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) "cmpl %[expect2], %[v2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) "jnz %l[error3]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) "movl %[newv], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) "movl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) /* cmp2 input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) [v2] "m" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) [expect2] "r" (expect2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) [newv] "m" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) , error1, error2, error3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) rseq_bug("1st expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) error3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) rseq_bug("2nd expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) /* TODO: implement a faster memcpy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) uint32_t rseq_scratch[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) "movl %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) "movl %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) "movl %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) "movl %[expect], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) "cmpl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) "jnz 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) "movl %[expect], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) "cmpl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) "jnz 7f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) /* try memcpy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) "test %[len], %[len]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) "jz 333f\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) "222:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) "movb (%[src]), %%al\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) "movb %%al, (%[dst])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) "inc %[src]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) "inc %[dst]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) "dec %[len]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) "jnz 222b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) "333:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) "movl %[newv], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) "movl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) "movl %[rseq_scratch0], %[src]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) RSEQ_ASM_DEFINE_ABORT(4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) RSEQ_ASM_DEFINE_CMPFAIL(5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) cmpfail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) RSEQ_ASM_DEFINE_CMPFAIL(6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) error1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) RSEQ_ASM_DEFINE_CMPFAIL(7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) error2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) [expect] "m" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) [newv] "m" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) /* try memcpy input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) [len] "r" (len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) [rseq_scratch0] "m" (rseq_scratch[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) [rseq_scratch1] "m" (rseq_scratch[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) [rseq_scratch2] "m" (rseq_scratch[2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) /* TODO: implement a faster memcpy. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) uint32_t rseq_scratch[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) "movl %[src], %[rseq_scratch0]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) "movl %[dst], %[rseq_scratch1]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) "movl %[len], %[rseq_scratch2]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) /* Start rseq by storing table entry pointer into rseq_cs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) "movl %[expect], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) "cmpl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) "jnz 5f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) "movl %[expect], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) "cmpl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) "jnz 7f\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) /* try memcpy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) "test %[len], %[len]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) "jz 333f\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) "222:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) "movb (%[src]), %%al\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) "movb %%al, (%[dst])\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) "inc %[src]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) "inc %[dst]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) "dec %[len]\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) "jnz 222b\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) "333:\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) "lock; addl $0,-128(%%esp)\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) "movl %[newv], %%eax\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) /* final store */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) "movl %%eax, %[v]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) "2:\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) /* teardown */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) "movl %[rseq_scratch0], %[src]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) RSEQ_ASM_DEFINE_ABORT(4,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) RSEQ_ASM_DEFINE_CMPFAIL(5,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) cmpfail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) RSEQ_ASM_DEFINE_CMPFAIL(6,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) error1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) RSEQ_ASM_DEFINE_CMPFAIL(7,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) "movl %[rseq_scratch2], %[len]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) "movl %[rseq_scratch1], %[dst]\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) "movl %[rseq_scratch0], %[src]\n\t",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) error2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) [rseq_abi] "r" (&__rseq_abi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) /* final store input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) [v] "m" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) [expect] "m" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) [newv] "m" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) /* try memcpy input */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) [len] "r" (len),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) [rseq_scratch0] "m" (rseq_scratch[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) [rseq_scratch1] "m" (rseq_scratch[1]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) [rseq_scratch2] "m" (rseq_scratch[2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) : "memory", "cc", "eax"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) RSEQ_INJECT_CLOBBER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) #endif /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) #endif
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags