^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: LGPL-2.1 OR MIT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * rseq-arm64.h
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * aarch64 -mbig-endian generates mixed endianness code vs data:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * matches code endianness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #ifdef __AARCH64EB__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #define RSEQ_SIG_DATA RSEQ_SIG_CODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define RSEQ_SIG RSEQ_SIG_DATA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define rseq_smp_load_acquire(p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) __extension__ ({ \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) __typeof(*p) ____p1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) switch (sizeof(*p)) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) case 1: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) asm volatile ("ldarb %w0, %1" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) : "=r" (*(__u8 *)p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) : "Q" (*p) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) case 2: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) asm volatile ("ldarh %w0, %1" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) : "=r" (*(__u16 *)p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) : "Q" (*p) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) case 4: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) asm volatile ("ldar %w0, %1" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) : "=r" (*(__u32 *)p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) : "Q" (*p) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) case 8: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) asm volatile ("ldar %0, %1" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) : "=r" (*(__u64 *)p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) : "Q" (*p) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) ____p1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) })
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #define rseq_smp_store_release(p, v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) switch (sizeof(*p)) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) case 1: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) asm volatile ("stlrb %w1, %0" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) : "=Q" (*p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) : "r" ((__u8)v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) case 2: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) asm volatile ("stlrh %w1, %0" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) : "=Q" (*p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) : "r" ((__u16)v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) case 4: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) asm volatile ("stlr %w1, %0" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) : "=Q" (*p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) : "r" ((__u32)v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) case 8: \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) asm volatile ("stlr %1, %0" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) : "=Q" (*p) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) : "r" ((__u64)v) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) : "memory"); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) break; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #ifdef RSEQ_SKIP_FASTPATH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #include "rseq-skip.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #else /* !RSEQ_SKIP_FASTPATH */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define RSEQ_ASM_TMP_REG32 "w15"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) #define RSEQ_ASM_TMP_REG "x15"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #define RSEQ_ASM_TMP_REG_2 "x14"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) post_commit_offset, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) " .pushsection __rseq_cs, \"aw\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) " .balign 32\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) __rseq_str(label) ":\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) " .quad " __rseq_str(start_ip) ", " \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) __rseq_str(post_commit_offset) ", " \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) __rseq_str(abort_ip) "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) " .popsection\n\t" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) " .quad " __rseq_str(label) "b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) " .popsection\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) (post_commit_ip - start_ip), abort_ip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * Exit points of a rseq critical section consist of all instructions outside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * of the critical section where a critical section can either branch to or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * reach through the normal course of its execution. The abort IP and the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * post-commit IP are already part of the __rseq_cs section and should not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * explicitly defined as additional exit points. Knowing all exit points is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * useful to assist debuggers stepping over the critical section.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) " .pushsection __rseq_exit_point_array, \"aw\"\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) " .popsection\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) RSEQ_INJECT_ASM(1) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ", :lo12:" __rseq_str(cs_label) "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) __rseq_str(label) ":\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) " b 222f\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) __rseq_str(label) ":\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) " b %l[" __rseq_str(abort_label) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) "222:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) #define RSEQ_ASM_OP_STORE(value, var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) RSEQ_ASM_OP_STORE(value, var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) __rseq_str(post_commit_label) ":\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) RSEQ_ASM_OP_STORE_RELEASE(value, var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) __rseq_str(post_commit_label) ":\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) ", %[" __rseq_str(expect) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) ", %w[" __rseq_str(expect) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #define RSEQ_ASM_OP_CMPNE(var, expect, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) ", %[" __rseq_str(expect) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) RSEQ_INJECT_ASM(2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define RSEQ_ASM_OP_R_LOAD(var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #define RSEQ_ASM_OP_R_STORE(var) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) #define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) ", %[" __rseq_str(offset) "]]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) #define RSEQ_ASM_OP_R_ADD(count) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) ", %[" __rseq_str(count) "]\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) __rseq_str(post_commit_label) ":\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) " cbz %[" __rseq_str(len) "], 333f\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) ", " RSEQ_ASM_TMP_REG_2 "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) ", " RSEQ_ASM_TMP_REG_2 "]\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) "333:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) : "memory", RSEQ_ASM_TMP_REG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) off_t voffp, intptr_t *load, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) RSEQ_ASM_OP_R_LOAD(v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) RSEQ_ASM_OP_R_STORE(load)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) RSEQ_ASM_OP_R_LOAD_OFF(voffp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) [expectnot] "r" (expectnot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) [load] "Qo" (*load),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) [voffp] "r" (voffp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) : "memory", RSEQ_ASM_TMP_REG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) int rseq_addv(intptr_t *v, intptr_t count, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) RSEQ_ASM_OP_R_LOAD(v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) RSEQ_ASM_OP_R_ADD(count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) [count] "r" (count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) : "memory", RSEQ_ASM_TMP_REG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) : abort
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) , error1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) RSEQ_ASM_OP_STORE(newv2, v2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) [v2] "Qo" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) [newv2] "r" (newv2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) : "memory", RSEQ_ASM_TMP_REG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) intptr_t *v2, intptr_t newv2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) RSEQ_ASM_OP_STORE(newv2, v2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) [v2] "Qo" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) [newv2] "r" (newv2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) : "memory", RSEQ_ASM_TMP_REG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) intptr_t *v2, intptr_t expect2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) [v2] "Qo" (*v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) [expect2] "r" (expect2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) [newv] "r" (newv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) : "memory", RSEQ_ASM_TMP_REG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) , error1, error2, error3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) error3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) rseq_bug("2nd expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) [len] "r" (len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) static inline __attribute__((always_inline))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) void *dst, void *src, size_t len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) intptr_t newv, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) RSEQ_INJECT_C(9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) __asm__ __volatile__ goto (
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) RSEQ_INJECT_ASM(3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) RSEQ_INJECT_ASM(4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) RSEQ_INJECT_ASM(5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) RSEQ_INJECT_ASM(6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) RSEQ_ASM_DEFINE_ABORT(4, abort)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) : /* gcc asm goto does not allow outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) : [cpu_id] "r" (cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) [rseq_cs] "m" (__rseq_abi.rseq_cs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) [expect] "r" (expect),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) [v] "Qo" (*v),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) [newv] "r" (newv),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) [dst] "r" (dst),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) [src] "r" (src),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) [len] "r" (len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) RSEQ_INJECT_INPUT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) : abort, cmpfail
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) , error1, error2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) );
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) abort:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) RSEQ_INJECT_FAILED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) return -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) cmpfail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) #ifdef RSEQ_COMPARE_TWICE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) error1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) rseq_bug("cpu_id comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) error2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) rseq_bug("expected value comparison failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) #endif /* !RSEQ_SKIP_FASTPATH */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags