^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2020 ARM Ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <asm/asm-uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <asm/assembler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/mte.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/sysreg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) .arch armv8.5-a+memtag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * multitag_transfer_size - set \reg to the block size that is accessed by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * LDGM/STGM instructions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) .macro multitag_transfer_size, reg, tmp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) mrs_s \reg, SYS_GMID_EL1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) mov \tmp, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) lsl \reg, \tmp, \reg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * Clear the tags in a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * x0 - address of the page to be cleared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) SYM_FUNC_START(mte_clear_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) multitag_transfer_size x1, x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) 1: stgm xzr, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) add x0, x0, x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) tst x0, #(PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) SYM_FUNC_END(mte_clear_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * Zero the page and tags at the same time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * Parameters:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * x0 - address to the beginning of the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) SYM_FUNC_START(mte_zero_clear_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) mrs x1, dczid_el0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) tbnz x1, #4, 2f // Branch if DC GZVA is prohibited
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) and w1, w1, #0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) mov x2, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) lsl x1, x2, x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) 1: dc gzva, x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) add x0, x0, x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) tst x0, #(PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) 2: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) tst x0, #(PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) b.ne 2b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) SYM_FUNC_END(mte_zero_clear_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * Copy the tags from the source page to the destination one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * x0 - address of the destination page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * x1 - address of the source page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) SYM_FUNC_START(mte_copy_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) mov x2, x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) mov x3, x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) multitag_transfer_size x5, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) 1: ldgm x4, [x3]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) stgm x4, [x2]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) add x2, x2, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) add x3, x3, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) tst x2, #(PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) SYM_FUNC_END(mte_copy_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Read tags from a user buffer (one tag per byte) and set the corresponding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * tags at the given kernel address. Used by PTRACE_POKEMTETAGS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * x0 - kernel address (to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * x1 - user buffer (from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * x2 - number of tags/bytes (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * x0 - number of tags read/set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) SYM_FUNC_START(mte_copy_tags_from_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) mov x3, x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) cbz x2, 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) uao_user_alternative 2f, ldrb, ldtrb, w4, x1, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) lsl x4, x4, #MTE_TAG_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) stg x4, [x0], #MTE_GRANULE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) add x1, x1, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) subs x2, x2, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) // exception handling and function return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 2: sub x0, x1, x3 // update the number of tags set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) SYM_FUNC_END(mte_copy_tags_from_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * Get the tags from a kernel address range and write the tag values to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * x0 - user buffer (to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * x1 - kernel address (from)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * x2 - number of tags/bytes (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * x0 - number of tags read/set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) SYM_FUNC_START(mte_copy_tags_to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) mov x3, x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) cbz x2, 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) ldg x4, [x1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) uao_user_alternative 2f, strb, sttrb, w4, x0, 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) add x0, x0, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) add x1, x1, #MTE_GRANULE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) subs x2, x2, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) // exception handling and function return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 2: sub x0, x0, x3 // update the number of tags copied
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) SYM_FUNC_END(mte_copy_tags_to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * Save the tags in a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) * x0 - page address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * x1 - tag storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) SYM_FUNC_START(mte_save_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) multitag_transfer_size x7, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) mov x2, #0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) ldgm x5, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) orr x2, x2, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) add x0, x0, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) tst x0, #0xFF // 16 tag values fit in a register,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) b.ne 2b // which is 16*16=256 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) str x2, [x1], #8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) tst x0, #(PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) SYM_FUNC_END(mte_save_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) * Restore the tags in a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * x0 - page address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * x1 - tag storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) SYM_FUNC_START(mte_restore_page_tags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) multitag_transfer_size x7, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) ldr x2, [x1], #8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) stgm x2, [x0]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) add x0, x0, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) tst x0, #0xFF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) b.ne 2b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) tst x0, #(PAGE_SIZE - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) b.ne 1b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) SYM_FUNC_END(mte_restore_page_tags)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags