^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0-only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Accelerated CRC32(C) using AArch64 CRC instructions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <asm/alternative.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/assembler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) .arch armv8-a+crc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) .macro __crc32, c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) cmp x2, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) b.lt 8f // less than 16 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) and x7, x2, #0x1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) and x2, x2, #~0x1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) cbz x7, 32f // multiple of 32 bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) and x8, x7, #0xf
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) ldp x3, x4, [x1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) add x8, x8, x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) add x1, x1, x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) ldp x5, x6, [x8]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) CPU_BE( rev x3, x3 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) CPU_BE( rev x4, x4 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) CPU_BE( rev x5, x5 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) CPU_BE( rev x6, x6 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) tst x7, #8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) crc32\c\()x w8, w0, x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) csel x3, x3, x4, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) csel w0, w0, w8, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) tst x7, #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) lsr x4, x3, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) crc32\c\()w w8, w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) csel x3, x3, x4, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) csel w0, w0, w8, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) tst x7, #2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) lsr w4, w3, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) crc32\c\()h w8, w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) csel w3, w3, w4, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) csel w0, w0, w8, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) tst x7, #1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) crc32\c\()b w8, w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) csel w0, w0, w8, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) tst x7, #16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) crc32\c\()x w8, w0, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) crc32\c\()x w8, w8, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) csel w0, w0, w8, eq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) cbz x2, 0f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) 32: ldp x3, x4, [x1], #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) sub x2, x2, #32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) ldp x5, x6, [x1, #-16]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) CPU_BE( rev x3, x3 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) CPU_BE( rev x4, x4 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) CPU_BE( rev x5, x5 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) CPU_BE( rev x6, x6 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) crc32\c\()x w0, w0, x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) crc32\c\()x w0, w0, x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) crc32\c\()x w0, w0, x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) crc32\c\()x w0, w0, x6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) cbnz x2, 32b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) 0: ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) 8: tbz x2, #3, 4f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) ldr x3, [x1], #8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) CPU_BE( rev x3, x3 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) crc32\c\()x w0, w0, x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) 4: tbz x2, #2, 2f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) ldr w3, [x1], #4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) CPU_BE( rev w3, w3 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) crc32\c\()w w0, w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) 2: tbz x2, #1, 1f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ldrh w3, [x1], #2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) CPU_BE( rev16 w3, w3 )
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) crc32\c\()h w0, w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) 1: tbz x2, #0, 0f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) ldrb w3, [x1]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) crc32\c\()b w0, w0, w3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) 0: ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) .endm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) .align 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) SYM_FUNC_START(crc32_le)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) alternative_if_not ARM64_HAS_CRC32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) b crc32_le_base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) alternative_else_nop_endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) __crc32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) SYM_FUNC_END(crc32_le)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) .align 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) SYM_FUNC_START(__crc32c_le)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) alternative_if_not ARM64_HAS_CRC32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) b __crc32c_le_base
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) alternative_else_nop_endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) __crc32 c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) SYM_FUNC_END(__crc32c_le)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags