^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Checksum functions for Hexagon
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2010-2011, The Linux Foundation. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) /* This was derived from arch/alpha/lib/checksum.c */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <asm/byteorder.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <net/checksum.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <asm/intrinsics.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) /* Vector value operations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #define SIGN(x, y) ((0x8000ULL*x)<<y)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #define CARRY(x, y) ((0x0002ULL*x)<<y)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #define SELECT(x, y) ((0x0001ULL*x)<<y)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define VR_NEGATE(a, b, c, d) (SIGN(a, 48) + SIGN(b, 32) + SIGN(c, 16) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) + SIGN(d, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define VR_CARRY(a, b, c, d) (CARRY(a, 48) + CARRY(b, 32) + CARRY(c, 16) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) + CARRY(d, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define VR_SELECT(a, b, c, d) (SELECT(a, 48) + SELECT(b, 32) + SELECT(c, 16) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) + SELECT(d, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) /* optimized HEXAGON V3 intrinsic version */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) static inline unsigned short from64to16(u64 x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) u64 sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) sum = HEXAGON_P_vrmpyh_PP(x^VR_NEGATE(1, 1, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) sum += VR_CARRY(0, 0, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) sum = HEXAGON_P_vrmpyh_PP(sum, VR_SELECT(0, 0, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return 0xFFFF & sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * computes the checksum of the TCP/UDP pseudo-header
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * returns a 16-bit checksum, already complemented.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) __sum16 csum_tcpudp_magic(__be32 saddr, __be32 daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) __u32 len, __u8 proto, __wsum sum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) return (__force __sum16)~from64to16(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) (__force u64)saddr + (__force u64)daddr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) (__force u64)sum + ((len + proto) << 8));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) __u32 len, __u8 proto, __wsum sum)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) u64 result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) result = (__force u64)saddr + (__force u64)daddr +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) (__force u64)sum + ((len + proto) << 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* Fold down to 32-bits so we don't lose in the typedef-less
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) network stack. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) /* 64 to 33 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) result = (result & 0xffffffffUL) + (result >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) /* 33 to 32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) result = (result & 0xffffffffUL) + (result >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return (__force __wsum)result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) EXPORT_SYMBOL(csum_tcpudp_nofold);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * Do a 64-bit checksum on an arbitrary memory area..
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * This isn't a great routine, but it's not _horrible_ either. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * inner loop could be unrolled a bit further, and there are better
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * ways to do the carry, but this is reasonable.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) /* optimized HEXAGON intrinsic version, with over read fixed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) unsigned int do_csum(const void *voidptr, int len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) u64 sum0, sum1, x0, x1, *ptr8_o, *ptr8_e, *ptr8;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) int i, start, mid, end, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) const char *ptr = voidptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) unsigned short *ptr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) unsigned int *ptr4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) if (len <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) start = 0xF & (16-(((int) ptr) & 0xF)) ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) mask = 0x7fffffffUL >> HEXAGON_R_cl0_R(len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) start = start & mask ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) mid = len - start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) end = mid & 0xF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) mid = mid>>4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) sum0 = mid << 18;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) sum1 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (start & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) sum0 += (u64) (ptr[0] << 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) ptr2 = (unsigned short *) &ptr[start & 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (start & 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) sum1 += (u64) ptr2[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) ptr4 = (unsigned int *) &ptr[start & 3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) if (start & 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) sum0 = HEXAGON_P_vrmpyhacc_PP(sum0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) VR_NEGATE(0, 0, 1, 1)^((u64)ptr4[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) VR_SELECT(0, 0, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) sum0 += VR_SELECT(0, 0, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) ptr8 = (u64 *) &ptr[start & 7];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) if (start & 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) sum1 = HEXAGON_P_vrmpyhacc_PP(sum1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) VR_NEGATE(1, 1, 1, 1)^(ptr8[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) sum1 += VR_CARRY(0, 0, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) ptr8_o = (u64 *) (ptr + start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) ptr8_e = (u64 *) (ptr + start + 8);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (mid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) x0 = *ptr8_e; ptr8_e += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) x1 = *ptr8_o; ptr8_o += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (mid > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) for (i = 0; i < mid-1; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) sum0 = HEXAGON_P_vrmpyhacc_PP(sum0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) x0^VR_NEGATE(1, 1, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) sum1 = HEXAGON_P_vrmpyhacc_PP(sum1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) x1^VR_NEGATE(1, 1, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) x0 = *ptr8_e; ptr8_e += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) x1 = *ptr8_o; ptr8_o += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) sum0 = HEXAGON_P_vrmpyhacc_PP(sum0, x0^VR_NEGATE(1, 1, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) sum1 = HEXAGON_P_vrmpyhacc_PP(sum1, x1^VR_NEGATE(1, 1, 1, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) ptr4 = (unsigned int *) &ptr[start + (mid * 16) + (end & 8)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) if (end & 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) sum1 = HEXAGON_P_vrmpyhacc_PP(sum1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) VR_NEGATE(0, 0, 1, 1)^((u64)ptr4[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) VR_SELECT(0, 0, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) sum1 += VR_SELECT(0, 0, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) ptr2 = (unsigned short *) &ptr[start + (mid * 16) + (end & 12)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (end & 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) sum0 += (u64) ptr2[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) if (end & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) sum1 += (u64) ptr[start + (mid * 16) + (end & 14)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) ptr8 = (u64 *) &ptr[start + (mid * 16)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (end & 8) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) sum0 = HEXAGON_P_vrmpyhacc_PP(sum0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) VR_NEGATE(1, 1, 1, 1)^(ptr8[0]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) VR_SELECT(1, 1, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) sum0 += VR_CARRY(0, 0, 1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) sum0 = HEXAGON_P_vrmpyh_PP((sum0+sum1)^VR_NEGATE(0, 0, 0, 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) VR_SELECT(0, 0, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) sum0 += VR_NEGATE(0, 0, 0, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) sum0 = HEXAGON_P_vrmpyh_PP(sum0, VR_SELECT(0, 0, 1, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if (start & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) sum0 = (sum0 << 8) | (0xFF & (sum0 >> 8));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) return 0xFFFF & sum0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags