^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0 OR MIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This is based in part on Andrew Moon's poly1305-donna, which is in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * public domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <crypto/internal/poly1305.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) typedef __uint128_t u128;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) void poly1305_core_setkey(struct poly1305_core_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) const u8 raw_key[POLY1305_BLOCK_SIZE])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) u64 t0, t1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) t0 = get_unaligned_le64(&raw_key[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) t1 = get_unaligned_le64(&raw_key[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) key->key.r64[0] = t0 & 0xffc0fffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /* s = 20*r */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) key->precomputed_s.r64[0] = key->key.r64[1] * 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) key->precomputed_s.r64[1] = key->key.r64[2] * 20;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) EXPORT_SYMBOL(poly1305_core_setkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) void poly1305_core_blocks(struct poly1305_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) const struct poly1305_core_key *key, const void *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) unsigned int nblocks, u32 hibit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) const u8 *input = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) u64 hibit64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) u64 r0, r1, r2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) u64 s1, s2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) u64 h0, h1, h2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) u64 c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) u128 d0, d1, d2, d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) if (!nblocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) hibit64 = ((u64)hibit) << 40;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) r0 = key->key.r64[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) r1 = key->key.r64[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) r2 = key->key.r64[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) h0 = state->h64[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) h1 = state->h64[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) h2 = state->h64[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) s1 = key->precomputed_s.r64[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) s2 = key->precomputed_s.r64[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) u64 t0, t1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* h += m[i] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) t0 = get_unaligned_le64(&input[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) t1 = get_unaligned_le64(&input[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) h0 += t0 & 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* h *= r */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) d0 = (u128)h0 * r0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) d = (u128)h1 * s2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) d0 += d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) d = (u128)h2 * s1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) d0 += d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) d1 = (u128)h0 * r1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) d = (u128)h1 * r0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) d1 += d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) d = (u128)h2 * s2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) d1 += d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) d2 = (u128)h0 * r2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) d = (u128)h1 * r1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) d2 += d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) d = (u128)h2 * r0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) d2 += d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /* (partial) h %= p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) c = (u64)(d0 >> 44);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) h0 = (u64)d0 & 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) d1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) c = (u64)(d1 >> 44);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) h1 = (u64)d1 & 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) d2 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) c = (u64)(d2 >> 42);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) h2 = (u64)d2 & 0x3ffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) h0 += c * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) c = h0 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) h0 = h0 & 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) h1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) input += POLY1305_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) } while (--nblocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) state->h64[0] = h0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) state->h64[1] = h1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) state->h64[2] = h2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) EXPORT_SYMBOL(poly1305_core_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) void *dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) u8 *mac = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) u64 h0, h1, h2, c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) u64 g0, g1, g2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) u64 t0, t1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) /* fully carry h */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) h0 = state->h64[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) h1 = state->h64[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) h2 = state->h64[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) c = h1 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) h1 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) h2 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) c = h2 >> 42;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) h2 &= 0x3ffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) h0 += c * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) c = h0 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) h0 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) h1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) c = h1 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) h1 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) h2 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) c = h2 >> 42;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) h2 &= 0x3ffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) h0 += c * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) c = h0 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) h0 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) h1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) /* compute h + -p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) g0 = h0 + 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) c = g0 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) g0 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) g1 = h1 + c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) c = g1 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) g1 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) g2 = h2 + c - (1ULL << 42);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /* select h if h < p, or h + -p if h >= p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) g0 &= c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) g1 &= c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) g2 &= c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) c = ~c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) h0 = (h0 & c) | g0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) h1 = (h1 & c) | g1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) h2 = (h2 & c) | g2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (likely(nonce)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* h = (h + nonce) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) t0 = ((u64)nonce[1] << 32) | nonce[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) t1 = ((u64)nonce[3] << 32) | nonce[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) h0 += t0 & 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) c = h0 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) h0 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) c = h1 >> 44;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) h1 &= 0xfffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) h2 &= 0x3ffffffffffULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) /* mac = h % (2^128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) h0 = h0 | (h1 << 44);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) h1 = (h1 >> 20) | (h2 << 24);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) put_unaligned_le64(h0, &mac[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) put_unaligned_le64(h1, &mac[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) EXPORT_SYMBOL(poly1305_core_emit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags