^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0 OR MIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * This is based in part on Andrew Moon's poly1305-donna, which is in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * public domain.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <asm/unaligned.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <crypto/internal/poly1305.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) void poly1305_core_setkey(struct poly1305_core_key *key,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) const u8 raw_key[POLY1305_BLOCK_SIZE])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) key->key.r[0] = (get_unaligned_le32(&raw_key[0])) & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) key->key.r[1] = (get_unaligned_le32(&raw_key[3]) >> 2) & 0x3ffff03;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) key->key.r[2] = (get_unaligned_le32(&raw_key[6]) >> 4) & 0x3ffc0ff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) key->key.r[3] = (get_unaligned_le32(&raw_key[9]) >> 6) & 0x3f03fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) key->key.r[4] = (get_unaligned_le32(&raw_key[12]) >> 8) & 0x00fffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) /* s = 5*r */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) key->precomputed_s.r[0] = key->key.r[1] * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) key->precomputed_s.r[1] = key->key.r[2] * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) key->precomputed_s.r[2] = key->key.r[3] * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) key->precomputed_s.r[3] = key->key.r[4] * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) EXPORT_SYMBOL(poly1305_core_setkey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) void poly1305_core_blocks(struct poly1305_state *state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) const struct poly1305_core_key *key, const void *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) unsigned int nblocks, u32 hibit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) const u8 *input = src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) u32 r0, r1, r2, r3, r4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) u32 s1, s2, s3, s4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) u32 h0, h1, h2, h3, h4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) u64 d0, d1, d2, d3, d4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) u32 c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) if (!nblocks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) hibit <<= 24;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) r0 = key->key.r[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) r1 = key->key.r[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) r2 = key->key.r[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) r3 = key->key.r[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) r4 = key->key.r[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) s1 = key->precomputed_s.r[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) s2 = key->precomputed_s.r[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) s3 = key->precomputed_s.r[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) s4 = key->precomputed_s.r[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) h0 = state->h[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) h1 = state->h[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) h2 = state->h[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) h3 = state->h[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) h4 = state->h[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /* h += m[i] */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) h0 += (get_unaligned_le32(&input[0])) & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) h1 += (get_unaligned_le32(&input[3]) >> 2) & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) h2 += (get_unaligned_le32(&input[6]) >> 4) & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) h3 += (get_unaligned_le32(&input[9]) >> 6) & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) h4 += (get_unaligned_le32(&input[12]) >> 8) | hibit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) /* h *= r */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) d0 = ((u64)h0 * r0) + ((u64)h1 * s4) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) ((u64)h2 * s3) + ((u64)h3 * s2) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) ((u64)h4 * s1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) d1 = ((u64)h0 * r1) + ((u64)h1 * r0) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) ((u64)h2 * s4) + ((u64)h3 * s3) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) ((u64)h4 * s2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) d2 = ((u64)h0 * r2) + ((u64)h1 * r1) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) ((u64)h2 * r0) + ((u64)h3 * s4) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ((u64)h4 * s3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) d3 = ((u64)h0 * r3) + ((u64)h1 * r2) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) ((u64)h2 * r1) + ((u64)h3 * r0) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) ((u64)h4 * s4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) d4 = ((u64)h0 * r4) + ((u64)h1 * r3) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) ((u64)h2 * r2) + ((u64)h3 * r1) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) ((u64)h4 * r0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) /* (partial) h %= p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) c = (u32)(d0 >> 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) h0 = (u32)d0 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) d1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) c = (u32)(d1 >> 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) h1 = (u32)d1 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) d2 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) c = (u32)(d2 >> 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) h2 = (u32)d2 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) d3 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) c = (u32)(d3 >> 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) h3 = (u32)d3 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) d4 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) c = (u32)(d4 >> 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) h4 = (u32)d4 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) h0 += c * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) c = (h0 >> 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) h0 = h0 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) h1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) input += POLY1305_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) } while (--nblocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) state->h[0] = h0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) state->h[1] = h1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) state->h[2] = h2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) state->h[3] = h3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) state->h[4] = h4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) EXPORT_SYMBOL(poly1305_core_blocks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) void *dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) u8 *mac = dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) u32 h0, h1, h2, h3, h4, c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) u32 g0, g1, g2, g3, g4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) u64 f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) u32 mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* fully carry h */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) h0 = state->h[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) h1 = state->h[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) h2 = state->h[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) h3 = state->h[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) h4 = state->h[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) c = h1 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) h1 = h1 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) h2 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) c = h2 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) h2 = h2 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) h3 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) c = h3 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) h3 = h3 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) h4 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) c = h4 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) h4 = h4 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) h0 += c * 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) c = h0 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) h0 = h0 & 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) h1 += c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* compute h + -p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) g0 = h0 + 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) c = g0 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) g0 &= 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) g1 = h1 + c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) c = g1 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) g1 &= 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) g2 = h2 + c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) c = g2 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) g2 &= 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) g3 = h3 + c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) c = g3 >> 26;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) g3 &= 0x3ffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) g4 = h4 + c - (1UL << 26);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) /* select h if h < p, or h + -p if h >= p */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) mask = (g4 >> ((sizeof(u32) * 8) - 1)) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) g0 &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) g1 &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) g2 &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) g3 &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) g4 &= mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) mask = ~mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) h0 = (h0 & mask) | g0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) h1 = (h1 & mask) | g1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) h2 = (h2 & mask) | g2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) h3 = (h3 & mask) | g3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) h4 = (h4 & mask) | g4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /* h = h % (2^128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) h0 = ((h0) | (h1 << 26)) & 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) h1 = ((h1 >> 6) | (h2 << 20)) & 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) h3 = ((h3 >> 18) | (h4 << 8)) & 0xffffffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (likely(nonce)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) /* mac = (h + nonce) % (2^128) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) f = (u64)h0 + nonce[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) h0 = (u32)f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) f = (u64)h1 + nonce[1] + (f >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) h1 = (u32)f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) f = (u64)h2 + nonce[2] + (f >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) h2 = (u32)f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) f = (u64)h3 + nonce[3] + (f >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) h3 = (u32)f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) put_unaligned_le32(h0, &mac[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) put_unaligned_le32(h1, &mac[4]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) put_unaligned_le32(h2, &mac[8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) put_unaligned_le32(h3, &mac[12]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) EXPORT_SYMBOL(poly1305_core_emit);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags