/src/boringssl/crypto/poly1305/poly1305_vec.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* Copyright (c) 2014, Google Inc. |
2 | | * |
3 | | * Permission to use, copy, modify, and/or distribute this software for any |
4 | | * purpose with or without fee is hereby granted, provided that the above |
5 | | * copyright notice and this permission notice appear in all copies. |
6 | | * |
7 | | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES |
8 | | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF |
9 | | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY |
10 | | * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES |
11 | | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION |
12 | | * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN |
13 | | * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ |
14 | | |
15 | | // This implementation of poly1305 is by Andrew Moon |
16 | | // (https://github.com/floodyberry/poly1305-donna) and released as public |
17 | | // domain. It implements SIMD vectorization based on the algorithm described in |
18 | | // http://cr.yp.to/papers.html#neoncrypto. Unrolled to 2 powers, i.e. 64 byte |
19 | | // block size |
20 | | |
21 | | #include <openssl/poly1305.h> |
22 | | |
23 | | #include <assert.h> |
24 | | |
25 | | #include "../internal.h" |
26 | | |
27 | | |
28 | | #if defined(BORINGSSL_HAS_UINT128) && defined(OPENSSL_X86_64) |
29 | | |
30 | | #include <emmintrin.h> |
31 | | |
32 | | typedef __m128i xmmi; |
33 | | |
34 | | alignas(16) static const uint32_t poly1305_x64_sse2_message_mask[4] = { |
35 | | (1 << 26) - 1, 0, (1 << 26) - 1, 0}; |
36 | | alignas(16) static const uint32_t poly1305_x64_sse2_5[4] = {5, 0, 5, 0}; |
37 | | alignas(16) static const uint32_t poly1305_x64_sse2_1shl128[4] = {(1 << 24), 0, |
38 | | (1 << 24), 0}; |
39 | | |
40 | 1.60k | static inline uint128_t add128(uint128_t a, uint128_t b) { return a + b; } |
41 | | |
42 | 734 | static inline uint128_t add128_64(uint128_t a, uint64_t b) { return a + b; } |
43 | | |
44 | 2.70k | static inline uint128_t mul64x64_128(uint64_t a, uint64_t b) { |
45 | 2.70k | return (uint128_t)a * b; |
46 | 2.70k | } |
47 | | |
48 | 1.10k | static inline uint64_t lo128(uint128_t a) { return (uint64_t)a; } |
49 | | |
50 | 1.10k | static inline uint64_t shr128(uint128_t v, const int shift) { |
51 | 1.10k | return (uint64_t)(v >> shift); |
52 | 1.10k | } |
53 | | |
54 | 321 | static inline uint64_t shr128_pair(uint64_t hi, uint64_t lo, const int shift) { |
55 | 321 | return (uint64_t)((((uint128_t)hi << 64) | lo) >> shift); |
56 | 321 | } |
57 | | |
58 | | typedef struct poly1305_power_t { |
59 | | union { |
60 | | xmmi v; |
61 | | uint64_t u[2]; |
62 | | uint32_t d[4]; |
63 | | } R20, R21, R22, R23, R24, S21, S22, S23, S24; |
64 | | } poly1305_power; |
65 | | |
66 | | typedef struct poly1305_state_internal_t { |
67 | | poly1305_power P[2]; /* 288 bytes, top 32 bit halves unused = 144 |
68 | | bytes of free storage */ |
69 | | union { |
70 | | xmmi H[5]; // 80 bytes |
71 | | uint64_t HH[10]; |
72 | | }; |
73 | | // uint64_t r0,r1,r2; [24 bytes] |
74 | | // uint64_t pad0,pad1; [16 bytes] |
75 | | uint64_t started; // 8 bytes |
76 | | uint64_t leftover; // 8 bytes |
77 | | uint8_t buffer[64]; // 64 bytes |
78 | | } poly1305_state_internal; /* 448 bytes total + 63 bytes for |
79 | | alignment = 511 bytes raw */ |
80 | | |
81 | | static_assert(sizeof(struct poly1305_state_internal_t) + 63 <= |
82 | | sizeof(poly1305_state), |
83 | | "poly1305_state isn't large enough to hold aligned " |
84 | | "poly1305_state_internal_t"); |
85 | | |
86 | | static inline poly1305_state_internal *poly1305_aligned_state( |
87 | 1.27k | poly1305_state *state) { |
88 | 1.27k | return (poly1305_state_internal *)(((uint64_t)state + 63) & ~63); |
89 | 1.27k | } |
90 | | |
91 | 702 | static inline size_t poly1305_min(size_t a, size_t b) { |
92 | 702 | return (a < b) ? a : b; |
93 | 702 | } |
94 | | |
95 | 152 | void CRYPTO_poly1305_init(poly1305_state *state, const uint8_t key[32]) { |
96 | 152 | poly1305_state_internal *st = poly1305_aligned_state(state); |
97 | 152 | poly1305_power *p; |
98 | 152 | uint64_t r0, r1, r2; |
99 | 152 | uint64_t t0, t1; |
100 | | |
101 | | // clamp key |
102 | 152 | t0 = CRYPTO_load_u64_le(key + 0); |
103 | 152 | t1 = CRYPTO_load_u64_le(key + 8); |
104 | 152 | r0 = t0 & 0xffc0fffffff; |
105 | 152 | t0 >>= 44; |
106 | 152 | t0 |= t1 << 20; |
107 | 152 | r1 = t0 & 0xfffffc0ffff; |
108 | 152 | t1 >>= 24; |
109 | 152 | r2 = t1 & 0x00ffffffc0f; |
110 | | |
111 | | // store r in un-used space of st->P[1] |
112 | 152 | p = &st->P[1]; |
113 | 152 | p->R20.d[1] = (uint32_t)(r0); |
114 | 152 | p->R20.d[3] = (uint32_t)(r0 >> 32); |
115 | 152 | p->R21.d[1] = (uint32_t)(r1); |
116 | 152 | p->R21.d[3] = (uint32_t)(r1 >> 32); |
117 | 152 | p->R22.d[1] = (uint32_t)(r2); |
118 | 152 | p->R22.d[3] = (uint32_t)(r2 >> 32); |
119 | | |
120 | | // store pad |
121 | 152 | p->R23.d[1] = CRYPTO_load_u32_le(key + 16); |
122 | 152 | p->R23.d[3] = CRYPTO_load_u32_le(key + 20); |
123 | 152 | p->R24.d[1] = CRYPTO_load_u32_le(key + 24); |
124 | 152 | p->R24.d[3] = CRYPTO_load_u32_le(key + 28); |
125 | | |
126 | | // H = 0 |
127 | 152 | st->H[0] = _mm_setzero_si128(); |
128 | 152 | st->H[1] = _mm_setzero_si128(); |
129 | 152 | st->H[2] = _mm_setzero_si128(); |
130 | 152 | st->H[3] = _mm_setzero_si128(); |
131 | 152 | st->H[4] = _mm_setzero_si128(); |
132 | | |
133 | 152 | st->started = 0; |
134 | 152 | st->leftover = 0; |
135 | 152 | } |
136 | | |
137 | | static void poly1305_first_block(poly1305_state_internal *st, |
138 | 99 | const uint8_t *m) { |
139 | 99 | const xmmi MMASK = |
140 | 99 | _mm_load_si128((const xmmi *)poly1305_x64_sse2_message_mask); |
141 | 99 | const xmmi FIVE = _mm_load_si128((const xmmi *)poly1305_x64_sse2_5); |
142 | 99 | const xmmi HIBIT = _mm_load_si128((const xmmi *)poly1305_x64_sse2_1shl128); |
143 | 99 | xmmi T5, T6; |
144 | 99 | poly1305_power *p; |
145 | 99 | uint128_t d[3]; |
146 | 99 | uint64_t r0, r1, r2; |
147 | 99 | uint64_t r20, r21, r22, s22; |
148 | 99 | uint64_t pad0, pad1; |
149 | 99 | uint64_t c; |
150 | 99 | uint64_t i; |
151 | | |
152 | | // pull out stored info |
153 | 99 | p = &st->P[1]; |
154 | | |
155 | 99 | r0 = ((uint64_t)p->R20.d[3] << 32) | (uint64_t)p->R20.d[1]; |
156 | 99 | r1 = ((uint64_t)p->R21.d[3] << 32) | (uint64_t)p->R21.d[1]; |
157 | 99 | r2 = ((uint64_t)p->R22.d[3] << 32) | (uint64_t)p->R22.d[1]; |
158 | 99 | pad0 = ((uint64_t)p->R23.d[3] << 32) | (uint64_t)p->R23.d[1]; |
159 | 99 | pad1 = ((uint64_t)p->R24.d[3] << 32) | (uint64_t)p->R24.d[1]; |
160 | | |
161 | | // compute powers r^2,r^4 |
162 | 99 | r20 = r0; |
163 | 99 | r21 = r1; |
164 | 99 | r22 = r2; |
165 | 297 | for (i = 0; i < 2; i++) { |
166 | 198 | s22 = r22 * (5 << 2); |
167 | | |
168 | 198 | d[0] = add128(mul64x64_128(r20, r20), mul64x64_128(r21 * 2, s22)); |
169 | 198 | d[1] = add128(mul64x64_128(r22, s22), mul64x64_128(r20 * 2, r21)); |
170 | 198 | d[2] = add128(mul64x64_128(r21, r21), mul64x64_128(r22 * 2, r20)); |
171 | | |
172 | 198 | r20 = lo128(d[0]) & 0xfffffffffff; |
173 | 198 | c = shr128(d[0], 44); |
174 | 198 | d[1] = add128_64(d[1], c); |
175 | 198 | r21 = lo128(d[1]) & 0xfffffffffff; |
176 | 198 | c = shr128(d[1], 44); |
177 | 198 | d[2] = add128_64(d[2], c); |
178 | 198 | r22 = lo128(d[2]) & 0x3ffffffffff; |
179 | 198 | c = shr128(d[2], 42); |
180 | 198 | r20 += c * 5; |
181 | 198 | c = (r20 >> 44); |
182 | 198 | r20 = r20 & 0xfffffffffff; |
183 | 198 | r21 += c; |
184 | | |
185 | 198 | p->R20.v = _mm_shuffle_epi32(_mm_cvtsi32_si128((uint32_t)(r20) & 0x3ffffff), |
186 | 198 | _MM_SHUFFLE(1, 0, 1, 0)); |
187 | 198 | p->R21.v = _mm_shuffle_epi32( |
188 | 198 | _mm_cvtsi32_si128((uint32_t)((r20 >> 26) | (r21 << 18)) & 0x3ffffff), |
189 | 198 | _MM_SHUFFLE(1, 0, 1, 0)); |
190 | 198 | p->R22.v = |
191 | 198 | _mm_shuffle_epi32(_mm_cvtsi32_si128((uint32_t)((r21 >> 8)) & 0x3ffffff), |
192 | 198 | _MM_SHUFFLE(1, 0, 1, 0)); |
193 | 198 | p->R23.v = _mm_shuffle_epi32( |
194 | 198 | _mm_cvtsi32_si128((uint32_t)((r21 >> 34) | (r22 << 10)) & 0x3ffffff), |
195 | 198 | _MM_SHUFFLE(1, 0, 1, 0)); |
196 | 198 | p->R24.v = _mm_shuffle_epi32(_mm_cvtsi32_si128((uint32_t)((r22 >> 16))), |
197 | 198 | _MM_SHUFFLE(1, 0, 1, 0)); |
198 | 198 | p->S21.v = _mm_mul_epu32(p->R21.v, FIVE); |
199 | 198 | p->S22.v = _mm_mul_epu32(p->R22.v, FIVE); |
200 | 198 | p->S23.v = _mm_mul_epu32(p->R23.v, FIVE); |
201 | 198 | p->S24.v = _mm_mul_epu32(p->R24.v, FIVE); |
202 | 198 | p--; |
203 | 198 | } |
204 | | |
205 | | // put saved info back |
206 | 99 | p = &st->P[1]; |
207 | 99 | p->R20.d[1] = (uint32_t)(r0); |
208 | 99 | p->R20.d[3] = (uint32_t)(r0 >> 32); |
209 | 99 | p->R21.d[1] = (uint32_t)(r1); |
210 | 99 | p->R21.d[3] = (uint32_t)(r1 >> 32); |
211 | 99 | p->R22.d[1] = (uint32_t)(r2); |
212 | 99 | p->R22.d[3] = (uint32_t)(r2 >> 32); |
213 | 99 | p->R23.d[1] = (uint32_t)(pad0); |
214 | 99 | p->R23.d[3] = (uint32_t)(pad0 >> 32); |
215 | 99 | p->R24.d[1] = (uint32_t)(pad1); |
216 | 99 | p->R24.d[3] = (uint32_t)(pad1 >> 32); |
217 | | |
218 | | // H = [Mx,My] |
219 | 99 | T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 0)), |
220 | 99 | _mm_loadl_epi64((const xmmi *)(m + 16))); |
221 | 99 | T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 8)), |
222 | 99 | _mm_loadl_epi64((const xmmi *)(m + 24))); |
223 | 99 | st->H[0] = _mm_and_si128(MMASK, T5); |
224 | 99 | st->H[1] = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
225 | 99 | T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12)); |
226 | 99 | st->H[2] = _mm_and_si128(MMASK, T5); |
227 | 99 | st->H[3] = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
228 | 99 | st->H[4] = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT); |
229 | 99 | } |
230 | | |
231 | | static void poly1305_blocks(poly1305_state_internal *st, const uint8_t *m, |
232 | 104 | size_t bytes) { |
233 | 104 | const xmmi MMASK = |
234 | 104 | _mm_load_si128((const xmmi *)poly1305_x64_sse2_message_mask); |
235 | 104 | const xmmi FIVE = _mm_load_si128((const xmmi *)poly1305_x64_sse2_5); |
236 | 104 | const xmmi HIBIT = _mm_load_si128((const xmmi *)poly1305_x64_sse2_1shl128); |
237 | | |
238 | 104 | poly1305_power *p; |
239 | 104 | xmmi H0, H1, H2, H3, H4; |
240 | 104 | xmmi T0, T1, T2, T3, T4, T5, T6; |
241 | 104 | xmmi M0, M1, M2, M3, M4; |
242 | 104 | xmmi C1, C2; |
243 | | |
244 | 104 | H0 = st->H[0]; |
245 | 104 | H1 = st->H[1]; |
246 | 104 | H2 = st->H[2]; |
247 | 104 | H3 = st->H[3]; |
248 | 104 | H4 = st->H[4]; |
249 | | |
250 | 208 | while (bytes >= 64) { |
251 | | // H *= [r^4,r^4] |
252 | 104 | p = &st->P[0]; |
253 | 104 | T0 = _mm_mul_epu32(H0, p->R20.v); |
254 | 104 | T1 = _mm_mul_epu32(H0, p->R21.v); |
255 | 104 | T2 = _mm_mul_epu32(H0, p->R22.v); |
256 | 104 | T3 = _mm_mul_epu32(H0, p->R23.v); |
257 | 104 | T4 = _mm_mul_epu32(H0, p->R24.v); |
258 | 104 | T5 = _mm_mul_epu32(H1, p->S24.v); |
259 | 104 | T6 = _mm_mul_epu32(H1, p->R20.v); |
260 | 104 | T0 = _mm_add_epi64(T0, T5); |
261 | 104 | T1 = _mm_add_epi64(T1, T6); |
262 | 104 | T5 = _mm_mul_epu32(H2, p->S23.v); |
263 | 104 | T6 = _mm_mul_epu32(H2, p->S24.v); |
264 | 104 | T0 = _mm_add_epi64(T0, T5); |
265 | 104 | T1 = _mm_add_epi64(T1, T6); |
266 | 104 | T5 = _mm_mul_epu32(H3, p->S22.v); |
267 | 104 | T6 = _mm_mul_epu32(H3, p->S23.v); |
268 | 104 | T0 = _mm_add_epi64(T0, T5); |
269 | 104 | T1 = _mm_add_epi64(T1, T6); |
270 | 104 | T5 = _mm_mul_epu32(H4, p->S21.v); |
271 | 104 | T6 = _mm_mul_epu32(H4, p->S22.v); |
272 | 104 | T0 = _mm_add_epi64(T0, T5); |
273 | 104 | T1 = _mm_add_epi64(T1, T6); |
274 | 104 | T5 = _mm_mul_epu32(H1, p->R21.v); |
275 | 104 | T6 = _mm_mul_epu32(H1, p->R22.v); |
276 | 104 | T2 = _mm_add_epi64(T2, T5); |
277 | 104 | T3 = _mm_add_epi64(T3, T6); |
278 | 104 | T5 = _mm_mul_epu32(H2, p->R20.v); |
279 | 104 | T6 = _mm_mul_epu32(H2, p->R21.v); |
280 | 104 | T2 = _mm_add_epi64(T2, T5); |
281 | 104 | T3 = _mm_add_epi64(T3, T6); |
282 | 104 | T5 = _mm_mul_epu32(H3, p->S24.v); |
283 | 104 | T6 = _mm_mul_epu32(H3, p->R20.v); |
284 | 104 | T2 = _mm_add_epi64(T2, T5); |
285 | 104 | T3 = _mm_add_epi64(T3, T6); |
286 | 104 | T5 = _mm_mul_epu32(H4, p->S23.v); |
287 | 104 | T6 = _mm_mul_epu32(H4, p->S24.v); |
288 | 104 | T2 = _mm_add_epi64(T2, T5); |
289 | 104 | T3 = _mm_add_epi64(T3, T6); |
290 | 104 | T5 = _mm_mul_epu32(H1, p->R23.v); |
291 | 104 | T4 = _mm_add_epi64(T4, T5); |
292 | 104 | T5 = _mm_mul_epu32(H2, p->R22.v); |
293 | 104 | T4 = _mm_add_epi64(T4, T5); |
294 | 104 | T5 = _mm_mul_epu32(H3, p->R21.v); |
295 | 104 | T4 = _mm_add_epi64(T4, T5); |
296 | 104 | T5 = _mm_mul_epu32(H4, p->R20.v); |
297 | 104 | T4 = _mm_add_epi64(T4, T5); |
298 | | |
299 | | // H += [Mx,My]*[r^2,r^2] |
300 | 104 | T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 0)), |
301 | 104 | _mm_loadl_epi64((const xmmi *)(m + 16))); |
302 | 104 | T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 8)), |
303 | 104 | _mm_loadl_epi64((const xmmi *)(m + 24))); |
304 | 104 | M0 = _mm_and_si128(MMASK, T5); |
305 | 104 | M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
306 | 104 | T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12)); |
307 | 104 | M2 = _mm_and_si128(MMASK, T5); |
308 | 104 | M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
309 | 104 | M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT); |
310 | | |
311 | 104 | p = &st->P[1]; |
312 | 104 | T5 = _mm_mul_epu32(M0, p->R20.v); |
313 | 104 | T6 = _mm_mul_epu32(M0, p->R21.v); |
314 | 104 | T0 = _mm_add_epi64(T0, T5); |
315 | 104 | T1 = _mm_add_epi64(T1, T6); |
316 | 104 | T5 = _mm_mul_epu32(M1, p->S24.v); |
317 | 104 | T6 = _mm_mul_epu32(M1, p->R20.v); |
318 | 104 | T0 = _mm_add_epi64(T0, T5); |
319 | 104 | T1 = _mm_add_epi64(T1, T6); |
320 | 104 | T5 = _mm_mul_epu32(M2, p->S23.v); |
321 | 104 | T6 = _mm_mul_epu32(M2, p->S24.v); |
322 | 104 | T0 = _mm_add_epi64(T0, T5); |
323 | 104 | T1 = _mm_add_epi64(T1, T6); |
324 | 104 | T5 = _mm_mul_epu32(M3, p->S22.v); |
325 | 104 | T6 = _mm_mul_epu32(M3, p->S23.v); |
326 | 104 | T0 = _mm_add_epi64(T0, T5); |
327 | 104 | T1 = _mm_add_epi64(T1, T6); |
328 | 104 | T5 = _mm_mul_epu32(M4, p->S21.v); |
329 | 104 | T6 = _mm_mul_epu32(M4, p->S22.v); |
330 | 104 | T0 = _mm_add_epi64(T0, T5); |
331 | 104 | T1 = _mm_add_epi64(T1, T6); |
332 | 104 | T5 = _mm_mul_epu32(M0, p->R22.v); |
333 | 104 | T6 = _mm_mul_epu32(M0, p->R23.v); |
334 | 104 | T2 = _mm_add_epi64(T2, T5); |
335 | 104 | T3 = _mm_add_epi64(T3, T6); |
336 | 104 | T5 = _mm_mul_epu32(M1, p->R21.v); |
337 | 104 | T6 = _mm_mul_epu32(M1, p->R22.v); |
338 | 104 | T2 = _mm_add_epi64(T2, T5); |
339 | 104 | T3 = _mm_add_epi64(T3, T6); |
340 | 104 | T5 = _mm_mul_epu32(M2, p->R20.v); |
341 | 104 | T6 = _mm_mul_epu32(M2, p->R21.v); |
342 | 104 | T2 = _mm_add_epi64(T2, T5); |
343 | 104 | T3 = _mm_add_epi64(T3, T6); |
344 | 104 | T5 = _mm_mul_epu32(M3, p->S24.v); |
345 | 104 | T6 = _mm_mul_epu32(M3, p->R20.v); |
346 | 104 | T2 = _mm_add_epi64(T2, T5); |
347 | 104 | T3 = _mm_add_epi64(T3, T6); |
348 | 104 | T5 = _mm_mul_epu32(M4, p->S23.v); |
349 | 104 | T6 = _mm_mul_epu32(M4, p->S24.v); |
350 | 104 | T2 = _mm_add_epi64(T2, T5); |
351 | 104 | T3 = _mm_add_epi64(T3, T6); |
352 | 104 | T5 = _mm_mul_epu32(M0, p->R24.v); |
353 | 104 | T4 = _mm_add_epi64(T4, T5); |
354 | 104 | T5 = _mm_mul_epu32(M1, p->R23.v); |
355 | 104 | T4 = _mm_add_epi64(T4, T5); |
356 | 104 | T5 = _mm_mul_epu32(M2, p->R22.v); |
357 | 104 | T4 = _mm_add_epi64(T4, T5); |
358 | 104 | T5 = _mm_mul_epu32(M3, p->R21.v); |
359 | 104 | T4 = _mm_add_epi64(T4, T5); |
360 | 104 | T5 = _mm_mul_epu32(M4, p->R20.v); |
361 | 104 | T4 = _mm_add_epi64(T4, T5); |
362 | | |
363 | | // H += [Mx,My] |
364 | 104 | T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 32)), |
365 | 104 | _mm_loadl_epi64((const xmmi *)(m + 48))); |
366 | 104 | T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 40)), |
367 | 104 | _mm_loadl_epi64((const xmmi *)(m + 56))); |
368 | 104 | M0 = _mm_and_si128(MMASK, T5); |
369 | 104 | M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
370 | 104 | T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12)); |
371 | 104 | M2 = _mm_and_si128(MMASK, T5); |
372 | 104 | M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
373 | 104 | M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT); |
374 | | |
375 | 104 | T0 = _mm_add_epi64(T0, M0); |
376 | 104 | T1 = _mm_add_epi64(T1, M1); |
377 | 104 | T2 = _mm_add_epi64(T2, M2); |
378 | 104 | T3 = _mm_add_epi64(T3, M3); |
379 | 104 | T4 = _mm_add_epi64(T4, M4); |
380 | | |
381 | | // reduce |
382 | 104 | C1 = _mm_srli_epi64(T0, 26); |
383 | 104 | C2 = _mm_srli_epi64(T3, 26); |
384 | 104 | T0 = _mm_and_si128(T0, MMASK); |
385 | 104 | T3 = _mm_and_si128(T3, MMASK); |
386 | 104 | T1 = _mm_add_epi64(T1, C1); |
387 | 104 | T4 = _mm_add_epi64(T4, C2); |
388 | 104 | C1 = _mm_srli_epi64(T1, 26); |
389 | 104 | C2 = _mm_srli_epi64(T4, 26); |
390 | 104 | T1 = _mm_and_si128(T1, MMASK); |
391 | 104 | T4 = _mm_and_si128(T4, MMASK); |
392 | 104 | T2 = _mm_add_epi64(T2, C1); |
393 | 104 | T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE)); |
394 | 104 | C1 = _mm_srli_epi64(T2, 26); |
395 | 104 | C2 = _mm_srli_epi64(T0, 26); |
396 | 104 | T2 = _mm_and_si128(T2, MMASK); |
397 | 104 | T0 = _mm_and_si128(T0, MMASK); |
398 | 104 | T3 = _mm_add_epi64(T3, C1); |
399 | 104 | T1 = _mm_add_epi64(T1, C2); |
400 | 104 | C1 = _mm_srli_epi64(T3, 26); |
401 | 104 | T3 = _mm_and_si128(T3, MMASK); |
402 | 104 | T4 = _mm_add_epi64(T4, C1); |
403 | | |
404 | | // H = (H*[r^4,r^4] + [Mx,My]*[r^2,r^2] + [Mx,My]) |
405 | 104 | H0 = T0; |
406 | 104 | H1 = T1; |
407 | 104 | H2 = T2; |
408 | 104 | H3 = T3; |
409 | 104 | H4 = T4; |
410 | | |
411 | 104 | m += 64; |
412 | 104 | bytes -= 64; |
413 | 104 | } |
414 | | |
415 | 104 | st->H[0] = H0; |
416 | 104 | st->H[1] = H1; |
417 | 104 | st->H[2] = H2; |
418 | 104 | st->H[3] = H3; |
419 | 104 | st->H[4] = H4; |
420 | 104 | } |
421 | | |
422 | | static size_t poly1305_combine(poly1305_state_internal *st, const uint8_t *m, |
423 | 99 | size_t bytes) { |
424 | 99 | const xmmi MMASK = |
425 | 99 | _mm_load_si128((const xmmi *)poly1305_x64_sse2_message_mask); |
426 | 99 | const xmmi HIBIT = _mm_load_si128((const xmmi *)poly1305_x64_sse2_1shl128); |
427 | 99 | const xmmi FIVE = _mm_load_si128((const xmmi *)poly1305_x64_sse2_5); |
428 | | |
429 | 99 | poly1305_power *p; |
430 | 99 | xmmi H0, H1, H2, H3, H4; |
431 | 99 | xmmi M0, M1, M2, M3, M4; |
432 | 99 | xmmi T0, T1, T2, T3, T4, T5, T6; |
433 | 99 | xmmi C1, C2; |
434 | | |
435 | 99 | uint64_t r0, r1, r2; |
436 | 99 | uint64_t t0, t1, t2, t3, t4; |
437 | 99 | uint64_t c; |
438 | 99 | size_t consumed = 0; |
439 | | |
440 | 99 | H0 = st->H[0]; |
441 | 99 | H1 = st->H[1]; |
442 | 99 | H2 = st->H[2]; |
443 | 99 | H3 = st->H[3]; |
444 | 99 | H4 = st->H[4]; |
445 | | |
446 | | // p = [r^2,r^2] |
447 | 99 | p = &st->P[1]; |
448 | | |
449 | 99 | if (bytes >= 32) { |
450 | | // H *= [r^2,r^2] |
451 | 32 | T0 = _mm_mul_epu32(H0, p->R20.v); |
452 | 32 | T1 = _mm_mul_epu32(H0, p->R21.v); |
453 | 32 | T2 = _mm_mul_epu32(H0, p->R22.v); |
454 | 32 | T3 = _mm_mul_epu32(H0, p->R23.v); |
455 | 32 | T4 = _mm_mul_epu32(H0, p->R24.v); |
456 | 32 | T5 = _mm_mul_epu32(H1, p->S24.v); |
457 | 32 | T6 = _mm_mul_epu32(H1, p->R20.v); |
458 | 32 | T0 = _mm_add_epi64(T0, T5); |
459 | 32 | T1 = _mm_add_epi64(T1, T6); |
460 | 32 | T5 = _mm_mul_epu32(H2, p->S23.v); |
461 | 32 | T6 = _mm_mul_epu32(H2, p->S24.v); |
462 | 32 | T0 = _mm_add_epi64(T0, T5); |
463 | 32 | T1 = _mm_add_epi64(T1, T6); |
464 | 32 | T5 = _mm_mul_epu32(H3, p->S22.v); |
465 | 32 | T6 = _mm_mul_epu32(H3, p->S23.v); |
466 | 32 | T0 = _mm_add_epi64(T0, T5); |
467 | 32 | T1 = _mm_add_epi64(T1, T6); |
468 | 32 | T5 = _mm_mul_epu32(H4, p->S21.v); |
469 | 32 | T6 = _mm_mul_epu32(H4, p->S22.v); |
470 | 32 | T0 = _mm_add_epi64(T0, T5); |
471 | 32 | T1 = _mm_add_epi64(T1, T6); |
472 | 32 | T5 = _mm_mul_epu32(H1, p->R21.v); |
473 | 32 | T6 = _mm_mul_epu32(H1, p->R22.v); |
474 | 32 | T2 = _mm_add_epi64(T2, T5); |
475 | 32 | T3 = _mm_add_epi64(T3, T6); |
476 | 32 | T5 = _mm_mul_epu32(H2, p->R20.v); |
477 | 32 | T6 = _mm_mul_epu32(H2, p->R21.v); |
478 | 32 | T2 = _mm_add_epi64(T2, T5); |
479 | 32 | T3 = _mm_add_epi64(T3, T6); |
480 | 32 | T5 = _mm_mul_epu32(H3, p->S24.v); |
481 | 32 | T6 = _mm_mul_epu32(H3, p->R20.v); |
482 | 32 | T2 = _mm_add_epi64(T2, T5); |
483 | 32 | T3 = _mm_add_epi64(T3, T6); |
484 | 32 | T5 = _mm_mul_epu32(H4, p->S23.v); |
485 | 32 | T6 = _mm_mul_epu32(H4, p->S24.v); |
486 | 32 | T2 = _mm_add_epi64(T2, T5); |
487 | 32 | T3 = _mm_add_epi64(T3, T6); |
488 | 32 | T5 = _mm_mul_epu32(H1, p->R23.v); |
489 | 32 | T4 = _mm_add_epi64(T4, T5); |
490 | 32 | T5 = _mm_mul_epu32(H2, p->R22.v); |
491 | 32 | T4 = _mm_add_epi64(T4, T5); |
492 | 32 | T5 = _mm_mul_epu32(H3, p->R21.v); |
493 | 32 | T4 = _mm_add_epi64(T4, T5); |
494 | 32 | T5 = _mm_mul_epu32(H4, p->R20.v); |
495 | 32 | T4 = _mm_add_epi64(T4, T5); |
496 | | |
497 | | // H += [Mx,My] |
498 | 32 | T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 0)), |
499 | 32 | _mm_loadl_epi64((const xmmi *)(m + 16))); |
500 | 32 | T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((const xmmi *)(m + 8)), |
501 | 32 | _mm_loadl_epi64((const xmmi *)(m + 24))); |
502 | 32 | M0 = _mm_and_si128(MMASK, T5); |
503 | 32 | M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
504 | 32 | T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12)); |
505 | 32 | M2 = _mm_and_si128(MMASK, T5); |
506 | 32 | M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26)); |
507 | 32 | M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT); |
508 | | |
509 | 32 | T0 = _mm_add_epi64(T0, M0); |
510 | 32 | T1 = _mm_add_epi64(T1, M1); |
511 | 32 | T2 = _mm_add_epi64(T2, M2); |
512 | 32 | T3 = _mm_add_epi64(T3, M3); |
513 | 32 | T4 = _mm_add_epi64(T4, M4); |
514 | | |
515 | | // reduce |
516 | 32 | C1 = _mm_srli_epi64(T0, 26); |
517 | 32 | C2 = _mm_srli_epi64(T3, 26); |
518 | 32 | T0 = _mm_and_si128(T0, MMASK); |
519 | 32 | T3 = _mm_and_si128(T3, MMASK); |
520 | 32 | T1 = _mm_add_epi64(T1, C1); |
521 | 32 | T4 = _mm_add_epi64(T4, C2); |
522 | 32 | C1 = _mm_srli_epi64(T1, 26); |
523 | 32 | C2 = _mm_srli_epi64(T4, 26); |
524 | 32 | T1 = _mm_and_si128(T1, MMASK); |
525 | 32 | T4 = _mm_and_si128(T4, MMASK); |
526 | 32 | T2 = _mm_add_epi64(T2, C1); |
527 | 32 | T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE)); |
528 | 32 | C1 = _mm_srli_epi64(T2, 26); |
529 | 32 | C2 = _mm_srli_epi64(T0, 26); |
530 | 32 | T2 = _mm_and_si128(T2, MMASK); |
531 | 32 | T0 = _mm_and_si128(T0, MMASK); |
532 | 32 | T3 = _mm_add_epi64(T3, C1); |
533 | 32 | T1 = _mm_add_epi64(T1, C2); |
534 | 32 | C1 = _mm_srli_epi64(T3, 26); |
535 | 32 | T3 = _mm_and_si128(T3, MMASK); |
536 | 32 | T4 = _mm_add_epi64(T4, C1); |
537 | | |
538 | | // H = (H*[r^2,r^2] + [Mx,My]) |
539 | 32 | H0 = T0; |
540 | 32 | H1 = T1; |
541 | 32 | H2 = T2; |
542 | 32 | H3 = T3; |
543 | 32 | H4 = T4; |
544 | | |
545 | 32 | consumed = 32; |
546 | 32 | } |
547 | | |
548 | | // finalize, H *= [r^2,r] |
549 | 99 | r0 = ((uint64_t)p->R20.d[3] << 32) | (uint64_t)p->R20.d[1]; |
550 | 99 | r1 = ((uint64_t)p->R21.d[3] << 32) | (uint64_t)p->R21.d[1]; |
551 | 99 | r2 = ((uint64_t)p->R22.d[3] << 32) | (uint64_t)p->R22.d[1]; |
552 | | |
553 | 99 | p->R20.d[2] = (uint32_t)(r0) & 0x3ffffff; |
554 | 99 | p->R21.d[2] = (uint32_t)((r0 >> 26) | (r1 << 18)) & 0x3ffffff; |
555 | 99 | p->R22.d[2] = (uint32_t)((r1 >> 8)) & 0x3ffffff; |
556 | 99 | p->R23.d[2] = (uint32_t)((r1 >> 34) | (r2 << 10)) & 0x3ffffff; |
557 | 99 | p->R24.d[2] = (uint32_t)((r2 >> 16)); |
558 | 99 | p->S21.d[2] = p->R21.d[2] * 5; |
559 | 99 | p->S22.d[2] = p->R22.d[2] * 5; |
560 | 99 | p->S23.d[2] = p->R23.d[2] * 5; |
561 | 99 | p->S24.d[2] = p->R24.d[2] * 5; |
562 | | |
563 | | // H *= [r^2,r] |
564 | 99 | T0 = _mm_mul_epu32(H0, p->R20.v); |
565 | 99 | T1 = _mm_mul_epu32(H0, p->R21.v); |
566 | 99 | T2 = _mm_mul_epu32(H0, p->R22.v); |
567 | 99 | T3 = _mm_mul_epu32(H0, p->R23.v); |
568 | 99 | T4 = _mm_mul_epu32(H0, p->R24.v); |
569 | 99 | T5 = _mm_mul_epu32(H1, p->S24.v); |
570 | 99 | T6 = _mm_mul_epu32(H1, p->R20.v); |
571 | 99 | T0 = _mm_add_epi64(T0, T5); |
572 | 99 | T1 = _mm_add_epi64(T1, T6); |
573 | 99 | T5 = _mm_mul_epu32(H2, p->S23.v); |
574 | 99 | T6 = _mm_mul_epu32(H2, p->S24.v); |
575 | 99 | T0 = _mm_add_epi64(T0, T5); |
576 | 99 | T1 = _mm_add_epi64(T1, T6); |
577 | 99 | T5 = _mm_mul_epu32(H3, p->S22.v); |
578 | 99 | T6 = _mm_mul_epu32(H3, p->S23.v); |
579 | 99 | T0 = _mm_add_epi64(T0, T5); |
580 | 99 | T1 = _mm_add_epi64(T1, T6); |
581 | 99 | T5 = _mm_mul_epu32(H4, p->S21.v); |
582 | 99 | T6 = _mm_mul_epu32(H4, p->S22.v); |
583 | 99 | T0 = _mm_add_epi64(T0, T5); |
584 | 99 | T1 = _mm_add_epi64(T1, T6); |
585 | 99 | T5 = _mm_mul_epu32(H1, p->R21.v); |
586 | 99 | T6 = _mm_mul_epu32(H1, p->R22.v); |
587 | 99 | T2 = _mm_add_epi64(T2, T5); |
588 | 99 | T3 = _mm_add_epi64(T3, T6); |
589 | 99 | T5 = _mm_mul_epu32(H2, p->R20.v); |
590 | 99 | T6 = _mm_mul_epu32(H2, p->R21.v); |
591 | 99 | T2 = _mm_add_epi64(T2, T5); |
592 | 99 | T3 = _mm_add_epi64(T3, T6); |
593 | 99 | T5 = _mm_mul_epu32(H3, p->S24.v); |
594 | 99 | T6 = _mm_mul_epu32(H3, p->R20.v); |
595 | 99 | T2 = _mm_add_epi64(T2, T5); |
596 | 99 | T3 = _mm_add_epi64(T3, T6); |
597 | 99 | T5 = _mm_mul_epu32(H4, p->S23.v); |
598 | 99 | T6 = _mm_mul_epu32(H4, p->S24.v); |
599 | 99 | T2 = _mm_add_epi64(T2, T5); |
600 | 99 | T3 = _mm_add_epi64(T3, T6); |
601 | 99 | T5 = _mm_mul_epu32(H1, p->R23.v); |
602 | 99 | T4 = _mm_add_epi64(T4, T5); |
603 | 99 | T5 = _mm_mul_epu32(H2, p->R22.v); |
604 | 99 | T4 = _mm_add_epi64(T4, T5); |
605 | 99 | T5 = _mm_mul_epu32(H3, p->R21.v); |
606 | 99 | T4 = _mm_add_epi64(T4, T5); |
607 | 99 | T5 = _mm_mul_epu32(H4, p->R20.v); |
608 | 99 | T4 = _mm_add_epi64(T4, T5); |
609 | | |
610 | 99 | C1 = _mm_srli_epi64(T0, 26); |
611 | 99 | C2 = _mm_srli_epi64(T3, 26); |
612 | 99 | T0 = _mm_and_si128(T0, MMASK); |
613 | 99 | T3 = _mm_and_si128(T3, MMASK); |
614 | 99 | T1 = _mm_add_epi64(T1, C1); |
615 | 99 | T4 = _mm_add_epi64(T4, C2); |
616 | 99 | C1 = _mm_srli_epi64(T1, 26); |
617 | 99 | C2 = _mm_srli_epi64(T4, 26); |
618 | 99 | T1 = _mm_and_si128(T1, MMASK); |
619 | 99 | T4 = _mm_and_si128(T4, MMASK); |
620 | 99 | T2 = _mm_add_epi64(T2, C1); |
621 | 99 | T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE)); |
622 | 99 | C1 = _mm_srli_epi64(T2, 26); |
623 | 99 | C2 = _mm_srli_epi64(T0, 26); |
624 | 99 | T2 = _mm_and_si128(T2, MMASK); |
625 | 99 | T0 = _mm_and_si128(T0, MMASK); |
626 | 99 | T3 = _mm_add_epi64(T3, C1); |
627 | 99 | T1 = _mm_add_epi64(T1, C2); |
628 | 99 | C1 = _mm_srli_epi64(T3, 26); |
629 | 99 | T3 = _mm_and_si128(T3, MMASK); |
630 | 99 | T4 = _mm_add_epi64(T4, C1); |
631 | | |
632 | | // H = H[0]+H[1] |
633 | 99 | H0 = _mm_add_epi64(T0, _mm_srli_si128(T0, 8)); |
634 | 99 | H1 = _mm_add_epi64(T1, _mm_srli_si128(T1, 8)); |
635 | 99 | H2 = _mm_add_epi64(T2, _mm_srli_si128(T2, 8)); |
636 | 99 | H3 = _mm_add_epi64(T3, _mm_srli_si128(T3, 8)); |
637 | 99 | H4 = _mm_add_epi64(T4, _mm_srli_si128(T4, 8)); |
638 | | |
639 | 99 | t0 = _mm_cvtsi128_si32(H0); |
640 | 99 | c = (t0 >> 26); |
641 | 99 | t0 &= 0x3ffffff; |
642 | 99 | t1 = _mm_cvtsi128_si32(H1) + c; |
643 | 99 | c = (t1 >> 26); |
644 | 99 | t1 &= 0x3ffffff; |
645 | 99 | t2 = _mm_cvtsi128_si32(H2) + c; |
646 | 99 | c = (t2 >> 26); |
647 | 99 | t2 &= 0x3ffffff; |
648 | 99 | t3 = _mm_cvtsi128_si32(H3) + c; |
649 | 99 | c = (t3 >> 26); |
650 | 99 | t3 &= 0x3ffffff; |
651 | 99 | t4 = _mm_cvtsi128_si32(H4) + c; |
652 | 99 | c = (t4 >> 26); |
653 | 99 | t4 &= 0x3ffffff; |
654 | 99 | t0 = t0 + (c * 5); |
655 | 99 | c = (t0 >> 26); |
656 | 99 | t0 &= 0x3ffffff; |
657 | 99 | t1 = t1 + c; |
658 | | |
659 | 99 | st->HH[0] = ((t0) | (t1 << 26)) & UINT64_C(0xfffffffffff); |
660 | 99 | st->HH[1] = ((t1 >> 18) | (t2 << 8) | (t3 << 34)) & UINT64_C(0xfffffffffff); |
661 | 99 | st->HH[2] = ((t3 >> 10) | (t4 << 16)) & UINT64_C(0x3ffffffffff); |
662 | | |
663 | 99 | return consumed; |
664 | 99 | } |
665 | | |
666 | | void CRYPTO_poly1305_update(poly1305_state *state, const uint8_t *m, |
667 | 975 | size_t bytes) { |
668 | 975 | poly1305_state_internal *st = poly1305_aligned_state(state); |
669 | 975 | size_t want; |
670 | | |
671 | | // Work around a C language bug. See https://crbug.com/1019588. |
672 | 975 | if (bytes == 0) { |
673 | 205 | return; |
674 | 205 | } |
675 | | |
676 | | // need at least 32 initial bytes to start the accelerated branch |
677 | 770 | if (!st->started) { |
678 | 435 | if ((st->leftover == 0) && (bytes > 32)) { |
679 | 68 | poly1305_first_block(st, m); |
680 | 68 | m += 32; |
681 | 68 | bytes -= 32; |
682 | 367 | } else { |
683 | 367 | want = poly1305_min(32 - st->leftover, bytes); |
684 | 367 | OPENSSL_memcpy(st->buffer + st->leftover, m, want); |
685 | 367 | bytes -= want; |
686 | 367 | m += want; |
687 | 367 | st->leftover += want; |
688 | 367 | if ((st->leftover < 32) || (bytes == 0)) { |
689 | 336 | return; |
690 | 336 | } |
691 | 31 | poly1305_first_block(st, st->buffer); |
692 | 31 | st->leftover = 0; |
693 | 31 | } |
694 | 99 | st->started = 1; |
695 | 99 | } |
696 | | |
697 | | // handle leftover |
698 | 434 | if (st->leftover) { |
699 | 335 | want = poly1305_min(64 - st->leftover, bytes); |
700 | 335 | OPENSSL_memcpy(st->buffer + st->leftover, m, want); |
701 | 335 | bytes -= want; |
702 | 335 | m += want; |
703 | 335 | st->leftover += want; |
704 | 335 | if (st->leftover < 64) { |
705 | 299 | return; |
706 | 299 | } |
707 | 36 | poly1305_blocks(st, st->buffer, 64); |
708 | 36 | st->leftover = 0; |
709 | 36 | } |
710 | | |
711 | | // process 64 byte blocks |
712 | 135 | if (bytes >= 64) { |
713 | 68 | want = (bytes & ~63); |
714 | 68 | poly1305_blocks(st, m, want); |
715 | 68 | m += want; |
716 | 68 | bytes -= want; |
717 | 68 | } |
718 | | |
719 | 135 | if (bytes) { |
720 | 99 | OPENSSL_memcpy(st->buffer + st->leftover, m, bytes); |
721 | 99 | st->leftover += bytes; |
722 | 99 | } |
723 | 135 | } |
724 | | |
725 | 152 | void CRYPTO_poly1305_finish(poly1305_state *state, uint8_t mac[16]) { |
726 | 152 | poly1305_state_internal *st = poly1305_aligned_state(state); |
727 | 152 | size_t leftover = st->leftover; |
728 | 152 | uint8_t *m = st->buffer; |
729 | 152 | uint128_t d[3]; |
730 | 152 | uint64_t h0, h1, h2; |
731 | 152 | uint64_t t0, t1; |
732 | 152 | uint64_t g0, g1, g2, c, nc; |
733 | 152 | uint64_t r0, r1, r2, s1, s2; |
734 | 152 | poly1305_power *p; |
735 | | |
736 | 152 | if (st->started) { |
737 | 99 | size_t consumed = poly1305_combine(st, m, leftover); |
738 | 99 | leftover -= consumed; |
739 | 99 | m += consumed; |
740 | 99 | } |
741 | | |
742 | | // st->HH will either be 0 or have the combined result |
743 | 152 | h0 = st->HH[0]; |
744 | 152 | h1 = st->HH[1]; |
745 | 152 | h2 = st->HH[2]; |
746 | | |
747 | 152 | p = &st->P[1]; |
748 | 152 | r0 = ((uint64_t)p->R20.d[3] << 32) | (uint64_t)p->R20.d[1]; |
749 | 152 | r1 = ((uint64_t)p->R21.d[3] << 32) | (uint64_t)p->R21.d[1]; |
750 | 152 | r2 = ((uint64_t)p->R22.d[3] << 32) | (uint64_t)p->R22.d[1]; |
751 | 152 | s1 = r1 * (5 << 2); |
752 | 152 | s2 = r2 * (5 << 2); |
753 | | |
754 | 152 | if (leftover < 16) { |
755 | 36 | goto poly1305_donna_atmost15bytes; |
756 | 36 | } |
757 | | |
758 | 169 | poly1305_donna_atleast16bytes: |
759 | 169 | t0 = CRYPTO_load_u64_le(m + 0); |
760 | 169 | t1 = CRYPTO_load_u64_le(m + 8); |
761 | 169 | h0 += t0 & 0xfffffffffff; |
762 | 169 | t0 = shr128_pair(t1, t0, 44); |
763 | 169 | h1 += t0 & 0xfffffffffff; |
764 | 169 | h2 += (t1 >> 24) | ((uint64_t)1 << 40); |
765 | | |
766 | 169 | poly1305_donna_mul: |
767 | 169 | d[0] = add128(add128(mul64x64_128(h0, r0), mul64x64_128(h1, s2)), |
768 | 169 | mul64x64_128(h2, s1)); |
769 | 169 | d[1] = add128(add128(mul64x64_128(h0, r1), mul64x64_128(h1, r0)), |
770 | 169 | mul64x64_128(h2, s2)); |
771 | 169 | d[2] = add128(add128(mul64x64_128(h0, r2), mul64x64_128(h1, r1)), |
772 | 169 | mul64x64_128(h2, r0)); |
773 | 169 | h0 = lo128(d[0]) & 0xfffffffffff; |
774 | 169 | c = shr128(d[0], 44); |
775 | 169 | d[1] = add128_64(d[1], c); |
776 | 169 | h1 = lo128(d[1]) & 0xfffffffffff; |
777 | 169 | c = shr128(d[1], 44); |
778 | 169 | d[2] = add128_64(d[2], c); |
779 | 169 | h2 = lo128(d[2]) & 0x3ffffffffff; |
780 | 169 | c = shr128(d[2], 42); |
781 | 169 | h0 += c * 5; |
782 | | |
783 | 169 | m += 16; |
784 | 169 | leftover -= 16; |
785 | 169 | if (leftover >= 16) { |
786 | 53 | goto poly1305_donna_atleast16bytes; |
787 | 53 | } |
788 | | |
789 | | // final bytes |
790 | 152 | poly1305_donna_atmost15bytes: |
791 | 152 | if (!leftover) { |
792 | 152 | goto poly1305_donna_finish; |
793 | 152 | } |
794 | | |
795 | 0 | m[leftover++] = 1; |
796 | 0 | OPENSSL_memset(m + leftover, 0, 16 - leftover); |
797 | 0 | leftover = 16; |
798 | |
|
799 | 0 | t0 = CRYPTO_load_u64_le(m + 0); |
800 | 0 | t1 = CRYPTO_load_u64_le(m + 8); |
801 | 0 | h0 += t0 & 0xfffffffffff; |
802 | 0 | t0 = shr128_pair(t1, t0, 44); |
803 | 0 | h1 += t0 & 0xfffffffffff; |
804 | 0 | h2 += (t1 >> 24); |
805 | |
|
806 | 0 | goto poly1305_donna_mul; |
807 | | |
808 | 152 | poly1305_donna_finish: |
809 | 152 | c = (h0 >> 44); |
810 | 152 | h0 &= 0xfffffffffff; |
811 | 152 | h1 += c; |
812 | 152 | c = (h1 >> 44); |
813 | 152 | h1 &= 0xfffffffffff; |
814 | 152 | h2 += c; |
815 | 152 | c = (h2 >> 42); |
816 | 152 | h2 &= 0x3ffffffffff; |
817 | 152 | h0 += c * 5; |
818 | | |
819 | 152 | g0 = h0 + 5; |
820 | 152 | c = (g0 >> 44); |
821 | 152 | g0 &= 0xfffffffffff; |
822 | 152 | g1 = h1 + c; |
823 | 152 | c = (g1 >> 44); |
824 | 152 | g1 &= 0xfffffffffff; |
825 | 152 | g2 = h2 + c - ((uint64_t)1 << 42); |
826 | | |
827 | 152 | c = (g2 >> 63) - 1; |
828 | 152 | nc = ~c; |
829 | 152 | h0 = (h0 & nc) | (g0 & c); |
830 | 152 | h1 = (h1 & nc) | (g1 & c); |
831 | 152 | h2 = (h2 & nc) | (g2 & c); |
832 | | |
833 | | // pad |
834 | 152 | t0 = ((uint64_t)p->R23.d[3] << 32) | (uint64_t)p->R23.d[1]; |
835 | 152 | t1 = ((uint64_t)p->R24.d[3] << 32) | (uint64_t)p->R24.d[1]; |
836 | 152 | h0 += (t0 & 0xfffffffffff); |
837 | 152 | c = (h0 >> 44); |
838 | 152 | h0 &= 0xfffffffffff; |
839 | 152 | t0 = shr128_pair(t1, t0, 44); |
840 | 152 | h1 += (t0 & 0xfffffffffff) + c; |
841 | 152 | c = (h1 >> 44); |
842 | 152 | h1 &= 0xfffffffffff; |
843 | 152 | t1 = (t1 >> 24); |
844 | 152 | h2 += (t1) + c; |
845 | | |
846 | 152 | CRYPTO_store_u64_le(mac + 0, ((h0) | (h1 << 44))); |
847 | 152 | CRYPTO_store_u64_le(mac + 8, ((h1 >> 20) | (h2 << 24))); |
848 | 152 | } |
849 | | |
850 | | #endif // BORINGSSL_HAS_UINT128 && OPENSSL_X86_64 |