/src/boringssl/crypto/fipsmodule/modes/gcm.c.inc
Line | Count | Source (jump to first uncovered line) |
1 | | /* ==================================================================== |
2 | | * Copyright (c) 2008 The OpenSSL Project. All rights reserved. |
3 | | * |
4 | | * Redistribution and use in source and binary forms, with or without |
5 | | * modification, are permitted provided that the following conditions |
6 | | * are met: |
7 | | * |
8 | | * 1. Redistributions of source code must retain the above copyright |
9 | | * notice, this list of conditions and the following disclaimer. |
10 | | * |
11 | | * 2. Redistributions in binary form must reproduce the above copyright |
12 | | * notice, this list of conditions and the following disclaimer in |
13 | | * the documentation and/or other materials provided with the |
14 | | * distribution. |
15 | | * |
16 | | * 3. All advertising materials mentioning features or use of this |
17 | | * software must display the following acknowledgment: |
18 | | * "This product includes software developed by the OpenSSL Project |
19 | | * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" |
20 | | * |
21 | | * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to |
22 | | * endorse or promote products derived from this software without |
23 | | * prior written permission. For written permission, please contact |
24 | | * openssl-core@openssl.org. |
25 | | * |
26 | | * 5. Products derived from this software may not be called "OpenSSL" |
27 | | * nor may "OpenSSL" appear in their names without prior written |
28 | | * permission of the OpenSSL Project. |
29 | | * |
30 | | * 6. Redistributions of any form whatsoever must retain the following |
31 | | * acknowledgment: |
32 | | * "This product includes software developed by the OpenSSL Project |
33 | | * for use in the OpenSSL Toolkit (http://www.openssl.org/)" |
34 | | * |
35 | | * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY |
36 | | * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE |
37 | | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
38 | | * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR |
39 | | * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
40 | | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
41 | | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
42 | | * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
43 | | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, |
44 | | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) |
45 | | * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED |
46 | | * OF THE POSSIBILITY OF SUCH DAMAGE. |
47 | | * ==================================================================== */ |
48 | | |
49 | | #include <openssl/base.h> |
50 | | |
51 | | #include <assert.h> |
52 | | #include <string.h> |
53 | | |
54 | | #include <openssl/mem.h> |
55 | | |
56 | | #include "internal.h" |
57 | | #include "../../internal.h" |
58 | | |
59 | | |
60 | | // kSizeTWithoutLower4Bits is a mask that can be used to zero the lower four |
61 | | // bits of a |size_t|. |
62 | | static const size_t kSizeTWithoutLower4Bits = (size_t) -16; |
63 | | |
64 | | |
65 | 4.58k | #define GCM_MUL(ctx, Xi) gcm_gmult_nohw((ctx)->Xi, (ctx)->gcm_key.Htable) |
66 | | #define GHASH(ctx, in, len) \ |
67 | 727 | gcm_ghash_nohw((ctx)->Xi, (ctx)->gcm_key.Htable, in, len) |
68 | | // GHASH_CHUNK is "stride parameter" missioned to mitigate cache |
69 | | // trashing effect. In other words idea is to hash data while it's |
70 | | // still in L1 cache after encryption pass... |
71 | 727 | #define GHASH_CHUNK (3 * 1024) |
72 | | |
73 | | #if defined(GHASH_ASM_X86_64) || defined(GHASH_ASM_X86) |
74 | 0 | static inline void gcm_reduce_1bit(u128 *V) { |
75 | 0 | if (sizeof(crypto_word_t) == 8) { |
76 | 0 | uint64_t T = UINT64_C(0xe100000000000000) & (0 - (V->hi & 1)); |
77 | 0 | V->hi = (V->lo << 63) | (V->hi >> 1); |
78 | 0 | V->lo = (V->lo >> 1) ^ T; |
79 | 0 | } else { |
80 | 0 | uint32_t T = 0xe1000000U & (0 - (uint32_t)(V->hi & 1)); |
81 | 0 | V->hi = (V->lo << 63) | (V->hi >> 1); |
82 | 0 | V->lo = (V->lo >> 1) ^ ((uint64_t)T << 32); |
83 | 0 | } |
84 | 0 | } |
85 | | |
86 | 0 | void gcm_init_ssse3(u128 Htable[16], const uint64_t H[2]) { |
87 | 0 | Htable[0].hi = 0; |
88 | 0 | Htable[0].lo = 0; |
89 | 0 | u128 V; |
90 | 0 | V.hi = H[1]; |
91 | 0 | V.lo = H[0]; |
92 | |
|
93 | 0 | Htable[8] = V; |
94 | 0 | gcm_reduce_1bit(&V); |
95 | 0 | Htable[4] = V; |
96 | 0 | gcm_reduce_1bit(&V); |
97 | 0 | Htable[2] = V; |
98 | 0 | gcm_reduce_1bit(&V); |
99 | 0 | Htable[1] = V; |
100 | 0 | Htable[3].hi = V.hi ^ Htable[2].hi, Htable[3].lo = V.lo ^ Htable[2].lo; |
101 | 0 | V = Htable[4]; |
102 | 0 | Htable[5].hi = V.hi ^ Htable[1].hi, Htable[5].lo = V.lo ^ Htable[1].lo; |
103 | 0 | Htable[6].hi = V.hi ^ Htable[2].hi, Htable[6].lo = V.lo ^ Htable[2].lo; |
104 | 0 | Htable[7].hi = V.hi ^ Htable[3].hi, Htable[7].lo = V.lo ^ Htable[3].lo; |
105 | 0 | V = Htable[8]; |
106 | 0 | Htable[9].hi = V.hi ^ Htable[1].hi, Htable[9].lo = V.lo ^ Htable[1].lo; |
107 | 0 | Htable[10].hi = V.hi ^ Htable[2].hi, Htable[10].lo = V.lo ^ Htable[2].lo; |
108 | 0 | Htable[11].hi = V.hi ^ Htable[3].hi, Htable[11].lo = V.lo ^ Htable[3].lo; |
109 | 0 | Htable[12].hi = V.hi ^ Htable[4].hi, Htable[12].lo = V.lo ^ Htable[4].lo; |
110 | 0 | Htable[13].hi = V.hi ^ Htable[5].hi, Htable[13].lo = V.lo ^ Htable[5].lo; |
111 | 0 | Htable[14].hi = V.hi ^ Htable[6].hi, Htable[14].lo = V.lo ^ Htable[6].lo; |
112 | 0 | Htable[15].hi = V.hi ^ Htable[7].hi, Htable[15].lo = V.lo ^ Htable[7].lo; |
113 | | |
114 | | // Treat |Htable| as a 16x16 byte table and transpose it. Thus, Htable[i] |
115 | | // contains the i'th byte of j*H for all j. |
116 | 0 | uint8_t *Hbytes = (uint8_t *)Htable; |
117 | 0 | for (int i = 0; i < 16; i++) { |
118 | 0 | for (int j = 0; j < i; j++) { |
119 | 0 | uint8_t tmp = Hbytes[16*i + j]; |
120 | 0 | Hbytes[16*i + j] = Hbytes[16*j + i]; |
121 | 0 | Hbytes[16*j + i] = tmp; |
122 | 0 | } |
123 | 0 | } |
124 | 0 | } |
125 | | #endif // GHASH_ASM_X86_64 || GHASH_ASM_X86 |
126 | | |
127 | | #ifdef GCM_FUNCREF |
128 | | #undef GCM_MUL |
129 | | #define GCM_MUL(ctx, Xi) (*gcm_gmult_p)((ctx)->Xi, (ctx)->gcm_key.Htable) |
130 | | #undef GHASH |
131 | | #define GHASH(ctx, in, len) \ |
132 | | (*gcm_ghash_p)((ctx)->Xi, (ctx)->gcm_key.Htable, in, len) |
133 | | #endif // GCM_FUNCREF |
134 | | |
135 | | #if defined(HW_GCM) && defined(OPENSSL_X86_64) |
136 | | static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
137 | | const AES_KEY *key, uint8_t ivec[16], |
138 | 82 | uint8_t Xi[16], const u128 Htable[16]) { |
139 | 82 | return aesni_gcm_encrypt(in, out, len, key, ivec, Htable, Xi); |
140 | 82 | } |
141 | | |
142 | | static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len, |
143 | | const AES_KEY *key, uint8_t ivec[16], |
144 | 42 | uint8_t Xi[16], const u128 Htable[16]) { |
145 | 42 | return aesni_gcm_decrypt(in, out, len, key, ivec, Htable, Xi); |
146 | 42 | } |
147 | | #endif // HW_GCM && X86_64 |
148 | | |
149 | | #if defined(HW_GCM) && defined(OPENSSL_AARCH64) |
150 | | |
151 | | static size_t hw_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
152 | | const AES_KEY *key, uint8_t ivec[16], |
153 | | uint8_t Xi[16], const u128 Htable[16]) { |
154 | | const size_t len_blocks = len & kSizeTWithoutLower4Bits; |
155 | | if (!len_blocks) { |
156 | | return 0; |
157 | | } |
158 | | aes_gcm_enc_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable); |
159 | | return len_blocks; |
160 | | } |
161 | | |
162 | | static size_t hw_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len, |
163 | | const AES_KEY *key, uint8_t ivec[16], |
164 | | uint8_t Xi[16], const u128 Htable[16]) { |
165 | | const size_t len_blocks = len & kSizeTWithoutLower4Bits; |
166 | | if (!len_blocks) { |
167 | | return 0; |
168 | | } |
169 | | aes_gcm_dec_kernel(in, len_blocks * 8, out, Xi, ivec, key, Htable); |
170 | | return len_blocks; |
171 | | } |
172 | | |
173 | | #endif // HW_GCM && AARCH64 |
174 | | |
175 | | void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash, |
176 | | u128 out_table[16], int *out_is_avx, |
177 | 694 | const uint8_t gcm_key[16]) { |
178 | 694 | *out_is_avx = 0; |
179 | | |
180 | | // H is passed to |gcm_init_*| as a pair of byte-swapped, 64-bit values. |
181 | 694 | uint64_t H[2] = {CRYPTO_load_u64_be(gcm_key), |
182 | 694 | CRYPTO_load_u64_be(gcm_key + 8)}; |
183 | | |
184 | | #if defined(GHASH_ASM_X86_64) |
185 | | if (crypto_gcm_clmul_enabled()) { |
186 | | if (CRYPTO_is_AVX_capable() && CRYPTO_is_MOVBE_capable()) { |
187 | | gcm_init_avx(out_table, H); |
188 | | *out_mult = gcm_gmult_avx; |
189 | | *out_hash = gcm_ghash_avx; |
190 | | *out_is_avx = 1; |
191 | | return; |
192 | | } |
193 | | gcm_init_clmul(out_table, H); |
194 | | *out_mult = gcm_gmult_clmul; |
195 | | *out_hash = gcm_ghash_clmul; |
196 | | return; |
197 | | } |
198 | | if (CRYPTO_is_SSSE3_capable()) { |
199 | | gcm_init_ssse3(out_table, H); |
200 | | *out_mult = gcm_gmult_ssse3; |
201 | | *out_hash = gcm_ghash_ssse3; |
202 | | return; |
203 | | } |
204 | | #elif defined(GHASH_ASM_X86) |
205 | | if (crypto_gcm_clmul_enabled()) { |
206 | | gcm_init_clmul(out_table, H); |
207 | | *out_mult = gcm_gmult_clmul; |
208 | | *out_hash = gcm_ghash_clmul; |
209 | | return; |
210 | | } |
211 | | if (CRYPTO_is_SSSE3_capable()) { |
212 | | gcm_init_ssse3(out_table, H); |
213 | | *out_mult = gcm_gmult_ssse3; |
214 | | *out_hash = gcm_ghash_ssse3; |
215 | | return; |
216 | | } |
217 | | #elif defined(GHASH_ASM_ARM) |
218 | | if (gcm_pmull_capable()) { |
219 | | gcm_init_v8(out_table, H); |
220 | | *out_mult = gcm_gmult_v8; |
221 | | *out_hash = gcm_ghash_v8; |
222 | | return; |
223 | | } |
224 | | |
225 | | if (gcm_neon_capable()) { |
226 | | gcm_init_neon(out_table, H); |
227 | | *out_mult = gcm_gmult_neon; |
228 | | *out_hash = gcm_ghash_neon; |
229 | | return; |
230 | | } |
231 | | #endif |
232 | | |
233 | 694 | gcm_init_nohw(out_table, H); |
234 | 694 | *out_mult = gcm_gmult_nohw; |
235 | 694 | *out_hash = gcm_ghash_nohw; |
236 | 694 | } |
237 | | |
238 | | void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key, const AES_KEY *aes_key, |
239 | 835 | block128_f block, int block_is_hwaes) { |
240 | 835 | OPENSSL_memset(gcm_key, 0, sizeof(*gcm_key)); |
241 | 835 | gcm_key->block = block; |
242 | | |
243 | 835 | uint8_t ghash_key[16]; |
244 | 835 | OPENSSL_memset(ghash_key, 0, sizeof(ghash_key)); |
245 | 835 | (*block)(ghash_key, ghash_key, aes_key); |
246 | | |
247 | 835 | int is_avx; |
248 | 835 | CRYPTO_ghash_init(&gcm_key->gmult, &gcm_key->ghash, gcm_key->Htable, &is_avx, |
249 | 835 | ghash_key); |
250 | | |
251 | | #if defined(OPENSSL_AARCH64) && !defined(OPENSSL_NO_ASM) |
252 | | gcm_key->use_hw_gcm_crypt = (gcm_pmull_capable() && block_is_hwaes) ? 1 : 0; |
253 | | #else |
254 | 835 | gcm_key->use_hw_gcm_crypt = (is_avx && block_is_hwaes) ? 1 : 0; |
255 | 835 | #endif |
256 | 835 | } |
257 | | |
258 | | void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key, |
259 | 793 | const uint8_t *iv, size_t len) { |
260 | | #ifdef GCM_FUNCREF |
261 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
262 | | ctx->gcm_key.gmult; |
263 | | #endif |
264 | | |
265 | 793 | OPENSSL_memset(&ctx->Yi, 0, sizeof(ctx->Yi)); |
266 | 793 | OPENSSL_memset(&ctx->Xi, 0, sizeof(ctx->Xi)); |
267 | 793 | ctx->len.aad = 0; |
268 | 793 | ctx->len.msg = 0; |
269 | 793 | ctx->ares = 0; |
270 | 793 | ctx->mres = 0; |
271 | | |
272 | 793 | uint32_t ctr; |
273 | 793 | if (len == 12) { |
274 | 112 | OPENSSL_memcpy(ctx->Yi, iv, 12); |
275 | 112 | ctx->Yi[15] = 1; |
276 | 112 | ctr = 1; |
277 | 681 | } else { |
278 | 681 | uint64_t len0 = len; |
279 | | |
280 | 2.79k | while (len >= 16) { |
281 | 2.11k | CRYPTO_xor16(ctx->Yi, ctx->Yi, iv); |
282 | 2.11k | GCM_MUL(ctx, Yi); |
283 | 2.11k | iv += 16; |
284 | 2.11k | len -= 16; |
285 | 2.11k | } |
286 | 681 | if (len) { |
287 | 2.01k | for (size_t i = 0; i < len; ++i) { |
288 | 1.79k | ctx->Yi[i] ^= iv[i]; |
289 | 1.79k | } |
290 | 225 | GCM_MUL(ctx, Yi); |
291 | 225 | } |
292 | | |
293 | 681 | uint8_t len_block[16]; |
294 | 681 | OPENSSL_memset(len_block, 0, 8); |
295 | 681 | CRYPTO_store_u64_be(len_block + 8, len0 << 3); |
296 | 681 | CRYPTO_xor16(ctx->Yi, ctx->Yi, len_block); |
297 | | |
298 | 681 | GCM_MUL(ctx, Yi); |
299 | 681 | ctr = CRYPTO_load_u32_be(ctx->Yi + 12); |
300 | 681 | } |
301 | | |
302 | 793 | (*ctx->gcm_key.block)(ctx->Yi, ctx->EK0, key); |
303 | 793 | ++ctr; |
304 | 793 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
305 | 793 | } |
306 | | |
307 | 425 | int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad, size_t len) { |
308 | | #ifdef GCM_FUNCREF |
309 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
310 | | ctx->gcm_key.gmult; |
311 | | void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, |
312 | | size_t len) = ctx->gcm_key.ghash; |
313 | | #endif |
314 | | |
315 | 425 | if (ctx->len.msg != 0) { |
316 | | // The caller must have finished the AAD before providing other input. |
317 | 0 | return 0; |
318 | 0 | } |
319 | | |
320 | 425 | uint64_t alen = ctx->len.aad + len; |
321 | 425 | if (alen > (UINT64_C(1) << 61) || (sizeof(len) == 8 && alen < len)) { |
322 | 0 | return 0; |
323 | 0 | } |
324 | 425 | ctx->len.aad = alen; |
325 | | |
326 | 425 | unsigned n = ctx->ares; |
327 | 425 | if (n) { |
328 | 0 | while (n && len) { |
329 | 0 | ctx->Xi[n] ^= *(aad++); |
330 | 0 | --len; |
331 | 0 | n = (n + 1) % 16; |
332 | 0 | } |
333 | 0 | if (n == 0) { |
334 | 0 | GCM_MUL(ctx, Xi); |
335 | 0 | } else { |
336 | 0 | ctx->ares = n; |
337 | 0 | return 1; |
338 | 0 | } |
339 | 0 | } |
340 | | |
341 | | // Process a whole number of blocks. |
342 | 425 | size_t len_blocks = len & kSizeTWithoutLower4Bits; |
343 | 425 | if (len_blocks != 0) { |
344 | 225 | GHASH(ctx, aad, len_blocks); |
345 | 225 | aad += len_blocks; |
346 | 225 | len -= len_blocks; |
347 | 225 | } |
348 | | |
349 | | // Process the remainder. |
350 | 425 | if (len != 0) { |
351 | 287 | n = (unsigned int)len; |
352 | 2.92k | for (size_t i = 0; i < len; ++i) { |
353 | 2.63k | ctx->Xi[i] ^= aad[i]; |
354 | 2.63k | } |
355 | 287 | } |
356 | | |
357 | 425 | ctx->ares = n; |
358 | 425 | return 1; |
359 | 425 | } |
360 | | |
361 | | int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx, const AES_KEY *key, |
362 | | const uint8_t *in, uint8_t *out, size_t len) { |
363 | | block128_f block = ctx->gcm_key.block; |
364 | | #ifdef GCM_FUNCREF |
365 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
366 | | ctx->gcm_key.gmult; |
367 | | void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, |
368 | | size_t len) = ctx->gcm_key.ghash; |
369 | | #endif |
370 | | |
371 | | uint64_t mlen = ctx->len.msg + len; |
372 | | if (mlen > ((UINT64_C(1) << 36) - 32) || |
373 | | (sizeof(len) == 8 && mlen < len)) { |
374 | | return 0; |
375 | | } |
376 | | ctx->len.msg = mlen; |
377 | | |
378 | | if (ctx->ares) { |
379 | | // First call to encrypt finalizes GHASH(AAD) |
380 | | GCM_MUL(ctx, Xi); |
381 | | ctx->ares = 0; |
382 | | } |
383 | | |
384 | | unsigned n = ctx->mres; |
385 | | if (n) { |
386 | | while (n && len) { |
387 | | ctx->Xi[n] ^= *(out++) = *(in++) ^ ctx->EKi[n]; |
388 | | --len; |
389 | | n = (n + 1) % 16; |
390 | | } |
391 | | if (n == 0) { |
392 | | GCM_MUL(ctx, Xi); |
393 | | } else { |
394 | | ctx->mres = n; |
395 | | return 1; |
396 | | } |
397 | | } |
398 | | |
399 | | uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12); |
400 | | while (len >= GHASH_CHUNK) { |
401 | | size_t j = GHASH_CHUNK; |
402 | | |
403 | | while (j) { |
404 | | (*block)(ctx->Yi, ctx->EKi, key); |
405 | | ++ctr; |
406 | | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
407 | | CRYPTO_xor16(out, in, ctx->EKi); |
408 | | out += 16; |
409 | | in += 16; |
410 | | j -= 16; |
411 | | } |
412 | | GHASH(ctx, out - GHASH_CHUNK, GHASH_CHUNK); |
413 | | len -= GHASH_CHUNK; |
414 | | } |
415 | | size_t len_blocks = len & kSizeTWithoutLower4Bits; |
416 | | if (len_blocks != 0) { |
417 | | while (len >= 16) { |
418 | | (*block)(ctx->Yi, ctx->EKi, key); |
419 | | ++ctr; |
420 | | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
421 | | CRYPTO_xor16(out, in, ctx->EKi); |
422 | | out += 16; |
423 | | in += 16; |
424 | | len -= 16; |
425 | | } |
426 | | GHASH(ctx, out - len_blocks, len_blocks); |
427 | | } |
428 | | if (len) { |
429 | | (*block)(ctx->Yi, ctx->EKi, key); |
430 | | ++ctr; |
431 | | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
432 | | while (len--) { |
433 | | ctx->Xi[n] ^= out[n] = in[n] ^ ctx->EKi[n]; |
434 | | ++n; |
435 | | } |
436 | | } |
437 | | |
438 | | ctx->mres = n; |
439 | | return 1; |
440 | | } |
441 | | |
442 | | int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx, const AES_KEY *key, |
443 | | const unsigned char *in, unsigned char *out, |
444 | | size_t len) { |
445 | | block128_f block = ctx->gcm_key.block; |
446 | | #ifdef GCM_FUNCREF |
447 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
448 | | ctx->gcm_key.gmult; |
449 | | void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, |
450 | | size_t len) = ctx->gcm_key.ghash; |
451 | | #endif |
452 | | |
453 | | uint64_t mlen = ctx->len.msg + len; |
454 | | if (mlen > ((UINT64_C(1) << 36) - 32) || |
455 | | (sizeof(len) == 8 && mlen < len)) { |
456 | | return 0; |
457 | | } |
458 | | ctx->len.msg = mlen; |
459 | | |
460 | | if (ctx->ares) { |
461 | | // First call to decrypt finalizes GHASH(AAD) |
462 | | GCM_MUL(ctx, Xi); |
463 | | ctx->ares = 0; |
464 | | } |
465 | | |
466 | | unsigned n = ctx->mres; |
467 | | if (n) { |
468 | | while (n && len) { |
469 | | uint8_t c = *(in++); |
470 | | *(out++) = c ^ ctx->EKi[n]; |
471 | | ctx->Xi[n] ^= c; |
472 | | --len; |
473 | | n = (n + 1) % 16; |
474 | | } |
475 | | if (n == 0) { |
476 | | GCM_MUL(ctx, Xi); |
477 | | } else { |
478 | | ctx->mres = n; |
479 | | return 1; |
480 | | } |
481 | | } |
482 | | |
483 | | uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12); |
484 | | while (len >= GHASH_CHUNK) { |
485 | | size_t j = GHASH_CHUNK; |
486 | | |
487 | | GHASH(ctx, in, GHASH_CHUNK); |
488 | | while (j) { |
489 | | (*block)(ctx->Yi, ctx->EKi, key); |
490 | | ++ctr; |
491 | | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
492 | | CRYPTO_xor16(out, in, ctx->EKi); |
493 | | out += 16; |
494 | | in += 16; |
495 | | j -= 16; |
496 | | } |
497 | | len -= GHASH_CHUNK; |
498 | | } |
499 | | size_t len_blocks = len & kSizeTWithoutLower4Bits; |
500 | | if (len_blocks != 0) { |
501 | | GHASH(ctx, in, len_blocks); |
502 | | while (len >= 16) { |
503 | | (*block)(ctx->Yi, ctx->EKi, key); |
504 | | ++ctr; |
505 | | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
506 | | CRYPTO_xor16(out, in, ctx->EKi); |
507 | | out += 16; |
508 | | in += 16; |
509 | | len -= 16; |
510 | | } |
511 | | } |
512 | | if (len) { |
513 | | (*block)(ctx->Yi, ctx->EKi, key); |
514 | | ++ctr; |
515 | | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
516 | | while (len--) { |
517 | | uint8_t c = in[n]; |
518 | | ctx->Xi[n] ^= c; |
519 | | out[n] = c ^ ctx->EKi[n]; |
520 | | ++n; |
521 | | } |
522 | | } |
523 | | |
524 | | ctx->mres = n; |
525 | | return 1; |
526 | | } |
527 | | |
528 | | int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key, |
529 | | const uint8_t *in, uint8_t *out, size_t len, |
530 | 581 | ctr128_f stream) { |
531 | | #ifdef GCM_FUNCREF |
532 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
533 | | ctx->gcm_key.gmult; |
534 | | void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, |
535 | | size_t len) = ctx->gcm_key.ghash; |
536 | | #endif |
537 | | |
538 | 581 | uint64_t mlen = ctx->len.msg + len; |
539 | 581 | if (mlen > ((UINT64_C(1) << 36) - 32) || |
540 | 581 | (sizeof(len) == 8 && mlen < len)) { |
541 | 0 | return 0; |
542 | 0 | } |
543 | 581 | ctx->len.msg = mlen; |
544 | | |
545 | 581 | if (ctx->ares) { |
546 | | // First call to encrypt finalizes GHASH(AAD) |
547 | 120 | GCM_MUL(ctx, Xi); |
548 | 120 | ctx->ares = 0; |
549 | 120 | } |
550 | | |
551 | 581 | unsigned n = ctx->mres; |
552 | 581 | if (n) { |
553 | 380 | while (n && len) { |
554 | 165 | ctx->Xi[n] ^= *(out++) = *(in++) ^ ctx->EKi[n]; |
555 | 165 | --len; |
556 | 165 | n = (n + 1) % 16; |
557 | 165 | } |
558 | 215 | if (n == 0) { |
559 | 28 | GCM_MUL(ctx, Xi); |
560 | 187 | } else { |
561 | 187 | ctx->mres = n; |
562 | 187 | return 1; |
563 | 187 | } |
564 | 215 | } |
565 | | |
566 | | #if defined(HW_GCM) |
567 | | // Check |len| to work around a C language bug. See https://crbug.com/1019588. |
568 | | if (ctx->gcm_key.use_hw_gcm_crypt && len > 0) { |
569 | | // |hw_gcm_encrypt| may not process all the input given to it. It may |
570 | | // not process *any* of its input if it is deemed too small. |
571 | | size_t bulk = hw_gcm_encrypt(in, out, len, key, ctx->Yi, ctx->Xi, |
572 | | ctx->gcm_key.Htable); |
573 | | in += bulk; |
574 | | out += bulk; |
575 | | len -= bulk; |
576 | | } |
577 | | #endif |
578 | | |
579 | 394 | uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12); |
580 | 398 | while (len >= GHASH_CHUNK) { |
581 | 4 | (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi); |
582 | 4 | ctr += GHASH_CHUNK / 16; |
583 | 4 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
584 | 4 | GHASH(ctx, out, GHASH_CHUNK); |
585 | 4 | out += GHASH_CHUNK; |
586 | 4 | in += GHASH_CHUNK; |
587 | 4 | len -= GHASH_CHUNK; |
588 | 4 | } |
589 | 394 | size_t len_blocks = len & kSizeTWithoutLower4Bits; |
590 | 394 | if (len_blocks != 0) { |
591 | 301 | size_t j = len_blocks / 16; |
592 | | |
593 | 301 | (*stream)(in, out, j, key, ctx->Yi); |
594 | 301 | ctr += (unsigned int)j; |
595 | 301 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
596 | 301 | in += len_blocks; |
597 | 301 | len -= len_blocks; |
598 | 301 | GHASH(ctx, out, len_blocks); |
599 | 301 | out += len_blocks; |
600 | 301 | } |
601 | 394 | if (len) { |
602 | 338 | (*ctx->gcm_key.block)(ctx->Yi, ctx->EKi, key); |
603 | 338 | ++ctr; |
604 | 338 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
605 | 4.26k | while (len--) { |
606 | 3.92k | ctx->Xi[n] ^= out[n] = in[n] ^ ctx->EKi[n]; |
607 | 3.92k | ++n; |
608 | 3.92k | } |
609 | 338 | } |
610 | | |
611 | 394 | ctx->mres = n; |
612 | 394 | return 1; |
613 | 581 | } |
614 | | |
615 | | int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx, const AES_KEY *key, |
616 | | const uint8_t *in, uint8_t *out, size_t len, |
617 | 1.54k | ctr128_f stream) { |
618 | | #ifdef GCM_FUNCREF |
619 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
620 | | ctx->gcm_key.gmult; |
621 | | void (*gcm_ghash_p)(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, |
622 | | size_t len) = ctx->gcm_key.ghash; |
623 | | #endif |
624 | | |
625 | 1.54k | uint64_t mlen = ctx->len.msg + len; |
626 | 1.54k | if (mlen > ((UINT64_C(1) << 36) - 32) || |
627 | 1.54k | (sizeof(len) == 8 && mlen < len)) { |
628 | 0 | return 0; |
629 | 0 | } |
630 | 1.54k | ctx->len.msg = mlen; |
631 | | |
632 | 1.54k | if (ctx->ares) { |
633 | | // First call to decrypt finalizes GHASH(AAD) |
634 | 98 | GCM_MUL(ctx, Xi); |
635 | 98 | ctx->ares = 0; |
636 | 98 | } |
637 | | |
638 | 1.54k | unsigned n = ctx->mres; |
639 | 1.54k | if (n) { |
640 | 1.51k | while (n && len) { |
641 | 279 | uint8_t c = *(in++); |
642 | 279 | *(out++) = c ^ ctx->EKi[n]; |
643 | 279 | ctx->Xi[n] ^= c; |
644 | 279 | --len; |
645 | 279 | n = (n + 1) % 16; |
646 | 279 | } |
647 | 1.24k | if (n == 0) { |
648 | 0 | GCM_MUL(ctx, Xi); |
649 | 1.24k | } else { |
650 | 1.24k | ctx->mres = n; |
651 | 1.24k | return 1; |
652 | 1.24k | } |
653 | 1.24k | } |
654 | | |
655 | | #if defined(HW_GCM) |
656 | | // Check |len| to work around a C language bug. See https://crbug.com/1019588. |
657 | | if (ctx->gcm_key.use_hw_gcm_crypt && len > 0) { |
658 | | // |hw_gcm_decrypt| may not process all the input given to it. It may |
659 | | // not process *any* of its input if it is deemed too small. |
660 | | size_t bulk = hw_gcm_decrypt(in, out, len, key, ctx->Yi, ctx->Xi, |
661 | | ctx->gcm_key.Htable); |
662 | | in += bulk; |
663 | | out += bulk; |
664 | | len -= bulk; |
665 | | } |
666 | | #endif |
667 | | |
668 | 309 | uint32_t ctr = CRYPTO_load_u32_be(ctx->Yi + 12); |
669 | 309 | while (len >= GHASH_CHUNK) { |
670 | 0 | GHASH(ctx, in, GHASH_CHUNK); |
671 | 0 | (*stream)(in, out, GHASH_CHUNK / 16, key, ctx->Yi); |
672 | 0 | ctr += GHASH_CHUNK / 16; |
673 | 0 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
674 | 0 | out += GHASH_CHUNK; |
675 | 0 | in += GHASH_CHUNK; |
676 | 0 | len -= GHASH_CHUNK; |
677 | 0 | } |
678 | 309 | size_t len_blocks = len & kSizeTWithoutLower4Bits; |
679 | 309 | if (len_blocks != 0) { |
680 | 197 | size_t j = len_blocks / 16; |
681 | | |
682 | 197 | GHASH(ctx, in, len_blocks); |
683 | 197 | (*stream)(in, out, j, key, ctx->Yi); |
684 | 197 | ctr += (unsigned int)j; |
685 | 197 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
686 | 197 | out += len_blocks; |
687 | 197 | in += len_blocks; |
688 | 197 | len -= len_blocks; |
689 | 197 | } |
690 | 309 | if (len) { |
691 | 202 | (*ctx->gcm_key.block)(ctx->Yi, ctx->EKi, key); |
692 | 202 | ++ctr; |
693 | 202 | CRYPTO_store_u32_be(ctx->Yi + 12, ctr); |
694 | 2.22k | while (len--) { |
695 | 2.02k | uint8_t c = in[n]; |
696 | 2.02k | ctx->Xi[n] ^= c; |
697 | 2.02k | out[n] = c ^ ctx->EKi[n]; |
698 | 2.02k | ++n; |
699 | 2.02k | } |
700 | 202 | } |
701 | | |
702 | 309 | ctx->mres = n; |
703 | 309 | return 1; |
704 | 1.54k | } |
705 | | |
706 | 725 | int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag, size_t len) { |
707 | | #ifdef GCM_FUNCREF |
708 | | void (*gcm_gmult_p)(uint8_t Xi[16], const u128 Htable[16]) = |
709 | | ctx->gcm_key.gmult; |
710 | | #endif |
711 | | |
712 | 725 | if (ctx->mres || ctx->ares) { |
713 | 599 | GCM_MUL(ctx, Xi); |
714 | 599 | } |
715 | | |
716 | 725 | uint8_t len_block[16]; |
717 | 725 | CRYPTO_store_u64_be(len_block, ctx->len.aad << 3); |
718 | 725 | CRYPTO_store_u64_be(len_block + 8, ctx->len.msg << 3); |
719 | 725 | CRYPTO_xor16(ctx->Xi, ctx->Xi, len_block); |
720 | 725 | GCM_MUL(ctx, Xi); |
721 | 725 | CRYPTO_xor16(ctx->Xi, ctx->Xi, ctx->EK0); |
722 | | |
723 | 725 | if (tag && len <= sizeof(ctx->Xi)) { |
724 | 2 | return CRYPTO_memcmp(ctx->Xi, tag, len) == 0; |
725 | 723 | } else { |
726 | 723 | return 0; |
727 | 723 | } |
728 | 725 | } |
729 | | |
730 | 742 | void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, unsigned char *tag, size_t len) { |
731 | 742 | CRYPTO_gcm128_finish(ctx, NULL, 0); |
732 | 742 | OPENSSL_memcpy(tag, ctx->Xi, len <= sizeof(ctx->Xi) ? len : sizeof(ctx->Xi)); |
733 | 742 | } |
734 | | |
735 | | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
736 | 141 | int crypto_gcm_clmul_enabled(void) { |
737 | 141 | #if defined(GHASH_ASM_X86) || defined(GHASH_ASM_X86_64) |
738 | 141 | return CRYPTO_is_FXSR_capable() && CRYPTO_is_PCLMUL_capable(); |
739 | | #else |
740 | | return 0; |
741 | | #endif |
742 | 141 | } |
743 | | #endif |