/src/boringssl/crypto/fipsmodule/aes/internal.h
Line | Count | Source |
1 | | // Copyright 2017 The BoringSSL Authors |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // https://www.apache.org/licenses/LICENSE-2.0 |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | #ifndef OPENSSL_HEADER_CRYPTO_FIPSMODULE_AES_INTERNAL_H |
16 | | #define OPENSSL_HEADER_CRYPTO_FIPSMODULE_AES_INTERNAL_H |
17 | | |
18 | | #include <stdlib.h> |
19 | | |
20 | | #include "../../internal.h" |
21 | | #include "../bcm_interface.h" |
22 | | |
23 | | |
24 | | BSSL_NAMESPACE_BEGIN |
25 | | |
26 | | // block128_f is the type of an AES block cipher implementation. |
27 | | // |
28 | | // Unlike upstream OpenSSL, it and the other functions in this file hard-code |
29 | | // |AES_KEY|. It is undefined in C to call a function pointer with anything |
30 | | // other than the original type. Thus we either must match |block128_f| to the |
31 | | // type signature of |BCM_aes_encrypt| and friends or pass in |void*| wrapper |
32 | | // functions. |
33 | | // |
34 | | // These functions are called exclusively with AES, so we use the former. |
35 | | typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16], |
36 | | const AES_KEY *key); |
37 | | |
38 | | // ctr128_f is the type of a function that performs CTR-mode encryption. |
39 | | typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks, |
40 | | const AES_KEY *key, const uint8_t ivec[16]); |
41 | | |
42 | | // aes_ctr_set_key initialises |*aes_key| using |key_bytes| bytes from |key|, |
43 | | // where |key_bytes| must either be 16, 24 or 32. If not NULL, |*out_block| is |
44 | | // set to a function that encrypts single blocks. If not NULL, |*out_is_hwaes| |
45 | | // is set to whether the hardware AES implementation was used. It returns a |
46 | | // function for optimised CTR-mode. |
47 | | ctr128_f aes_ctr_set_key(AES_KEY *aes_key, int *out_is_hwaes, |
48 | | block128_f *out_block, const uint8_t *key, |
49 | | size_t key_bytes); |
50 | | |
51 | | |
52 | | // AES implementations. |
53 | | |
54 | | #if !defined(OPENSSL_NO_ASM) |
55 | | |
56 | | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
57 | | #define HWAES |
58 | | #define HWAES_ECB |
59 | | |
60 | 10.8M | inline int hwaes_capable() { return CRYPTO_is_AESNI_capable(); } |
61 | | |
62 | | #define VPAES |
63 | | #define VPAES_CBC |
64 | 0 | inline int vpaes_capable() { return CRYPTO_is_SSSE3_capable(); } |
65 | | |
66 | | #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) |
67 | | #define HWAES |
68 | | |
69 | | inline int hwaes_capable() { return CRYPTO_is_ARMv8_AES_capable(); } |
70 | | |
71 | | #if defined(OPENSSL_ARM) |
72 | | #define BSAES |
73 | | #define VPAES |
74 | | inline int bsaes_capable() { return CRYPTO_is_NEON_capable(); } |
75 | | inline int vpaes_capable() { return CRYPTO_is_NEON_capable(); } |
76 | | #endif |
77 | | |
78 | | #if defined(OPENSSL_AARCH64) |
79 | | #define VPAES |
80 | | #define VPAES_CBC |
81 | | inline int vpaes_capable() { return CRYPTO_is_NEON_capable(); } |
82 | | #endif |
83 | | |
84 | | #endif |
85 | | |
86 | | #endif // !NO_ASM |
87 | | |
88 | | |
89 | | #if defined(HWAES) |
90 | | |
91 | | extern "C" int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits, |
92 | | AES_KEY *key); |
93 | | extern "C" int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits, |
94 | | AES_KEY *key); |
95 | | extern "C" void aes_hw_encrypt(const uint8_t *in, uint8_t *out, |
96 | | const AES_KEY *key); |
97 | | extern "C" void aes_hw_decrypt(const uint8_t *in, uint8_t *out, |
98 | | const AES_KEY *key); |
99 | | extern "C" void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, |
100 | | size_t length, const AES_KEY *key, |
101 | | uint8_t *ivec, int enc); |
102 | | extern "C" void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, |
103 | | size_t len, const AES_KEY *key, |
104 | | const uint8_t ivec[16]); |
105 | | |
106 | | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
107 | | // On x86 and x86_64, |aes_hw_set_decrypt_key| is implemented in terms of |
108 | | // |aes_hw_set_encrypt_key| and a conversion function. |
109 | | extern "C" void aes_hw_encrypt_key_to_decrypt_key(AES_KEY *key); |
110 | | |
111 | | // There are two variants of this function, one which uses aeskeygenassist |
112 | | // ("base") and one which uses aesenclast + pshufb ("alt"). aesenclast is |
113 | | // overall faster but is slower on some older processors. It doesn't use AVX, |
114 | | // but AVX is used as a proxy to detecting this. See |
115 | | // https://groups.google.com/g/mailing.openssl.dev/c/OuFXwW4NfO8/m/7d2ZXVjkxVkJ |
116 | | // |
117 | | // TODO(davidben): It is unclear if the aeskeygenassist version is still |
118 | | // worthwhile. However, the aesenclast version requires SSSE3. SSSE3 long |
119 | | // predates AES-NI, but it's not clear if AES-NI implies SSSE3. In OpenSSL, the |
120 | | // CCM AES-NI assembly seems to assume it does. |
121 | 0 | inline int aes_hw_set_encrypt_key_alt_capable() { |
122 | 0 | return hwaes_capable() && CRYPTO_is_SSSE3_capable(); |
123 | 0 | } |
124 | 1.94M | inline int aes_hw_set_encrypt_key_alt_preferred() { |
125 | 1.94M | return hwaes_capable() && CRYPTO_is_AVX_capable(); |
126 | 1.94M | } |
127 | | extern "C" int aes_hw_set_encrypt_key_base(const uint8_t *user_key, int bits, |
128 | | AES_KEY *key); |
129 | | extern "C" int aes_hw_set_encrypt_key_alt(const uint8_t *user_key, int bits, |
130 | | AES_KEY *key); |
131 | | #endif // OPENSSL_X86 || OPENSSL_X86_64 |
132 | | |
133 | | #else |
134 | | |
135 | | // If HWAES isn't defined then we provide dummy functions for each of the hwaes |
136 | | // functions. |
137 | | inline int hwaes_capable() { return 0; } |
138 | | |
139 | | inline int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits, |
140 | | AES_KEY *key) { |
141 | | abort(); |
142 | | } |
143 | | |
144 | | inline int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits, |
145 | | AES_KEY *key) { |
146 | | abort(); |
147 | | } |
148 | | |
149 | | inline void aes_hw_encrypt(const uint8_t *in, uint8_t *out, |
150 | | const AES_KEY *key) { |
151 | | abort(); |
152 | | } |
153 | | |
154 | | inline void aes_hw_decrypt(const uint8_t *in, uint8_t *out, |
155 | | const AES_KEY *key) { |
156 | | abort(); |
157 | | } |
158 | | |
159 | | inline void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, |
160 | | const AES_KEY *key, uint8_t *ivec, int enc) { |
161 | | abort(); |
162 | | } |
163 | | |
164 | | inline void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, |
165 | | size_t len, const AES_KEY *key, |
166 | | const uint8_t ivec[16]) { |
167 | | abort(); |
168 | | } |
169 | | |
170 | | #endif // !HWAES |
171 | | |
172 | | |
173 | | #if defined(HWAES_ECB) |
174 | | extern "C" void aes_hw_ecb_encrypt(const uint8_t *in, uint8_t *out, |
175 | | size_t length, const AES_KEY *key, int enc); |
176 | | #endif // HWAES_ECB |
177 | | |
178 | | |
179 | | #if defined(BSAES) |
180 | | // Note |bsaes_cbc_encrypt| requires |enc| to be zero. |
181 | | extern "C" void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out, |
182 | | size_t length, const AES_KEY *key, |
183 | | uint8_t ivec[16], int enc); |
184 | | extern "C" void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, |
185 | | size_t len, const AES_KEY *key, |
186 | | const uint8_t ivec[16]); |
187 | | // VPAES to BSAES conversions are available on all BSAES platforms. |
188 | | extern "C" void vpaes_encrypt_key_to_bsaes(AES_KEY *out_bsaes, |
189 | | const AES_KEY *vpaes); |
190 | | extern "C" void vpaes_decrypt_key_to_bsaes(AES_KEY *out_bsaes, |
191 | | const AES_KEY *vpaes); |
192 | | void vpaes_ctr32_encrypt_blocks_with_bsaes(const uint8_t *in, uint8_t *out, |
193 | | size_t blocks, const AES_KEY *key, |
194 | | const uint8_t ivec[16]); |
195 | | #endif // !BSAES |
196 | | |
197 | | |
198 | | #if defined(VPAES) |
199 | | // On platforms where VPAES gets defined (just above), then these functions are |
200 | | // provided by asm. |
201 | | extern "C" int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, |
202 | | AES_KEY *key); |
203 | | extern "C" int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, |
204 | | AES_KEY *key); |
205 | | |
206 | | extern "C" void vpaes_encrypt(const uint8_t *in, uint8_t *out, |
207 | | const AES_KEY *key); |
208 | | extern "C" void vpaes_decrypt(const uint8_t *in, uint8_t *out, |
209 | | const AES_KEY *key); |
210 | | |
211 | | #if defined(VPAES_CBC) |
212 | | extern "C" void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, |
213 | | size_t length, const AES_KEY *key, |
214 | | uint8_t *ivec, int enc); |
215 | | #endif |
216 | | extern "C" void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, |
217 | | size_t len, const AES_KEY *key, |
218 | | const uint8_t ivec[16]); |
219 | | #else |
220 | | inline int vpaes_capable() { return 0; } |
221 | | |
222 | | // On other platforms, vpaes_capable() will always return false and so the |
223 | | // following will never be called. |
224 | | inline int vpaes_set_encrypt_key(const uint8_t *userKey, int bits, |
225 | | AES_KEY *key) { |
226 | | abort(); |
227 | | } |
228 | | inline int vpaes_set_decrypt_key(const uint8_t *userKey, int bits, |
229 | | AES_KEY *key) { |
230 | | abort(); |
231 | | } |
232 | | inline void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { |
233 | | abort(); |
234 | | } |
235 | | inline void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { |
236 | | abort(); |
237 | | } |
238 | | inline void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, |
239 | | const AES_KEY *key, uint8_t *ivec, int enc) { |
240 | | abort(); |
241 | | } |
242 | | inline void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, |
243 | | size_t len, const AES_KEY *key, |
244 | | const uint8_t ivec[16]) { |
245 | | abort(); |
246 | | } |
247 | | #endif // !VPAES |
248 | | |
249 | | |
250 | | int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits, |
251 | | AES_KEY *aeskey); |
252 | | int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits, |
253 | | AES_KEY *aeskey); |
254 | | void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); |
255 | | void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); |
256 | | void aes_nohw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, |
257 | | size_t blocks, const AES_KEY *key, |
258 | | const uint8_t ivec[16]); |
259 | | void aes_nohw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
260 | | const AES_KEY *key, uint8_t *ivec, int enc); |
261 | | |
262 | | // Modes |
263 | | |
264 | | inline void CRYPTO_xor16(uint8_t out[16], const uint8_t a[16], |
265 | 5.52M | const uint8_t b[16]) { |
266 | | // TODO(davidben): Ideally we'd leave this to the compiler, which could use |
267 | | // vector registers, etc. But the compiler doesn't know that |in| and |out| |
268 | | // cannot partially alias. |restrict| is slightly two strict (we allow exact |
269 | | // aliasing), but perhaps in-place could be a separate function? |
270 | 5.52M | static_assert(16 % sizeof(crypto_word_t) == 0, |
271 | 5.52M | "block cannot be evenly divided into words"); |
272 | 16.5M | for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) { |
273 | 11.0M | CRYPTO_store_word_le( |
274 | 11.0M | out + i, CRYPTO_load_word_le(a + i) ^ CRYPTO_load_word_le(b + i)); |
275 | 11.0M | } |
276 | 5.52M | } |
277 | | |
278 | | |
279 | | // CTR. |
280 | | |
281 | | // CRYPTO_ctr128_encrypt_ctr32 encrypts (or decrypts, it's the same in CTR mode) |
282 | | // |len| bytes from |in| to |out| using |block| in counter mode. There's no |
283 | | // requirement that |len| be a multiple of any value and any partial blocks are |
284 | | // stored in |ecount_buf| and |*num|, which must be zeroed before the initial |
285 | | // call. The counter is a 128-bit, big-endian value in |ivec| and is |
286 | | // incremented by this function. If the counter overflows, it wraps around. |
287 | | // |ctr| must be a function that performs CTR mode but only deals with the lower |
288 | | // 32 bits of the counter. |
289 | | void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len, |
290 | | const AES_KEY *key, uint8_t ivec[16], |
291 | | uint8_t ecount_buf[16], unsigned *num, |
292 | | ctr128_f ctr); |
293 | | |
294 | | |
295 | | // GCM. |
296 | | // |
297 | | // This API differs from the upstream API slightly. The |GCM128_CONTEXT| does |
298 | | // not have a |key| pointer that points to the key as upstream's version does. |
299 | | // Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT| |
300 | | // can be safely copied. Additionally, |gcm_key| is split into a separate |
301 | | // struct. |
302 | | |
303 | | // gcm_impl_t specifies an assembly implementation of AES-GCM. |
304 | | enum gcm_impl_t { |
305 | | gcm_separate = 0, // No combined AES-GCM, but may have AES-CTR and GHASH. |
306 | | gcm_x86_aesni, |
307 | | gcm_x86_vaes_avx2, |
308 | | gcm_x86_vaes_avx512, |
309 | | gcm_arm64_aes, |
310 | | gcm_arm64_aes_eor3, |
311 | | }; |
312 | | |
313 | | typedef struct { uint64_t hi,lo; } u128; |
314 | | |
315 | | // gmult_func multiplies |Xi| by the GCM key and writes the result back to |
316 | | // |Xi|. |
317 | | typedef void (*gmult_func)(uint8_t Xi[16], const u128 Htable[16]); |
318 | | |
319 | | // ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from |
320 | | // |inp|. The result is written back to |Xi| and the |len| argument must be a |
321 | | // multiple of 16. |
322 | | typedef void (*ghash_func)(uint8_t Xi[16], const u128 Htable[16], |
323 | | const uint8_t *inp, size_t len); |
324 | | |
325 | | typedef struct gcm128_key_st { |
326 | | u128 Htable[16]; |
327 | | gmult_func gmult; |
328 | | ghash_func ghash; |
329 | | AES_KEY aes; |
330 | | |
331 | | ctr128_f ctr; |
332 | | block128_f block; |
333 | | enum gcm_impl_t impl; |
334 | | } GCM128_KEY; |
335 | | |
336 | | // GCM128_CONTEXT contains state for a single GCM operation. The structure |
337 | | // should be zero-initialized before use. |
338 | | typedef struct { |
339 | | // The following 5 names follow names in GCM specification |
340 | | uint8_t Yi[16]; |
341 | | uint8_t EKi[16]; |
342 | | uint8_t EK0[16]; |
343 | | struct { |
344 | | uint64_t aad; |
345 | | uint64_t msg; |
346 | | } len; |
347 | | uint8_t Xi[16]; |
348 | | unsigned mres, ares; |
349 | | } GCM128_CONTEXT; |
350 | | |
351 | | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
352 | | // crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is |
353 | | // used. |
354 | | int crypto_gcm_clmul_enabled(); |
355 | | #endif |
356 | | |
357 | | // CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to |
358 | | // |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware |
359 | | // accelerated) functions for performing operations in the GHASH field. |
360 | | void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash, |
361 | | u128 out_table[16], const uint8_t gcm_key[16]); |
362 | | |
363 | | // CRYPTO_gcm128_init_aes_key initialises |gcm_key| to with AES key |key|. |
364 | | void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key, |
365 | | size_t key_bytes); |
366 | | |
367 | | // CRYPTO_gcm128_init_ctx initializes |ctx| to encrypt with |key| and |iv|. |
368 | | void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx, |
369 | | const uint8_t *iv, size_t iv_len); |
370 | | |
371 | | // CRYPTO_gcm128_aad adds to the authenticated data for an instance of GCM. |
372 | | // This must be called before and data is encrypted. |key| must be the same |
373 | | // value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on success |
374 | | // and zero otherwise. |
375 | | int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx, |
376 | | const uint8_t *aad, size_t aad_len); |
377 | | |
378 | | // CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. |key| must be |
379 | | // the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on |
380 | | // success and zero otherwise. |
381 | | int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx, |
382 | | const uint8_t *in, uint8_t *out, size_t len); |
383 | | |
384 | | // CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. |key| must be |
385 | | // the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on |
386 | | // success and zero otherwise. |
387 | | int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx, |
388 | | const uint8_t *in, uint8_t *out, size_t len); |
389 | | |
390 | | // CRYPTO_gcm128_finish calculates the authenticator and compares it against |
391 | | // |len| bytes of |tag|. |key| must be the same value that was passed to |
392 | | // |CRYPTO_gcm128_init_ctx|. It returns one on success and zero otherwise. |
393 | | int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx, |
394 | | const uint8_t *tag, size_t len); |
395 | | |
396 | | // CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|. |
397 | | // The minimum of |len| and 16 bytes are copied into |tag|. |key| must be the |
398 | | // same value that was passed to |CRYPTO_gcm128_init_ctx|. |
399 | | void CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx, uint8_t *tag, |
400 | | size_t len); |
401 | | |
402 | | |
403 | | // GCM assembly. |
404 | | |
405 | | void gcm_init_nohw(u128 Htable[16], const uint64_t H[2]); |
406 | | void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]); |
407 | | void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp, |
408 | | size_t len); |
409 | | |
410 | | #if !defined(OPENSSL_NO_ASM) |
411 | | |
412 | | #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) |
413 | | #define GCM_FUNCREF |
414 | | extern "C" void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]); |
415 | | extern "C" void gcm_gmult_clmul(uint8_t Xi[16], const u128 Htable[16]); |
416 | | extern "C" void gcm_ghash_clmul(uint8_t Xi[16], const u128 Htable[16], |
417 | | const uint8_t *inp, size_t len); |
418 | | |
419 | | void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]); |
420 | | extern "C" void gcm_gmult_ssse3(uint8_t Xi[16], const u128 Htable[16]); |
421 | | extern "C" void gcm_ghash_ssse3(uint8_t Xi[16], const u128 Htable[16], |
422 | | const uint8_t *in, size_t len); |
423 | | |
424 | | #if defined(OPENSSL_X86_64) |
425 | | #define GHASH_ASM_X86_64 |
426 | | extern "C" void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]); |
427 | | extern "C" void gcm_gmult_avx(uint8_t Xi[16], const u128 Htable[16]); |
428 | | extern "C" void gcm_ghash_avx(uint8_t Xi[16], const u128 Htable[16], |
429 | | const uint8_t *in, size_t len); |
430 | | |
431 | | #define HW_GCM |
432 | | extern "C" size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
433 | | const AES_KEY *key, uint8_t ivec[16], |
434 | | const u128 Htable[16], uint8_t Xi[16]); |
435 | | extern "C" size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len, |
436 | | const AES_KEY *key, uint8_t ivec[16], |
437 | | const u128 Htable[16], uint8_t Xi[16]); |
438 | | |
439 | | extern "C" void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]); |
440 | | extern "C" void gcm_gmult_vpclmulqdq_avx2(uint8_t Xi[16], |
441 | | const u128 Htable[16]); |
442 | | extern "C" void gcm_ghash_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16], |
443 | | const uint8_t *in, size_t len); |
444 | | extern "C" void aes_gcm_enc_update_vaes_avx2(const uint8_t *in, uint8_t *out, |
445 | | size_t len, const AES_KEY *key, |
446 | | const uint8_t ivec[16], |
447 | | const u128 Htable[16], |
448 | | uint8_t Xi[16]); |
449 | | extern "C" void aes_gcm_dec_update_vaes_avx2(const uint8_t *in, uint8_t *out, |
450 | | size_t len, const AES_KEY *key, |
451 | | const uint8_t ivec[16], |
452 | | const u128 Htable[16], |
453 | | uint8_t Xi[16]); |
454 | | |
455 | | extern "C" void gcm_init_vpclmulqdq_avx512(u128 Htable[16], |
456 | | const uint64_t H[2]); |
457 | | extern "C" void gcm_gmult_vpclmulqdq_avx512(uint8_t Xi[16], |
458 | | const u128 Htable[16]); |
459 | | extern "C" void gcm_ghash_vpclmulqdq_avx512(uint8_t Xi[16], |
460 | | const u128 Htable[16], |
461 | | const uint8_t *in, size_t len); |
462 | | extern "C" void aes_gcm_enc_update_vaes_avx512(const uint8_t *in, uint8_t *out, |
463 | | size_t len, const AES_KEY *key, |
464 | | const uint8_t ivec[16], |
465 | | const u128 Htable[16], |
466 | | uint8_t Xi[16]); |
467 | | extern "C" void aes_gcm_dec_update_vaes_avx512(const uint8_t *in, uint8_t *out, |
468 | | size_t len, const AES_KEY *key, |
469 | | const uint8_t ivec[16], |
470 | | const u128 Htable[16], |
471 | | uint8_t Xi[16]); |
472 | | |
473 | | #endif // OPENSSL_X86_64 |
474 | | |
475 | | #if defined(OPENSSL_X86) |
476 | | #define GHASH_ASM_X86 |
477 | | #endif // OPENSSL_X86 |
478 | | |
479 | | #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) |
480 | | |
481 | | #define GHASH_ASM_ARM |
482 | | #define GCM_FUNCREF |
483 | | |
484 | | inline int gcm_pmull_capable() { return CRYPTO_is_ARMv8_PMULL_capable(); } |
485 | | inline int gcm_sha3_capable() { return CRYPTO_is_ARMv8_SHA3_capable(); } |
486 | | |
487 | | extern "C" void gcm_init_v8(u128 Htable[16], const uint64_t H[2]); |
488 | | extern "C" void gcm_gmult_v8(uint8_t Xi[16], const u128 Htable[16]); |
489 | | extern "C" void gcm_ghash_v8(uint8_t Xi[16], const u128 Htable[16], |
490 | | const uint8_t *inp, size_t len); |
491 | | |
492 | | inline int gcm_neon_capable() { return CRYPTO_is_NEON_capable(); } |
493 | | |
494 | | extern "C" void gcm_init_neon(u128 Htable[16], const uint64_t H[2]); |
495 | | extern "C" void gcm_gmult_neon(uint8_t Xi[16], const u128 Htable[16]); |
496 | | extern "C" void gcm_ghash_neon(uint8_t Xi[16], const u128 Htable[16], |
497 | | const uint8_t *inp, size_t len); |
498 | | |
499 | | #if defined(OPENSSL_AARCH64) |
500 | | #define HW_GCM |
501 | | // These functions are defined in aesv8-gcm-armv8.pl. |
502 | | extern "C" void aes_gcm_enc_kernel(const uint8_t *in, uint64_t in_bits, |
503 | | void *out, void *Xi, uint8_t *ivec, |
504 | | const AES_KEY *key, const u128 Htable[16]); |
505 | | extern "C" void aes_gcm_dec_kernel(const uint8_t *in, uint64_t in_bits, |
506 | | void *out, void *Xi, uint8_t *ivec, |
507 | | const AES_KEY *key, const u128 Htable[16]); |
508 | | extern "C" void aes_gcm_enc_kernel_eor3(const uint8_t *in, uint64_t in_bits, |
509 | | void *out, void *Xi, uint8_t *ivec, |
510 | | const AES_KEY *key, const u128 Htable[16]); |
511 | | extern "C" void aes_gcm_dec_kernel_eor3(const uint8_t *in, uint64_t in_bits, |
512 | | void *out, void *Xi, uint8_t *ivec, |
513 | | const AES_KEY *key, const u128 Htable[16]); |
514 | | #endif |
515 | | |
516 | | #endif |
517 | | #endif // OPENSSL_NO_ASM |
518 | | |
519 | | |
520 | | // CBC. |
521 | | |
522 | | // cbc128_f is the type of a function that performs CBC-mode encryption. |
523 | | typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len, |
524 | | const AES_KEY *key, uint8_t ivec[16], int enc); |
525 | | |
526 | | // CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the |
527 | | // given IV and block cipher in CBC mode. The input need not be a multiple of |
528 | | // 128 bits long, but the output will round up to the nearest 128 bit multiple, |
529 | | // zero padding the input if needed. The IV will be updated on return. |
530 | | void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
531 | | const AES_KEY *key, uint8_t ivec[16], |
532 | | block128_f block); |
533 | | |
534 | | // CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the |
535 | | // given IV and block cipher in CBC mode. If |len| is not a multiple of 128 |
536 | | // bits then only that many bytes will be written, but a multiple of 128 bits |
537 | | // is always read from |in|. The IV will be updated on return. |
538 | | void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len, |
539 | | const AES_KEY *key, uint8_t ivec[16], |
540 | | block128_f block); |
541 | | |
542 | | |
543 | | // OFB. |
544 | | |
545 | | // CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode) |
546 | | // |len| bytes from |in| to |out| using |block| in OFB mode. There's no |
547 | | // requirement that |len| be a multiple of any value and any partial blocks are |
548 | | // stored in |ivec| and |*num|, the latter must be zero before the initial |
549 | | // call. |
550 | | void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
551 | | const AES_KEY *key, uint8_t ivec[16], unsigned *num, |
552 | | block128_f block); |
553 | | |
554 | | |
555 | | // CFB. |
556 | | |
557 | | // CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes |
558 | | // from |in| to |out| using |block| in CFB mode. There's no requirement that |
559 | | // |len| be a multiple of any value and any partial blocks are stored in |ivec| |
560 | | // and |*num|, the latter must be zero before the initial call. |
561 | | void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
562 | | const AES_KEY *key, uint8_t ivec[16], unsigned *num, |
563 | | int enc, block128_f block); |
564 | | |
565 | | // CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes |
566 | | // from |in| to |out| using |block| in CFB-8 mode. Prior to the first call |
567 | | // |num| should be set to zero. |
568 | | void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len, |
569 | | const AES_KEY *key, uint8_t ivec[16], |
570 | | unsigned *num, int enc, block128_f block); |
571 | | |
572 | | // CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes |
573 | | // from |in| to |out| using |block| in CFB-1 mode. Prior to the first call |
574 | | // |num| should be set to zero. |
575 | | void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits, |
576 | | const AES_KEY *key, uint8_t ivec[16], |
577 | | unsigned *num, int enc, block128_f block); |
578 | | |
579 | | size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len, |
580 | | const AES_KEY *key, uint8_t ivec[16], |
581 | | block128_f block); |
582 | | |
583 | | BSSL_NAMESPACE_END |
584 | | |
585 | | #endif // OPENSSL_HEADER_CRYPTO_FIPSMODULE_AES_INTERNAL_H |