Coverage Report

Created: 2026-04-02 07:09

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/boringssl/crypto/fipsmodule/aes/internal.h
Line
Count
Source
1
// Copyright 2017 The BoringSSL Authors
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     https://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
#ifndef OPENSSL_HEADER_CRYPTO_FIPSMODULE_AES_INTERNAL_H
16
#define OPENSSL_HEADER_CRYPTO_FIPSMODULE_AES_INTERNAL_H
17
18
#include <stdlib.h>
19
20
#include "../../internal.h"
21
#include "../bcm_interface.h"
22
23
24
BSSL_NAMESPACE_BEGIN
25
26
// block128_f is the type of an AES block cipher implementation.
27
//
28
// Unlike upstream OpenSSL, it and the other functions in this file hard-code
29
// |AES_KEY|. It is undefined in C to call a function pointer with anything
30
// other than the original type. Thus we either must match |block128_f| to the
31
// type signature of |BCM_aes_encrypt| and friends or pass in |void*| wrapper
32
// functions.
33
//
34
// These functions are called exclusively with AES, so we use the former.
35
typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16],
36
                           const AES_KEY *key);
37
38
// ctr128_f is the type of a function that performs CTR-mode encryption.
39
typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks,
40
                         const AES_KEY *key, const uint8_t ivec[16]);
41
42
// aes_ctr_set_key initialises |*aes_key| using |key_bytes| bytes from |key|,
43
// where |key_bytes| must either be 16, 24 or 32. If not NULL, |*out_block| is
44
// set to a function that encrypts single blocks. If not NULL, |*out_is_hwaes|
45
// is set to whether the hardware AES implementation was used. It returns a
46
// function for optimised CTR-mode.
47
ctr128_f aes_ctr_set_key(AES_KEY *aes_key, int *out_is_hwaes,
48
                         block128_f *out_block, const uint8_t *key,
49
                         size_t key_bytes);
50
51
52
// AES implementations.
53
54
#if !defined(OPENSSL_NO_ASM)
55
56
#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
57
#define HWAES
58
#define HWAES_ECB
59
60
10.8M
inline int hwaes_capable() { return CRYPTO_is_AESNI_capable(); }
61
62
#define VPAES
63
#define VPAES_CBC
64
0
inline int vpaes_capable() { return CRYPTO_is_SSSE3_capable(); }
65
66
#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
67
#define HWAES
68
69
inline int hwaes_capable() { return CRYPTO_is_ARMv8_AES_capable(); }
70
71
#if defined(OPENSSL_ARM)
72
#define BSAES
73
#define VPAES
74
inline int bsaes_capable() { return CRYPTO_is_NEON_capable(); }
75
inline int vpaes_capable() { return CRYPTO_is_NEON_capable(); }
76
#endif
77
78
#if defined(OPENSSL_AARCH64)
79
#define VPAES
80
#define VPAES_CBC
81
inline int vpaes_capable() { return CRYPTO_is_NEON_capable(); }
82
#endif
83
84
#endif
85
86
#endif  // !NO_ASM
87
88
89
#if defined(HWAES)
90
91
extern "C" int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits,
92
                                      AES_KEY *key);
93
extern "C" int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits,
94
                                      AES_KEY *key);
95
extern "C" void aes_hw_encrypt(const uint8_t *in, uint8_t *out,
96
                               const AES_KEY *key);
97
extern "C" void aes_hw_decrypt(const uint8_t *in, uint8_t *out,
98
                               const AES_KEY *key);
99
extern "C" void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out,
100
                                   size_t length, const AES_KEY *key,
101
                                   uint8_t *ivec, int enc);
102
extern "C" void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
103
                                            size_t len, const AES_KEY *key,
104
                                            const uint8_t ivec[16]);
105
106
#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
107
// On x86 and x86_64, |aes_hw_set_decrypt_key| is implemented in terms of
108
// |aes_hw_set_encrypt_key| and a conversion function.
109
extern "C" void aes_hw_encrypt_key_to_decrypt_key(AES_KEY *key);
110
111
// There are two variants of this function, one which uses aeskeygenassist
112
// ("base") and one which uses aesenclast + pshufb ("alt"). aesenclast is
113
// overall faster but is slower on some older processors. It doesn't use AVX,
114
// but AVX is used as a proxy to detecting this. See
115
// https://groups.google.com/g/mailing.openssl.dev/c/OuFXwW4NfO8/m/7d2ZXVjkxVkJ
116
//
117
// TODO(davidben): It is unclear if the aeskeygenassist version is still
118
// worthwhile. However, the aesenclast version requires SSSE3. SSSE3 long
119
// predates AES-NI, but it's not clear if AES-NI implies SSSE3. In OpenSSL, the
120
// CCM AES-NI assembly seems to assume it does.
121
0
inline int aes_hw_set_encrypt_key_alt_capable() {
122
0
  return hwaes_capable() && CRYPTO_is_SSSE3_capable();
123
0
}
124
1.94M
inline int aes_hw_set_encrypt_key_alt_preferred() {
125
1.94M
  return hwaes_capable() && CRYPTO_is_AVX_capable();
126
1.94M
}
127
extern "C" int aes_hw_set_encrypt_key_base(const uint8_t *user_key, int bits,
128
                                           AES_KEY *key);
129
extern "C" int aes_hw_set_encrypt_key_alt(const uint8_t *user_key, int bits,
130
                                          AES_KEY *key);
131
#endif  // OPENSSL_X86 || OPENSSL_X86_64
132
133
#else
134
135
// If HWAES isn't defined then we provide dummy functions for each of the hwaes
136
// functions.
137
inline int hwaes_capable() { return 0; }
138
139
inline int aes_hw_set_encrypt_key(const uint8_t *user_key, int bits,
140
                                  AES_KEY *key) {
141
  abort();
142
}
143
144
inline int aes_hw_set_decrypt_key(const uint8_t *user_key, int bits,
145
                                  AES_KEY *key) {
146
  abort();
147
}
148
149
inline void aes_hw_encrypt(const uint8_t *in, uint8_t *out,
150
                           const AES_KEY *key) {
151
  abort();
152
}
153
154
inline void aes_hw_decrypt(const uint8_t *in, uint8_t *out,
155
                           const AES_KEY *key) {
156
  abort();
157
}
158
159
inline void aes_hw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
160
                               const AES_KEY *key, uint8_t *ivec, int enc) {
161
  abort();
162
}
163
164
inline void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
165
                                        size_t len, const AES_KEY *key,
166
                                        const uint8_t ivec[16]) {
167
  abort();
168
}
169
170
#endif  // !HWAES
171
172
173
#if defined(HWAES_ECB)
174
extern "C" void aes_hw_ecb_encrypt(const uint8_t *in, uint8_t *out,
175
                                   size_t length, const AES_KEY *key, int enc);
176
#endif  // HWAES_ECB
177
178
179
#if defined(BSAES)
180
// Note |bsaes_cbc_encrypt| requires |enc| to be zero.
181
extern "C" void bsaes_cbc_encrypt(const uint8_t *in, uint8_t *out,
182
                                  size_t length, const AES_KEY *key,
183
                                  uint8_t ivec[16], int enc);
184
extern "C" void bsaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
185
                                           size_t len, const AES_KEY *key,
186
                                           const uint8_t ivec[16]);
187
// VPAES to BSAES conversions are available on all BSAES platforms.
188
extern "C" void vpaes_encrypt_key_to_bsaes(AES_KEY *out_bsaes,
189
                                           const AES_KEY *vpaes);
190
extern "C" void vpaes_decrypt_key_to_bsaes(AES_KEY *out_bsaes,
191
                                           const AES_KEY *vpaes);
192
void vpaes_ctr32_encrypt_blocks_with_bsaes(const uint8_t *in, uint8_t *out,
193
                                           size_t blocks, const AES_KEY *key,
194
                                           const uint8_t ivec[16]);
195
#endif  // !BSAES
196
197
198
#if defined(VPAES)
199
// On platforms where VPAES gets defined (just above), then these functions are
200
// provided by asm.
201
extern "C" int vpaes_set_encrypt_key(const uint8_t *userKey, int bits,
202
                                     AES_KEY *key);
203
extern "C" int vpaes_set_decrypt_key(const uint8_t *userKey, int bits,
204
                                     AES_KEY *key);
205
206
extern "C" void vpaes_encrypt(const uint8_t *in, uint8_t *out,
207
                              const AES_KEY *key);
208
extern "C" void vpaes_decrypt(const uint8_t *in, uint8_t *out,
209
                              const AES_KEY *key);
210
211
#if defined(VPAES_CBC)
212
extern "C" void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out,
213
                                  size_t length, const AES_KEY *key,
214
                                  uint8_t *ivec, int enc);
215
#endif
216
extern "C" void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
217
                                           size_t len, const AES_KEY *key,
218
                                           const uint8_t ivec[16]);
219
#else
220
inline int vpaes_capable() { return 0; }
221
222
// On other platforms, vpaes_capable() will always return false and so the
223
// following will never be called.
224
inline int vpaes_set_encrypt_key(const uint8_t *userKey, int bits,
225
                                 AES_KEY *key) {
226
  abort();
227
}
228
inline int vpaes_set_decrypt_key(const uint8_t *userKey, int bits,
229
                                 AES_KEY *key) {
230
  abort();
231
}
232
inline void vpaes_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
233
  abort();
234
}
235
inline void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
236
  abort();
237
}
238
inline void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
239
                              const AES_KEY *key, uint8_t *ivec, int enc) {
240
  abort();
241
}
242
inline void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
243
                                       size_t len, const AES_KEY *key,
244
                                       const uint8_t ivec[16]) {
245
  abort();
246
}
247
#endif  // !VPAES
248
249
250
int aes_nohw_set_encrypt_key(const uint8_t *key, unsigned bits,
251
                             AES_KEY *aeskey);
252
int aes_nohw_set_decrypt_key(const uint8_t *key, unsigned bits,
253
                             AES_KEY *aeskey);
254
void aes_nohw_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
255
void aes_nohw_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
256
void aes_nohw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
257
                                   size_t blocks, const AES_KEY *key,
258
                                   const uint8_t ivec[16]);
259
void aes_nohw_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len,
260
                          const AES_KEY *key, uint8_t *ivec, int enc);
261
262
// Modes
263
264
inline void CRYPTO_xor16(uint8_t out[16], const uint8_t a[16],
265
5.52M
                         const uint8_t b[16]) {
266
  // TODO(davidben): Ideally we'd leave this to the compiler, which could use
267
  // vector registers, etc. But the compiler doesn't know that |in| and |out|
268
  // cannot partially alias. |restrict| is slightly two strict (we allow exact
269
  // aliasing), but perhaps in-place could be a separate function?
270
5.52M
  static_assert(16 % sizeof(crypto_word_t) == 0,
271
5.52M
                "block cannot be evenly divided into words");
272
16.5M
  for (size_t i = 0; i < 16; i += sizeof(crypto_word_t)) {
273
11.0M
    CRYPTO_store_word_le(
274
11.0M
        out + i, CRYPTO_load_word_le(a + i) ^ CRYPTO_load_word_le(b + i));
275
11.0M
  }
276
5.52M
}
277
278
279
// CTR.
280
281
// CRYPTO_ctr128_encrypt_ctr32 encrypts (or decrypts, it's the same in CTR mode)
282
// |len| bytes from |in| to |out| using |block| in counter mode. There's no
283
// requirement that |len| be a multiple of any value and any partial blocks are
284
// stored in |ecount_buf| and |*num|, which must be zeroed before the initial
285
// call. The counter is a 128-bit, big-endian value in |ivec| and is
286
// incremented by this function. If the counter overflows, it wraps around.
287
// |ctr| must be a function that performs CTR mode but only deals with the lower
288
// 32 bits of the counter.
289
void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
290
                                 const AES_KEY *key, uint8_t ivec[16],
291
                                 uint8_t ecount_buf[16], unsigned *num,
292
                                 ctr128_f ctr);
293
294
295
// GCM.
296
//
297
// This API differs from the upstream API slightly. The |GCM128_CONTEXT| does
298
// not have a |key| pointer that points to the key as upstream's version does.
299
// Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT|
300
// can be safely copied. Additionally, |gcm_key| is split into a separate
301
// struct.
302
303
// gcm_impl_t specifies an assembly implementation of AES-GCM.
304
enum gcm_impl_t {
305
  gcm_separate = 0,  // No combined AES-GCM, but may have AES-CTR and GHASH.
306
  gcm_x86_aesni,
307
  gcm_x86_vaes_avx2,
308
  gcm_x86_vaes_avx512,
309
  gcm_arm64_aes,
310
  gcm_arm64_aes_eor3,
311
};
312
313
typedef struct { uint64_t hi,lo; } u128;
314
315
// gmult_func multiplies |Xi| by the GCM key and writes the result back to
316
// |Xi|.
317
typedef void (*gmult_func)(uint8_t Xi[16], const u128 Htable[16]);
318
319
// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from
320
// |inp|. The result is written back to |Xi| and the |len| argument must be a
321
// multiple of 16.
322
typedef void (*ghash_func)(uint8_t Xi[16], const u128 Htable[16],
323
                           const uint8_t *inp, size_t len);
324
325
typedef struct gcm128_key_st {
326
  u128 Htable[16];
327
  gmult_func gmult;
328
  ghash_func ghash;
329
  AES_KEY aes;
330
331
  ctr128_f ctr;
332
  block128_f block;
333
  enum gcm_impl_t impl;
334
} GCM128_KEY;
335
336
// GCM128_CONTEXT contains state for a single GCM operation. The structure
337
// should be zero-initialized before use.
338
typedef struct {
339
  // The following 5 names follow names in GCM specification
340
  uint8_t Yi[16];
341
  uint8_t EKi[16];
342
  uint8_t EK0[16];
343
  struct {
344
    uint64_t aad;
345
    uint64_t msg;
346
  } len;
347
  uint8_t Xi[16];
348
  unsigned mres, ares;
349
} GCM128_CONTEXT;
350
351
#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
352
// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is
353
// used.
354
int crypto_gcm_clmul_enabled();
355
#endif
356
357
// CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to
358
// |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware
359
// accelerated) functions for performing operations in the GHASH field.
360
void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
361
                       u128 out_table[16], const uint8_t gcm_key[16]);
362
363
// CRYPTO_gcm128_init_aes_key initialises |gcm_key| to with AES key |key|.
364
void CRYPTO_gcm128_init_aes_key(GCM128_KEY *gcm_key, const uint8_t *key,
365
                                size_t key_bytes);
366
367
// CRYPTO_gcm128_init_ctx initializes |ctx| to encrypt with |key| and |iv|.
368
void CRYPTO_gcm128_init_ctx(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
369
                            const uint8_t *iv, size_t iv_len);
370
371
// CRYPTO_gcm128_aad adds to the authenticated data for an instance of GCM.
372
// This must be called before and data is encrypted. |key| must be the same
373
// value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on success
374
// and zero otherwise.
375
int CRYPTO_gcm128_aad(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
376
                      const uint8_t *aad, size_t aad_len);
377
378
// CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. |key| must be
379
// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on
380
// success and zero otherwise.
381
int CRYPTO_gcm128_encrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
382
                          const uint8_t *in, uint8_t *out, size_t len);
383
384
// CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. |key| must be
385
// the same value that was passed to |CRYPTO_gcm128_init_ctx|. It returns one on
386
// success and zero otherwise.
387
int CRYPTO_gcm128_decrypt(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
388
                          const uint8_t *in, uint8_t *out, size_t len);
389
390
// CRYPTO_gcm128_finish calculates the authenticator and compares it against
391
// |len| bytes of |tag|. |key| must be the same value that was passed to
392
// |CRYPTO_gcm128_init_ctx|. It returns one on success and zero otherwise.
393
int CRYPTO_gcm128_finish(const GCM128_KEY *key, GCM128_CONTEXT *ctx,
394
                         const uint8_t *tag, size_t len);
395
396
// CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|.
397
// The minimum of |len| and 16 bytes are copied into |tag|. |key| must be the
398
// same value that was passed to |CRYPTO_gcm128_init_ctx|.
399
void CRYPTO_gcm128_tag(const GCM128_KEY *key, GCM128_CONTEXT *ctx, uint8_t *tag,
400
                       size_t len);
401
402
403
// GCM assembly.
404
405
void gcm_init_nohw(u128 Htable[16], const uint64_t H[2]);
406
void gcm_gmult_nohw(uint8_t Xi[16], const u128 Htable[16]);
407
void gcm_ghash_nohw(uint8_t Xi[16], const u128 Htable[16], const uint8_t *inp,
408
                    size_t len);
409
410
#if !defined(OPENSSL_NO_ASM)
411
412
#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
413
#define GCM_FUNCREF
414
extern "C" void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]);
415
extern "C" void gcm_gmult_clmul(uint8_t Xi[16], const u128 Htable[16]);
416
extern "C" void gcm_ghash_clmul(uint8_t Xi[16], const u128 Htable[16],
417
                                const uint8_t *inp, size_t len);
418
419
void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]);
420
extern "C" void gcm_gmult_ssse3(uint8_t Xi[16], const u128 Htable[16]);
421
extern "C" void gcm_ghash_ssse3(uint8_t Xi[16], const u128 Htable[16],
422
                                const uint8_t *in, size_t len);
423
424
#if defined(OPENSSL_X86_64)
425
#define GHASH_ASM_X86_64
426
extern "C" void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
427
extern "C" void gcm_gmult_avx(uint8_t Xi[16], const u128 Htable[16]);
428
extern "C" void gcm_ghash_avx(uint8_t Xi[16], const u128 Htable[16],
429
                              const uint8_t *in, size_t len);
430
431
#define HW_GCM
432
extern "C" size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
433
                                    const AES_KEY *key, uint8_t ivec[16],
434
                                    const u128 Htable[16], uint8_t Xi[16]);
435
extern "C" size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
436
                                    const AES_KEY *key, uint8_t ivec[16],
437
                                    const u128 Htable[16], uint8_t Xi[16]);
438
439
extern "C" void gcm_init_vpclmulqdq_avx2(u128 Htable[16], const uint64_t H[2]);
440
extern "C" void gcm_gmult_vpclmulqdq_avx2(uint8_t Xi[16],
441
                                          const u128 Htable[16]);
442
extern "C" void gcm_ghash_vpclmulqdq_avx2(uint8_t Xi[16], const u128 Htable[16],
443
                                          const uint8_t *in, size_t len);
444
extern "C" void aes_gcm_enc_update_vaes_avx2(const uint8_t *in, uint8_t *out,
445
                                             size_t len, const AES_KEY *key,
446
                                             const uint8_t ivec[16],
447
                                             const u128 Htable[16],
448
                                             uint8_t Xi[16]);
449
extern "C" void aes_gcm_dec_update_vaes_avx2(const uint8_t *in, uint8_t *out,
450
                                             size_t len, const AES_KEY *key,
451
                                             const uint8_t ivec[16],
452
                                             const u128 Htable[16],
453
                                             uint8_t Xi[16]);
454
455
extern "C" void gcm_init_vpclmulqdq_avx512(u128 Htable[16],
456
                                           const uint64_t H[2]);
457
extern "C" void gcm_gmult_vpclmulqdq_avx512(uint8_t Xi[16],
458
                                            const u128 Htable[16]);
459
extern "C" void gcm_ghash_vpclmulqdq_avx512(uint8_t Xi[16],
460
                                            const u128 Htable[16],
461
                                            const uint8_t *in, size_t len);
462
extern "C" void aes_gcm_enc_update_vaes_avx512(const uint8_t *in, uint8_t *out,
463
                                               size_t len, const AES_KEY *key,
464
                                               const uint8_t ivec[16],
465
                                               const u128 Htable[16],
466
                                               uint8_t Xi[16]);
467
extern "C" void aes_gcm_dec_update_vaes_avx512(const uint8_t *in, uint8_t *out,
468
                                               size_t len, const AES_KEY *key,
469
                                               const uint8_t ivec[16],
470
                                               const u128 Htable[16],
471
                                               uint8_t Xi[16]);
472
473
#endif  // OPENSSL_X86_64
474
475
#if defined(OPENSSL_X86)
476
#define GHASH_ASM_X86
477
#endif  // OPENSSL_X86
478
479
#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
480
481
#define GHASH_ASM_ARM
482
#define GCM_FUNCREF
483
484
inline int gcm_pmull_capable() { return CRYPTO_is_ARMv8_PMULL_capable(); }
485
inline int gcm_sha3_capable() { return CRYPTO_is_ARMv8_SHA3_capable(); }
486
487
extern "C" void gcm_init_v8(u128 Htable[16], const uint64_t H[2]);
488
extern "C" void gcm_gmult_v8(uint8_t Xi[16], const u128 Htable[16]);
489
extern "C" void gcm_ghash_v8(uint8_t Xi[16], const u128 Htable[16],
490
                             const uint8_t *inp, size_t len);
491
492
inline int gcm_neon_capable() { return CRYPTO_is_NEON_capable(); }
493
494
extern "C" void gcm_init_neon(u128 Htable[16], const uint64_t H[2]);
495
extern "C" void gcm_gmult_neon(uint8_t Xi[16], const u128 Htable[16]);
496
extern "C" void gcm_ghash_neon(uint8_t Xi[16], const u128 Htable[16],
497
                               const uint8_t *inp, size_t len);
498
499
#if defined(OPENSSL_AARCH64)
500
#define HW_GCM
501
// These functions are defined in aesv8-gcm-armv8.pl.
502
extern "C" void aes_gcm_enc_kernel(const uint8_t *in, uint64_t in_bits,
503
                                   void *out, void *Xi, uint8_t *ivec,
504
                                   const AES_KEY *key, const u128 Htable[16]);
505
extern "C" void aes_gcm_dec_kernel(const uint8_t *in, uint64_t in_bits,
506
                                   void *out, void *Xi, uint8_t *ivec,
507
                                   const AES_KEY *key, const u128 Htable[16]);
508
extern "C" void aes_gcm_enc_kernel_eor3(const uint8_t *in, uint64_t in_bits,
509
                                   void *out, void *Xi, uint8_t *ivec,
510
                                   const AES_KEY *key, const u128 Htable[16]);
511
extern "C" void aes_gcm_dec_kernel_eor3(const uint8_t *in, uint64_t in_bits,
512
                                   void *out, void *Xi, uint8_t *ivec,
513
                                   const AES_KEY *key, const u128 Htable[16]);
514
#endif
515
516
#endif
517
#endif  // OPENSSL_NO_ASM
518
519
520
// CBC.
521
522
// cbc128_f is the type of a function that performs CBC-mode encryption.
523
typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len,
524
                         const AES_KEY *key, uint8_t ivec[16], int enc);
525
526
// CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the
527
// given IV and block cipher in CBC mode. The input need not be a multiple of
528
// 128 bits long, but the output will round up to the nearest 128 bit multiple,
529
// zero padding the input if needed. The IV will be updated on return.
530
void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
531
                           const AES_KEY *key, uint8_t ivec[16],
532
                           block128_f block);
533
534
// CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the
535
// given IV and block cipher in CBC mode. If |len| is not a multiple of 128
536
// bits then only that many bytes will be written, but a multiple of 128 bits
537
// is always read from |in|. The IV will be updated on return.
538
void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len,
539
                           const AES_KEY *key, uint8_t ivec[16],
540
                           block128_f block);
541
542
543
// OFB.
544
545
// CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode)
546
// |len| bytes from |in| to |out| using |block| in OFB mode. There's no
547
// requirement that |len| be a multiple of any value and any partial blocks are
548
// stored in |ivec| and |*num|, the latter must be zero before the initial
549
// call.
550
void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
551
                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
552
                           block128_f block);
553
554
555
// CFB.
556
557
// CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
558
// from |in| to |out| using |block| in CFB mode. There's no requirement that
559
// |len| be a multiple of any value and any partial blocks are stored in |ivec|
560
// and |*num|, the latter must be zero before the initial call.
561
void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
562
                           const AES_KEY *key, uint8_t ivec[16], unsigned *num,
563
                           int enc, block128_f block);
564
565
// CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
566
// from |in| to |out| using |block| in CFB-8 mode. Prior to the first call
567
// |num| should be set to zero.
568
void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len,
569
                             const AES_KEY *key, uint8_t ivec[16],
570
                             unsigned *num, int enc, block128_f block);
571
572
// CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
573
// from |in| to |out| using |block| in CFB-1 mode. Prior to the first call
574
// |num| should be set to zero.
575
void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
576
                             const AES_KEY *key, uint8_t ivec[16],
577
                             unsigned *num, int enc, block128_f block);
578
579
size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len,
580
                                   const AES_KEY *key, uint8_t ivec[16],
581
                                   block128_f block);
582
583
BSSL_NAMESPACE_END
584
585
#endif  // OPENSSL_HEADER_CRYPTO_FIPSMODULE_AES_INTERNAL_H