Coverage Report

Created: 2024-11-21 07:03

/src/SymCrypt/lib/aes-default.c
Line
Count
Source (jump to first uncovered line)
1
//
2
// aes-default.c   code for AES implementation
3
//
4
// Copyright (c) Microsoft Corporation. Licensed under the MIT license.
5
//
6
// This is the interface for the default AES implementation.
7
// On each platform, this is the fastest AES implementation irrespective of code size.
8
// It uses assembler, XMM, or any other trick.
9
//
10
11
12
#include "precomp.h"
13
14
//
15
// Virtual table for generic functions
16
// This allows us to default to generic implementations for some modes without pulling in all the
17
// dedicated functions.
18
// We use this when we cannot use the optimized implementations for some reason.
19
//
20
const SYMCRYPT_BLOCKCIPHER SymCryptAesBlockCipherNoOpt = {
21
    &SymCryptAesExpandKey,
22
#if SYMCRYPT_CPU_AMD64 | SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_ARM
23
    &SymCryptAesEncryptAsm,
24
    &SymCryptAesDecryptAsm,
25
#else
26
    &SymCryptAesEncryptC,
27
    &SymCryptAesDecryptC,
28
#endif
29
    NULL,
30
    NULL,
31
    NULL,
32
    NULL,
33
    NULL,
34
    NULL,
35
    NULL,
36
    NULL,
37
38
    SYMCRYPT_AES_BLOCK_SIZE,
39
    sizeof( SYMCRYPT_AES_EXPANDED_KEY ),
40
};
41
42
VOID
43
SYMCRYPT_CALL
44
SymCryptAes4Sbox( _In_reads_(4) PCBYTE pIn, _Out_writes_(4) PBYTE pOut, BOOL UseSimd )
45
0
{
46
0
#if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64
47
0
    if( UseSimd )
48
0
    {
49
0
        SymCryptAes4SboxXmm( pIn, pOut );
50
0
    } else {
51
0
        SymCryptAes4SboxC( pIn, pOut );
52
0
    }
53
#elif SYMCRYPT_CPU_ARM64
54
    if( UseSimd )
55
    {
56
        SymCryptAes4SboxNeon( pIn, pOut );
57
    } else {
58
        SymCryptAes4SboxC( pIn, pOut );
59
    }
60
#else
61
    UNREFERENCED_PARAMETER( UseSimd );
62
    SymCryptAes4SboxC( pIn, pOut );         // never use XMM on SaveXmm arch, save/restore overhead is too large.
63
#endif
64
0
}
65
66
VOID
67
SYMCRYPT_CALL
68
SymCryptAesCreateDecryptionRoundKey(
69
    _In_reads_(16)      PCBYTE  pEncryptionRoundKey,
70
    _Out_writes_(16)    PBYTE   pDecryptionRoundKey,
71
                        BOOL    UseSimd )
72
0
{
73
0
#if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64
74
0
    if( UseSimd )
75
0
    {
76
0
        SymCryptAesCreateDecryptionRoundKeyXmm( pEncryptionRoundKey, pDecryptionRoundKey );
77
0
    } else {
78
0
        SymCryptAesCreateDecryptionRoundKeyC( pEncryptionRoundKey, pDecryptionRoundKey );
79
0
    }
80
#elif SYMCRYPT_CPU_ARM64
81
    if( UseSimd )
82
    {
83
        SymCryptAesCreateDecryptionRoundKeyNeon( pEncryptionRoundKey, pDecryptionRoundKey );
84
    } else {
85
        SymCryptAesCreateDecryptionRoundKeyC( pEncryptionRoundKey, pDecryptionRoundKey );
86
    }
87
#else
88
    UNREFERENCED_PARAMETER( UseSimd );
89
    SymCryptAesCreateDecryptionRoundKeyC( pEncryptionRoundKey, pDecryptionRoundKey );   // never use XMM on SaveXmm arch, save/restore overhead is too large.
90
#endif
91
0
}
92
93
VOID
94
SYMCRYPT_CALL
95
SymCryptAesEncrypt(
96
    _In_                                    PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
97
    _In_reads_(SYMCRYPT_AES_BLOCK_SIZE)     PCBYTE                      pbSrc,
98
    _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE)   PBYTE                       pbDst )
99
0
{
100
0
#if SYMCRYPT_CPU_AMD64
101
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
102
0
    {
103
0
        SymCryptAesEncryptXmm( pExpandedKey, pbSrc, pbDst );
104
0
    } else {
105
0
        SymCryptAesEncryptAsm( pExpandedKey, pbSrc, pbDst );
106
0
    }
107
#elif SYMCRYPT_CPU_X86
108
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
109
110
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
111
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
112
    {
113
        SymCryptAesEncryptXmm( pExpandedKey, pbSrc, pbDst );
114
        SymCryptRestoreXmm( &SaveData );
115
    } else {
116
        SymCryptAesEncryptAsm( pExpandedKey, pbSrc, pbDst );
117
    }
118
#elif SYMCRYPT_CPU_ARM
119
    SymCryptAesEncryptAsm( pExpandedKey, pbSrc, pbDst );
120
#elif SYMCRYPT_CPU_ARM64
121
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
122
    {
123
        SymCryptAesEncryptNeon( pExpandedKey, pbSrc, pbDst );
124
    } else {
125
        SymCryptAesEncryptC( pExpandedKey, pbSrc, pbDst );
126
    }
127
#else
128
    SymCryptAesEncryptC( pExpandedKey, pbSrc, pbDst );
129
#endif
130
0
}
131
132
VOID
133
SYMCRYPT_CALL
134
SymCryptAesDecrypt(
135
    _In_                                    PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
136
    _In_reads_(SYMCRYPT_AES_BLOCK_SIZE)     PCBYTE                      pbSrc,
137
    _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE)   PBYTE                       pbDst )
138
0
{
139
0
#if SYMCRYPT_CPU_AMD64
140
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
141
0
    {
142
0
        SymCryptAesDecryptXmm( pExpandedKey, pbSrc, pbDst );
143
0
    } else {
144
0
        SymCryptAesDecryptAsm( pExpandedKey, pbSrc, pbDst );
145
0
    }
146
#elif SYMCRYPT_CPU_X86
147
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
148
149
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
150
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
151
    {
152
        SymCryptAesDecryptXmm( pExpandedKey, pbSrc, pbDst );
153
        SymCryptRestoreXmm( &SaveData );
154
    } else {
155
        SymCryptAesDecryptAsm( pExpandedKey, pbSrc, pbDst );
156
    }
157
#elif SYMCRYPT_CPU_ARM
158
    SymCryptAesDecryptAsm( pExpandedKey, pbSrc, pbDst );
159
#elif SYMCRYPT_CPU_ARM64
160
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
161
    {
162
        SymCryptAesDecryptNeon( pExpandedKey, pbSrc, pbDst );
163
    } else {
164
        SymCryptAesDecryptC( pExpandedKey, pbSrc, pbDst );
165
    }
166
#else
167
    SymCryptAesDecryptC( pExpandedKey, pbSrc, pbDst );
168
#endif
169
0
}
170
171
VOID
172
SYMCRYPT_CALL
173
SymCryptAesCbcEncrypt(
174
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
175
    _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE )  PBYTE                       pbChainingValue,
176
    _In_reads_( cbData )                        PCBYTE                      pbSrc,
177
    _Out_writes_( cbData )                      PBYTE                       pbDst,
178
                                                SIZE_T                      cbData )
179
0
{
180
0
#if SYMCRYPT_CPU_AMD64
181
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
182
0
    {
183
0
        SymCryptAesCbcEncryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
184
0
    } else {
185
0
        SymCryptAesCbcEncryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
186
0
    }
187
#elif SYMCRYPT_CPU_X86
188
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
189
190
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
191
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
192
    {
193
        SymCryptAesCbcEncryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
194
        SymCryptRestoreXmm( &SaveData );
195
    } else {
196
        SymCryptAesCbcEncryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
197
    }
198
#elif SYMCRYPT_CPU_ARM
199
    SymCryptAesCbcEncryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
200
#elif SYMCRYPT_CPU_ARM64
201
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
202
    {
203
        SymCryptAesCbcEncryptNeon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
204
    } else {
205
        SymCryptCbcEncrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
206
    }
207
#else
208
    SymCryptCbcEncrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
209
#endif
210
0
}
211
212
VOID
213
SYMCRYPT_CALL
214
SymCryptAesCbcDecrypt(
215
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
216
    _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE )  PBYTE                       pbChainingValue,
217
    _In_reads_( cbData )                        PCBYTE                      pbSrc,
218
    _Out_writes_( cbData )                      PBYTE                       pbDst,
219
                                                SIZE_T                      cbData )
220
0
{
221
0
#if SYMCRYPT_CPU_AMD64
222
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
223
0
    {
224
0
        SymCryptAesCbcDecryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
225
0
    } else {
226
0
        SymCryptAesCbcDecryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
227
0
    }
228
#elif SYMCRYPT_CPU_X86
229
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
230
231
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
232
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
233
    {
234
        SymCryptAesCbcDecryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
235
        SymCryptRestoreXmm( &SaveData );
236
    } else {
237
        SymCryptAesCbcDecryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
238
    }
239
#elif SYMCRYPT_CPU_ARM
240
    SymCryptAesCbcDecryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
241
#elif SYMCRYPT_CPU_ARM64
242
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
243
    {
244
        SymCryptAesCbcDecryptNeon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
245
    } else {
246
        SymCryptCbcDecrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
247
    }
248
#else
249
    SymCryptCbcDecrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
250
#endif
251
0
}
252
253
VOID
254
SYMCRYPT_CALL
255
SymCryptAesEcbEncrypt(
256
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
257
    _In_reads_( cbData )                        PCBYTE                      pbSrc,
258
    _Out_writes_( cbData )                      PBYTE                       pbDst,
259
                                                SIZE_T                      cbData )
260
0
{
261
0
#if SYMCRYPT_CPU_AMD64
262
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
263
0
    {
264
0
        SymCryptAesEcbEncryptXmm( pExpandedKey, pbSrc, pbDst, cbData );
265
0
    } else {
266
0
        SymCryptAesEcbEncryptAsm( pExpandedKey, pbSrc, pbDst, cbData );
267
0
    }
268
#elif SYMCRYPT_CPU_X86
269
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
270
271
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
272
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
273
    {
274
        SymCryptAesEcbEncryptXmm( pExpandedKey, pbSrc, pbDst, cbData );
275
        SymCryptRestoreXmm( &SaveData );
276
    } else {
277
        SymCryptAesEcbEncryptAsm( pExpandedKey, pbSrc, pbDst, cbData );
278
    }
279
#elif SYMCRYPT_CPU_ARM
280
    SymCryptAesEcbEncryptAsm( pExpandedKey, pbSrc, pbDst, cbData );
281
#elif SYMCRYPT_CPU_ARM64
282
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
283
    {
284
        SymCryptAesEcbEncryptNeon( pExpandedKey, pbSrc, pbDst, cbData );
285
    } else {
286
        SymCryptAesEcbEncryptC( pExpandedKey, pbSrc, pbDst, cbData );
287
    }
288
#else
289
    SymCryptAesEcbEncryptC( pExpandedKey, pbSrc, pbDst, cbData );
290
#endif
291
0
}
292
293
//
294
// NOTE: There is no reason that SymCryptAesEcbDecrypt could not have unrolled versions similar to
295
// SymCryptAesEcbEncrypt if a real use case requiring large scale Ecb decryption is found.
296
// For now just decrypt 1 block at a time to reduce code size.
297
//
298
VOID
299
SYMCRYPT_CALL
300
SymCryptAesEcbDecrypt(
301
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
302
    _In_reads_( cbData )                        PCBYTE                      pbSrc,
303
    _Out_writes_( cbData )                      PBYTE                       pbDst,
304
                                                SIZE_T                      cbData )
305
0
{
306
0
    while( cbData >= SYMCRYPT_AES_BLOCK_SIZE )
307
0
    {
308
0
        SymCryptAesDecrypt( pExpandedKey, pbSrc, pbDst );
309
0
        pbSrc += SYMCRYPT_AES_BLOCK_SIZE;
310
0
        pbDst += SYMCRYPT_AES_BLOCK_SIZE;
311
0
        cbData -= SYMCRYPT_AES_BLOCK_SIZE;
312
0
    }
313
0
}
314
315
VOID
316
SYMCRYPT_CALL
317
SymCryptAesCbcMac(
318
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
319
    _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE )  PBYTE                       pbChainingValue,
320
    _In_reads_( cbData )                        PCBYTE                      pbData,
321
                                                SIZE_T                      cbData )
322
0
{
323
0
#if SYMCRYPT_CPU_AMD64
324
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
325
0
    {
326
0
        SymCryptAesCbcMacXmm( pExpandedKey, pbChainingValue, pbData, cbData );
327
0
    } else {
328
0
        SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE );
329
0
        SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData );
330
0
    }
331
#elif SYMCRYPT_CPU_X86
332
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
333
334
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
335
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
336
    {
337
        SymCryptAesCbcMacXmm( pExpandedKey, pbChainingValue, pbData, cbData );
338
        SymCryptRestoreXmm( &SaveData );
339
    } else {
340
        SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE );
341
        SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData );
342
    }
343
#elif SYMCRYPT_CPU_ARM64
344
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
345
    {
346
        SymCryptAesCbcMacNeon( pExpandedKey, pbChainingValue, pbData, cbData );
347
    } else {
348
        SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE );
349
        SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData );
350
    }
351
#else
352
    SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE );
353
    SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData );
354
#endif
355
0
}
356
357
VOID
358
SYMCRYPT_CALL
359
SymCryptAesCtrMsb32(
360
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
361
    _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE )  PBYTE                       pbChainingValue,
362
    _In_reads_( cbData )                        PCBYTE                      pbSrc,
363
    _Out_writes_( cbData )                      PBYTE                       pbDst,
364
                                                SIZE_T                      cbData )
365
0
{
366
0
#if SYMCRYPT_CPU_AMD64
367
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
368
0
    {
369
0
        SymCryptAesCtrMsb32Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
370
0
    } else {
371
0
        SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy
372
0
        SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
373
0
    }
374
375
#elif SYMCRYPT_CPU_X86
376
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
377
378
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
379
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
380
    {
381
        SymCryptAesCtrMsb32Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
382
        SymCryptRestoreXmm( &SaveData );
383
    } else {
384
        SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy
385
        SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
386
    }
387
388
#elif SYMCRYPT_CPU_ARM64
389
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
390
    {
391
        SymCryptAesCtrMsb32Neon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
392
    } else {
393
        SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
394
    }
395
396
#else
397
    SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy
398
    SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
399
#endif
400
0
}
401
402
VOID
403
SYMCRYPT_CALL
404
SymCryptAesCtrMsb64(
405
    _In_                                        PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey,
406
    _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE )  PBYTE                       pbChainingValue,
407
    _In_reads_( cbData )                        PCBYTE                      pbSrc,
408
    _Out_writes_( cbData )                      PBYTE                       pbDst,
409
                                                SIZE_T                      cbData )
410
0
{
411
0
#if SYMCRYPT_CPU_AMD64
412
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) )
413
0
    {
414
0
        SymCryptAesCtrMsb64Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
415
0
    } else {
416
0
        SymCryptAesCtrMsb64Asm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
417
0
    }
418
419
#elif SYMCRYPT_CPU_X86
420
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
421
422
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) &&
423
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
424
    {
425
        SymCryptAesCtrMsb64Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
426
        SymCryptRestoreXmm( &SaveData );
427
    } else {
428
        SymCryptAesCtrMsb64Asm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
429
    }
430
431
#elif SYMCRYPT_CPU_ARM
432
    SymCryptAesCtrMsb64Asm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
433
434
#elif SYMCRYPT_CPU_ARM64
435
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) )
436
    {
437
        SymCryptAesCtrMsb64Neon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
438
    } else {
439
        SymCryptCtrMsb64( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
440
    }
441
442
#else
443
    SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE );        // keep Prefast happy
444
    SymCryptCtrMsb64( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData );
445
#endif
446
0
}
447
448
VOID
449
SYMCRYPT_CALL
450
SymCryptAesGcmEncryptPartOnePass(
451
    _Inout_                 PSYMCRYPT_GCM_STATE pState,
452
    _In_reads_( cbData )    PCBYTE              pbSrc,
453
    _Out_writes_( cbData )  PBYTE               pbDst,
454
                            SIZE_T              cbData )
455
0
{
456
0
    SIZE_T      bytesToProcess;
457
0
#if SYMCRYPT_CPU_AMD64
458
0
    SYMCRYPT_EXTENDED_SAVE_DATA SaveData;
459
0
#endif
460
461
    //
462
    // We have entered the encrypt phase, the AAD has been padded to be a multiple of block size
463
    // We know that the bytes still to use in the key stream buffer and the bytes left to fill the
464
    // macBlock will be the same in the context of this function
465
    //
466
0
    SYMCRYPT_ASSERT( (pState->cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == pState->bytesInMacBlock );
467
468
    //
469
    // We update pState->cbData once before we modify cbData.
470
    // pState->cbData is not used in the rest of this function
471
    //
472
0
    SYMCRYPT_ASSERT( pState->cbData + cbData <= SYMCRYPT_GCM_MAX_DATA_SIZE );
473
0
    pState->cbData += cbData;
474
475
0
    if( pState->bytesInMacBlock > 0 )
476
0
    {
477
0
        bytesToProcess = SYMCRYPT_MIN( cbData, SYMCRYPT_GCM_BLOCK_SIZE - pState->bytesInMacBlock );
478
0
        SymCryptXorBytes(
479
0
            pbSrc,
480
0
            &pState->keystreamBlock[pState->bytesInMacBlock],
481
0
            &pState->macBlock[pState->bytesInMacBlock],
482
0
            bytesToProcess );
483
0
        memcpy( pbDst, &pState->macBlock[pState->bytesInMacBlock], bytesToProcess );
484
0
        pbSrc += bytesToProcess;
485
0
        pbDst += bytesToProcess;
486
0
        cbData -= bytesToProcess;
487
0
        pState->bytesInMacBlock += bytesToProcess;
488
489
0
        if( pState->bytesInMacBlock == SYMCRYPT_GCM_BLOCK_SIZE )
490
0
        {
491
0
            SymCryptGHashAppendData(    &pState->pKey->ghashKey,
492
0
                                        &pState->ghashState,
493
0
                                        &pState->macBlock[0],
494
0
                                        SYMCRYPT_GCM_BLOCK_SIZE );
495
0
            pState->bytesInMacBlock = 0;
496
0
        }
497
498
        //
499
        // If there are bytes left in the key stream buffer, then cbData == 0 and we're done.
500
        // If we used up all the bytes, then we are fine, no need to compute the next key stream block
501
        //
502
0
    }
503
504
0
    if( cbData >= SYMCRYPT_GCM_BLOCK_SIZE )
505
0
    {
506
0
        bytesToProcess = cbData & SYMCRYPT_GCM_BLOCK_ROUND_MASK;
507
508
        //
509
        // We use a Gcm function that increments the CTR by 64 bits, rather than the 32 bits that GCM requires.
510
        // As we only support 12-byte nonces, the 32-bit counter never overflows, and we can safely use
511
        // the 64-bit incrementing primitive.
512
        // If we ever support other nonce sizes this is going to be a big problem.
513
        // You can't fake a 32-bit counter using a 64-bit counter function without side-channels that expose
514
        // information about the current counter value.
515
        // With other nonce sizes the actual counter value itself is not public, so we can't expose that.
516
        // We can do two things:
517
        // - create SymCryptAesGcmEncryptXXX32
518
        // - Accept that we leak information about the counter value; after all it is not treated as a
519
        //   secret when the nonce is 12 bytes.
520
        //
521
0
        SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE );
522
523
0
#if SYMCRYPT_CPU_AMD64
524
0
        if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_VAES_256_CODE ) &&
525
0
            (bytesToProcess >= GCM_YMM_MINBLOCKS * SYMCRYPT_GCM_BLOCK_SIZE) &&
526
0
            SymCryptSaveYmm( &SaveData ) == SYMCRYPT_NO_ERROR )
527
0
        {
528
0
            SymCryptAesGcmEncryptStitchedYmm_2048(
529
0
                &pState->pKey->blockcipherKey.aes,
530
0
                &pState->counterBlock[0],
531
0
                &pState->pKey->ghashKey.table[0],
532
0
                &pState->ghashState,
533
0
                pbSrc,
534
0
                pbDst,
535
0
                bytesToProcess );
536
537
0
            SymCryptRestoreYmm( &SaveData );
538
0
        } else {
539
0
            SymCryptAesGcmEncryptStitchedXmm(
540
0
                &pState->pKey->blockcipherKey.aes,
541
0
                &pState->counterBlock[0],
542
0
                &pState->pKey->ghashKey.table[0],
543
0
                &pState->ghashState,
544
0
                pbSrc,
545
0
                pbDst,
546
0
                bytesToProcess );
547
0
        }
548
549
#elif SYMCRYPT_CPU_X86
550
        SymCryptAesGcmEncryptStitchedXmm(
551
            &pState->pKey->blockcipherKey.aes,
552
            &pState->counterBlock[0],
553
            (PSYMCRYPT_GF128_ELEMENT)&pState->pKey->ghashKey.tableSpace[pState->pKey->ghashKey.tableOffset],
554
            &pState->ghashState,
555
            pbSrc,
556
            pbDst,
557
            bytesToProcess );
558
559
#elif SYMCRYPT_CPU_ARM64
560
        SymCryptAesGcmEncryptStitchedNeon(
561
            &pState->pKey->blockcipherKey.aes,
562
            &pState->counterBlock[0],
563
            &pState->pKey->ghashKey.table[0],
564
            &pState->ghashState,
565
            pbSrc,
566
            pbDst,
567
            bytesToProcess );
568
569
#else
570
        SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes,
571
                            &pState->counterBlock[0],
572
                            pbSrc,
573
                            pbDst,
574
                            cbData );
575
        //
576
        // We break the read-once/write once rule here by reading the pbDst data back.
577
        // In this particular situation this is safe, and avoiding it is expensive as it
578
        // requires an extra copy and an extra memory buffer.
579
        // The first write exposes the GCM key stream, independent of the underlying data that
580
        // we are processing. From an attacking point of view we can think of this as literally
581
        // handing over the key stream. So encryption consists of two steps:
582
        // - hand over the key stream
583
        // - MAC some ciphertext
584
        // In this view (which has equivalent security properties to GCM) is obviously doesn't
585
        // matter that we read pbDst back.
586
        //
587
        SymCryptGHashAppendData(&pState->pKey->ghashKey,
588
                                &pState->ghashState,
589
                                pbDst,
590
                                cbData );
591
592
#endif
593
594
0
        pbSrc += bytesToProcess;
595
0
        pbDst += bytesToProcess;
596
0
        cbData -= bytesToProcess;
597
0
    }
598
599
0
    if( cbData > 0 )
600
0
    {
601
0
        SymCryptWipeKnownSize( &pState->keystreamBlock[0], SYMCRYPT_GCM_BLOCK_SIZE );
602
603
0
        SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE );
604
0
        SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes,
605
0
                            &pState->counterBlock[0],
606
0
                            &pState->keystreamBlock[0],
607
0
                            &pState->keystreamBlock[0],
608
0
                            SYMCRYPT_GCM_BLOCK_SIZE );
609
610
0
        SymCryptXorBytes( &pState->keystreamBlock[0], pbSrc, &pState->macBlock[0], cbData );
611
0
        memcpy( pbDst, &pState->macBlock[0], cbData );
612
0
        pState->bytesInMacBlock = cbData;
613
614
        //
615
        // pState->cbData contains the data length after this call already, so it knows how many
616
        // bytes are left in the keystream block
617
        //
618
0
    }
619
0
}
620
621
VOID
622
SYMCRYPT_CALL
623
SymCryptAesGcmDecryptPartOnePass(
624
    _Inout_                 PSYMCRYPT_GCM_STATE pState,
625
    _In_reads_( cbData )    PCBYTE              pbSrc,
626
    _Out_writes_( cbData )  PBYTE               pbDst,
627
                            SIZE_T              cbData )
628
0
{
629
0
    SIZE_T      bytesToProcess;
630
0
#if SYMCRYPT_CPU_AMD64
631
0
    SYMCRYPT_EXTENDED_SAVE_DATA SaveData;
632
0
#endif
633
634
    //
635
    // We have entered the decrypt phase, the AAD has been padded to be a multiple of block size
636
    // We know that the bytes still to use in the key stream buffer and the bytes left to fill the
637
    // macBlock will be the same in the context of this function
638
    //
639
0
    SYMCRYPT_ASSERT( (pState->cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == pState->bytesInMacBlock );
640
641
    //
642
    // We update pState->cbData once before we modify cbData.
643
    // pState->cbData is not used in the rest of this function
644
    //
645
0
    SYMCRYPT_ASSERT( pState->cbData + cbData <= SYMCRYPT_GCM_MAX_DATA_SIZE );
646
0
    pState->cbData += cbData;
647
648
0
    if( pState->bytesInMacBlock > 0 )
649
0
    {
650
0
        bytesToProcess = SYMCRYPT_MIN( cbData, SYMCRYPT_GCM_BLOCK_SIZE - pState->bytesInMacBlock );
651
0
        memcpy( &pState->macBlock[pState->bytesInMacBlock], pbSrc, bytesToProcess );
652
0
        SymCryptXorBytes(
653
0
            &pState->keystreamBlock[pState->bytesInMacBlock],
654
0
            &pState->macBlock[pState->bytesInMacBlock],
655
0
            pbDst,
656
0
            bytesToProcess );
657
658
0
        pbSrc += bytesToProcess;
659
0
        pbDst += bytesToProcess;
660
0
        cbData -= bytesToProcess;
661
0
        pState->bytesInMacBlock += bytesToProcess;
662
663
0
        if( pState->bytesInMacBlock == SYMCRYPT_GCM_BLOCK_SIZE )
664
0
        {
665
0
            SymCryptGHashAppendData(    &pState->pKey->ghashKey,
666
0
                                        &pState->ghashState,
667
0
                                        &pState->macBlock[0],
668
0
                                        SYMCRYPT_GCM_BLOCK_SIZE );
669
0
            pState->bytesInMacBlock = 0;
670
0
        }
671
672
        //
673
        // If there are bytes left in the key stream buffer, then cbData == 0 and we're done.
674
        // If we used up all the bytes, then we are fine, no need to compute the next key stream block
675
        //
676
0
    }
677
678
0
    if( cbData >= SYMCRYPT_GCM_BLOCK_SIZE )
679
0
    {
680
0
        bytesToProcess = cbData & SYMCRYPT_GCM_BLOCK_ROUND_MASK;
681
682
        //
683
        // We use a Gcm function that increments the CTR by 64 bits, rather than the 32 bits that GCM requires.
684
        // As we only support 12-byte nonces, the 32-bit counter never overflows, and we can safely use
685
        // the 64-bit incrementing primitive.
686
        // If we ever support other nonce sizes this is going to be a big problem.
687
        // You can't fake a 32-bit counter using a 64-bit counter function without side-channels that expose
688
        // information about the current counter value.
689
        // With other nonce sizes the actual counter value itself is not public, so we can't expose that.
690
        // We can do two things:
691
        // - create SymCryptAesGcmDecryptXXX32
692
        // - Accept that we leak information about the counter value; after all it is not treated as a
693
        //   secret when the nonce is 12 bytes.
694
        //
695
0
        SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE );
696
697
0
#if SYMCRYPT_CPU_AMD64
698
0
        if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_VAES_256_CODE ) &&
699
0
            (bytesToProcess >= GCM_YMM_MINBLOCKS * SYMCRYPT_GCM_BLOCK_SIZE) &&
700
0
            SymCryptSaveYmm( &SaveData ) == SYMCRYPT_NO_ERROR )
701
0
        {
702
0
            SymCryptAesGcmDecryptStitchedYmm_2048(
703
0
                &pState->pKey->blockcipherKey.aes,
704
0
                &pState->counterBlock[0],
705
0
                &pState->pKey->ghashKey.table[0],
706
0
                &pState->ghashState,
707
0
                pbSrc,
708
0
                pbDst,
709
0
                bytesToProcess );
710
711
0
            SymCryptRestoreYmm( &SaveData );
712
0
        } else {
713
0
            SymCryptAesGcmDecryptStitchedXmm(
714
0
                &pState->pKey->blockcipherKey.aes,
715
0
                &pState->counterBlock[0],
716
0
                &pState->pKey->ghashKey.table[0],
717
0
                &pState->ghashState,
718
0
                pbSrc,
719
0
                pbDst,
720
0
                bytesToProcess );
721
0
        }
722
723
#elif SYMCRYPT_CPU_X86
724
        SymCryptAesGcmDecryptStitchedXmm(
725
            &pState->pKey->blockcipherKey.aes,
726
            &pState->counterBlock[0],
727
            (PSYMCRYPT_GF128_ELEMENT)&pState->pKey->ghashKey.tableSpace[pState->pKey->ghashKey.tableOffset],
728
            &pState->ghashState,
729
            pbSrc,
730
            pbDst,
731
            bytesToProcess );
732
733
#elif SYMCRYPT_CPU_ARM64
734
        SymCryptAesGcmDecryptStitchedNeon(
735
            &pState->pKey->blockcipherKey.aes,
736
            &pState->counterBlock[0],
737
            &pState->pKey->ghashKey.table[0],
738
            &pState->ghashState,
739
            pbSrc,
740
            pbDst,
741
            bytesToProcess );
742
743
#else
744
        SymCryptGHashAppendData(&pState->pKey->ghashKey,
745
                                &pState->ghashState,
746
                                pbSrc,
747
                                cbData );
748
        //
749
        // Do the actual decryption
750
        // This violates the read-once rule, but it is safe for the same reasons as above
751
        // in the encryption case.
752
        //
753
        SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes,
754
                            &pState->counterBlock[0],
755
                            pbSrc,
756
                            pbDst,
757
                            cbData );
758
759
#endif
760
0
        pbSrc += bytesToProcess;
761
0
        pbDst += bytesToProcess;
762
0
        cbData -= bytesToProcess;
763
0
    }
764
765
0
    if( cbData > 0 )
766
0
    {
767
0
        SymCryptWipeKnownSize( &pState->keystreamBlock[0], SYMCRYPT_GCM_BLOCK_SIZE );
768
769
0
        SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE );
770
0
        SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes,
771
0
                            &pState->counterBlock[0],
772
0
                            &pState->keystreamBlock[0],
773
0
                            &pState->keystreamBlock[0],
774
0
                            SYMCRYPT_GCM_BLOCK_SIZE );
775
776
0
        memcpy( &pState->macBlock[0], pbSrc, cbData );
777
0
        SymCryptXorBytes(
778
0
            &pState->keystreamBlock[0],
779
0
            &pState->macBlock[0],
780
0
            pbDst,
781
0
            cbData );
782
783
0
        pState->bytesInMacBlock = cbData;
784
785
        //
786
        // pState->cbData contains the data length after this call already, so it knows how many
787
        // bytes are left in the keystream block
788
        //
789
0
    }
790
0
}
791
792
VOID
793
SYMCRYPT_CALL
794
SymCryptAesGcmEncryptPart(
795
    _Inout_                 PSYMCRYPT_GCM_STATE pState,
796
    _In_reads_( cbData )    PCBYTE              pbSrc,
797
    _Out_writes_( cbData )  PBYTE               pbDst,
798
                            SIZE_T              cbData )
799
0
{
800
0
#if SYMCRYPT_CPU_AMD64
801
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) )
802
0
    {
803
0
        SymCryptAesGcmEncryptPartOnePass( pState, pbSrc, pbDst, cbData );
804
0
    } else {
805
0
        SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData );
806
0
    }
807
808
#elif SYMCRYPT_CPU_X86
809
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
810
811
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) &&
812
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
813
    {
814
        SymCryptAesGcmEncryptPartOnePass( pState, pbSrc, pbDst, cbData );
815
        SymCryptRestoreXmm( &SaveData );
816
    } else {
817
        SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData );
818
    }
819
820
#elif SYMCRYPT_CPU_ARM64
821
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES | SYMCRYPT_CPU_FEATURE_NEON_PMULL ) )
822
    {
823
        SymCryptAesGcmEncryptPartOnePass( pState, pbSrc, pbDst, cbData );
824
    } else {
825
        SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData );
826
    }
827
828
#else
829
    SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData );
830
#endif
831
0
}
832
833
VOID
834
SYMCRYPT_CALL
835
SymCryptAesGcmDecryptPart(
836
    _Inout_                 PSYMCRYPT_GCM_STATE pState,
837
    _In_reads_( cbData )    PCBYTE              pbSrc,
838
    _Out_writes_( cbData )  PBYTE               pbDst,
839
                            SIZE_T              cbData )
840
0
{
841
0
#if SYMCRYPT_CPU_AMD64
842
0
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) )
843
0
    {
844
0
        SymCryptAesGcmDecryptPartOnePass( pState, pbSrc, pbDst, cbData );
845
0
    } else {
846
0
        SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData );
847
0
    }
848
849
#elif SYMCRYPT_CPU_X86
850
    SYMCRYPT_EXTENDED_SAVE_DATA  SaveData;
851
852
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) &&
853
        SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR )
854
    {
855
        SymCryptAesGcmDecryptPartOnePass( pState, pbSrc, pbDst, cbData );
856
        SymCryptRestoreXmm( &SaveData );
857
    } else {
858
        SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData );
859
    }
860
861
#elif SYMCRYPT_CPU_ARM64
862
    if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES | SYMCRYPT_CPU_FEATURE_NEON_PMULL ) )
863
    {
864
        SymCryptAesGcmDecryptPartOnePass( pState, pbSrc, pbDst, cbData );
865
    } else {
866
        SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData );
867
    }
868
869
#else
870
    SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData );
871
#endif
872
0
}