/src/SymCrypt/lib/aes-default.c
Line | Count | Source (jump to first uncovered line) |
1 | | // |
2 | | // aes-default.c code for AES implementation |
3 | | // |
4 | | // Copyright (c) Microsoft Corporation. Licensed under the MIT license. |
5 | | // |
6 | | // This is the interface for the default AES implementation. |
7 | | // On each platform, this is the fastest AES implementation irrespective of code size. |
8 | | // It uses assembler, XMM, or any other trick. |
9 | | // |
10 | | |
11 | | |
12 | | #include "precomp.h" |
13 | | |
14 | | // |
15 | | // Virtual table for generic functions |
16 | | // This allows us to default to generic implementations for some modes without pulling in all the |
17 | | // dedicated functions. |
18 | | // We use this when we cannot use the optimized implementations for some reason. |
19 | | // |
20 | | const SYMCRYPT_BLOCKCIPHER SymCryptAesBlockCipherNoOpt = { |
21 | | &SymCryptAesExpandKey, |
22 | | #if SYMCRYPT_CPU_AMD64 | SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_ARM |
23 | | &SymCryptAesEncryptAsm, |
24 | | &SymCryptAesDecryptAsm, |
25 | | #else |
26 | | &SymCryptAesEncryptC, |
27 | | &SymCryptAesDecryptC, |
28 | | #endif |
29 | | NULL, |
30 | | NULL, |
31 | | NULL, |
32 | | NULL, |
33 | | NULL, |
34 | | NULL, |
35 | | NULL, |
36 | | NULL, |
37 | | |
38 | | SYMCRYPT_AES_BLOCK_SIZE, |
39 | | sizeof( SYMCRYPT_AES_EXPANDED_KEY ), |
40 | | }; |
41 | | |
42 | | VOID |
43 | | SYMCRYPT_CALL |
44 | | SymCryptAes4Sbox( _In_reads_(4) PCBYTE pIn, _Out_writes_(4) PBYTE pOut, BOOL UseSimd ) |
45 | 0 | { |
46 | 0 | #if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64 |
47 | 0 | if( UseSimd ) |
48 | 0 | { |
49 | 0 | SymCryptAes4SboxXmm( pIn, pOut ); |
50 | 0 | } else { |
51 | 0 | SymCryptAes4SboxC( pIn, pOut ); |
52 | 0 | } |
53 | | #elif SYMCRYPT_CPU_ARM64 |
54 | | if( UseSimd ) |
55 | | { |
56 | | SymCryptAes4SboxNeon( pIn, pOut ); |
57 | | } else { |
58 | | SymCryptAes4SboxC( pIn, pOut ); |
59 | | } |
60 | | #else |
61 | | UNREFERENCED_PARAMETER( UseSimd ); |
62 | | SymCryptAes4SboxC( pIn, pOut ); // never use XMM on SaveXmm arch, save/restore overhead is too large. |
63 | | #endif |
64 | 0 | } |
65 | | |
66 | | VOID |
67 | | SYMCRYPT_CALL |
68 | | SymCryptAesCreateDecryptionRoundKey( |
69 | | _In_reads_(16) PCBYTE pEncryptionRoundKey, |
70 | | _Out_writes_(16) PBYTE pDecryptionRoundKey, |
71 | | BOOL UseSimd ) |
72 | 0 | { |
73 | 0 | #if SYMCRYPT_CPU_X86 | SYMCRYPT_CPU_AMD64 |
74 | 0 | if( UseSimd ) |
75 | 0 | { |
76 | 0 | SymCryptAesCreateDecryptionRoundKeyXmm( pEncryptionRoundKey, pDecryptionRoundKey ); |
77 | 0 | } else { |
78 | 0 | SymCryptAesCreateDecryptionRoundKeyC( pEncryptionRoundKey, pDecryptionRoundKey ); |
79 | 0 | } |
80 | | #elif SYMCRYPT_CPU_ARM64 |
81 | | if( UseSimd ) |
82 | | { |
83 | | SymCryptAesCreateDecryptionRoundKeyNeon( pEncryptionRoundKey, pDecryptionRoundKey ); |
84 | | } else { |
85 | | SymCryptAesCreateDecryptionRoundKeyC( pEncryptionRoundKey, pDecryptionRoundKey ); |
86 | | } |
87 | | #else |
88 | | UNREFERENCED_PARAMETER( UseSimd ); |
89 | | SymCryptAesCreateDecryptionRoundKeyC( pEncryptionRoundKey, pDecryptionRoundKey ); // never use XMM on SaveXmm arch, save/restore overhead is too large. |
90 | | #endif |
91 | 0 | } |
92 | | |
93 | | VOID |
94 | | SYMCRYPT_CALL |
95 | | SymCryptAesEncrypt( |
96 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
97 | | _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PCBYTE pbSrc, |
98 | | _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbDst ) |
99 | 0 | { |
100 | 0 | #if SYMCRYPT_CPU_AMD64 |
101 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
102 | 0 | { |
103 | 0 | SymCryptAesEncryptXmm( pExpandedKey, pbSrc, pbDst ); |
104 | 0 | } else { |
105 | 0 | SymCryptAesEncryptAsm( pExpandedKey, pbSrc, pbDst ); |
106 | 0 | } |
107 | | #elif SYMCRYPT_CPU_X86 |
108 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
109 | | |
110 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
111 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
112 | | { |
113 | | SymCryptAesEncryptXmm( pExpandedKey, pbSrc, pbDst ); |
114 | | SymCryptRestoreXmm( &SaveData ); |
115 | | } else { |
116 | | SymCryptAesEncryptAsm( pExpandedKey, pbSrc, pbDst ); |
117 | | } |
118 | | #elif SYMCRYPT_CPU_ARM |
119 | | SymCryptAesEncryptAsm( pExpandedKey, pbSrc, pbDst ); |
120 | | #elif SYMCRYPT_CPU_ARM64 |
121 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
122 | | { |
123 | | SymCryptAesEncryptNeon( pExpandedKey, pbSrc, pbDst ); |
124 | | } else { |
125 | | SymCryptAesEncryptC( pExpandedKey, pbSrc, pbDst ); |
126 | | } |
127 | | #else |
128 | | SymCryptAesEncryptC( pExpandedKey, pbSrc, pbDst ); |
129 | | #endif |
130 | 0 | } |
131 | | |
132 | | VOID |
133 | | SYMCRYPT_CALL |
134 | | SymCryptAesDecrypt( |
135 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
136 | | _In_reads_(SYMCRYPT_AES_BLOCK_SIZE) PCBYTE pbSrc, |
137 | | _Out_writes_(SYMCRYPT_AES_BLOCK_SIZE) PBYTE pbDst ) |
138 | 0 | { |
139 | 0 | #if SYMCRYPT_CPU_AMD64 |
140 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
141 | 0 | { |
142 | 0 | SymCryptAesDecryptXmm( pExpandedKey, pbSrc, pbDst ); |
143 | 0 | } else { |
144 | 0 | SymCryptAesDecryptAsm( pExpandedKey, pbSrc, pbDst ); |
145 | 0 | } |
146 | | #elif SYMCRYPT_CPU_X86 |
147 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
148 | | |
149 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
150 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
151 | | { |
152 | | SymCryptAesDecryptXmm( pExpandedKey, pbSrc, pbDst ); |
153 | | SymCryptRestoreXmm( &SaveData ); |
154 | | } else { |
155 | | SymCryptAesDecryptAsm( pExpandedKey, pbSrc, pbDst ); |
156 | | } |
157 | | #elif SYMCRYPT_CPU_ARM |
158 | | SymCryptAesDecryptAsm( pExpandedKey, pbSrc, pbDst ); |
159 | | #elif SYMCRYPT_CPU_ARM64 |
160 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
161 | | { |
162 | | SymCryptAesDecryptNeon( pExpandedKey, pbSrc, pbDst ); |
163 | | } else { |
164 | | SymCryptAesDecryptC( pExpandedKey, pbSrc, pbDst ); |
165 | | } |
166 | | #else |
167 | | SymCryptAesDecryptC( pExpandedKey, pbSrc, pbDst ); |
168 | | #endif |
169 | 0 | } |
170 | | |
171 | | VOID |
172 | | SYMCRYPT_CALL |
173 | | SymCryptAesCbcEncrypt( |
174 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
175 | | _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue, |
176 | | _In_reads_( cbData ) PCBYTE pbSrc, |
177 | | _Out_writes_( cbData ) PBYTE pbDst, |
178 | | SIZE_T cbData ) |
179 | 0 | { |
180 | 0 | #if SYMCRYPT_CPU_AMD64 |
181 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
182 | 0 | { |
183 | 0 | SymCryptAesCbcEncryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
184 | 0 | } else { |
185 | 0 | SymCryptAesCbcEncryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
186 | 0 | } |
187 | | #elif SYMCRYPT_CPU_X86 |
188 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
189 | | |
190 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
191 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
192 | | { |
193 | | SymCryptAesCbcEncryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
194 | | SymCryptRestoreXmm( &SaveData ); |
195 | | } else { |
196 | | SymCryptAesCbcEncryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
197 | | } |
198 | | #elif SYMCRYPT_CPU_ARM |
199 | | SymCryptAesCbcEncryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
200 | | #elif SYMCRYPT_CPU_ARM64 |
201 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
202 | | { |
203 | | SymCryptAesCbcEncryptNeon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
204 | | } else { |
205 | | SymCryptCbcEncrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
206 | | } |
207 | | #else |
208 | | SymCryptCbcEncrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
209 | | #endif |
210 | 0 | } |
211 | | |
212 | | VOID |
213 | | SYMCRYPT_CALL |
214 | | SymCryptAesCbcDecrypt( |
215 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
216 | | _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue, |
217 | | _In_reads_( cbData ) PCBYTE pbSrc, |
218 | | _Out_writes_( cbData ) PBYTE pbDst, |
219 | | SIZE_T cbData ) |
220 | 0 | { |
221 | 0 | #if SYMCRYPT_CPU_AMD64 |
222 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
223 | 0 | { |
224 | 0 | SymCryptAesCbcDecryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
225 | 0 | } else { |
226 | 0 | SymCryptAesCbcDecryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
227 | 0 | } |
228 | | #elif SYMCRYPT_CPU_X86 |
229 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
230 | | |
231 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
232 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
233 | | { |
234 | | SymCryptAesCbcDecryptXmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
235 | | SymCryptRestoreXmm( &SaveData ); |
236 | | } else { |
237 | | SymCryptAesCbcDecryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
238 | | } |
239 | | #elif SYMCRYPT_CPU_ARM |
240 | | SymCryptAesCbcDecryptAsm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
241 | | #elif SYMCRYPT_CPU_ARM64 |
242 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
243 | | { |
244 | | SymCryptAesCbcDecryptNeon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
245 | | } else { |
246 | | SymCryptCbcDecrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
247 | | } |
248 | | #else |
249 | | SymCryptCbcDecrypt( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
250 | | #endif |
251 | 0 | } |
252 | | |
253 | | VOID |
254 | | SYMCRYPT_CALL |
255 | | SymCryptAesEcbEncrypt( |
256 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
257 | | _In_reads_( cbData ) PCBYTE pbSrc, |
258 | | _Out_writes_( cbData ) PBYTE pbDst, |
259 | | SIZE_T cbData ) |
260 | 0 | { |
261 | 0 | #if SYMCRYPT_CPU_AMD64 |
262 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
263 | 0 | { |
264 | 0 | SymCryptAesEcbEncryptXmm( pExpandedKey, pbSrc, pbDst, cbData ); |
265 | 0 | } else { |
266 | 0 | SymCryptAesEcbEncryptAsm( pExpandedKey, pbSrc, pbDst, cbData ); |
267 | 0 | } |
268 | | #elif SYMCRYPT_CPU_X86 |
269 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
270 | | |
271 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
272 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
273 | | { |
274 | | SymCryptAesEcbEncryptXmm( pExpandedKey, pbSrc, pbDst, cbData ); |
275 | | SymCryptRestoreXmm( &SaveData ); |
276 | | } else { |
277 | | SymCryptAesEcbEncryptAsm( pExpandedKey, pbSrc, pbDst, cbData ); |
278 | | } |
279 | | #elif SYMCRYPT_CPU_ARM |
280 | | SymCryptAesEcbEncryptAsm( pExpandedKey, pbSrc, pbDst, cbData ); |
281 | | #elif SYMCRYPT_CPU_ARM64 |
282 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
283 | | { |
284 | | SymCryptAesEcbEncryptNeon( pExpandedKey, pbSrc, pbDst, cbData ); |
285 | | } else { |
286 | | SymCryptAesEcbEncryptC( pExpandedKey, pbSrc, pbDst, cbData ); |
287 | | } |
288 | | #else |
289 | | SymCryptAesEcbEncryptC( pExpandedKey, pbSrc, pbDst, cbData ); |
290 | | #endif |
291 | 0 | } |
292 | | |
293 | | // |
294 | | // NOTE: There is no reason that SymCryptAesEcbDecrypt could not have unrolled versions similar to |
295 | | // SymCryptAesEcbEncrypt if a real use case requiring large scale Ecb decryption is found. |
296 | | // For now just decrypt 1 block at a time to reduce code size. |
297 | | // |
298 | | VOID |
299 | | SYMCRYPT_CALL |
300 | | SymCryptAesEcbDecrypt( |
301 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
302 | | _In_reads_( cbData ) PCBYTE pbSrc, |
303 | | _Out_writes_( cbData ) PBYTE pbDst, |
304 | | SIZE_T cbData ) |
305 | 0 | { |
306 | 0 | while( cbData >= SYMCRYPT_AES_BLOCK_SIZE ) |
307 | 0 | { |
308 | 0 | SymCryptAesDecrypt( pExpandedKey, pbSrc, pbDst ); |
309 | 0 | pbSrc += SYMCRYPT_AES_BLOCK_SIZE; |
310 | 0 | pbDst += SYMCRYPT_AES_BLOCK_SIZE; |
311 | 0 | cbData -= SYMCRYPT_AES_BLOCK_SIZE; |
312 | 0 | } |
313 | 0 | } |
314 | | |
315 | | VOID |
316 | | SYMCRYPT_CALL |
317 | | SymCryptAesCbcMac( |
318 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
319 | | _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue, |
320 | | _In_reads_( cbData ) PCBYTE pbData, |
321 | | SIZE_T cbData ) |
322 | 0 | { |
323 | 0 | #if SYMCRYPT_CPU_AMD64 |
324 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
325 | 0 | { |
326 | 0 | SymCryptAesCbcMacXmm( pExpandedKey, pbChainingValue, pbData, cbData ); |
327 | 0 | } else { |
328 | 0 | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); |
329 | 0 | SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData ); |
330 | 0 | } |
331 | | #elif SYMCRYPT_CPU_X86 |
332 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
333 | | |
334 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
335 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
336 | | { |
337 | | SymCryptAesCbcMacXmm( pExpandedKey, pbChainingValue, pbData, cbData ); |
338 | | SymCryptRestoreXmm( &SaveData ); |
339 | | } else { |
340 | | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); |
341 | | SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData ); |
342 | | } |
343 | | #elif SYMCRYPT_CPU_ARM64 |
344 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
345 | | { |
346 | | SymCryptAesCbcMacNeon( pExpandedKey, pbChainingValue, pbData, cbData ); |
347 | | } else { |
348 | | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); |
349 | | SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData ); |
350 | | } |
351 | | #else |
352 | | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); |
353 | | SymCryptCbcMac( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbData, cbData ); |
354 | | #endif |
355 | 0 | } |
356 | | |
357 | | VOID |
358 | | SYMCRYPT_CALL |
359 | | SymCryptAesCtrMsb32( |
360 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
361 | | _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue, |
362 | | _In_reads_( cbData ) PCBYTE pbSrc, |
363 | | _Out_writes_( cbData ) PBYTE pbDst, |
364 | | SIZE_T cbData ) |
365 | 0 | { |
366 | 0 | #if SYMCRYPT_CPU_AMD64 |
367 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
368 | 0 | { |
369 | 0 | SymCryptAesCtrMsb32Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
370 | 0 | } else { |
371 | 0 | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy |
372 | 0 | SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
373 | 0 | } |
374 | |
|
375 | | #elif SYMCRYPT_CPU_X86 |
376 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
377 | | |
378 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
379 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
380 | | { |
381 | | SymCryptAesCtrMsb32Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
382 | | SymCryptRestoreXmm( &SaveData ); |
383 | | } else { |
384 | | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy |
385 | | SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
386 | | } |
387 | | |
388 | | #elif SYMCRYPT_CPU_ARM64 |
389 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
390 | | { |
391 | | SymCryptAesCtrMsb32Neon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
392 | | } else { |
393 | | SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
394 | | } |
395 | | |
396 | | #else |
397 | | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy |
398 | | SymCryptCtrMsb32( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
399 | | #endif |
400 | 0 | } |
401 | | |
402 | | VOID |
403 | | SYMCRYPT_CALL |
404 | | SymCryptAesCtrMsb64( |
405 | | _In_ PCSYMCRYPT_AES_EXPANDED_KEY pExpandedKey, |
406 | | _Inout_updates_( SYMCRYPT_AES_BLOCK_SIZE ) PBYTE pbChainingValue, |
407 | | _In_reads_( cbData ) PCBYTE pbSrc, |
408 | | _Out_writes_( cbData ) PBYTE pbDst, |
409 | | SIZE_T cbData ) |
410 | 0 | { |
411 | 0 | #if SYMCRYPT_CPU_AMD64 |
412 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) ) |
413 | 0 | { |
414 | 0 | SymCryptAesCtrMsb64Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
415 | 0 | } else { |
416 | 0 | SymCryptAesCtrMsb64Asm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
417 | 0 | } |
418 | |
|
419 | | #elif SYMCRYPT_CPU_X86 |
420 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
421 | | |
422 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_CODE ) && |
423 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
424 | | { |
425 | | SymCryptAesCtrMsb64Xmm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
426 | | SymCryptRestoreXmm( &SaveData ); |
427 | | } else { |
428 | | SymCryptAesCtrMsb64Asm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
429 | | } |
430 | | |
431 | | #elif SYMCRYPT_CPU_ARM |
432 | | SymCryptAesCtrMsb64Asm( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
433 | | |
434 | | #elif SYMCRYPT_CPU_ARM64 |
435 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES ) ) |
436 | | { |
437 | | SymCryptAesCtrMsb64Neon( pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
438 | | } else { |
439 | | SymCryptCtrMsb64( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
440 | | } |
441 | | |
442 | | #else |
443 | | SYMCRYPT_ASSERT( SymCryptAesBlockCipherNoOpt.blockSize == SYMCRYPT_AES_BLOCK_SIZE ); // keep Prefast happy |
444 | | SymCryptCtrMsb64( &SymCryptAesBlockCipherNoOpt, pExpandedKey, pbChainingValue, pbSrc, pbDst, cbData ); |
445 | | #endif |
446 | 0 | } |
447 | | |
448 | | VOID |
449 | | SYMCRYPT_CALL |
450 | | SymCryptAesGcmEncryptPartOnePass( |
451 | | _Inout_ PSYMCRYPT_GCM_STATE pState, |
452 | | _In_reads_( cbData ) PCBYTE pbSrc, |
453 | | _Out_writes_( cbData ) PBYTE pbDst, |
454 | | SIZE_T cbData ) |
455 | 0 | { |
456 | 0 | SIZE_T bytesToProcess; |
457 | 0 | #if SYMCRYPT_CPU_AMD64 |
458 | 0 | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
459 | 0 | #endif |
460 | | |
461 | | // |
462 | | // We have entered the encrypt phase, the AAD has been padded to be a multiple of block size |
463 | | // We know that the bytes still to use in the key stream buffer and the bytes left to fill the |
464 | | // macBlock will be the same in the context of this function |
465 | | // |
466 | 0 | SYMCRYPT_ASSERT( (pState->cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == pState->bytesInMacBlock ); |
467 | | |
468 | | // |
469 | | // We update pState->cbData once before we modify cbData. |
470 | | // pState->cbData is not used in the rest of this function |
471 | | // |
472 | 0 | SYMCRYPT_ASSERT( pState->cbData + cbData <= SYMCRYPT_GCM_MAX_DATA_SIZE ); |
473 | 0 | pState->cbData += cbData; |
474 | |
|
475 | 0 | if( pState->bytesInMacBlock > 0 ) |
476 | 0 | { |
477 | 0 | bytesToProcess = SYMCRYPT_MIN( cbData, SYMCRYPT_GCM_BLOCK_SIZE - pState->bytesInMacBlock ); |
478 | 0 | SymCryptXorBytes( |
479 | 0 | pbSrc, |
480 | 0 | &pState->keystreamBlock[pState->bytesInMacBlock], |
481 | 0 | &pState->macBlock[pState->bytesInMacBlock], |
482 | 0 | bytesToProcess ); |
483 | 0 | memcpy( pbDst, &pState->macBlock[pState->bytesInMacBlock], bytesToProcess ); |
484 | 0 | pbSrc += bytesToProcess; |
485 | 0 | pbDst += bytesToProcess; |
486 | 0 | cbData -= bytesToProcess; |
487 | 0 | pState->bytesInMacBlock += bytesToProcess; |
488 | |
|
489 | 0 | if( pState->bytesInMacBlock == SYMCRYPT_GCM_BLOCK_SIZE ) |
490 | 0 | { |
491 | 0 | SymCryptGHashAppendData( &pState->pKey->ghashKey, |
492 | 0 | &pState->ghashState, |
493 | 0 | &pState->macBlock[0], |
494 | 0 | SYMCRYPT_GCM_BLOCK_SIZE ); |
495 | 0 | pState->bytesInMacBlock = 0; |
496 | 0 | } |
497 | | |
498 | | // |
499 | | // If there are bytes left in the key stream buffer, then cbData == 0 and we're done. |
500 | | // If we used up all the bytes, then we are fine, no need to compute the next key stream block |
501 | | // |
502 | 0 | } |
503 | |
|
504 | 0 | if( cbData >= SYMCRYPT_GCM_BLOCK_SIZE ) |
505 | 0 | { |
506 | 0 | bytesToProcess = cbData & SYMCRYPT_GCM_BLOCK_ROUND_MASK; |
507 | | |
508 | | // |
509 | | // We use a Gcm function that increments the CTR by 64 bits, rather than the 32 bits that GCM requires. |
510 | | // As we only support 12-byte nonces, the 32-bit counter never overflows, and we can safely use |
511 | | // the 64-bit incrementing primitive. |
512 | | // If we ever support other nonce sizes this is going to be a big problem. |
513 | | // You can't fake a 32-bit counter using a 64-bit counter function without side-channels that expose |
514 | | // information about the current counter value. |
515 | | // With other nonce sizes the actual counter value itself is not public, so we can't expose that. |
516 | | // We can do two things: |
517 | | // - create SymCryptAesGcmEncryptXXX32 |
518 | | // - Accept that we leak information about the counter value; after all it is not treated as a |
519 | | // secret when the nonce is 12 bytes. |
520 | | // |
521 | 0 | SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE ); |
522 | |
|
523 | 0 | #if SYMCRYPT_CPU_AMD64 |
524 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_VAES_256_CODE ) && |
525 | 0 | (bytesToProcess >= GCM_YMM_MINBLOCKS * SYMCRYPT_GCM_BLOCK_SIZE) && |
526 | 0 | SymCryptSaveYmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
527 | 0 | { |
528 | 0 | SymCryptAesGcmEncryptStitchedYmm_2048( |
529 | 0 | &pState->pKey->blockcipherKey.aes, |
530 | 0 | &pState->counterBlock[0], |
531 | 0 | &pState->pKey->ghashKey.table[0], |
532 | 0 | &pState->ghashState, |
533 | 0 | pbSrc, |
534 | 0 | pbDst, |
535 | 0 | bytesToProcess ); |
536 | |
|
537 | 0 | SymCryptRestoreYmm( &SaveData ); |
538 | 0 | } else { |
539 | 0 | SymCryptAesGcmEncryptStitchedXmm( |
540 | 0 | &pState->pKey->blockcipherKey.aes, |
541 | 0 | &pState->counterBlock[0], |
542 | 0 | &pState->pKey->ghashKey.table[0], |
543 | 0 | &pState->ghashState, |
544 | 0 | pbSrc, |
545 | 0 | pbDst, |
546 | 0 | bytesToProcess ); |
547 | 0 | } |
548 | |
|
549 | | #elif SYMCRYPT_CPU_X86 |
550 | | SymCryptAesGcmEncryptStitchedXmm( |
551 | | &pState->pKey->blockcipherKey.aes, |
552 | | &pState->counterBlock[0], |
553 | | (PSYMCRYPT_GF128_ELEMENT)&pState->pKey->ghashKey.tableSpace[pState->pKey->ghashKey.tableOffset], |
554 | | &pState->ghashState, |
555 | | pbSrc, |
556 | | pbDst, |
557 | | bytesToProcess ); |
558 | | |
559 | | #elif SYMCRYPT_CPU_ARM64 |
560 | | SymCryptAesGcmEncryptStitchedNeon( |
561 | | &pState->pKey->blockcipherKey.aes, |
562 | | &pState->counterBlock[0], |
563 | | &pState->pKey->ghashKey.table[0], |
564 | | &pState->ghashState, |
565 | | pbSrc, |
566 | | pbDst, |
567 | | bytesToProcess ); |
568 | | |
569 | | #else |
570 | | SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes, |
571 | | &pState->counterBlock[0], |
572 | | pbSrc, |
573 | | pbDst, |
574 | | cbData ); |
575 | | // |
576 | | // We break the read-once/write once rule here by reading the pbDst data back. |
577 | | // In this particular situation this is safe, and avoiding it is expensive as it |
578 | | // requires an extra copy and an extra memory buffer. |
579 | | // The first write exposes the GCM key stream, independent of the underlying data that |
580 | | // we are processing. From an attacking point of view we can think of this as literally |
581 | | // handing over the key stream. So encryption consists of two steps: |
582 | | // - hand over the key stream |
583 | | // - MAC some ciphertext |
584 | | // In this view (which has equivalent security properties to GCM) is obviously doesn't |
585 | | // matter that we read pbDst back. |
586 | | // |
587 | | SymCryptGHashAppendData(&pState->pKey->ghashKey, |
588 | | &pState->ghashState, |
589 | | pbDst, |
590 | | cbData ); |
591 | | |
592 | | #endif |
593 | |
|
594 | 0 | pbSrc += bytesToProcess; |
595 | 0 | pbDst += bytesToProcess; |
596 | 0 | cbData -= bytesToProcess; |
597 | 0 | } |
598 | |
|
599 | 0 | if( cbData > 0 ) |
600 | 0 | { |
601 | 0 | SymCryptWipeKnownSize( &pState->keystreamBlock[0], SYMCRYPT_GCM_BLOCK_SIZE ); |
602 | |
|
603 | 0 | SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE ); |
604 | 0 | SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes, |
605 | 0 | &pState->counterBlock[0], |
606 | 0 | &pState->keystreamBlock[0], |
607 | 0 | &pState->keystreamBlock[0], |
608 | 0 | SYMCRYPT_GCM_BLOCK_SIZE ); |
609 | |
|
610 | 0 | SymCryptXorBytes( &pState->keystreamBlock[0], pbSrc, &pState->macBlock[0], cbData ); |
611 | 0 | memcpy( pbDst, &pState->macBlock[0], cbData ); |
612 | 0 | pState->bytesInMacBlock = cbData; |
613 | | |
614 | | // |
615 | | // pState->cbData contains the data length after this call already, so it knows how many |
616 | | // bytes are left in the keystream block |
617 | | // |
618 | 0 | } |
619 | 0 | } |
620 | | |
621 | | VOID |
622 | | SYMCRYPT_CALL |
623 | | SymCryptAesGcmDecryptPartOnePass( |
624 | | _Inout_ PSYMCRYPT_GCM_STATE pState, |
625 | | _In_reads_( cbData ) PCBYTE pbSrc, |
626 | | _Out_writes_( cbData ) PBYTE pbDst, |
627 | | SIZE_T cbData ) |
628 | 0 | { |
629 | 0 | SIZE_T bytesToProcess; |
630 | 0 | #if SYMCRYPT_CPU_AMD64 |
631 | 0 | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
632 | 0 | #endif |
633 | | |
634 | | // |
635 | | // We have entered the decrypt phase, the AAD has been padded to be a multiple of block size |
636 | | // We know that the bytes still to use in the key stream buffer and the bytes left to fill the |
637 | | // macBlock will be the same in the context of this function |
638 | | // |
639 | 0 | SYMCRYPT_ASSERT( (pState->cbData & SYMCRYPT_GCM_BLOCK_MOD_MASK) == pState->bytesInMacBlock ); |
640 | | |
641 | | // |
642 | | // We update pState->cbData once before we modify cbData. |
643 | | // pState->cbData is not used in the rest of this function |
644 | | // |
645 | 0 | SYMCRYPT_ASSERT( pState->cbData + cbData <= SYMCRYPT_GCM_MAX_DATA_SIZE ); |
646 | 0 | pState->cbData += cbData; |
647 | |
|
648 | 0 | if( pState->bytesInMacBlock > 0 ) |
649 | 0 | { |
650 | 0 | bytesToProcess = SYMCRYPT_MIN( cbData, SYMCRYPT_GCM_BLOCK_SIZE - pState->bytesInMacBlock ); |
651 | 0 | memcpy( &pState->macBlock[pState->bytesInMacBlock], pbSrc, bytesToProcess ); |
652 | 0 | SymCryptXorBytes( |
653 | 0 | &pState->keystreamBlock[pState->bytesInMacBlock], |
654 | 0 | &pState->macBlock[pState->bytesInMacBlock], |
655 | 0 | pbDst, |
656 | 0 | bytesToProcess ); |
657 | |
|
658 | 0 | pbSrc += bytesToProcess; |
659 | 0 | pbDst += bytesToProcess; |
660 | 0 | cbData -= bytesToProcess; |
661 | 0 | pState->bytesInMacBlock += bytesToProcess; |
662 | |
|
663 | 0 | if( pState->bytesInMacBlock == SYMCRYPT_GCM_BLOCK_SIZE ) |
664 | 0 | { |
665 | 0 | SymCryptGHashAppendData( &pState->pKey->ghashKey, |
666 | 0 | &pState->ghashState, |
667 | 0 | &pState->macBlock[0], |
668 | 0 | SYMCRYPT_GCM_BLOCK_SIZE ); |
669 | 0 | pState->bytesInMacBlock = 0; |
670 | 0 | } |
671 | | |
672 | | // |
673 | | // If there are bytes left in the key stream buffer, then cbData == 0 and we're done. |
674 | | // If we used up all the bytes, then we are fine, no need to compute the next key stream block |
675 | | // |
676 | 0 | } |
677 | |
|
678 | 0 | if( cbData >= SYMCRYPT_GCM_BLOCK_SIZE ) |
679 | 0 | { |
680 | 0 | bytesToProcess = cbData & SYMCRYPT_GCM_BLOCK_ROUND_MASK; |
681 | | |
682 | | // |
683 | | // We use a Gcm function that increments the CTR by 64 bits, rather than the 32 bits that GCM requires. |
684 | | // As we only support 12-byte nonces, the 32-bit counter never overflows, and we can safely use |
685 | | // the 64-bit incrementing primitive. |
686 | | // If we ever support other nonce sizes this is going to be a big problem. |
687 | | // You can't fake a 32-bit counter using a 64-bit counter function without side-channels that expose |
688 | | // information about the current counter value. |
689 | | // With other nonce sizes the actual counter value itself is not public, so we can't expose that. |
690 | | // We can do two things: |
691 | | // - create SymCryptAesGcmDecryptXXX32 |
692 | | // - Accept that we leak information about the counter value; after all it is not treated as a |
693 | | // secret when the nonce is 12 bytes. |
694 | | // |
695 | 0 | SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE ); |
696 | |
|
697 | 0 | #if SYMCRYPT_CPU_AMD64 |
698 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_VAES_256_CODE ) && |
699 | 0 | (bytesToProcess >= GCM_YMM_MINBLOCKS * SYMCRYPT_GCM_BLOCK_SIZE) && |
700 | 0 | SymCryptSaveYmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
701 | 0 | { |
702 | 0 | SymCryptAesGcmDecryptStitchedYmm_2048( |
703 | 0 | &pState->pKey->blockcipherKey.aes, |
704 | 0 | &pState->counterBlock[0], |
705 | 0 | &pState->pKey->ghashKey.table[0], |
706 | 0 | &pState->ghashState, |
707 | 0 | pbSrc, |
708 | 0 | pbDst, |
709 | 0 | bytesToProcess ); |
710 | |
|
711 | 0 | SymCryptRestoreYmm( &SaveData ); |
712 | 0 | } else { |
713 | 0 | SymCryptAesGcmDecryptStitchedXmm( |
714 | 0 | &pState->pKey->blockcipherKey.aes, |
715 | 0 | &pState->counterBlock[0], |
716 | 0 | &pState->pKey->ghashKey.table[0], |
717 | 0 | &pState->ghashState, |
718 | 0 | pbSrc, |
719 | 0 | pbDst, |
720 | 0 | bytesToProcess ); |
721 | 0 | } |
722 | |
|
723 | | #elif SYMCRYPT_CPU_X86 |
724 | | SymCryptAesGcmDecryptStitchedXmm( |
725 | | &pState->pKey->blockcipherKey.aes, |
726 | | &pState->counterBlock[0], |
727 | | (PSYMCRYPT_GF128_ELEMENT)&pState->pKey->ghashKey.tableSpace[pState->pKey->ghashKey.tableOffset], |
728 | | &pState->ghashState, |
729 | | pbSrc, |
730 | | pbDst, |
731 | | bytesToProcess ); |
732 | | |
733 | | #elif SYMCRYPT_CPU_ARM64 |
734 | | SymCryptAesGcmDecryptStitchedNeon( |
735 | | &pState->pKey->blockcipherKey.aes, |
736 | | &pState->counterBlock[0], |
737 | | &pState->pKey->ghashKey.table[0], |
738 | | &pState->ghashState, |
739 | | pbSrc, |
740 | | pbDst, |
741 | | bytesToProcess ); |
742 | | |
743 | | #else |
744 | | SymCryptGHashAppendData(&pState->pKey->ghashKey, |
745 | | &pState->ghashState, |
746 | | pbSrc, |
747 | | cbData ); |
748 | | // |
749 | | // Do the actual decryption |
750 | | // This violates the read-once rule, but it is safe for the same reasons as above |
751 | | // in the encryption case. |
752 | | // |
753 | | SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes, |
754 | | &pState->counterBlock[0], |
755 | | pbSrc, |
756 | | pbDst, |
757 | | cbData ); |
758 | | |
759 | | #endif |
760 | 0 | pbSrc += bytesToProcess; |
761 | 0 | pbDst += bytesToProcess; |
762 | 0 | cbData -= bytesToProcess; |
763 | 0 | } |
764 | |
|
765 | 0 | if( cbData > 0 ) |
766 | 0 | { |
767 | 0 | SymCryptWipeKnownSize( &pState->keystreamBlock[0], SYMCRYPT_GCM_BLOCK_SIZE ); |
768 | |
|
769 | 0 | SYMCRYPT_ASSERT( pState->pKey->pBlockCipher->blockSize == SYMCRYPT_GCM_BLOCK_SIZE ); |
770 | 0 | SymCryptAesCtrMsb32(&pState->pKey->blockcipherKey.aes, |
771 | 0 | &pState->counterBlock[0], |
772 | 0 | &pState->keystreamBlock[0], |
773 | 0 | &pState->keystreamBlock[0], |
774 | 0 | SYMCRYPT_GCM_BLOCK_SIZE ); |
775 | |
|
776 | 0 | memcpy( &pState->macBlock[0], pbSrc, cbData ); |
777 | 0 | SymCryptXorBytes( |
778 | 0 | &pState->keystreamBlock[0], |
779 | 0 | &pState->macBlock[0], |
780 | 0 | pbDst, |
781 | 0 | cbData ); |
782 | |
|
783 | 0 | pState->bytesInMacBlock = cbData; |
784 | | |
785 | | // |
786 | | // pState->cbData contains the data length after this call already, so it knows how many |
787 | | // bytes are left in the keystream block |
788 | | // |
789 | 0 | } |
790 | 0 | } |
791 | | |
792 | | VOID |
793 | | SYMCRYPT_CALL |
794 | | SymCryptAesGcmEncryptPart( |
795 | | _Inout_ PSYMCRYPT_GCM_STATE pState, |
796 | | _In_reads_( cbData ) PCBYTE pbSrc, |
797 | | _Out_writes_( cbData ) PBYTE pbDst, |
798 | | SIZE_T cbData ) |
799 | 0 | { |
800 | 0 | #if SYMCRYPT_CPU_AMD64 |
801 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) ) |
802 | 0 | { |
803 | 0 | SymCryptAesGcmEncryptPartOnePass( pState, pbSrc, pbDst, cbData ); |
804 | 0 | } else { |
805 | 0 | SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
806 | 0 | } |
807 | |
|
808 | | #elif SYMCRYPT_CPU_X86 |
809 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
810 | | |
811 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) && |
812 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
813 | | { |
814 | | SymCryptAesGcmEncryptPartOnePass( pState, pbSrc, pbDst, cbData ); |
815 | | SymCryptRestoreXmm( &SaveData ); |
816 | | } else { |
817 | | SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
818 | | } |
819 | | |
820 | | #elif SYMCRYPT_CPU_ARM64 |
821 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES | SYMCRYPT_CPU_FEATURE_NEON_PMULL ) ) |
822 | | { |
823 | | SymCryptAesGcmEncryptPartOnePass( pState, pbSrc, pbDst, cbData ); |
824 | | } else { |
825 | | SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
826 | | } |
827 | | |
828 | | #else |
829 | | SymCryptGcmEncryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
830 | | #endif |
831 | 0 | } |
832 | | |
833 | | VOID |
834 | | SYMCRYPT_CALL |
835 | | SymCryptAesGcmDecryptPart( |
836 | | _Inout_ PSYMCRYPT_GCM_STATE pState, |
837 | | _In_reads_( cbData ) PCBYTE pbSrc, |
838 | | _Out_writes_( cbData ) PBYTE pbDst, |
839 | | SIZE_T cbData ) |
840 | 0 | { |
841 | 0 | #if SYMCRYPT_CPU_AMD64 |
842 | 0 | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) ) |
843 | 0 | { |
844 | 0 | SymCryptAesGcmDecryptPartOnePass( pState, pbSrc, pbDst, cbData ); |
845 | 0 | } else { |
846 | 0 | SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
847 | 0 | } |
848 | |
|
849 | | #elif SYMCRYPT_CPU_X86 |
850 | | SYMCRYPT_EXTENDED_SAVE_DATA SaveData; |
851 | | |
852 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURES_FOR_AESNI_PCLMULQDQ_CODE ) && |
853 | | SymCryptSaveXmm( &SaveData ) == SYMCRYPT_NO_ERROR ) |
854 | | { |
855 | | SymCryptAesGcmDecryptPartOnePass( pState, pbSrc, pbDst, cbData ); |
856 | | SymCryptRestoreXmm( &SaveData ); |
857 | | } else { |
858 | | SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
859 | | } |
860 | | |
861 | | #elif SYMCRYPT_CPU_ARM64 |
862 | | if( SYMCRYPT_CPU_FEATURES_PRESENT( SYMCRYPT_CPU_FEATURE_NEON_AES | SYMCRYPT_CPU_FEATURE_NEON_PMULL ) ) |
863 | | { |
864 | | SymCryptAesGcmDecryptPartOnePass( pState, pbSrc, pbDst, cbData ); |
865 | | } else { |
866 | | SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
867 | | } |
868 | | |
869 | | #else |
870 | | SymCryptGcmDecryptPartTwoPass( pState, pbSrc, pbDst, cbData ); |
871 | | #endif |
872 | 0 | } |