/src/openssl/crypto/aes/aes_vaes512_intrinsics.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright 2025-2026 The OpenSSL Project Authors. All Rights Reserved. |
3 | | * Copyright (C) 2026, Advanced Micro Devices, all rights reserved. |
4 | | * |
5 | | * Licensed under the Apache License 2.0 (the "License"). You may not use |
6 | | * this file except in compliance with the License. You can obtain a copy |
7 | | * in the file LICENSE in the source distribution or at |
8 | | * https://www.openssl.org/source/license.html |
9 | | * |
10 | | * AVX-512/VAES-512 intrinsic implementations of AES modes. |
11 | | * |
12 | | * This translation unit gathers the VAES-accelerated AES mode |
13 | | * implementations together with the shared 512-bit primitives they rely on: |
14 | | * - portable compiler abstractions for ISA targeting and inlining, |
15 | | * - a macro that emits 1x/2x/4x parallel 512-bit AES round functions, |
16 | | * - macros to broadcast-load and securely scrub the round-key schedule, |
17 | | * - a runtime CPU capability check. |
18 | | * |
19 | | * Currently implemented: AES-CTR encryption and AES-CBC decryption. |
20 | | * |
21 | | * CBC encryption is inherently serial (each ciphertext block depends on the |
22 | | * previous one), so VAES provides no benefit there -- that path falls back to |
23 | | * the aesni_cbc_encrypt assembly routine. CBC decryption is parallel: all |
24 | | * blocks are decrypted independently, then XORed with the preceding ciphertext |
25 | | * block (or the IV for the first). |
26 | | */ |
27 | | |
28 | | #include "internal/deprecated.h" |
29 | | |
30 | | #include <openssl/opensslconf.h> |
31 | | #include "internal/cryptlib.h" |
32 | | #include <openssl/aes.h> |
33 | | #include "crypto/modes.h" |
34 | | #include "crypto/aes_platform.h" |
35 | | |
36 | | #if VAES512_ELIGIBLE |
37 | | |
38 | | #include <openssl/e_os2.h> |
39 | | #include <openssl/modes.h> |
40 | | #include <immintrin.h> |
41 | | |
42 | | /* Function prototypes */ |
43 | | void ossl_aes_ctr_vaes(const unsigned char *in, unsigned char *out, |
44 | | size_t length, const AES_KEY *key, |
45 | | unsigned char *counter, |
46 | | unsigned char *ecount_buf, unsigned int *num); |
47 | | int ossl_aes_ctr_vaes_eligible(void); |
48 | | void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out, |
49 | | size_t len, const void *key, |
50 | | unsigned char ivec[16], int enc); |
51 | | int ossl_aes_cbc_vaes_eligible(void); |
52 | | |
53 | | /* Forward declarations — defined in aesni-x86_64.pl assembly */ |
54 | | void aesni_encrypt(const unsigned char *in, unsigned char *out, const AES_KEY *key); |
55 | | void aesni_cbc_encrypt(const unsigned char *in, unsigned char *out, |
56 | | size_t len, const AES_KEY *key, |
57 | | unsigned char *ivec, int enc); |
58 | | void aesni_decrypt(const unsigned char *in, unsigned char *out, |
59 | | const AES_KEY *key); |
60 | | |
61 | | /* |
62 | | * Keep the cleanup out of line in assembly. Its opacity to C and LTO |
63 | | * optimizers is what prevents the cleanup call and stores from being |
64 | | * eliminated as dead. Do not replace it with a compiler-visible C function. |
65 | | */ |
66 | | void ossl_aes_vaes_cleanup(void *key_schedule, size_t num_keys); |
67 | | int ossl_aes_vaes_cleanup_eligible(void); |
68 | | |
69 | | /* Portable compiler abstractions for inlining and ISA target selection */ |
70 | | #define OSSL_VAES512_STRINGIFY_IMPL_(a) #a |
71 | | #define OSSL_VAES512_STRINGIFY_(a) OSSL_VAES512_STRINGIFY_IMPL_(a) |
72 | | |
73 | | #ifdef __clang__ |
74 | | #define OPENSSL_TARGET_VAES512 \ |
75 | | _Pragma(OSSL_VAES512_STRINGIFY_(clang attribute push( \ |
76 | | __attribute__((target("avx512f,avx512dq,avx512bw,vaes,aes"))), \ |
77 | | apply_to = function))) |
78 | | #define OPENSSL_UNTARGET_VAES512 _Pragma("clang attribute pop") |
79 | | #elif defined(__GNUC__) |
80 | | #define OPENSSL_TARGET_VAES512 \ |
81 | | _Pragma("GCC push_options") \ |
82 | | _Pragma(OSSL_VAES512_STRINGIFY_(GCC target("avx512f,avx512dq,avx512bw,vaes,aes"))) |
83 | | #define OPENSSL_UNTARGET_VAES512 _Pragma("GCC pop_options") |
84 | | #else |
85 | | /* MSVC: all intrinsics are always available via <immintrin.h>. */ |
86 | | #define OPENSSL_TARGET_VAES512 |
87 | | #define OPENSSL_UNTARGET_VAES512 |
88 | | #endif |
89 | | |
90 | | #if defined(__GNUC__) || defined(__clang__) |
91 | | #define OSSL_FUNC_ALWAYS_INLINE static inline __attribute__((always_inline)) |
92 | | #define OSSL_FUNC_NOINLINE __attribute__((noinline)) |
93 | | #elif defined(_MSC_VER) |
94 | | #define OSSL_FUNC_ALWAYS_INLINE static __forceinline |
95 | | #define OSSL_FUNC_NOINLINE __declspec(noinline) |
96 | | #else |
97 | | #define OSSL_FUNC_ALWAYS_INLINE static inline |
98 | | #define OSSL_FUNC_NOINLINE |
99 | | #endif |
100 | | |
101 | | /* |
102 | | * Runtime CPU capability check shared by all VAES-512 mode helpers. |
103 | | * Uses only OPENSSL_ia32cap_P bit tests, so it requires no special ISA |
104 | | * target and is safe to define outside an OPENSSL_TARGET_VAES512 region. |
105 | | */ |
106 | | static ossl_inline int ossl_vaes512_cpu_capable(void) |
107 | 32 | { |
108 | 32 | return (OPENSSL_ia32cap_P[2] & (1 << 16)) /* AVX512F */ |
109 | 0 | && (OPENSSL_ia32cap_P[2] & (1 << 17)) /* AVX512DQ */ |
110 | 0 | && (OPENSSL_ia32cap_P[2] & (1 << 30)) /* AVX512BW */ |
111 | 0 | && (OPENSSL_ia32cap_P[3] & (1 << 9)); /* AVX512VAES */ |
112 | 32 | } |
113 | | |
114 | | /* |
115 | | * Emit the 1x/2x/4x parallel 512-bit AES round functions for a given round |
116 | | * count. TAG names the generated functions (e.g. AesEnc, AesDec); AESOP and |
117 | | * AESLAST select the middle-round and last-round intrinsics (encrypt vs |
118 | | * decrypt). Each __m512i packs four independent 128-bit AES blocks, and |
119 | | * always_inline keeps the round keys resident in ZMM registers. |
120 | | * |
121 | | * Must be expanded inside an OPENSSL_TARGET_VAES512 region. |
122 | | */ |
123 | | #define OSSL_VAES512_DEFINE_ROUNDS(TAG, ROUNDS, AESOP, AESLAST) \ |
124 | | OSSL_FUNC_ALWAYS_INLINE \ |
125 | | void TAG##_4x512_##ROUNDS( \ |
126 | | __m512i *b1, __m512i *b2, __m512i *b3, __m512i *b4, \ |
127 | | const __m512i *rk) \ |
128 | 0 | { \ |
129 | 0 | *b1 = _mm512_xor_si512(*b1, rk[0]); \ |
130 | 0 | *b2 = _mm512_xor_si512(*b2, rk[0]); \ |
131 | 0 | *b3 = _mm512_xor_si512(*b3, rk[0]); \ |
132 | 0 | *b4 = _mm512_xor_si512(*b4, rk[0]); \ |
133 | 0 | for (int i = 1; i < ROUNDS; i++) { \ |
134 | 0 | *b1 = AESOP(*b1, rk[i]); \ |
135 | 0 | *b2 = AESOP(*b2, rk[i]); \ |
136 | 0 | *b3 = AESOP(*b3, rk[i]); \ |
137 | 0 | *b4 = AESOP(*b4, rk[i]); \ |
138 | 0 | } \ |
139 | 0 | *b1 = AESLAST(*b1, rk[ROUNDS]); \ |
140 | 0 | *b2 = AESLAST(*b2, rk[ROUNDS]); \ |
141 | 0 | *b3 = AESLAST(*b3, rk[ROUNDS]); \ |
142 | 0 | *b4 = AESLAST(*b4, rk[ROUNDS]); \ |
143 | 0 | } \ Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_4x512_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_4x512_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_4x512_14 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_4x512_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_4x512_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_4x512_14 |
144 | | \ |
145 | | OSSL_FUNC_ALWAYS_INLINE \ |
146 | | void TAG##_2x512_##ROUNDS( \ |
147 | | __m512i *b1, __m512i *b2, const __m512i *rk) \ |
148 | 0 | { \ |
149 | 0 | *b1 = _mm512_xor_si512(*b1, rk[0]); \ |
150 | 0 | *b2 = _mm512_xor_si512(*b2, rk[0]); \ |
151 | 0 | for (int i = 1; i < ROUNDS; i++) { \ |
152 | 0 | *b1 = AESOP(*b1, rk[i]); \ |
153 | 0 | *b2 = AESOP(*b2, rk[i]); \ |
154 | 0 | } \ |
155 | 0 | *b1 = AESLAST(*b1, rk[ROUNDS]); \ |
156 | 0 | *b2 = AESLAST(*b2, rk[ROUNDS]); \ |
157 | 0 | } \ Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_2x512_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_2x512_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_2x512_14 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_2x512_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_2x512_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_2x512_14 |
158 | | \ |
159 | | OSSL_FUNC_ALWAYS_INLINE \ |
160 | | void TAG##_1x512_##ROUNDS( \ |
161 | | __m512i *b1, const __m512i *rk) \ |
162 | 0 | { \ |
163 | 0 | *b1 = _mm512_xor_si512(*b1, rk[0]); \ |
164 | 0 | for (int i = 1; i < ROUNDS; i++) \ |
165 | 0 | *b1 = AESOP(*b1, rk[i]); \ |
166 | 0 | *b1 = AESLAST(*b1, rk[ROUNDS]); \ |
167 | 0 | } Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_1x512_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_1x512_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesEnc_1x512_14 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_1x512_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_1x512_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:AesDec_1x512_14 |
168 | | |
169 | | /* Emit the AES encryption round helpers (used by CTR, CFB, GCM, ...). */ |
170 | | #define OSSL_VAES512_DEFINE_ENCRYPT(ROUNDS) \ |
171 | | OSSL_VAES512_DEFINE_ROUNDS(AesEnc, ROUNDS, \ |
172 | | _mm512_aesenc_epi128, _mm512_aesenclast_epi128) |
173 | | |
174 | | /* Emit the AES decryption round helpers (used by CBC decrypt). */ |
175 | | #define OSSL_VAES512_DEFINE_DECRYPT(ROUNDS) \ |
176 | | OSSL_VAES512_DEFINE_ROUNDS(AesDec, ROUNDS, \ |
177 | | _mm512_aesdec_epi128, _mm512_aesdeclast_epi128) |
178 | | |
179 | | /* |
180 | | * Broadcast-load the AES round-key schedule into an array of ZMM registers, |
181 | | * replicating each 128-bit round key across all four lanes. key is an |
182 | | * AES_KEY *, rk an array of at least NR+1 __m512i. |
183 | | * |
184 | | * Must be expanded inside an OPENSSL_TARGET_VAES512 region. |
185 | | */ |
186 | | #define OSSL_VAES512_LOAD_ROUNDKEYS(rk, key, NR) \ |
187 | 0 | do { \ |
188 | 0 | const unsigned char *rk_bytes_ = (const unsigned char *)(key)->rd_key; \ |
189 | 0 | for (int i_ = 0; i_ <= (NR); i_++) { \ |
190 | 0 | __m128i t_ = _mm_loadu_si128( \ |
191 | 0 | (const __m128i *)(rk_bytes_ + i_ * 16)); \ |
192 | 0 | (rk)[i_] = _mm512_broadcast_i32x4(t_); \ |
193 | 0 | } \ |
194 | 0 | } while (0) |
195 | | |
196 | | OPENSSL_TARGET_VAES512 |
197 | | |
198 | | /* AES encryption round helpers (1x/2x/4x parallel 512-bit blocks). */ |
199 | | OSSL_VAES512_DEFINE_ENCRYPT(10) /* AES-128 */ |
200 | | OSSL_VAES512_DEFINE_ENCRYPT(12) /* AES-192 */ |
201 | | OSSL_VAES512_DEFINE_ENCRYPT(14) /* AES-256 */ |
202 | | |
203 | | /*- |
204 | | * Counter initialisation. |
205 | | * |
206 | | * Counters are kept in little-endian form (full 128-bit byte swap) and |
207 | | * incremented with 64-bit arithmetic. This is safe for all practical |
208 | | * counter values — encrypting data less than 2^68 bytes. |
209 | | */ |
210 | | static inline __m512i ctr_swap_mask(void) |
211 | 0 | { |
212 | 0 | return _mm512_set_epi32(0x00010203, 0x04050607, |
213 | 0 | 0x08090a0b, 0x0c0d0e0f, |
214 | 0 | 0x00010203, 0x04050607, |
215 | 0 | 0x08090a0b, 0x0c0d0e0f, |
216 | 0 | 0x00010203, 0x04050607, |
217 | 0 | 0x08090a0b, 0x0c0d0e0f, |
218 | 0 | 0x00010203, 0x04050607, |
219 | 0 | 0x08090a0b, 0x0c0d0e0f); |
220 | 0 | } |
221 | | |
222 | | static inline __m512i ctr_init4(const unsigned char *iv, __m512i swap) |
223 | 0 | { |
224 | | /* unaligned 128-bit load for iv and broadcast */ |
225 | 0 | __m128i iv128 = _mm_loadu_si128((const __m128i *)iv); |
226 | 0 | __m512i c = _mm512_broadcast_i64x2(iv128); |
227 | 0 | c = _mm512_shuffle_epi8(c, swap); |
228 | 0 | c = _mm512_add_epi64(c, _mm512_set_epi64(0, 3, 0, 2, 0, 1, 0, 0)); |
229 | 0 | return c; |
230 | 0 | } |
231 | | |
232 | | /*- |
233 | | * CTR-mode processing, templated per round count. |
234 | | * |
235 | | * Processes as many blocks as possible: |
236 | | * 16 blocks at a time (4×zmm = 4×4 = 16 blocks) |
237 | | * 8 blocks at a time (2×zmm) |
238 | | * 4 blocks at a time (1×zmm) |
239 | | * 0-3 tail blocks + partial residue via masked load/store |
240 | | */ |
241 | | #define DEFINE_CTR_BLOCK(NR) \ |
242 | | OSSL_FUNC_NOINLINE \ |
243 | | static void ctr_process_##NR( \ |
244 | | const unsigned char *in, unsigned char *out, \ |
245 | | size_t len, const AES_KEY *key, unsigned char *iv) \ |
246 | 0 | { \ |
247 | 0 | __m512i rk[NR + 1]; \ |
248 | 0 | OSSL_VAES512_LOAD_ROUNDKEYS(rk, key, NR); \ |
249 | 0 | \ |
250 | 0 | const __m512i *p_in = (const __m512i *)in; \ |
251 | 0 | __m512i *p_out = (__m512i *)out; \ |
252 | 0 | __m512i swap = ctr_swap_mask(); \ |
253 | 0 | __m512i c1 = ctr_init4(iv, swap); \ |
254 | 0 | \ |
255 | 0 | __m512i a1, a2, a3, a4; \ |
256 | 0 | __m512i b1, b2, b3, b4; \ |
257 | 0 | __m512i c2, c3, c4; \ |
258 | 0 | \ |
259 | 0 | size_t blocks = len / AES_BLOCK_SIZE; \ |
260 | 0 | size_t res = len % AES_BLOCK_SIZE; \ |
261 | 0 | \ |
262 | 0 | const __m512i inc4 = _mm512_set_epi64(0, 4, 0, 4, 0, 4, 0, 4); \ |
263 | 0 | const __m512i inc8 = _mm512_set_epi64(0, 8, 0, 8, 0, 8, 0, 8); \ |
264 | 0 | const __m512i inc12 = _mm512_set_epi64(0, 12, 0, 12, 0, 12, 0, 12); \ |
265 | 0 | const __m512i inc16 = _mm512_set_epi64(0, 16, 0, 16, 0, 16, 0, 16); \ |
266 | 0 | \ |
267 | 0 | /* --- 16-block (4×zmm) main loop --- */ \ |
268 | 0 | while (blocks >= 16) { \ |
269 | 0 | c2 = _mm512_add_epi64(c1, inc4); \ |
270 | 0 | c3 = _mm512_add_epi64(c1, inc8); \ |
271 | 0 | c4 = _mm512_add_epi64(c1, inc12); \ |
272 | 0 | \ |
273 | 0 | a1 = _mm512_loadu_si512(p_in); \ |
274 | 0 | a2 = _mm512_loadu_si512(p_in + 1); \ |
275 | 0 | a3 = _mm512_loadu_si512(p_in + 2); \ |
276 | 0 | a4 = _mm512_loadu_si512(p_in + 3); \ |
277 | 0 | \ |
278 | 0 | b1 = _mm512_shuffle_epi8(c1, swap); \ |
279 | 0 | b2 = _mm512_shuffle_epi8(c2, swap); \ |
280 | 0 | b3 = _mm512_shuffle_epi8(c3, swap); \ |
281 | 0 | b4 = _mm512_shuffle_epi8(c4, swap); \ |
282 | 0 | \ |
283 | 0 | AesEnc_4x512_##NR(&b1, &b2, &b3, &b4, rk); \ |
284 | 0 | \ |
285 | 0 | _mm512_storeu_si512(p_out, _mm512_xor_si512(b1, a1)); \ |
286 | 0 | _mm512_storeu_si512(p_out + 1, _mm512_xor_si512(b2, a2)); \ |
287 | 0 | _mm512_storeu_si512(p_out + 2, _mm512_xor_si512(b3, a3)); \ |
288 | 0 | _mm512_storeu_si512(p_out + 3, _mm512_xor_si512(b4, a4)); \ |
289 | 0 | \ |
290 | 0 | c1 = _mm512_add_epi64(c1, inc16); \ |
291 | 0 | p_in += 4; \ |
292 | 0 | p_out += 4; \ |
293 | 0 | blocks -= 16; \ |
294 | 0 | } \ |
295 | 0 | \ |
296 | 0 | /* --- 8-block (2×zmm) --- */ \ |
297 | 0 | if (blocks >= 8) { \ |
298 | 0 | c2 = _mm512_add_epi64(c1, inc4); \ |
299 | 0 | \ |
300 | 0 | a1 = _mm512_loadu_si512(p_in); \ |
301 | 0 | a2 = _mm512_loadu_si512(p_in + 1); \ |
302 | 0 | \ |
303 | 0 | b1 = _mm512_shuffle_epi8(c1, swap); \ |
304 | 0 | b2 = _mm512_shuffle_epi8(c2, swap); \ |
305 | 0 | \ |
306 | 0 | AesEnc_2x512_##NR(&b1, &b2, rk); \ |
307 | 0 | \ |
308 | 0 | _mm512_storeu_si512(p_out, _mm512_xor_si512(b1, a1)); \ |
309 | 0 | _mm512_storeu_si512(p_out + 1, _mm512_xor_si512(b2, a2)); \ |
310 | 0 | \ |
311 | 0 | c1 = _mm512_add_epi64(c1, inc8); \ |
312 | 0 | p_in += 2; \ |
313 | 0 | p_out += 2; \ |
314 | 0 | blocks -= 8; \ |
315 | 0 | } \ |
316 | 0 | \ |
317 | 0 | /* --- 4-block (1×zmm) --- */ \ |
318 | 0 | if (blocks >= 4) { \ |
319 | 0 | a1 = _mm512_loadu_si512(p_in); \ |
320 | 0 | b1 = _mm512_shuffle_epi8(c1, swap); \ |
321 | 0 | AesEnc_1x512_##NR(&b1, rk); \ |
322 | 0 | _mm512_storeu_si512(p_out, _mm512_xor_si512(b1, a1)); \ |
323 | 0 | \ |
324 | 0 | c1 = _mm512_add_epi64(c1, inc4); \ |
325 | 0 | p_in += 1; \ |
326 | 0 | p_out += 1; \ |
327 | 0 | blocks -= 4; \ |
328 | 0 | } \ |
329 | 0 | \ |
330 | 0 | /* --- Tail: 0-3 full blocks + residue partial block --- */ \ |
331 | 0 | { \ |
332 | 0 | size_t tail_bytes = (blocks * AES_BLOCK_SIZE) + res; \ |
333 | 0 | if (tail_bytes > 0) { \ |
334 | 0 | __mmask64 mask = (__mmask64)((1ULL << tail_bytes) - 1ULL); \ |
335 | 0 | a1 = _mm512_maskz_loadu_epi8(mask, p_in); \ |
336 | 0 | b1 = _mm512_shuffle_epi8(c1, swap); \ |
337 | 0 | AesEnc_1x512_##NR(&b1, rk); \ |
338 | 0 | _mm512_mask_storeu_epi8(p_out, mask, _mm512_xor_si512(b1, a1)); \ |
339 | 0 | \ |
340 | 0 | size_t adv = blocks + (res > 0 ? 1 : 0); \ |
341 | 0 | __m512i one_lo = _mm512_set_epi64(0, 0, 0, 0, 0, 0, 0, 1); \ |
342 | 0 | for (size_t i = 0; i < adv; i++) \ |
343 | 0 | c1 = _mm512_add_epi64(c1, one_lo); \ |
344 | 0 | } \ |
345 | 0 | } \ |
346 | 0 | \ |
347 | 0 | /* Write back updated counter (lane 0 only, byte-swapped to BE) */ \ |
348 | 0 | { \ |
349 | 0 | __m512i c_be = _mm512_shuffle_epi8(c1, swap); \ |
350 | 0 | _mm512_mask_storeu_epi64((__m128i *)iv, 0x03, c_be); \ |
351 | 0 | } \ |
352 | 0 | \ |
353 | 0 | /* Erase the broadcast schedule and the volatile vector register bank. */ \ |
354 | 0 | ossl_aes_vaes_cleanup(rk, NR + 1); \ |
355 | 0 | } Unexecuted instantiation: aes_vaes512_intrinsics.c:ctr_process_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:ctr_process_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:ctr_process_14 |
356 | | |
357 | | DEFINE_CTR_BLOCK(10) /* AES-128 */ |
358 | | DEFINE_CTR_BLOCK(12) /* AES-192 */ |
359 | | DEFINE_CTR_BLOCK(14) /* AES-256 */ |
360 | | |
361 | | /* Public entry point. */ |
362 | | void ossl_aes_ctr_vaes(const unsigned char *in, unsigned char *out, |
363 | | size_t length, const AES_KEY *key, |
364 | | unsigned char *counter, |
365 | | unsigned char *ecount_buf, unsigned int *num) |
366 | 0 | { |
367 | 0 | size_t n = *num; |
368 | 0 | size_t l = length; |
369 | 0 | int nr = key->rounds + 1; |
370 | | |
371 | | /* Drain leftover bytes from a previous partial block */ |
372 | 0 | if (n != 0) { |
373 | 0 | while (l > 0 && n < 16) { |
374 | 0 | *(out++) = *(in++) ^ ecount_buf[n]; |
375 | 0 | ++n; |
376 | 0 | l--; |
377 | 0 | } |
378 | 0 | *num = n % 16; |
379 | 0 | if (l == 0) |
380 | 0 | return; |
381 | 0 | } |
382 | | |
383 | | /* Process full 16-byte blocks with VAES. |
384 | | * |
385 | | * The VAES loop uses 64-bit counter arithmetic (no carry into the |
386 | | * upper 64 bits). If processing all requested blocks would overflow |
387 | | * the low 64 bits of the BE counter: |
388 | | * Phase 1 — VAES processes the safe blocks before the boundary. |
389 | | * Phase 2 — Scalar handles 1 block to cross the 64-bit carry. |
390 | | * Phase 3 — VAES resumes for the remaining bulk (>= 512 bytes), |
391 | | * since the counter low-64 is now near zero and safe. |
392 | | * Any final partial block is always handled by the scalar path. |
393 | | */ |
394 | 0 | { |
395 | 0 | size_t block_bytes = (l / 16) * 16; |
396 | 0 | if (block_bytes > 0) { |
397 | 0 | size_t total_blocks = block_bytes / 16; |
398 | | /* Read low 64 bits of the big-endian counter (bytes 8..15) */ |
399 | 0 | uint64_t ctr_lo = ((uint64_t)counter[8] << 56) |
400 | 0 | | ((uint64_t)counter[9] << 48) |
401 | 0 | | ((uint64_t)counter[10] << 40) |
402 | 0 | | ((uint64_t)counter[11] << 32) |
403 | 0 | | ((uint64_t)counter[12] << 24) |
404 | 0 | | ((uint64_t)counter[13] << 16) |
405 | 0 | | ((uint64_t)counter[14] << 8) |
406 | 0 | | ((uint64_t)counter[15]); |
407 | | |
408 | | /* Clamp to the number of blocks safe for 64-bit arithmetic */ |
409 | 0 | size_t safe_blocks = (ctr_lo <= UINT64_MAX - total_blocks) |
410 | 0 | ? total_blocks |
411 | 0 | : (size_t)(UINT64_MAX - ctr_lo); |
412 | 0 | size_t safe_bytes = safe_blocks * 16; |
413 | | |
414 | | /* Phase 1: VAES for the safe portion before the boundary */ |
415 | 0 | if (safe_bytes > 0) { |
416 | 0 | switch (nr) { |
417 | 0 | case 10: |
418 | 0 | ctr_process_10(in, out, safe_bytes, key, counter); |
419 | 0 | break; |
420 | 0 | case 12: |
421 | 0 | ctr_process_12(in, out, safe_bytes, key, counter); |
422 | 0 | break; |
423 | 0 | case 14: |
424 | 0 | ctr_process_14(in, out, safe_bytes, key, counter); |
425 | 0 | break; |
426 | 0 | default: /* invalid key size */ |
427 | 0 | CRYPTO_ctr128_encrypt(in, out, safe_bytes, key, counter, |
428 | 0 | ecount_buf, num, |
429 | 0 | (block128_f)aesni_encrypt); |
430 | 0 | break; |
431 | 0 | } |
432 | 0 | in += safe_bytes; |
433 | 0 | out += safe_bytes; |
434 | 0 | l -= safe_bytes; |
435 | 0 | } |
436 | | |
437 | | /* Phase 2 & 3: only entered when clamping actually occurred */ |
438 | 0 | if (safe_blocks < total_blocks && l > 0) { |
439 | | /* Phase 2: scalar encrypts 1 block across the carry */ |
440 | 0 | CRYPTO_ctr128_encrypt(in, out, 16, key, counter, |
441 | 0 | ecount_buf, num, |
442 | 0 | (block128_f)aesni_encrypt); |
443 | 0 | in += 16; |
444 | 0 | out += 16; |
445 | 0 | l -= 16; |
446 | | |
447 | | /* Phase 3: counter low-64 is now ~0 — VAES is safe again. |
448 | | * Resume VAES if enough data remains (>= 512 bytes). */ |
449 | 0 | if (l >= 512) { |
450 | 0 | size_t resume_bytes = (l / 16) * 16; |
451 | 0 | switch (nr) { |
452 | 0 | case 10: |
453 | 0 | ctr_process_10(in, out, resume_bytes, key, counter); |
454 | 0 | break; |
455 | 0 | case 12: |
456 | 0 | ctr_process_12(in, out, resume_bytes, key, counter); |
457 | 0 | break; |
458 | 0 | case 14: |
459 | 0 | ctr_process_14(in, out, resume_bytes, key, counter); |
460 | 0 | break; |
461 | 0 | default: /* invalid key size */ |
462 | 0 | CRYPTO_ctr128_encrypt(in, out, resume_bytes, key, |
463 | 0 | counter, ecount_buf, num, |
464 | 0 | (block128_f)aesni_encrypt); |
465 | 0 | break; |
466 | 0 | } |
467 | 0 | in += resume_bytes; |
468 | 0 | out += resume_bytes; |
469 | 0 | l -= resume_bytes; |
470 | 0 | } |
471 | 0 | } |
472 | 0 | } |
473 | 0 | } |
474 | | |
475 | | /* Handle any remaining bytes (partial block or small tail) */ |
476 | 0 | if (l > 0) |
477 | 0 | CRYPTO_ctr128_encrypt(in, out, l, key, counter, |
478 | 0 | ecount_buf, num, (block128_f)aesni_encrypt); |
479 | 0 | else |
480 | 0 | *num = 0; |
481 | 0 | } |
482 | | |
483 | | /* CPU feature check. */ |
484 | | int ossl_aes_ctr_vaes_eligible(void) |
485 | 32 | { |
486 | 32 | return ossl_aes_vaes_cleanup_eligible() |
487 | 32 | && ossl_vaes512_cpu_capable(); |
488 | 32 | } |
489 | | |
490 | | /* AES decryption round helpers (1x/2x/4x parallel 512-bit blocks). */ |
491 | | OSSL_VAES512_DEFINE_DECRYPT(10) /* AES-128 */ |
492 | | OSSL_VAES512_DEFINE_DECRYPT(12) /* AES-192 */ |
493 | | OSSL_VAES512_DEFINE_DECRYPT(14) /* AES-256 */ |
494 | | |
495 | | /*- |
496 | | * CBC-mode decryption -- templated per round count. |
497 | | * |
498 | | * Processes as many full blocks as possible: |
499 | | * 16 blocks at a time (4 x zmm = 4 x 4 = 16 blocks) |
500 | | * 8 blocks at a time (2 x zmm) |
501 | | * 4 blocks at a time (1 x zmm) |
502 | | * 1 block at a time for the remaining 0-3 blocks |
503 | | * |
504 | | * The chaining vector b1 packs [prev_ct[last] | ct[0] | ct[1] | ct[2]] so |
505 | | * that a single XOR after decryption applies the CBC feedback to all four |
506 | | * lanes simultaneously. |
507 | | */ |
508 | | #define DEFINE_CBC_DECRYPT(NR) \ |
509 | | OSSL_FUNC_NOINLINE \ |
510 | | static void cbc_decrypt_##NR( \ |
511 | | const unsigned char *in, unsigned char *out, size_t len, \ |
512 | | const AES_KEY *key, unsigned char *iv) \ |
513 | 0 | { \ |
514 | 0 | __m512i rk[NR + 1]; \ |
515 | 0 | OSSL_VAES512_LOAD_ROUNDKEYS(rk, key, NR); \ |
516 | 0 | \ |
517 | 0 | __m512i a1, a2, a3, a4; \ |
518 | 0 | __m512i b1, b2, b3, b4; \ |
519 | 0 | \ |
520 | 0 | const __m128i *pa = (const __m128i *)in; \ |
521 | 0 | __m512i *po = (__m512i *)out; \ |
522 | 0 | size_t blocks = len / AES_BLOCK_SIZE; \ |
523 | 0 | \ |
524 | 0 | /* Save last ciphertext block for IV update (in-place safe) */ \ |
525 | 0 | __m128i saved_iv = _mm_setzero_si128(); \ |
526 | 0 | int has_blocks = (blocks > 0); \ |
527 | 0 | if (has_blocks) \ |
528 | 0 | saved_iv = _mm_loadu_si128(pa + blocks - 1); \ |
529 | 0 | \ |
530 | 0 | if (blocks >= 4) { \ |
531 | 0 | /* Build b1 = [IV | ct[0] | ct[1] | ct[2]] */ \ |
532 | 0 | __m512i idx = _mm512_set_epi64(5, 4, 3, 2, 1, 0, 0, 0); \ |
533 | 0 | __m512i ct0; \ |
534 | 0 | \ |
535 | 0 | /* CBC C[0]=IV; 0x03 loads one 128-bit block (two 64-bit lanes). */ \ |
536 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, iv); \ |
537 | 0 | ct0 = _mm512_loadu_si512(pa); \ |
538 | 0 | ct0 = _mm512_permutexvar_epi64(idx, ct0); \ |
539 | 0 | b1 = _mm512_mask_blend_epi64(0xFC, b1, ct0); \ |
540 | 0 | \ |
541 | 0 | /* --- 16-block (4 x zmm) main loop --- */ \ |
542 | 0 | while (blocks >= 16) { \ |
543 | 0 | __m128i last; \ |
544 | 0 | \ |
545 | 0 | a1 = _mm512_loadu_si512(pa); \ |
546 | 0 | a2 = _mm512_loadu_si512(pa + 4); \ |
547 | 0 | a3 = _mm512_loadu_si512(pa + 8); \ |
548 | 0 | a4 = _mm512_loadu_si512(pa + 12); \ |
549 | 0 | \ |
550 | 0 | b2 = _mm512_loadu_si512(pa + 3); \ |
551 | 0 | b3 = _mm512_loadu_si512(pa + 7); \ |
552 | 0 | b4 = _mm512_loadu_si512(pa + 11); \ |
553 | 0 | \ |
554 | 0 | last = _mm_loadu_si128(pa + 15); \ |
555 | 0 | \ |
556 | 0 | AesDec_4x512_##NR(&a1, &a2, &a3, &a4, rk); \ |
557 | 0 | \ |
558 | 0 | a1 = _mm512_xor_si512(a1, b1); \ |
559 | 0 | a2 = _mm512_xor_si512(a2, b2); \ |
560 | 0 | a3 = _mm512_xor_si512(a3, b3); \ |
561 | 0 | a4 = _mm512_xor_si512(a4, b4); \ |
562 | 0 | \ |
563 | 0 | _mm512_storeu_si512(po, a1); \ |
564 | 0 | _mm512_storeu_si512(po + 1, a2); \ |
565 | 0 | _mm512_storeu_si512(po + 2, a3); \ |
566 | 0 | _mm512_storeu_si512(po + 3, a4); \ |
567 | 0 | \ |
568 | 0 | /* Build next b1 from last ciphertext block */ \ |
569 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, &last); \ |
570 | 0 | if (blocks > 16) { \ |
571 | 0 | size_t rem = blocks - 16; \ |
572 | 0 | /* Load only available lookahead blocks to avoid OOB read. */ \ |
573 | 0 | /* One AES block is 16 bytes, it maps to two 64-bit lanes. */ \ |
574 | 0 | /* 0x03 (00000011) for 1 block (2 lanes), */ \ |
575 | 0 | /* 0x0F (00001111) for 2 blocks (4 lanes), */ \ |
576 | 0 | /* 0x3F (00111111) for 3 or more blocks (6 lanes). */ \ |
577 | 0 | __mmask8 nxmask = (rem >= 3) ? 0x3F : (rem == 2 ? 0x0F : 0x03); \ |
578 | 0 | __m512i nx = _mm512_maskz_loadu_epi64(nxmask, pa + 16); \ |
579 | 0 | nx = _mm512_permutexvar_epi64(idx, nx); \ |
580 | 0 | b1 = _mm512_mask_blend_epi64(0xFC, b1, nx); \ |
581 | 0 | } \ |
582 | 0 | \ |
583 | 0 | pa += 16; \ |
584 | 0 | po += 4; \ |
585 | 0 | blocks -= 16; \ |
586 | 0 | } \ |
587 | 0 | \ |
588 | 0 | /* --- 8-block (2 x zmm) --- */ \ |
589 | 0 | if (blocks >= 8) { \ |
590 | 0 | __m128i last8; \ |
591 | 0 | \ |
592 | 0 | a1 = _mm512_loadu_si512(pa); \ |
593 | 0 | a2 = _mm512_loadu_si512(pa + 4); \ |
594 | 0 | b2 = _mm512_loadu_si512(pa + 3); \ |
595 | 0 | last8 = _mm_loadu_si128(pa + 7); \ |
596 | 0 | \ |
597 | 0 | AesDec_2x512_##NR(&a1, &a2, rk); \ |
598 | 0 | a1 = _mm512_xor_si512(a1, b1); \ |
599 | 0 | a2 = _mm512_xor_si512(a2, b2); \ |
600 | 0 | \ |
601 | 0 | _mm512_storeu_si512(po, a1); \ |
602 | 0 | _mm512_storeu_si512(po + 1, a2); \ |
603 | 0 | \ |
604 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, &last8); \ |
605 | 0 | pa += 8; \ |
606 | 0 | po += 2; \ |
607 | 0 | blocks -= 8; \ |
608 | 0 | \ |
609 | 0 | if (blocks >= 4) { \ |
610 | 0 | __m512i nx = _mm512_loadu_si512(pa); \ |
611 | 0 | nx = _mm512_permutexvar_epi64(idx, nx); \ |
612 | 0 | b1 = _mm512_mask_blend_epi64(0xFC, b1, nx); \ |
613 | 0 | } \ |
614 | 0 | } \ |
615 | 0 | \ |
616 | 0 | /* --- 4-block (1 x zmm) --- */ \ |
617 | 0 | if (blocks >= 4) { \ |
618 | 0 | __m128i last4; \ |
619 | 0 | \ |
620 | 0 | a1 = _mm512_loadu_si512(pa); \ |
621 | 0 | last4 = _mm_loadu_si128(pa + 3); \ |
622 | 0 | \ |
623 | 0 | AesDec_1x512_##NR(&a1, rk); \ |
624 | 0 | a1 = _mm512_xor_si512(a1, b1); \ |
625 | 0 | _mm512_storeu_si512(po, a1); \ |
626 | 0 | \ |
627 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, &last4); \ |
628 | 0 | pa += 4; \ |
629 | 0 | po += 1; \ |
630 | 0 | blocks -= 4; \ |
631 | 0 | } \ |
632 | 0 | \ |
633 | 0 | /* --- Remaining 1-3 blocks --- */ \ |
634 | 0 | { \ |
635 | 0 | __m128i *po128 = (__m128i *)po; \ |
636 | 0 | while (blocks > 0) { \ |
637 | 0 | __m128i ct = _mm_loadu_si128(pa); \ |
638 | 0 | a1 = _mm512_maskz_loadu_epi64(0x03, pa); \ |
639 | 0 | AesDec_1x512_##NR(&a1, rk); \ |
640 | 0 | a1 = _mm512_xor_si512(a1, b1); \ |
641 | 0 | _mm512_mask_storeu_epi64(po128, 0x03, a1); \ |
642 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, &ct); \ |
643 | 0 | pa++; \ |
644 | 0 | po128++; \ |
645 | 0 | blocks--; \ |
646 | 0 | } \ |
647 | 0 | } \ |
648 | 0 | } else { \ |
649 | 0 | /* Less than 4 blocks -- process individually */ \ |
650 | 0 | __m128i *po128 = (__m128i *)po; \ |
651 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, iv); \ |
652 | 0 | while (blocks > 0) { \ |
653 | 0 | __m128i ct = _mm_loadu_si128(pa); \ |
654 | 0 | a1 = _mm512_maskz_loadu_epi64(0x03, pa); \ |
655 | 0 | AesDec_1x512_##NR(&a1, rk); \ |
656 | 0 | a1 = _mm512_xor_si512(a1, b1); \ |
657 | 0 | _mm512_mask_storeu_epi64(po128, 0x03, a1); \ |
658 | 0 | b1 = _mm512_maskz_loadu_epi64(0x03, &ct); \ |
659 | 0 | pa++; \ |
660 | 0 | po128++; \ |
661 | 0 | blocks--; \ |
662 | 0 | } \ |
663 | 0 | } \ |
664 | 0 | \ |
665 | 0 | if (has_blocks) \ |
666 | 0 | _mm_storeu_si128((__m128i *)iv, saved_iv); \ |
667 | 0 | \ |
668 | 0 | /* Erase the broadcast schedule and the volatile vector register bank. */ \ |
669 | 0 | ossl_aes_vaes_cleanup(rk, NR + 1); \ |
670 | 0 | } Unexecuted instantiation: aes_vaes512_intrinsics.c:cbc_decrypt_10 Unexecuted instantiation: aes_vaes512_intrinsics.c:cbc_decrypt_12 Unexecuted instantiation: aes_vaes512_intrinsics.c:cbc_decrypt_14 |
671 | | |
672 | | DEFINE_CBC_DECRYPT(10) /* AES-128 */ |
673 | | DEFINE_CBC_DECRYPT(12) /* AES-192 */ |
674 | | DEFINE_CBC_DECRYPT(14) /* AES-256 */ |
675 | | |
676 | | /* Public entry point. */ |
677 | | void ossl_aes_cbc_vaes_decrypt(const unsigned char *in, unsigned char *out, |
678 | | size_t len, const void *key, |
679 | | unsigned char ivec[16], int enc) |
680 | 0 | { |
681 | 0 | size_t full_bytes; |
682 | 0 | int nr = ((const AES_KEY *)key)->rounds + 1; |
683 | |
|
684 | 0 | if (len == 0) |
685 | 0 | return; |
686 | | |
687 | | /* VAES path only optimises decryption; encrypt falls back to asm */ |
688 | 0 | if (enc) { |
689 | 0 | aesni_cbc_encrypt(in, out, len, (const AES_KEY *)key, ivec, enc); |
690 | 0 | return; |
691 | 0 | } |
692 | | |
693 | 0 | full_bytes = (len / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; |
694 | 0 | if (full_bytes > 0) { |
695 | 0 | switch (nr) { |
696 | 0 | case 10: |
697 | 0 | cbc_decrypt_10(in, out, full_bytes, (const AES_KEY *)key, ivec); |
698 | 0 | break; |
699 | 0 | case 12: |
700 | 0 | cbc_decrypt_12(in, out, full_bytes, (const AES_KEY *)key, ivec); |
701 | 0 | break; |
702 | 0 | case 14: |
703 | 0 | cbc_decrypt_14(in, out, full_bytes, (const AES_KEY *)key, ivec); |
704 | 0 | break; |
705 | 0 | default: /* invalid key size */ |
706 | 0 | aesni_cbc_encrypt(in, out, len, (const AES_KEY *)key, ivec, 0); |
707 | 0 | break; |
708 | 0 | } |
709 | 0 | } |
710 | 0 | } |
711 | | |
712 | | /* CPU feature check. */ |
713 | | int ossl_aes_cbc_vaes_eligible(void) |
714 | 0 | { |
715 | 0 | return ossl_aes_vaes_cleanup_eligible() |
716 | 0 | && ossl_vaes512_cpu_capable(); |
717 | 0 | } |
718 | | |
719 | | OPENSSL_UNTARGET_VAES512 |
720 | | |
721 | | #endif /* VAES512_ELIGIBLE */ |