Coverage Report

Created: 2024-11-21 07:03

/src/mbedtls/library/aesni.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 *  AES-NI support functions
3
 *
4
 *  Copyright The Mbed TLS Contributors
5
 *  SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6
 */
7
8
/*
9
 * [AES-WP] https://www.intel.com/content/www/us/en/developer/articles/tool/intel-advanced-encryption-standard-aes-instructions-set.html
10
 * [CLMUL-WP] https://www.intel.com/content/www/us/en/develop/download/intel-carry-less-multiplication-instruction-and-its-usage-for-computing-the-gcm-mode.html
11
 */
12
13
#include "common.h"
14
15
#if defined(MBEDTLS_AESNI_C)
16
17
#include "aesni.h"
18
19
#include <string.h>
20
21
#if defined(MBEDTLS_AESNI_HAVE_CODE)
22
23
#if MBEDTLS_AESNI_HAVE_CODE == 2
24
#if defined(__GNUC__)
25
#include <cpuid.h>
26
#elif defined(_MSC_VER)
27
#include <intrin.h>
28
#else
29
#error "`__cpuid` required by MBEDTLS_AESNI_C is not supported by the compiler"
30
#endif
31
#include <immintrin.h>
32
#endif
33
34
#if defined(MBEDTLS_ARCH_IS_X86)
35
#if defined(MBEDTLS_COMPILER_IS_GCC)
36
#pragma GCC push_options
37
#pragma GCC target ("pclmul,sse2,aes")
38
#define MBEDTLS_POP_TARGET_PRAGMA
39
#elif defined(__clang__) && (__clang_major__ >= 5)
40
#pragma clang attribute push (__attribute__((target("pclmul,sse2,aes"))), apply_to=function)
41
#define MBEDTLS_POP_TARGET_PRAGMA
42
#endif
43
#endif
44
45
#if !defined(MBEDTLS_AES_USE_HARDWARE_ONLY)
46
/*
47
 * AES-NI support detection routine
48
 */
49
int mbedtls_aesni_has_support(unsigned int what)
50
4.11k
{
51
4.11k
    static int done = 0;
52
4.11k
    static unsigned int c = 0;
53
54
4.11k
    if (!done) {
55
#if MBEDTLS_AESNI_HAVE_CODE == 2
56
        static int info[4] = { 0, 0, 0, 0 };
57
#if defined(_MSC_VER)
58
        __cpuid(info, 1);
59
#else
60
        __cpuid(1, info[0], info[1], info[2], info[3]);
61
#endif
62
        c = info[2];
63
#else /* AESNI using asm */
64
10
        asm ("movl  $1, %%eax   \n\t"
65
10
             "cpuid             \n\t"
66
10
             : "=c" (c)
67
10
             :
68
10
             : "eax", "ebx", "edx");
69
10
#endif /* MBEDTLS_AESNI_HAVE_CODE */
70
10
        done = 1;
71
10
    }
72
73
4.11k
    return (c & what) != 0;
74
4.11k
}
75
#endif /* !MBEDTLS_AES_USE_HARDWARE_ONLY */
76
77
#if MBEDTLS_AESNI_HAVE_CODE == 2
78
79
/*
80
 * AES-NI AES-ECB block en(de)cryption
81
 */
82
int mbedtls_aesni_crypt_ecb(mbedtls_aes_context *ctx,
83
                            int mode,
84
                            const unsigned char input[16],
85
                            unsigned char output[16])
86
{
87
    const __m128i *rk = (const __m128i *) (ctx->buf + ctx->rk_offset);
88
    unsigned nr = ctx->nr; // Number of remaining rounds
89
90
    // Load round key 0
91
    __m128i state;
92
    memcpy(&state, input, 16);
93
    state = _mm_xor_si128(state, rk[0]);  // state ^= *rk;
94
    ++rk;
95
    --nr;
96
97
#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
98
    if (mode == MBEDTLS_AES_DECRYPT) {
99
        while (nr != 0) {
100
            state = _mm_aesdec_si128(state, *rk);
101
            ++rk;
102
            --nr;
103
        }
104
        state = _mm_aesdeclast_si128(state, *rk);
105
    } else
106
#else
107
    (void) mode;
108
#endif
109
    {
110
        while (nr != 0) {
111
            state = _mm_aesenc_si128(state, *rk);
112
            ++rk;
113
            --nr;
114
        }
115
        state = _mm_aesenclast_si128(state, *rk);
116
    }
117
118
    memcpy(output, &state, 16);
119
    return 0;
120
}
121
122
/*
123
 * GCM multiplication: c = a times b in GF(2^128)
124
 * Based on [CLMUL-WP] algorithms 1 (with equation 27) and 5.
125
 */
126
127
static void gcm_clmul(const __m128i aa, const __m128i bb,
128
                      __m128i *cc, __m128i *dd)
129
{
130
    /*
131
     * Caryless multiplication dd:cc = aa * bb
132
     * using [CLMUL-WP] algorithm 1 (p. 12).
133
     */
134
    *cc = _mm_clmulepi64_si128(aa, bb, 0x00); // a0*b0 = c1:c0
135
    *dd = _mm_clmulepi64_si128(aa, bb, 0x11); // a1*b1 = d1:d0
136
    __m128i ee = _mm_clmulepi64_si128(aa, bb, 0x10); // a0*b1 = e1:e0
137
    __m128i ff = _mm_clmulepi64_si128(aa, bb, 0x01); // a1*b0 = f1:f0
138
    ff = _mm_xor_si128(ff, ee);                      // e1+f1:e0+f0
139
    ee = ff;                                         // e1+f1:e0+f0
140
    ff = _mm_srli_si128(ff, 8);                      // 0:e1+f1
141
    ee = _mm_slli_si128(ee, 8);                      // e0+f0:0
142
    *dd = _mm_xor_si128(*dd, ff);                    // d1:d0+e1+f1
143
    *cc = _mm_xor_si128(*cc, ee);                    // c1+e0+f0:c0
144
}
145
146
static void gcm_shift(__m128i *cc, __m128i *dd)
147
{
148
    /* [CMUCL-WP] Algorithm 5 Step 1: shift cc:dd one bit to the left,
149
     * taking advantage of [CLMUL-WP] eq 27 (p. 18). */
150
    //                                        // *cc = r1:r0
151
    //                                        // *dd = r3:r2
152
    __m128i cc_lo = _mm_slli_epi64(*cc, 1);   // r1<<1:r0<<1
153
    __m128i dd_lo = _mm_slli_epi64(*dd, 1);   // r3<<1:r2<<1
154
    __m128i cc_hi = _mm_srli_epi64(*cc, 63);  // r1>>63:r0>>63
155
    __m128i dd_hi = _mm_srli_epi64(*dd, 63);  // r3>>63:r2>>63
156
    __m128i xmm5 = _mm_srli_si128(cc_hi, 8);  // 0:r1>>63
157
    cc_hi = _mm_slli_si128(cc_hi, 8);         // r0>>63:0
158
    dd_hi = _mm_slli_si128(dd_hi, 8);         // 0:r1>>63
159
160
    *cc = _mm_or_si128(cc_lo, cc_hi);         // r1<<1|r0>>63:r0<<1
161
    *dd = _mm_or_si128(_mm_or_si128(dd_lo, dd_hi), xmm5); // r3<<1|r2>>62:r2<<1|r1>>63
162
}
163
164
static __m128i gcm_reduce(__m128i xx)
165
{
166
    //                                            // xx = x1:x0
167
    /* [CLMUL-WP] Algorithm 5 Step 2 */
168
    __m128i aa = _mm_slli_epi64(xx, 63);          // x1<<63:x0<<63 = stuff:a
169
    __m128i bb = _mm_slli_epi64(xx, 62);          // x1<<62:x0<<62 = stuff:b
170
    __m128i cc = _mm_slli_epi64(xx, 57);          // x1<<57:x0<<57 = stuff:c
171
    __m128i dd = _mm_slli_si128(_mm_xor_si128(_mm_xor_si128(aa, bb), cc), 8); // a+b+c:0
172
    return _mm_xor_si128(dd, xx);                 // x1+a+b+c:x0 = d:x0
173
}
174
175
static __m128i gcm_mix(__m128i dx)
176
{
177
    /* [CLMUL-WP] Algorithm 5 Steps 3 and 4 */
178
    __m128i ee = _mm_srli_epi64(dx, 1);           // e1:x0>>1 = e1:e0'
179
    __m128i ff = _mm_srli_epi64(dx, 2);           // f1:x0>>2 = f1:f0'
180
    __m128i gg = _mm_srli_epi64(dx, 7);           // g1:x0>>7 = g1:g0'
181
182
    // e0'+f0'+g0' is almost e0+f0+g0, except for some missing
183
    // bits carried from d. Now get those bits back in.
184
    __m128i eh = _mm_slli_epi64(dx, 63);          // d<<63:stuff
185
    __m128i fh = _mm_slli_epi64(dx, 62);          // d<<62:stuff
186
    __m128i gh = _mm_slli_epi64(dx, 57);          // d<<57:stuff
187
    __m128i hh = _mm_srli_si128(_mm_xor_si128(_mm_xor_si128(eh, fh), gh), 8); // 0:missing bits of d
188
189
    return _mm_xor_si128(_mm_xor_si128(_mm_xor_si128(_mm_xor_si128(ee, ff), gg), hh), dx);
190
}
191
192
void mbedtls_aesni_gcm_mult(unsigned char c[16],
193
                            const unsigned char a[16],
194
                            const unsigned char b[16])
195
{
196
    __m128i aa = { 0 }, bb = { 0 }, cc, dd;
197
198
    /* The inputs are in big-endian order, so byte-reverse them */
199
    for (size_t i = 0; i < 16; i++) {
200
        ((uint8_t *) &aa)[i] = a[15 - i];
201
        ((uint8_t *) &bb)[i] = b[15 - i];
202
    }
203
204
    gcm_clmul(aa, bb, &cc, &dd);
205
    gcm_shift(&cc, &dd);
206
    /*
207
     * Now reduce modulo the GCM polynomial x^128 + x^7 + x^2 + x + 1
208
     * using [CLMUL-WP] algorithm 5 (p. 18).
209
     * Currently dd:cc holds x3:x2:x1:x0 (already shifted).
210
     */
211
    __m128i dx = gcm_reduce(cc);
212
    __m128i xh = gcm_mix(dx);
213
    cc = _mm_xor_si128(xh, dd); // x3+h1:x2+h0
214
215
    /* Now byte-reverse the outputs */
216
    for (size_t i = 0; i < 16; i++) {
217
        c[i] = ((uint8_t *) &cc)[15 - i];
218
    }
219
220
    return;
221
}
222
223
/*
224
 * Compute decryption round keys from encryption round keys
225
 */
226
#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
227
void mbedtls_aesni_inverse_key(unsigned char *invkey,
228
                               const unsigned char *fwdkey, int nr)
229
{
230
    __m128i *ik = (__m128i *) invkey;
231
    const __m128i *fk = (const __m128i *) fwdkey + nr;
232
233
    *ik = *fk;
234
    for (--fk, ++ik; fk > (const __m128i *) fwdkey; --fk, ++ik) {
235
        *ik = _mm_aesimc_si128(*fk);
236
    }
237
    *ik = *fk;
238
}
239
#endif
240
241
/*
242
 * Key expansion, 128-bit case
243
 */
244
static __m128i aesni_set_rk_128(__m128i state, __m128i xword)
245
{
246
    /*
247
     * Finish generating the next round key.
248
     *
249
     * On entry state is r3:r2:r1:r0 and xword is X:stuff:stuff:stuff
250
     * with X = rot( sub( r3 ) ) ^ RCON (obtained with AESKEYGENASSIST).
251
     *
252
     * On exit, xword is r7:r6:r5:r4
253
     * with r4 = X + r0, r5 = r4 + r1, r6 = r5 + r2, r7 = r6 + r3
254
     * and this is returned, to be written to the round key buffer.
255
     */
256
    xword = _mm_shuffle_epi32(xword, 0xff);   // X:X:X:X
257
    xword = _mm_xor_si128(xword, state);      // X+r3:X+r2:X+r1:r4
258
    state = _mm_slli_si128(state, 4);         // r2:r1:r0:0
259
    xword = _mm_xor_si128(xword, state);      // X+r3+r2:X+r2+r1:r5:r4
260
    state = _mm_slli_si128(state, 4);         // r1:r0:0:0
261
    xword = _mm_xor_si128(xword, state);      // X+r3+r2+r1:r6:r5:r4
262
    state = _mm_slli_si128(state, 4);         // r0:0:0:0
263
    state = _mm_xor_si128(xword, state);      // r7:r6:r5:r4
264
    return state;
265
}
266
267
static void aesni_setkey_enc_128(unsigned char *rk_bytes,
268
                                 const unsigned char *key)
269
{
270
    __m128i *rk = (__m128i *) rk_bytes;
271
272
    memcpy(&rk[0], key, 16);
273
    rk[1] = aesni_set_rk_128(rk[0], _mm_aeskeygenassist_si128(rk[0], 0x01));
274
    rk[2] = aesni_set_rk_128(rk[1], _mm_aeskeygenassist_si128(rk[1], 0x02));
275
    rk[3] = aesni_set_rk_128(rk[2], _mm_aeskeygenassist_si128(rk[2], 0x04));
276
    rk[4] = aesni_set_rk_128(rk[3], _mm_aeskeygenassist_si128(rk[3], 0x08));
277
    rk[5] = aesni_set_rk_128(rk[4], _mm_aeskeygenassist_si128(rk[4], 0x10));
278
    rk[6] = aesni_set_rk_128(rk[5], _mm_aeskeygenassist_si128(rk[5], 0x20));
279
    rk[7] = aesni_set_rk_128(rk[6], _mm_aeskeygenassist_si128(rk[6], 0x40));
280
    rk[8] = aesni_set_rk_128(rk[7], _mm_aeskeygenassist_si128(rk[7], 0x80));
281
    rk[9] = aesni_set_rk_128(rk[8], _mm_aeskeygenassist_si128(rk[8], 0x1B));
282
    rk[10] = aesni_set_rk_128(rk[9], _mm_aeskeygenassist_si128(rk[9], 0x36));
283
}
284
285
/*
286
 * Key expansion, 192-bit case
287
 */
288
#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
289
static void aesni_set_rk_192(__m128i *state0, __m128i *state1, __m128i xword,
290
                             unsigned char *rk)
291
{
292
    /*
293
     * Finish generating the next 6 quarter-keys.
294
     *
295
     * On entry state0 is r3:r2:r1:r0, state1 is stuff:stuff:r5:r4
296
     * and xword is stuff:stuff:X:stuff with X = rot( sub( r3 ) ) ^ RCON
297
     * (obtained with AESKEYGENASSIST).
298
     *
299
     * On exit, state0 is r9:r8:r7:r6 and state1 is stuff:stuff:r11:r10
300
     * and those are written to the round key buffer.
301
     */
302
    xword = _mm_shuffle_epi32(xword, 0x55);   // X:X:X:X
303
    xword = _mm_xor_si128(xword, *state0);    // X+r3:X+r2:X+r1:X+r0
304
    *state0 = _mm_slli_si128(*state0, 4);     // r2:r1:r0:0
305
    xword = _mm_xor_si128(xword, *state0);    // X+r3+r2:X+r2+r1:X+r1+r0:X+r0
306
    *state0 = _mm_slli_si128(*state0, 4);     // r1:r0:0:0
307
    xword = _mm_xor_si128(xword, *state0);    // X+r3+r2+r1:X+r2+r1+r0:X+r1+r0:X+r0
308
    *state0 = _mm_slli_si128(*state0, 4);     // r0:0:0:0
309
    xword = _mm_xor_si128(xword, *state0);    // X+r3+r2+r1+r0:X+r2+r1+r0:X+r1+r0:X+r0
310
    *state0 = xword;                          // = r9:r8:r7:r6
311
312
    xword = _mm_shuffle_epi32(xword, 0xff);   // r9:r9:r9:r9
313
    xword = _mm_xor_si128(xword, *state1);    // stuff:stuff:r9+r5:r9+r4
314
    *state1 = _mm_slli_si128(*state1, 4);     // stuff:stuff:r4:0
315
    xword = _mm_xor_si128(xword, *state1);    // stuff:stuff:r9+r5+r4:r9+r4
316
    *state1 = xword;                          // = stuff:stuff:r11:r10
317
318
    /* Store state0 and the low half of state1 into rk, which is conceptually
319
     * an array of 24-byte elements. Since 24 is not a multiple of 16,
320
     * rk is not necessarily aligned so just `*rk = *state0` doesn't work. */
321
    memcpy(rk, state0, 16);
322
    memcpy(rk + 16, state1, 8);
323
}
324
325
static void aesni_setkey_enc_192(unsigned char *rk,
326
                                 const unsigned char *key)
327
{
328
    /* First round: use original key */
329
    memcpy(rk, key, 24);
330
    /* aes.c guarantees that rk is aligned on a 16-byte boundary. */
331
    __m128i state0 = ((__m128i *) rk)[0];
332
    __m128i state1 = _mm_loadl_epi64(((__m128i *) rk) + 1);
333
334
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x01), rk + 24 * 1);
335
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x02), rk + 24 * 2);
336
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x04), rk + 24 * 3);
337
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x08), rk + 24 * 4);
338
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x10), rk + 24 * 5);
339
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x20), rk + 24 * 6);
340
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x40), rk + 24 * 7);
341
    aesni_set_rk_192(&state0, &state1, _mm_aeskeygenassist_si128(state1, 0x80), rk + 24 * 8);
342
}
343
#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
344
345
/*
346
 * Key expansion, 256-bit case
347
 */
348
#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
349
static void aesni_set_rk_256(__m128i state0, __m128i state1, __m128i xword,
350
                             __m128i *rk0, __m128i *rk1)
351
{
352
    /*
353
     * Finish generating the next two round keys.
354
     *
355
     * On entry state0 is r3:r2:r1:r0, state1 is r7:r6:r5:r4 and
356
     * xword is X:stuff:stuff:stuff with X = rot( sub( r7 )) ^ RCON
357
     * (obtained with AESKEYGENASSIST).
358
     *
359
     * On exit, *rk0 is r11:r10:r9:r8 and *rk1 is r15:r14:r13:r12
360
     */
361
    xword = _mm_shuffle_epi32(xword, 0xff);
362
    xword = _mm_xor_si128(xword, state0);
363
    state0 = _mm_slli_si128(state0, 4);
364
    xword = _mm_xor_si128(xword, state0);
365
    state0 = _mm_slli_si128(state0, 4);
366
    xword = _mm_xor_si128(xword, state0);
367
    state0 = _mm_slli_si128(state0, 4);
368
    state0 = _mm_xor_si128(state0, xword);
369
    *rk0 = state0;
370
371
    /* Set xword to stuff:Y:stuff:stuff with Y = subword( r11 )
372
     * and proceed to generate next round key from there */
373
    xword = _mm_aeskeygenassist_si128(state0, 0x00);
374
    xword = _mm_shuffle_epi32(xword, 0xaa);
375
    xword = _mm_xor_si128(xword, state1);
376
    state1 = _mm_slli_si128(state1, 4);
377
    xword = _mm_xor_si128(xword, state1);
378
    state1 = _mm_slli_si128(state1, 4);
379
    xword = _mm_xor_si128(xword, state1);
380
    state1 = _mm_slli_si128(state1, 4);
381
    state1 = _mm_xor_si128(state1, xword);
382
    *rk1 = state1;
383
}
384
385
static void aesni_setkey_enc_256(unsigned char *rk_bytes,
386
                                 const unsigned char *key)
387
{
388
    __m128i *rk = (__m128i *) rk_bytes;
389
390
    memcpy(&rk[0], key, 16);
391
    memcpy(&rk[1], key + 16, 16);
392
393
    /*
394
     * Main "loop" - Generating one more key than necessary,
395
     * see definition of mbedtls_aes_context.buf
396
     */
397
    aesni_set_rk_256(rk[0], rk[1], _mm_aeskeygenassist_si128(rk[1], 0x01), &rk[2], &rk[3]);
398
    aesni_set_rk_256(rk[2], rk[3], _mm_aeskeygenassist_si128(rk[3], 0x02), &rk[4], &rk[5]);
399
    aesni_set_rk_256(rk[4], rk[5], _mm_aeskeygenassist_si128(rk[5], 0x04), &rk[6], &rk[7]);
400
    aesni_set_rk_256(rk[6], rk[7], _mm_aeskeygenassist_si128(rk[7], 0x08), &rk[8], &rk[9]);
401
    aesni_set_rk_256(rk[8], rk[9], _mm_aeskeygenassist_si128(rk[9], 0x10), &rk[10], &rk[11]);
402
    aesni_set_rk_256(rk[10], rk[11], _mm_aeskeygenassist_si128(rk[11], 0x20), &rk[12], &rk[13]);
403
    aesni_set_rk_256(rk[12], rk[13], _mm_aeskeygenassist_si128(rk[13], 0x40), &rk[14], &rk[15]);
404
}
405
#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
406
407
#if defined(MBEDTLS_POP_TARGET_PRAGMA)
408
#if defined(__clang__)
409
#pragma clang attribute pop
410
#elif defined(__GNUC__)
411
#pragma GCC pop_options
412
#endif
413
#undef MBEDTLS_POP_TARGET_PRAGMA
414
#endif
415
416
#else /* MBEDTLS_AESNI_HAVE_CODE == 1 */
417
418
#if defined(__has_feature)
419
#if __has_feature(memory_sanitizer)
420
#warning \
421
    "MBEDTLS_AESNI_C is known to cause spurious error reports with some memory sanitizers as they do not understand the assembly code."
422
#endif
423
#endif
424
425
/*
426
 * Binutils needs to be at least 2.19 to support AES-NI instructions.
427
 * Unfortunately, a lot of users have a lower version now (2014-04).
428
 * Emit bytecode directly in order to support "old" version of gas.
429
 *
430
 * Opcodes from the Intel architecture reference manual, vol. 3.
431
 * We always use registers, so we don't need prefixes for memory operands.
432
 * Operand macros are in gas order (src, dst) as opposed to Intel order
433
 * (dst, src) in order to blend better into the surrounding assembly code.
434
 */
435
#define AESDEC(regs)      ".byte 0x66,0x0F,0x38,0xDE," regs "\n\t"
436
#define AESDECLAST(regs)  ".byte 0x66,0x0F,0x38,0xDF," regs "\n\t"
437
#define AESENC(regs)      ".byte 0x66,0x0F,0x38,0xDC," regs "\n\t"
438
#define AESENCLAST(regs)  ".byte 0x66,0x0F,0x38,0xDD," regs "\n\t"
439
#define AESIMC(regs)      ".byte 0x66,0x0F,0x38,0xDB," regs "\n\t"
440
#define AESKEYGENA(regs, imm)  ".byte 0x66,0x0F,0x3A,0xDF," regs "," imm "\n\t"
441
#define PCLMULQDQ(regs, imm)   ".byte 0x66,0x0F,0x3A,0x44," regs "," imm "\n\t"
442
443
#define xmm0_xmm0   "0xC0"
444
#define xmm0_xmm1   "0xC8"
445
#define xmm0_xmm2   "0xD0"
446
#define xmm0_xmm3   "0xD8"
447
#define xmm0_xmm4   "0xE0"
448
#define xmm1_xmm0   "0xC1"
449
#define xmm1_xmm2   "0xD1"
450
451
/*
452
 * AES-NI AES-ECB block en(de)cryption
453
 */
454
int mbedtls_aesni_crypt_ecb(mbedtls_aes_context *ctx,
455
                            int mode,
456
                            const unsigned char input[16],
457
                            unsigned char output[16])
458
3.59k
{
459
3.59k
    asm ("movdqu    (%3), %%xmm0    \n\t" // load input
460
3.59k
         "movdqu    (%1), %%xmm1    \n\t" // load round key 0
461
3.59k
         "pxor      %%xmm1, %%xmm0  \n\t" // round 0
462
3.59k
         "add       $16, %1         \n\t" // point to next round key
463
3.59k
         "subl      $1, %0          \n\t" // normal rounds = nr - 1
464
3.59k
         "test      %2, %2          \n\t" // mode?
465
3.59k
         "jz        2f              \n\t" // 0 = decrypt
466
467
3.59k
         "1:                        \n\t" // encryption loop
468
3.59k
         "movdqu    (%1), %%xmm1    \n\t" // load round key
469
         AESENC(xmm1_xmm0)                // do round
470
3.59k
         "add       $16, %1         \n\t" // point to next round key
471
3.59k
         "subl      $1, %0          \n\t" // loop
472
3.59k
         "jnz       1b              \n\t"
473
3.59k
         "movdqu    (%1), %%xmm1    \n\t" // load round key
474
         AESENCLAST(xmm1_xmm0)            // last round
475
3.59k
#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
476
3.59k
         "jmp       3f              \n\t"
477
478
3.59k
         "2:                        \n\t" // decryption loop
479
3.59k
         "movdqu    (%1), %%xmm1    \n\t"
480
         AESDEC(xmm1_xmm0)                // do round
481
3.59k
         "add       $16, %1         \n\t"
482
3.59k
         "subl      $1, %0          \n\t"
483
3.59k
         "jnz       2b              \n\t"
484
3.59k
         "movdqu    (%1), %%xmm1    \n\t" // load round key
485
         AESDECLAST(xmm1_xmm0)            // last round
486
3.59k
#endif
487
488
3.59k
         "3:                        \n\t"
489
3.59k
         "movdqu    %%xmm0, (%4)    \n\t" // export output
490
3.59k
         :
491
3.59k
         : "r" (ctx->nr), "r" (ctx->buf + ctx->rk_offset), "r" (mode), "r" (input), "r" (output)
492
3.59k
         : "memory", "cc", "xmm0", "xmm1");
493
494
495
3.59k
    return 0;
496
3.59k
}
497
498
/*
499
 * GCM multiplication: c = a times b in GF(2^128)
500
 * Based on [CLMUL-WP] algorithms 1 (with equation 27) and 5.
501
 */
502
void mbedtls_aesni_gcm_mult(unsigned char c[16],
503
                            const unsigned char a[16],
504
                            const unsigned char b[16])
505
8.89k
{
506
8.89k
    unsigned char aa[16], bb[16], cc[16];
507
8.89k
    size_t i;
508
509
    /* The inputs are in big-endian order, so byte-reverse them */
510
151k
    for (i = 0; i < 16; i++) {
511
142k
        aa[i] = a[15 - i];
512
142k
        bb[i] = b[15 - i];
513
142k
    }
514
515
8.89k
    asm ("movdqu (%0), %%xmm0               \n\t" // a1:a0
516
8.89k
         "movdqu (%1), %%xmm1               \n\t" // b1:b0
517
518
         /*
519
          * Caryless multiplication xmm2:xmm1 = xmm0 * xmm1
520
          * using [CLMUL-WP] algorithm 1 (p. 12).
521
          */
522
8.89k
         "movdqa %%xmm1, %%xmm2             \n\t" // copy of b1:b0
523
8.89k
         "movdqa %%xmm1, %%xmm3             \n\t" // same
524
8.89k
         "movdqa %%xmm1, %%xmm4             \n\t" // same
525
         PCLMULQDQ(xmm0_xmm1, "0x00")             // a0*b0 = c1:c0
526
         PCLMULQDQ(xmm0_xmm2, "0x11")             // a1*b1 = d1:d0
527
         PCLMULQDQ(xmm0_xmm3, "0x10")             // a0*b1 = e1:e0
528
         PCLMULQDQ(xmm0_xmm4, "0x01")             // a1*b0 = f1:f0
529
8.89k
         "pxor %%xmm3, %%xmm4               \n\t" // e1+f1:e0+f0
530
8.89k
         "movdqa %%xmm4, %%xmm3             \n\t" // same
531
8.89k
         "psrldq $8, %%xmm4                 \n\t" // 0:e1+f1
532
8.89k
         "pslldq $8, %%xmm3                 \n\t" // e0+f0:0
533
8.89k
         "pxor %%xmm4, %%xmm2               \n\t" // d1:d0+e1+f1
534
8.89k
         "pxor %%xmm3, %%xmm1               \n\t" // c1+e0+f1:c0
535
536
         /*
537
          * Now shift the result one bit to the left,
538
          * taking advantage of [CLMUL-WP] eq 27 (p. 18)
539
          */
540
8.89k
         "movdqa %%xmm1, %%xmm3             \n\t" // r1:r0
541
8.89k
         "movdqa %%xmm2, %%xmm4             \n\t" // r3:r2
542
8.89k
         "psllq $1, %%xmm1                  \n\t" // r1<<1:r0<<1
543
8.89k
         "psllq $1, %%xmm2                  \n\t" // r3<<1:r2<<1
544
8.89k
         "psrlq $63, %%xmm3                 \n\t" // r1>>63:r0>>63
545
8.89k
         "psrlq $63, %%xmm4                 \n\t" // r3>>63:r2>>63
546
8.89k
         "movdqa %%xmm3, %%xmm5             \n\t" // r1>>63:r0>>63
547
8.89k
         "pslldq $8, %%xmm3                 \n\t" // r0>>63:0
548
8.89k
         "pslldq $8, %%xmm4                 \n\t" // r2>>63:0
549
8.89k
         "psrldq $8, %%xmm5                 \n\t" // 0:r1>>63
550
8.89k
         "por %%xmm3, %%xmm1                \n\t" // r1<<1|r0>>63:r0<<1
551
8.89k
         "por %%xmm4, %%xmm2                \n\t" // r3<<1|r2>>62:r2<<1
552
8.89k
         "por %%xmm5, %%xmm2                \n\t" // r3<<1|r2>>62:r2<<1|r1>>63
553
554
         /*
555
          * Now reduce modulo the GCM polynomial x^128 + x^7 + x^2 + x + 1
556
          * using [CLMUL-WP] algorithm 5 (p. 18).
557
          * Currently xmm2:xmm1 holds x3:x2:x1:x0 (already shifted).
558
          */
559
         /* Step 2 (1) */
560
8.89k
         "movdqa %%xmm1, %%xmm3             \n\t" // x1:x0
561
8.89k
         "movdqa %%xmm1, %%xmm4             \n\t" // same
562
8.89k
         "movdqa %%xmm1, %%xmm5             \n\t" // same
563
8.89k
         "psllq $63, %%xmm3                 \n\t" // x1<<63:x0<<63 = stuff:a
564
8.89k
         "psllq $62, %%xmm4                 \n\t" // x1<<62:x0<<62 = stuff:b
565
8.89k
         "psllq $57, %%xmm5                 \n\t" // x1<<57:x0<<57 = stuff:c
566
567
         /* Step 2 (2) */
568
8.89k
         "pxor %%xmm4, %%xmm3               \n\t" // stuff:a+b
569
8.89k
         "pxor %%xmm5, %%xmm3               \n\t" // stuff:a+b+c
570
8.89k
         "pslldq $8, %%xmm3                 \n\t" // a+b+c:0
571
8.89k
         "pxor %%xmm3, %%xmm1               \n\t" // x1+a+b+c:x0 = d:x0
572
573
         /* Steps 3 and 4 */
574
8.89k
         "movdqa %%xmm1,%%xmm0              \n\t" // d:x0
575
8.89k
         "movdqa %%xmm1,%%xmm4              \n\t" // same
576
8.89k
         "movdqa %%xmm1,%%xmm5              \n\t" // same
577
8.89k
         "psrlq $1, %%xmm0                  \n\t" // e1:x0>>1 = e1:e0'
578
8.89k
         "psrlq $2, %%xmm4                  \n\t" // f1:x0>>2 = f1:f0'
579
8.89k
         "psrlq $7, %%xmm5                  \n\t" // g1:x0>>7 = g1:g0'
580
8.89k
         "pxor %%xmm4, %%xmm0               \n\t" // e1+f1:e0'+f0'
581
8.89k
         "pxor %%xmm5, %%xmm0               \n\t" // e1+f1+g1:e0'+f0'+g0'
582
         // e0'+f0'+g0' is almost e0+f0+g0, ex\tcept for some missing
583
         // bits carried from d. Now get those\t bits back in.
584
8.89k
         "movdqa %%xmm1,%%xmm3              \n\t" // d:x0
585
8.89k
         "movdqa %%xmm1,%%xmm4              \n\t" // same
586
8.89k
         "movdqa %%xmm1,%%xmm5              \n\t" // same
587
8.89k
         "psllq $63, %%xmm3                 \n\t" // d<<63:stuff
588
8.89k
         "psllq $62, %%xmm4                 \n\t" // d<<62:stuff
589
8.89k
         "psllq $57, %%xmm5                 \n\t" // d<<57:stuff
590
8.89k
         "pxor %%xmm4, %%xmm3               \n\t" // d<<63+d<<62:stuff
591
8.89k
         "pxor %%xmm5, %%xmm3               \n\t" // missing bits of d:stuff
592
8.89k
         "psrldq $8, %%xmm3                 \n\t" // 0:missing bits of d
593
8.89k
         "pxor %%xmm3, %%xmm0               \n\t" // e1+f1+g1:e0+f0+g0
594
8.89k
         "pxor %%xmm1, %%xmm0               \n\t" // h1:h0
595
8.89k
         "pxor %%xmm2, %%xmm0               \n\t" // x3+h1:x2+h0
596
597
8.89k
         "movdqu %%xmm0, (%2)               \n\t" // done
598
8.89k
         :
599
8.89k
         : "r" (aa), "r" (bb), "r" (cc)
600
8.89k
         : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5");
601
602
    /* Now byte-reverse the outputs */
603
151k
    for (i = 0; i < 16; i++) {
604
142k
        c[i] = cc[15 - i];
605
142k
    }
606
607
8.89k
    return;
608
8.89k
}
609
610
/*
611
 * Compute decryption round keys from encryption round keys
612
 */
613
#if !defined(MBEDTLS_BLOCK_CIPHER_NO_DECRYPT)
614
void mbedtls_aesni_inverse_key(unsigned char *invkey,
615
                               const unsigned char *fwdkey, int nr)
616
22
{
617
22
    unsigned char *ik = invkey;
618
22
    const unsigned char *fk = fwdkey + 16 * nr;
619
620
22
    memcpy(ik, fk, 16);
621
622
238
    for (fk -= 16, ik += 16; fk > fwdkey; fk -= 16, ik += 16) {
623
216
        asm ("movdqu (%0), %%xmm0       \n\t"
624
216
             AESIMC(xmm0_xmm0)
625
216
             "movdqu %%xmm0, (%1)       \n\t"
626
216
             :
627
216
             : "r" (fk), "r" (ik)
628
216
             : "memory", "xmm0");
629
216
    }
630
631
22
    memcpy(ik, fk, 16);
632
22
}
633
#endif
634
635
/*
636
 * Key expansion, 128-bit case
637
 */
638
static void aesni_setkey_enc_128(unsigned char *rk,
639
                                 const unsigned char *key)
640
123
{
641
123
    asm ("movdqu (%1), %%xmm0               \n\t" // copy the original key
642
123
         "movdqu %%xmm0, (%0)               \n\t" // as round key 0
643
123
         "jmp 2f                            \n\t" // skip auxiliary routine
644
645
         /*
646
          * Finish generating the next round key.
647
          *
648
          * On entry xmm0 is r3:r2:r1:r0 and xmm1 is X:stuff:stuff:stuff
649
          * with X = rot( sub( r3 ) ) ^ RCON.
650
          *
651
          * On exit, xmm0 is r7:r6:r5:r4
652
          * with r4 = X + r0, r5 = r4 + r1, r6 = r5 + r2, r7 = r6 + r3
653
          * and those are written to the round key buffer.
654
          */
655
123
         "1:                                \n\t"
656
123
         "pshufd $0xff, %%xmm1, %%xmm1      \n\t" // X:X:X:X
657
123
         "pxor %%xmm0, %%xmm1               \n\t" // X+r3:X+r2:X+r1:r4
658
123
         "pslldq $4, %%xmm0                 \n\t" // r2:r1:r0:0
659
123
         "pxor %%xmm0, %%xmm1               \n\t" // X+r3+r2:X+r2+r1:r5:r4
660
123
         "pslldq $4, %%xmm0                 \n\t" // etc
661
123
         "pxor %%xmm0, %%xmm1               \n\t"
662
123
         "pslldq $4, %%xmm0                 \n\t"
663
123
         "pxor %%xmm1, %%xmm0               \n\t" // update xmm0 for next time!
664
123
         "add $16, %0                       \n\t" // point to next round key
665
123
         "movdqu %%xmm0, (%0)               \n\t" // write it
666
123
         "ret                               \n\t"
667
668
         /* Main "loop" */
669
123
         "2:                                \n\t"
670
123
         AESKEYGENA(xmm0_xmm1, "0x01")      "call 1b \n\t"
671
123
         AESKEYGENA(xmm0_xmm1, "0x02")      "call 1b \n\t"
672
123
         AESKEYGENA(xmm0_xmm1, "0x04")      "call 1b \n\t"
673
123
         AESKEYGENA(xmm0_xmm1, "0x08")      "call 1b \n\t"
674
123
         AESKEYGENA(xmm0_xmm1, "0x10")      "call 1b \n\t"
675
123
         AESKEYGENA(xmm0_xmm1, "0x20")      "call 1b \n\t"
676
123
         AESKEYGENA(xmm0_xmm1, "0x40")      "call 1b \n\t"
677
123
         AESKEYGENA(xmm0_xmm1, "0x80")      "call 1b \n\t"
678
123
         AESKEYGENA(xmm0_xmm1, "0x1B")      "call 1b \n\t"
679
123
         AESKEYGENA(xmm0_xmm1, "0x36")      "call 1b \n\t"
680
123
         :
681
123
         : "r" (rk), "r" (key)
682
123
         : "memory", "cc", "0");
683
123
}
684
685
/*
686
 * Key expansion, 192-bit case
687
 */
688
#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
689
static void aesni_setkey_enc_192(unsigned char *rk,
690
                                 const unsigned char *key)
691
72
{
692
72
    asm ("movdqu (%1), %%xmm0   \n\t" // copy original round key
693
72
         "movdqu %%xmm0, (%0)   \n\t"
694
72
         "add $16, %0           \n\t"
695
72
         "movq 16(%1), %%xmm1   \n\t"
696
72
         "movq %%xmm1, (%0)     \n\t"
697
72
         "add $8, %0            \n\t"
698
72
         "jmp 2f                \n\t" // skip auxiliary routine
699
700
         /*
701
          * Finish generating the next 6 quarter-keys.
702
          *
703
          * On entry xmm0 is r3:r2:r1:r0, xmm1 is stuff:stuff:r5:r4
704
          * and xmm2 is stuff:stuff:X:stuff with X = rot( sub( r3 ) ) ^ RCON.
705
          *
706
          * On exit, xmm0 is r9:r8:r7:r6 and xmm1 is stuff:stuff:r11:r10
707
          * and those are written to the round key buffer.
708
          */
709
72
         "1:                            \n\t"
710
72
         "pshufd $0x55, %%xmm2, %%xmm2  \n\t" // X:X:X:X
711
72
         "pxor %%xmm0, %%xmm2           \n\t" // X+r3:X+r2:X+r1:r4
712
72
         "pslldq $4, %%xmm0             \n\t" // etc
713
72
         "pxor %%xmm0, %%xmm2           \n\t"
714
72
         "pslldq $4, %%xmm0             \n\t"
715
72
         "pxor %%xmm0, %%xmm2           \n\t"
716
72
         "pslldq $4, %%xmm0             \n\t"
717
72
         "pxor %%xmm2, %%xmm0           \n\t" // update xmm0 = r9:r8:r7:r6
718
72
         "movdqu %%xmm0, (%0)           \n\t"
719
72
         "add $16, %0                   \n\t"
720
72
         "pshufd $0xff, %%xmm0, %%xmm2  \n\t" // r9:r9:r9:r9
721
72
         "pxor %%xmm1, %%xmm2           \n\t" // stuff:stuff:r9+r5:r10
722
72
         "pslldq $4, %%xmm1             \n\t" // r2:r1:r0:0
723
72
         "pxor %%xmm2, %%xmm1           \n\t" // xmm1 = stuff:stuff:r11:r10
724
72
         "movq %%xmm1, (%0)             \n\t"
725
72
         "add $8, %0                    \n\t"
726
72
         "ret                           \n\t"
727
728
72
         "2:                            \n\t"
729
72
         AESKEYGENA(xmm1_xmm2, "0x01")  "call 1b \n\t"
730
72
         AESKEYGENA(xmm1_xmm2, "0x02")  "call 1b \n\t"
731
72
         AESKEYGENA(xmm1_xmm2, "0x04")  "call 1b \n\t"
732
72
         AESKEYGENA(xmm1_xmm2, "0x08")  "call 1b \n\t"
733
72
         AESKEYGENA(xmm1_xmm2, "0x10")  "call 1b \n\t"
734
72
         AESKEYGENA(xmm1_xmm2, "0x20")  "call 1b \n\t"
735
72
         AESKEYGENA(xmm1_xmm2, "0x40")  "call 1b \n\t"
736
72
         AESKEYGENA(xmm1_xmm2, "0x80")  "call 1b \n\t"
737
738
72
         :
739
72
         : "r" (rk), "r" (key)
740
72
         : "memory", "cc", "0");
741
72
}
742
#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
743
744
/*
745
 * Key expansion, 256-bit case
746
 */
747
#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
748
static void aesni_setkey_enc_256(unsigned char *rk,
749
                                 const unsigned char *key)
750
115
{
751
115
    asm ("movdqu (%1), %%xmm0           \n\t"
752
115
         "movdqu %%xmm0, (%0)           \n\t"
753
115
         "add $16, %0                   \n\t"
754
115
         "movdqu 16(%1), %%xmm1         \n\t"
755
115
         "movdqu %%xmm1, (%0)           \n\t"
756
115
         "jmp 2f                        \n\t" // skip auxiliary routine
757
758
         /*
759
          * Finish generating the next two round keys.
760
          *
761
          * On entry xmm0 is r3:r2:r1:r0, xmm1 is r7:r6:r5:r4 and
762
          * xmm2 is X:stuff:stuff:stuff with X = rot( sub( r7 )) ^ RCON
763
          *
764
          * On exit, xmm0 is r11:r10:r9:r8 and xmm1 is r15:r14:r13:r12
765
          * and those have been written to the output buffer.
766
          */
767
115
         "1:                                \n\t"
768
115
         "pshufd $0xff, %%xmm2, %%xmm2      \n\t"
769
115
         "pxor %%xmm0, %%xmm2               \n\t"
770
115
         "pslldq $4, %%xmm0                 \n\t"
771
115
         "pxor %%xmm0, %%xmm2               \n\t"
772
115
         "pslldq $4, %%xmm0                 \n\t"
773
115
         "pxor %%xmm0, %%xmm2               \n\t"
774
115
         "pslldq $4, %%xmm0                 \n\t"
775
115
         "pxor %%xmm2, %%xmm0               \n\t"
776
115
         "add $16, %0                       \n\t"
777
115
         "movdqu %%xmm0, (%0)               \n\t"
778
779
         /* Set xmm2 to stuff:Y:stuff:stuff with Y = subword( r11 )
780
          * and proceed to generate next round key from there */
781
115
         AESKEYGENA(xmm0_xmm2, "0x00")
782
115
         "pshufd $0xaa, %%xmm2, %%xmm2      \n\t"
783
115
         "pxor %%xmm1, %%xmm2               \n\t"
784
115
         "pslldq $4, %%xmm1                 \n\t"
785
115
         "pxor %%xmm1, %%xmm2               \n\t"
786
115
         "pslldq $4, %%xmm1                 \n\t"
787
115
         "pxor %%xmm1, %%xmm2               \n\t"
788
115
         "pslldq $4, %%xmm1                 \n\t"
789
115
         "pxor %%xmm2, %%xmm1               \n\t"
790
115
         "add $16, %0                       \n\t"
791
115
         "movdqu %%xmm1, (%0)               \n\t"
792
115
         "ret                               \n\t"
793
794
         /*
795
          * Main "loop" - Generating one more key than necessary,
796
          * see definition of mbedtls_aes_context.buf
797
          */
798
115
         "2:                                \n\t"
799
115
         AESKEYGENA(xmm1_xmm2, "0x01")      "call 1b \n\t"
800
115
         AESKEYGENA(xmm1_xmm2, "0x02")      "call 1b \n\t"
801
115
         AESKEYGENA(xmm1_xmm2, "0x04")      "call 1b \n\t"
802
115
         AESKEYGENA(xmm1_xmm2, "0x08")      "call 1b \n\t"
803
115
         AESKEYGENA(xmm1_xmm2, "0x10")      "call 1b \n\t"
804
115
         AESKEYGENA(xmm1_xmm2, "0x20")      "call 1b \n\t"
805
115
         AESKEYGENA(xmm1_xmm2, "0x40")      "call 1b \n\t"
806
115
         :
807
115
         : "r" (rk), "r" (key)
808
115
         : "memory", "cc", "0");
809
115
}
810
#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
811
812
#endif  /* MBEDTLS_AESNI_HAVE_CODE */
813
814
/*
815
 * Key expansion, wrapper
816
 */
817
int mbedtls_aesni_setkey_enc(unsigned char *rk,
818
                             const unsigned char *key,
819
                             size_t bits)
820
310
{
821
310
    switch (bits) {
822
123
        case 128: aesni_setkey_enc_128(rk, key); break;
823
0
#if !defined(MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH)
824
72
        case 192: aesni_setkey_enc_192(rk, key); break;
825
115
        case 256: aesni_setkey_enc_256(rk, key); break;
826
0
#endif /* !MBEDTLS_AES_ONLY_128_BIT_KEY_LENGTH */
827
0
        default: return MBEDTLS_ERR_AES_INVALID_KEY_LENGTH;
828
310
    }
829
830
310
    return 0;
831
310
}
832
833
#endif /* MBEDTLS_AESNI_HAVE_CODE */
834
835
#endif /* MBEDTLS_AESNI_C */