Coverage Report

Created: 2024-11-21 06:47

/src/openssl/crypto/bn/asm/x86_64-gcc.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright 2002-2018 The OpenSSL Project Authors. All Rights Reserved.
3
 *
4
 * Licensed under the Apache License 2.0 (the "License").  You may not use
5
 * this file except in compliance with the License.  You can obtain a copy
6
 * in the file LICENSE in the source distribution or at
7
 * https://www.openssl.org/source/license.html
8
 */
9
10
#include "../bn_local.h"
11
#if !(defined(__GNUC__) && __GNUC__>=2)
12
# include "../bn_asm.c"         /* kind of dirty hack for Sun Studio */
13
#else
14
/*-
15
 * x86_64 BIGNUM accelerator version 0.1, December 2002.
16
 *
17
 * Implemented by Andy Polyakov <appro@openssl.org> for the OpenSSL
18
 * project.
19
 *
20
 * Rights for redistribution and usage in source and binary forms are
21
 * granted according to the License. Warranty of any kind is disclaimed.
22
 *
23
 * Q. Version 0.1? It doesn't sound like Andy, he used to assign real
24
 *    versions, like 1.0...
25
 * A. Well, that's because this code is basically a quick-n-dirty
26
 *    proof-of-concept hack. As you can see it's implemented with
27
 *    inline assembler, which means that you're bound to GCC and that
28
 *    there might be enough room for further improvement.
29
 *
30
 * Q. Why inline assembler?
31
 * A. x86_64 features own ABI which I'm not familiar with. This is
32
 *    why I decided to let the compiler take care of subroutine
33
 *    prologue/epilogue as well as register allocation. For reference.
34
 *    Win64 implements different ABI for AMD64, different from Linux.
35
 *
36
 * Q. How much faster does it get?
37
 * A. 'apps/openssl speed rsa dsa' output with no-asm:
38
 *
39
 *                        sign    verify    sign/s verify/s
40
 *      rsa  512 bits   0.0006s   0.0001s   1683.8  18456.2
41
 *      rsa 1024 bits   0.0028s   0.0002s    356.0   6407.0
42
 *      rsa 2048 bits   0.0172s   0.0005s     58.0   1957.8
43
 *      rsa 4096 bits   0.1155s   0.0018s      8.7    555.6
44
 *                        sign    verify    sign/s verify/s
45
 *      dsa  512 bits   0.0005s   0.0006s   2100.8   1768.3
46
 *      dsa 1024 bits   0.0014s   0.0018s    692.3    559.2
47
 *      dsa 2048 bits   0.0049s   0.0061s    204.7    165.0
48
 *
49
 *    'apps/openssl speed rsa dsa' output with this module:
50
 *
51
 *                        sign    verify    sign/s verify/s
52
 *      rsa  512 bits   0.0004s   0.0000s   2767.1  33297.9
53
 *      rsa 1024 bits   0.0012s   0.0001s    867.4  14674.7
54
 *      rsa 2048 bits   0.0061s   0.0002s    164.0   5270.0
55
 *      rsa 4096 bits   0.0384s   0.0006s     26.1   1650.8
56
 *                        sign    verify    sign/s verify/s
57
 *      dsa  512 bits   0.0002s   0.0003s   4442.2   3786.3
58
 *      dsa 1024 bits   0.0005s   0.0007s   1835.1   1497.4
59
 *      dsa 2048 bits   0.0016s   0.0020s    620.4    504.6
60
 *
61
 *    For the reference. IA-32 assembler implementation performs
62
 *    very much like 64-bit code compiled with no-asm on the same
63
 *    machine.
64
 */
65
66
# undef mul
67
# undef mul_add
68
69
/*-
70
 * "m"(a), "+m"(r)      is the way to favor DirectPath ยต-code;
71
 * "g"(0)               let the compiler to decide where does it
72
 *                      want to keep the value of zero;
73
 */
74
16.3G
# define mul_add(r,a,word,carry) do {   \
75
16.3G
        register BN_ULONG high,low;     \
76
16.3G
        asm ("mulq %3"                  \
77
16.3G
                : "=a"(low),"=d"(high)  \
78
16.3G
                : "a"(word),"m"(a)      \
79
16.3G
                : "cc");                \
80
16.3G
        asm ("addq %2,%0; adcq %3,%1"   \
81
16.3G
                : "+r"(carry),"+d"(high)\
82
16.3G
                : "a"(low),"g"(0)       \
83
16.3G
                : "cc");                \
84
16.3G
        asm ("addq %2,%0; adcq %3,%1"   \
85
16.3G
                : "+m"(r),"+d"(high)    \
86
16.3G
                : "r"(carry),"g"(0)     \
87
16.3G
                : "cc");                \
88
16.3G
        carry=high;                     \
89
16.3G
        } while (0)
90
91
29.0G
# define mul(r,a,word,carry) do {       \
92
29.0G
        register BN_ULONG high,low;     \
93
29.0G
        asm ("mulq %3"                  \
94
29.0G
                : "=a"(low),"=d"(high)  \
95
29.0G
                : "a"(word),"g"(a)      \
96
29.0G
                : "cc");                \
97
29.0G
        asm ("addq %2,%0; adcq %3,%1"   \
98
29.0G
                : "+r"(carry),"+d"(high)\
99
29.0G
                : "a"(low),"g"(0)       \
100
29.0G
                : "cc");                \
101
29.0G
        (r)=carry, carry=high;          \
102
29.0G
        } while (0)
103
# undef sqr
104
# define sqr(r0,r1,a)                   \
105
593M
        asm ("mulq %2"                  \
106
593M
                : "=a"(r0),"=d"(r1)     \
107
593M
                : "a"(a)                \
108
593M
                : "cc");
109
110
BN_ULONG bn_mul_add_words(BN_ULONG *rp, const BN_ULONG *ap, int num,
111
                          BN_ULONG w)
112
606M
{
113
606M
    BN_ULONG c1 = 0;
114
115
606M
    if (num <= 0)
116
0
        return c1;
117
118
4.46G
    while (num & ~3) {
119
3.85G
        mul_add(rp[0], ap[0], w, c1);
120
3.85G
        mul_add(rp[1], ap[1], w, c1);
121
3.85G
        mul_add(rp[2], ap[2], w, c1);
122
3.85G
        mul_add(rp[3], ap[3], w, c1);
123
3.85G
        ap += 4;
124
3.85G
        rp += 4;
125
3.85G
        num -= 4;
126
3.85G
    }
127
606M
    if (num) {
128
462M
        mul_add(rp[0], ap[0], w, c1);
129
462M
        if (--num == 0)
130
155M
            return c1;
131
306M
        mul_add(rp[1], ap[1], w, c1);
132
306M
        if (--num == 0)
133
149M
            return c1;
134
157M
        mul_add(rp[2], ap[2], w, c1);
135
157M
        return c1;
136
306M
    }
137
138
144M
    return c1;
139
606M
}
140
141
BN_ULONG bn_mul_words(BN_ULONG *rp, const BN_ULONG *ap, int num, BN_ULONG w)
142
526M
{
143
526M
    BN_ULONG c1 = 0;
144
145
526M
    if (num <= 0)
146
0
        return c1;
147
148
7.70G
    while (num & ~3) {
149
7.18G
        mul(rp[0], ap[0], w, c1);
150
7.18G
        mul(rp[1], ap[1], w, c1);
151
7.18G
        mul(rp[2], ap[2], w, c1);
152
7.18G
        mul(rp[3], ap[3], w, c1);
153
7.18G
        ap += 4;
154
7.18G
        rp += 4;
155
7.18G
        num -= 4;
156
7.18G
    }
157
526M
    if (num) {
158
188M
        mul(rp[0], ap[0], w, c1);
159
188M
        if (--num == 0)
160
97.7M
            return c1;
161
90.9M
        mul(rp[1], ap[1], w, c1);
162
90.9M
        if (--num == 0)
163
20.8M
            return c1;
164
70.1M
        mul(rp[2], ap[2], w, c1);
165
70.1M
    }
166
408M
    return c1;
167
526M
}
168
169
void bn_sqr_words(BN_ULONG *r, const BN_ULONG *a, int n)
170
12.7M
{
171
12.7M
    if (n <= 0)
172
0
        return;
173
174
156M
    while (n & ~3) {
175
143M
        sqr(r[0], r[1], a[0]);
176
143M
        sqr(r[2], r[3], a[1]);
177
143M
        sqr(r[4], r[5], a[2]);
178
143M
        sqr(r[6], r[7], a[3]);
179
143M
        a += 4;
180
143M
        r += 8;
181
143M
        n -= 4;
182
143M
    }
183
12.7M
    if (n) {
184
10.0M
        sqr(r[0], r[1], a[0]);
185
10.0M
        if (--n == 0)
186
4.15M
            return;
187
5.87M
        sqr(r[2], r[3], a[1]);
188
5.87M
        if (--n == 0)
189
903k
            return;
190
4.97M
        sqr(r[4], r[5], a[2]);
191
4.97M
    }
192
12.7M
}
193
194
BN_ULONG bn_div_words(BN_ULONG h, BN_ULONG l, BN_ULONG d)
195
922M
{
196
922M
    BN_ULONG ret, waste;
197
198
922M
 asm("divq      %4":"=a"(ret), "=d"(waste)
199
922M
 :     "a"(l), "d"(h), "r"(d)
200
922M
 :     "cc");
201
202
922M
    return ret;
203
922M
}
204
205
BN_ULONG bn_add_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
206
                      int n)
207
687M
{
208
687M
    BN_ULONG ret;
209
687M
    size_t i = 0;
210
211
687M
    if (n <= 0)
212
8.58M
        return 0;
213
214
679M
    asm volatile ("       subq    %0,%0           \n" /* clear carry */
215
679M
                  "       jmp     1f              \n"
216
679M
                  ".p2align 4                     \n"
217
679M
                  "1:     movq    (%4,%2,8),%0    \n"
218
679M
                  "       adcq    (%5,%2,8),%0    \n"
219
679M
                  "       movq    %0,(%3,%2,8)    \n"
220
679M
                  "       lea     1(%2),%2        \n"
221
679M
                  "       dec     %1              \n"
222
679M
                  "       jnz     1b              \n"
223
679M
                  "       sbbq    %0,%0           \n"
224
679M
                  :"=&r" (ret), "+c"(n), "+r"(i)
225
679M
                  :"r"(rp), "r"(ap), "r"(bp)
226
679M
                  :"cc", "memory");
227
228
679M
    return ret & 1;
229
687M
}
230
231
# ifndef SIMICS
232
BN_ULONG bn_sub_words(BN_ULONG *rp, const BN_ULONG *ap, const BN_ULONG *bp,
233
                      int n)
234
684M
{
235
684M
    BN_ULONG ret;
236
684M
    size_t i = 0;
237
238
684M
    if (n <= 0)
239
8.70M
        return 0;
240
241
676M
    asm volatile ("       subq    %0,%0           \n" /* clear borrow */
242
676M
                  "       jmp     1f              \n"
243
676M
                  ".p2align 4                     \n"
244
676M
                  "1:     movq    (%4,%2,8),%0    \n"
245
676M
                  "       sbbq    (%5,%2,8),%0    \n"
246
676M
                  "       movq    %0,(%3,%2,8)    \n"
247
676M
                  "       lea     1(%2),%2        \n"
248
676M
                  "       dec     %1              \n"
249
676M
                  "       jnz     1b              \n"
250
676M
                  "       sbbq    %0,%0           \n"
251
676M
                  :"=&r" (ret), "+c"(n), "+r"(i)
252
676M
                  :"r"(rp), "r"(ap), "r"(bp)
253
676M
                  :"cc", "memory");
254
255
676M
    return ret & 1;
256
684M
}
257
# else
258
/* Simics 1.4<7 has buggy sbbq:-( */
259
#  define BN_MASK2 0xffffffffffffffffL
260
BN_ULONG bn_sub_words(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b, int n)
261
{
262
    BN_ULONG t1, t2;
263
    int c = 0;
264
265
    if (n <= 0)
266
        return (BN_ULONG)0;
267
268
    for (;;) {
269
        t1 = a[0];
270
        t2 = b[0];
271
        r[0] = (t1 - t2 - c) & BN_MASK2;
272
        if (t1 != t2)
273
            c = (t1 < t2);
274
        if (--n <= 0)
275
            break;
276
277
        t1 = a[1];
278
        t2 = b[1];
279
        r[1] = (t1 - t2 - c) & BN_MASK2;
280
        if (t1 != t2)
281
            c = (t1 < t2);
282
        if (--n <= 0)
283
            break;
284
285
        t1 = a[2];
286
        t2 = b[2];
287
        r[2] = (t1 - t2 - c) & BN_MASK2;
288
        if (t1 != t2)
289
            c = (t1 < t2);
290
        if (--n <= 0)
291
            break;
292
293
        t1 = a[3];
294
        t2 = b[3];
295
        r[3] = (t1 - t2 - c) & BN_MASK2;
296
        if (t1 != t2)
297
            c = (t1 < t2);
298
        if (--n <= 0)
299
            break;
300
301
        a += 4;
302
        b += 4;
303
        r += 4;
304
    }
305
    return c;
306
}
307
# endif
308
309
/* mul_add_c(a,b,c0,c1,c2)  -- c+=a*b for three word number c=(c2,c1,c0) */
310
/* mul_add_c2(a,b,c0,c1,c2) -- c+=2*a*b for three word number c=(c2,c1,c0) */
311
/* sqr_add_c(a,i,c0,c1,c2)  -- c+=a[i]^2 for three word number c=(c2,c1,c0) */
312
/*
313
 * sqr_add_c2(a,i,c0,c1,c2) -- c+=2*a[i]*a[j] for three word number
314
 * c=(c2,c1,c0)
315
 */
316
317
/*
318
 * Keep in mind that carrying into high part of multiplication result
319
 * can not overflow, because it cannot be all-ones.
320
 */
321
# if 0
322
/* original macros are kept for reference purposes */
323
#  define mul_add_c(a,b,c0,c1,c2)       do {    \
324
        BN_ULONG ta = (a), tb = (b);            \
325
        BN_ULONG lo, hi;                        \
326
        BN_UMULT_LOHI(lo,hi,ta,tb);             \
327
        c0 += lo; hi += (c0<lo)?1:0;            \
328
        c1 += hi; c2 += (c1<hi)?1:0;            \
329
        } while(0)
330
331
#  define mul_add_c2(a,b,c0,c1,c2)      do {    \
332
        BN_ULONG ta = (a), tb = (b);            \
333
        BN_ULONG lo, hi, tt;                    \
334
        BN_UMULT_LOHI(lo,hi,ta,tb);             \
335
        c0 += lo; tt = hi+((c0<lo)?1:0);        \
336
        c1 += tt; c2 += (c1<tt)?1:0;            \
337
        c0 += lo; hi += (c0<lo)?1:0;            \
338
        c1 += hi; c2 += (c1<hi)?1:0;            \
339
        } while(0)
340
341
#  define sqr_add_c(a,i,c0,c1,c2)       do {    \
342
        BN_ULONG ta = (a)[i];                   \
343
        BN_ULONG lo, hi;                        \
344
        BN_UMULT_LOHI(lo,hi,ta,ta);             \
345
        c0 += lo; hi += (c0<lo)?1:0;            \
346
        c1 += hi; c2 += (c1<hi)?1:0;            \
347
        } while(0)
348
# else
349
7.94G
#  define mul_add_c(a,b,c0,c1,c2) do {  \
350
7.94G
        BN_ULONG t1,t2;                 \
351
7.94G
        asm ("mulq %3"                  \
352
7.94G
                : "=a"(t1),"=d"(t2)     \
353
7.94G
                : "a"(a),"m"(b)         \
354
7.94G
                : "cc");                \
355
7.94G
        asm ("addq %3,%0; adcq %4,%1; adcq %5,%2"       \
356
7.94G
                : "+r"(c0),"+r"(c1),"+r"(c2)            \
357
7.94G
                : "r"(t1),"r"(t2),"g"(0)                \
358
7.94G
                : "cc");                                \
359
7.94G
        } while (0)
360
361
44.1M
#  define sqr_add_c(a,i,c0,c1,c2) do {  \
362
44.1M
        BN_ULONG t1,t2;                 \
363
44.1M
        asm ("mulq %2"                  \
364
44.1M
                : "=a"(t1),"=d"(t2)     \
365
44.1M
                : "a"(a[i])             \
366
44.1M
                : "cc");                \
367
44.1M
        asm ("addq %3,%0; adcq %4,%1; adcq %5,%2"       \
368
44.1M
                : "+r"(c0),"+r"(c1),"+r"(c2)            \
369
44.1M
                : "r"(t1),"r"(t2),"g"(0)                \
370
44.1M
                : "cc");                                \
371
44.1M
        } while (0)
372
373
154M
#  define mul_add_c2(a,b,c0,c1,c2) do { \
374
154M
        BN_ULONG t1,t2;                 \
375
154M
        asm ("mulq %3"                  \
376
154M
                : "=a"(t1),"=d"(t2)     \
377
154M
                : "a"(a),"m"(b)         \
378
154M
                : "cc");                \
379
154M
        asm ("addq %3,%0; adcq %4,%1; adcq %5,%2"       \
380
154M
                : "+r"(c0),"+r"(c1),"+r"(c2)            \
381
154M
                : "r"(t1),"r"(t2),"g"(0)                \
382
154M
                : "cc");                                \
383
154M
        asm ("addq %3,%0; adcq %4,%1; adcq %5,%2"       \
384
154M
                : "+r"(c0),"+r"(c1),"+r"(c2)            \
385
154M
                : "r"(t1),"r"(t2),"g"(0)                \
386
154M
                : "cc");                                \
387
154M
        } while (0)
388
# endif
389
390
# define sqr_add_c2(a,i,j,c0,c1,c2)      \
391
154M
        mul_add_c2((a)[i],(a)[j],c0,c1,c2)
392
393
void bn_mul_comba8(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
394
124M
{
395
124M
    BN_ULONG c1, c2, c3;
396
397
124M
    c1 = 0;
398
124M
    c2 = 0;
399
124M
    c3 = 0;
400
124M
    mul_add_c(a[0], b[0], c1, c2, c3);
401
124M
    r[0] = c1;
402
124M
    c1 = 0;
403
124M
    mul_add_c(a[0], b[1], c2, c3, c1);
404
124M
    mul_add_c(a[1], b[0], c2, c3, c1);
405
124M
    r[1] = c2;
406
124M
    c2 = 0;
407
124M
    mul_add_c(a[2], b[0], c3, c1, c2);
408
124M
    mul_add_c(a[1], b[1], c3, c1, c2);
409
124M
    mul_add_c(a[0], b[2], c3, c1, c2);
410
124M
    r[2] = c3;
411
124M
    c3 = 0;
412
124M
    mul_add_c(a[0], b[3], c1, c2, c3);
413
124M
    mul_add_c(a[1], b[2], c1, c2, c3);
414
124M
    mul_add_c(a[2], b[1], c1, c2, c3);
415
124M
    mul_add_c(a[3], b[0], c1, c2, c3);
416
124M
    r[3] = c1;
417
124M
    c1 = 0;
418
124M
    mul_add_c(a[4], b[0], c2, c3, c1);
419
124M
    mul_add_c(a[3], b[1], c2, c3, c1);
420
124M
    mul_add_c(a[2], b[2], c2, c3, c1);
421
124M
    mul_add_c(a[1], b[3], c2, c3, c1);
422
124M
    mul_add_c(a[0], b[4], c2, c3, c1);
423
124M
    r[4] = c2;
424
124M
    c2 = 0;
425
124M
    mul_add_c(a[0], b[5], c3, c1, c2);
426
124M
    mul_add_c(a[1], b[4], c3, c1, c2);
427
124M
    mul_add_c(a[2], b[3], c3, c1, c2);
428
124M
    mul_add_c(a[3], b[2], c3, c1, c2);
429
124M
    mul_add_c(a[4], b[1], c3, c1, c2);
430
124M
    mul_add_c(a[5], b[0], c3, c1, c2);
431
124M
    r[5] = c3;
432
124M
    c3 = 0;
433
124M
    mul_add_c(a[6], b[0], c1, c2, c3);
434
124M
    mul_add_c(a[5], b[1], c1, c2, c3);
435
124M
    mul_add_c(a[4], b[2], c1, c2, c3);
436
124M
    mul_add_c(a[3], b[3], c1, c2, c3);
437
124M
    mul_add_c(a[2], b[4], c1, c2, c3);
438
124M
    mul_add_c(a[1], b[5], c1, c2, c3);
439
124M
    mul_add_c(a[0], b[6], c1, c2, c3);
440
124M
    r[6] = c1;
441
124M
    c1 = 0;
442
124M
    mul_add_c(a[0], b[7], c2, c3, c1);
443
124M
    mul_add_c(a[1], b[6], c2, c3, c1);
444
124M
    mul_add_c(a[2], b[5], c2, c3, c1);
445
124M
    mul_add_c(a[3], b[4], c2, c3, c1);
446
124M
    mul_add_c(a[4], b[3], c2, c3, c1);
447
124M
    mul_add_c(a[5], b[2], c2, c3, c1);
448
124M
    mul_add_c(a[6], b[1], c2, c3, c1);
449
124M
    mul_add_c(a[7], b[0], c2, c3, c1);
450
124M
    r[7] = c2;
451
124M
    c2 = 0;
452
124M
    mul_add_c(a[7], b[1], c3, c1, c2);
453
124M
    mul_add_c(a[6], b[2], c3, c1, c2);
454
124M
    mul_add_c(a[5], b[3], c3, c1, c2);
455
124M
    mul_add_c(a[4], b[4], c3, c1, c2);
456
124M
    mul_add_c(a[3], b[5], c3, c1, c2);
457
124M
    mul_add_c(a[2], b[6], c3, c1, c2);
458
124M
    mul_add_c(a[1], b[7], c3, c1, c2);
459
124M
    r[8] = c3;
460
124M
    c3 = 0;
461
124M
    mul_add_c(a[2], b[7], c1, c2, c3);
462
124M
    mul_add_c(a[3], b[6], c1, c2, c3);
463
124M
    mul_add_c(a[4], b[5], c1, c2, c3);
464
124M
    mul_add_c(a[5], b[4], c1, c2, c3);
465
124M
    mul_add_c(a[6], b[3], c1, c2, c3);
466
124M
    mul_add_c(a[7], b[2], c1, c2, c3);
467
124M
    r[9] = c1;
468
124M
    c1 = 0;
469
124M
    mul_add_c(a[7], b[3], c2, c3, c1);
470
124M
    mul_add_c(a[6], b[4], c2, c3, c1);
471
124M
    mul_add_c(a[5], b[5], c2, c3, c1);
472
124M
    mul_add_c(a[4], b[6], c2, c3, c1);
473
124M
    mul_add_c(a[3], b[7], c2, c3, c1);
474
124M
    r[10] = c2;
475
124M
    c2 = 0;
476
124M
    mul_add_c(a[4], b[7], c3, c1, c2);
477
124M
    mul_add_c(a[5], b[6], c3, c1, c2);
478
124M
    mul_add_c(a[6], b[5], c3, c1, c2);
479
124M
    mul_add_c(a[7], b[4], c3, c1, c2);
480
124M
    r[11] = c3;
481
124M
    c3 = 0;
482
124M
    mul_add_c(a[7], b[5], c1, c2, c3);
483
124M
    mul_add_c(a[6], b[6], c1, c2, c3);
484
124M
    mul_add_c(a[5], b[7], c1, c2, c3);
485
124M
    r[12] = c1;
486
124M
    c1 = 0;
487
124M
    mul_add_c(a[6], b[7], c2, c3, c1);
488
124M
    mul_add_c(a[7], b[6], c2, c3, c1);
489
124M
    r[13] = c2;
490
124M
    c2 = 0;
491
124M
    mul_add_c(a[7], b[7], c3, c1, c2);
492
124M
    r[14] = c3;
493
124M
    r[15] = c1;
494
124M
}
495
496
void bn_mul_comba4(BN_ULONG *r, BN_ULONG *a, BN_ULONG *b)
497
0
{
498
0
    BN_ULONG c1, c2, c3;
499
500
0
    c1 = 0;
501
0
    c2 = 0;
502
0
    c3 = 0;
503
0
    mul_add_c(a[0], b[0], c1, c2, c3);
504
0
    r[0] = c1;
505
0
    c1 = 0;
506
0
    mul_add_c(a[0], b[1], c2, c3, c1);
507
0
    mul_add_c(a[1], b[0], c2, c3, c1);
508
0
    r[1] = c2;
509
0
    c2 = 0;
510
0
    mul_add_c(a[2], b[0], c3, c1, c2);
511
0
    mul_add_c(a[1], b[1], c3, c1, c2);
512
0
    mul_add_c(a[0], b[2], c3, c1, c2);
513
0
    r[2] = c3;
514
0
    c3 = 0;
515
0
    mul_add_c(a[0], b[3], c1, c2, c3);
516
0
    mul_add_c(a[1], b[2], c1, c2, c3);
517
0
    mul_add_c(a[2], b[1], c1, c2, c3);
518
0
    mul_add_c(a[3], b[0], c1, c2, c3);
519
0
    r[3] = c1;
520
0
    c1 = 0;
521
0
    mul_add_c(a[3], b[1], c2, c3, c1);
522
0
    mul_add_c(a[2], b[2], c2, c3, c1);
523
0
    mul_add_c(a[1], b[3], c2, c3, c1);
524
0
    r[4] = c2;
525
0
    c2 = 0;
526
0
    mul_add_c(a[2], b[3], c3, c1, c2);
527
0
    mul_add_c(a[3], b[2], c3, c1, c2);
528
0
    r[5] = c3;
529
0
    c3 = 0;
530
0
    mul_add_c(a[3], b[3], c1, c2, c3);
531
0
    r[6] = c1;
532
0
    r[7] = c2;
533
0
}
534
535
void bn_sqr_comba8(BN_ULONG *r, const BN_ULONG *a)
536
5.48M
{
537
5.48M
    BN_ULONG c1, c2, c3;
538
539
5.48M
    c1 = 0;
540
5.48M
    c2 = 0;
541
5.48M
    c3 = 0;
542
5.48M
    sqr_add_c(a, 0, c1, c2, c3);
543
5.48M
    r[0] = c1;
544
5.48M
    c1 = 0;
545
5.48M
    sqr_add_c2(a, 1, 0, c2, c3, c1);
546
5.48M
    r[1] = c2;
547
5.48M
    c2 = 0;
548
5.48M
    sqr_add_c(a, 1, c3, c1, c2);
549
5.48M
    sqr_add_c2(a, 2, 0, c3, c1, c2);
550
5.48M
    r[2] = c3;
551
5.48M
    c3 = 0;
552
5.48M
    sqr_add_c2(a, 3, 0, c1, c2, c3);
553
5.48M
    sqr_add_c2(a, 2, 1, c1, c2, c3);
554
5.48M
    r[3] = c1;
555
5.48M
    c1 = 0;
556
5.48M
    sqr_add_c(a, 2, c2, c3, c1);
557
5.48M
    sqr_add_c2(a, 3, 1, c2, c3, c1);
558
5.48M
    sqr_add_c2(a, 4, 0, c2, c3, c1);
559
5.48M
    r[4] = c2;
560
5.48M
    c2 = 0;
561
5.48M
    sqr_add_c2(a, 5, 0, c3, c1, c2);
562
5.48M
    sqr_add_c2(a, 4, 1, c3, c1, c2);
563
5.48M
    sqr_add_c2(a, 3, 2, c3, c1, c2);
564
5.48M
    r[5] = c3;
565
5.48M
    c3 = 0;
566
5.48M
    sqr_add_c(a, 3, c1, c2, c3);
567
5.48M
    sqr_add_c2(a, 4, 2, c1, c2, c3);
568
5.48M
    sqr_add_c2(a, 5, 1, c1, c2, c3);
569
5.48M
    sqr_add_c2(a, 6, 0, c1, c2, c3);
570
5.48M
    r[6] = c1;
571
5.48M
    c1 = 0;
572
5.48M
    sqr_add_c2(a, 7, 0, c2, c3, c1);
573
5.48M
    sqr_add_c2(a, 6, 1, c2, c3, c1);
574
5.48M
    sqr_add_c2(a, 5, 2, c2, c3, c1);
575
5.48M
    sqr_add_c2(a, 4, 3, c2, c3, c1);
576
5.48M
    r[7] = c2;
577
5.48M
    c2 = 0;
578
5.48M
    sqr_add_c(a, 4, c3, c1, c2);
579
5.48M
    sqr_add_c2(a, 5, 3, c3, c1, c2);
580
5.48M
    sqr_add_c2(a, 6, 2, c3, c1, c2);
581
5.48M
    sqr_add_c2(a, 7, 1, c3, c1, c2);
582
5.48M
    r[8] = c3;
583
5.48M
    c3 = 0;
584
5.48M
    sqr_add_c2(a, 7, 2, c1, c2, c3);
585
5.48M
    sqr_add_c2(a, 6, 3, c1, c2, c3);
586
5.48M
    sqr_add_c2(a, 5, 4, c1, c2, c3);
587
5.48M
    r[9] = c1;
588
5.48M
    c1 = 0;
589
5.48M
    sqr_add_c(a, 5, c2, c3, c1);
590
5.48M
    sqr_add_c2(a, 6, 4, c2, c3, c1);
591
5.48M
    sqr_add_c2(a, 7, 3, c2, c3, c1);
592
5.48M
    r[10] = c2;
593
5.48M
    c2 = 0;
594
5.48M
    sqr_add_c2(a, 7, 4, c3, c1, c2);
595
5.48M
    sqr_add_c2(a, 6, 5, c3, c1, c2);
596
5.48M
    r[11] = c3;
597
5.48M
    c3 = 0;
598
5.48M
    sqr_add_c(a, 6, c1, c2, c3);
599
5.48M
    sqr_add_c2(a, 7, 5, c1, c2, c3);
600
5.48M
    r[12] = c1;
601
5.48M
    c1 = 0;
602
5.48M
    sqr_add_c2(a, 7, 6, c2, c3, c1);
603
5.48M
    r[13] = c2;
604
5.48M
    c2 = 0;
605
5.48M
    sqr_add_c(a, 7, c3, c1, c2);
606
5.48M
    r[14] = c3;
607
5.48M
    r[15] = c1;
608
5.48M
}
609
610
void bn_sqr_comba4(BN_ULONG *r, const BN_ULONG *a)
611
66.4k
{
612
66.4k
    BN_ULONG c1, c2, c3;
613
614
66.4k
    c1 = 0;
615
66.4k
    c2 = 0;
616
66.4k
    c3 = 0;
617
66.4k
    sqr_add_c(a, 0, c1, c2, c3);
618
66.4k
    r[0] = c1;
619
66.4k
    c1 = 0;
620
66.4k
    sqr_add_c2(a, 1, 0, c2, c3, c1);
621
66.4k
    r[1] = c2;
622
66.4k
    c2 = 0;
623
66.4k
    sqr_add_c(a, 1, c3, c1, c2);
624
66.4k
    sqr_add_c2(a, 2, 0, c3, c1, c2);
625
66.4k
    r[2] = c3;
626
66.4k
    c3 = 0;
627
66.4k
    sqr_add_c2(a, 3, 0, c1, c2, c3);
628
66.4k
    sqr_add_c2(a, 2, 1, c1, c2, c3);
629
66.4k
    r[3] = c1;
630
66.4k
    c1 = 0;
631
66.4k
    sqr_add_c(a, 2, c2, c3, c1);
632
66.4k
    sqr_add_c2(a, 3, 1, c2, c3, c1);
633
66.4k
    r[4] = c2;
634
66.4k
    c2 = 0;
635
66.4k
    sqr_add_c2(a, 3, 2, c3, c1, c2);
636
66.4k
    r[5] = c3;
637
66.4k
    c3 = 0;
638
66.4k
    sqr_add_c(a, 3, c1, c2, c3);
639
66.4k
    r[6] = c1;
640
66.4k
    r[7] = c2;
641
66.4k
}
642
#endif