Coverage Report

Created: 2024-11-21 07:03

/src/nss-nspr/nss/lib/freebl/deprecated/alg2268.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * alg2268.c - implementation of the algorithm in RFC 2268
3
 *
4
 * This Source Code Form is subject to the terms of the Mozilla Public
5
 * License, v. 2.0. If a copy of the MPL was not distributed with this
6
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7
8
#ifdef FREEBL_NO_DEPEND
9
#include "../stubs.h"
10
#endif
11
12
#include "../blapi.h"
13
#include "../blapii.h"
14
#include "secerr.h"
15
#ifdef XP_UNIX_XXX
16
#include <stddef.h> /* for ptrdiff_t */
17
#endif
18
19
/*
20
** RC2 symmetric block cypher
21
*/
22
23
typedef SECStatus(rc2Func)(RC2Context *cx, unsigned char *output,
24
                           const unsigned char *input, unsigned int inputLen);
25
26
/* forward declarations */
27
static rc2Func rc2_EncryptECB;
28
static rc2Func rc2_DecryptECB;
29
static rc2Func rc2_EncryptCBC;
30
static rc2Func rc2_DecryptCBC;
31
32
typedef union {
33
    PRUint32 l[2];
34
    PRUint16 s[4];
35
    PRUint8 b[8];
36
} RC2Block;
37
38
struct RC2ContextStr {
39
    union {
40
        PRUint8 Kb[128];
41
        PRUint16 Kw[64];
42
    } u;
43
    RC2Block iv;
44
    rc2Func *enc;
45
    rc2Func *dec;
46
};
47
48
0
#define B u.Kb
49
0
#define K u.Kw
50
#define BYTESWAP(x) ((x) << 8 | (x) >> 8)
51
#define SWAPK(i) cx->K[i] = (tmpS = cx->K[i], BYTESWAP(tmpS))
52
0
#define RC2_BLOCK_SIZE 8
53
54
#define LOAD_HARD(R)                           \
55
    R[0] = (PRUint16)input[1] << 8 | input[0]; \
56
    R[1] = (PRUint16)input[3] << 8 | input[2]; \
57
    R[2] = (PRUint16)input[5] << 8 | input[4]; \
58
    R[3] = (PRUint16)input[7] << 8 | input[6];
59
#define LOAD_EASY(R)               \
60
0
    R[0] = ((PRUint16 *)input)[0]; \
61
0
    R[1] = ((PRUint16 *)input)[1]; \
62
0
    R[2] = ((PRUint16 *)input)[2]; \
63
0
    R[3] = ((PRUint16 *)input)[3];
64
#define STORE_HARD(R)                 \
65
    output[0] = (PRUint8)(R[0]);      \
66
    output[1] = (PRUint8)(R[0] >> 8); \
67
    output[2] = (PRUint8)(R[1]);      \
68
    output[3] = (PRUint8)(R[1] >> 8); \
69
    output[4] = (PRUint8)(R[2]);      \
70
    output[5] = (PRUint8)(R[2] >> 8); \
71
    output[6] = (PRUint8)(R[3]);      \
72
    output[7] = (PRUint8)(R[3] >> 8);
73
#define STORE_EASY(R)               \
74
0
    ((PRUint16 *)output)[0] = R[0]; \
75
0
    ((PRUint16 *)output)[1] = R[1]; \
76
0
    ((PRUint16 *)output)[2] = R[2]; \
77
0
    ((PRUint16 *)output)[3] = R[3];
78
79
#if defined(NSS_X86_OR_X64)
80
0
#define LOAD(R) LOAD_EASY(R)
81
0
#define STORE(R) STORE_EASY(R)
82
#elif !defined(IS_LITTLE_ENDIAN)
83
#define LOAD(R) LOAD_HARD(R)
84
#define STORE(R) STORE_HARD(R)
85
#else
86
#define LOAD(R)                 \
87
    if ((ptrdiff_t)input & 1) { \
88
        LOAD_HARD(R)            \
89
    } else {                    \
90
        LOAD_EASY(R)            \
91
    }
92
#define STORE(R)                \
93
    if ((ptrdiff_t)input & 1) { \
94
        STORE_HARD(R)           \
95
    } else {                    \
96
        STORE_EASY(R)           \
97
    }
98
#endif
99
100
static const PRUint8 S[256] = {
101
    0331, 0170, 0371, 0304, 0031, 0335, 0265, 0355, 0050, 0351, 0375, 0171, 0112, 0240, 0330, 0235,
102
    0306, 0176, 0067, 0203, 0053, 0166, 0123, 0216, 0142, 0114, 0144, 0210, 0104, 0213, 0373, 0242,
103
    0027, 0232, 0131, 0365, 0207, 0263, 0117, 0023, 0141, 0105, 0155, 0215, 0011, 0201, 0175, 0062,
104
    0275, 0217, 0100, 0353, 0206, 0267, 0173, 0013, 0360, 0225, 0041, 0042, 0134, 0153, 0116, 0202,
105
    0124, 0326, 0145, 0223, 0316, 0140, 0262, 0034, 0163, 0126, 0300, 0024, 0247, 0214, 0361, 0334,
106
    0022, 0165, 0312, 0037, 0073, 0276, 0344, 0321, 0102, 0075, 0324, 0060, 0243, 0074, 0266, 0046,
107
    0157, 0277, 0016, 0332, 0106, 0151, 0007, 0127, 0047, 0362, 0035, 0233, 0274, 0224, 0103, 0003,
108
    0370, 0021, 0307, 0366, 0220, 0357, 0076, 0347, 0006, 0303, 0325, 0057, 0310, 0146, 0036, 0327,
109
    0010, 0350, 0352, 0336, 0200, 0122, 0356, 0367, 0204, 0252, 0162, 0254, 0065, 0115, 0152, 0052,
110
    0226, 0032, 0322, 0161, 0132, 0025, 0111, 0164, 0113, 0237, 0320, 0136, 0004, 0030, 0244, 0354,
111
    0302, 0340, 0101, 0156, 0017, 0121, 0313, 0314, 0044, 0221, 0257, 0120, 0241, 0364, 0160, 0071,
112
    0231, 0174, 0072, 0205, 0043, 0270, 0264, 0172, 0374, 0002, 0066, 0133, 0045, 0125, 0227, 0061,
113
    0055, 0135, 0372, 0230, 0343, 0212, 0222, 0256, 0005, 0337, 0051, 0020, 0147, 0154, 0272, 0311,
114
    0323, 0000, 0346, 0317, 0341, 0236, 0250, 0054, 0143, 0026, 0001, 0077, 0130, 0342, 0211, 0251,
115
    0015, 0070, 0064, 0033, 0253, 0063, 0377, 0260, 0273, 0110, 0014, 0137, 0271, 0261, 0315, 0056,
116
    0305, 0363, 0333, 0107, 0345, 0245, 0234, 0167, 0012, 0246, 0040, 0150, 0376, 0177, 0301, 0255
117
};
118
119
RC2Context *
120
RC2_AllocateContext(void)
121
0
{
122
0
    return PORT_ZNew(RC2Context);
123
0
}
124
SECStatus
125
RC2_InitContext(RC2Context *cx, const unsigned char *key, unsigned int len,
126
                const unsigned char *input, int mode, unsigned int efLen8,
127
                unsigned int unused)
128
0
{
129
0
    PRUint8 *L, *L2;
130
0
    int i;
131
#if !defined(IS_LITTLE_ENDIAN)
132
    PRUint16 tmpS;
133
#endif
134
0
    PRUint8 tmpB;
135
136
0
    if (!key || !cx || !len || len > (sizeof cx->B) ||
137
0
        efLen8 > (sizeof cx->B)) {
138
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
139
0
        return SECFailure;
140
0
    }
141
0
    if (mode == NSS_RC2) {
142
        /* groovy */
143
0
    } else if (mode == NSS_RC2_CBC) {
144
0
        if (!input) {
145
0
            PORT_SetError(SEC_ERROR_INVALID_ARGS);
146
0
            return SECFailure;
147
0
        }
148
0
    } else {
149
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
150
0
        return SECFailure;
151
0
    }
152
153
0
    if (mode == NSS_RC2_CBC) {
154
0
        cx->enc = &rc2_EncryptCBC;
155
0
        cx->dec = &rc2_DecryptCBC;
156
0
        LOAD(cx->iv.s);
157
0
    } else {
158
0
        cx->enc = &rc2_EncryptECB;
159
0
        cx->dec = &rc2_DecryptECB;
160
0
    }
161
162
    /* Step 0. Copy key into table. */
163
0
    memcpy(cx->B, key, len);
164
165
    /* Step 1. Compute all values to the right of the key. */
166
0
    L2 = cx->B;
167
0
    L = L2 + len;
168
0
    tmpB = L[-1];
169
0
    for (i = (sizeof cx->B) - len; i > 0; --i) {
170
0
        *L++ = tmpB = S[(PRUint8)(tmpB + *L2++)];
171
0
    }
172
173
    /* step 2. Adjust left most byte of effective key. */
174
0
    i = (sizeof cx->B) - efLen8;
175
0
    L = cx->B + i;
176
0
    *L = tmpB = S[*L]; /* mask is always 0xff */
177
178
    /* step 3. Recompute all values to the left of effective key. */
179
0
    L2 = --L + efLen8;
180
0
    while (L >= cx->B) {
181
0
        *L-- = tmpB = S[tmpB ^ *L2--];
182
0
    }
183
184
#if !defined(IS_LITTLE_ENDIAN)
185
    for (i = 63; i >= 0; --i) {
186
        SWAPK(i); /* candidate for unrolling */
187
    }
188
#endif
189
0
    return SECSuccess;
190
0
}
191
192
/*
193
** Create a new RC2 context suitable for RC2 encryption/decryption.
194
**  "key" raw key data
195
**  "len" the number of bytes of key data
196
**  "iv" is the CBC initialization vector (if mode is NSS_RC2_CBC)
197
**  "mode" one of NSS_RC2 or NSS_RC2_CBC
198
**  "effectiveKeyLen" in bytes, not bits.
199
**
200
** When mode is set to NSS_RC2_CBC the RC2 cipher is run in "cipher block
201
** chaining" mode.
202
*/
203
RC2Context *
204
RC2_CreateContext(const unsigned char *key, unsigned int len,
205
                  const unsigned char *iv, int mode, unsigned efLen8)
206
0
{
207
0
    RC2Context *cx = PORT_ZNew(RC2Context);
208
0
    if (cx) {
209
0
        SECStatus rv = RC2_InitContext(cx, key, len, iv, mode, efLen8, 0);
210
0
        if (rv != SECSuccess) {
211
0
            RC2_DestroyContext(cx, PR_TRUE);
212
0
            cx = NULL;
213
0
        }
214
0
    }
215
0
    return cx;
216
0
}
217
218
/*
219
** Destroy an RC2 encryption/decryption context.
220
**  "cx" the context
221
**  "freeit" if PR_TRUE then free the object as well as its sub-objects
222
*/
223
void
224
RC2_DestroyContext(RC2Context *cx, PRBool freeit)
225
0
{
226
0
    if (cx) {
227
0
        memset(cx, 0, sizeof *cx);
228
0
        if (freeit) {
229
0
            PORT_Free(cx);
230
0
        }
231
0
    }
232
0
}
233
234
0
#define ROL(x, k) (x << k | x >> (16 - k))
235
#define MIX(j)                                           \
236
0
    R0 = R0 + cx->K[4 * j + 0] + (R3 & R2) + (~R3 & R1); \
237
0
    R0 = ROL(R0, 1);                                     \
238
0
    R1 = R1 + cx->K[4 * j + 1] + (R0 & R3) + (~R0 & R2); \
239
0
    R1 = ROL(R1, 2);                                     \
240
0
    R2 = R2 + cx->K[4 * j + 2] + (R1 & R0) + (~R1 & R3); \
241
0
    R2 = ROL(R2, 3);                                     \
242
0
    R3 = R3 + cx->K[4 * j + 3] + (R2 & R1) + (~R2 & R0); \
243
0
    R3 = ROL(R3, 5)
244
#define MASH                  \
245
0
    R0 = R0 + cx->K[R3 & 63]; \
246
0
    R1 = R1 + cx->K[R0 & 63]; \
247
0
    R2 = R2 + cx->K[R1 & 63]; \
248
0
    R3 = R3 + cx->K[R2 & 63]
249
250
/* Encrypt one block */
251
static void
252
rc2_Encrypt1Block(RC2Context *cx, RC2Block *output, RC2Block *input)
253
0
{
254
0
    register PRUint16 R0, R1, R2, R3;
255
256
    /* step 1. Initialize input. */
257
0
    R0 = input->s[0];
258
0
    R1 = input->s[1];
259
0
    R2 = input->s[2];
260
0
    R3 = input->s[3];
261
262
    /* step 2.  Expand Key (already done, in context) */
263
    /* step 3.  j = 0 */
264
    /* step 4.  Perform 5 mixing rounds. */
265
266
0
    MIX(0);
267
0
    MIX(1);
268
0
    MIX(2);
269
0
    MIX(3);
270
0
    MIX(4);
271
272
    /* step 5. Perform 1 mashing round. */
273
0
    MASH;
274
275
    /* step 6. Perform 6 mixing rounds. */
276
277
0
    MIX(5);
278
0
    MIX(6);
279
0
    MIX(7);
280
0
    MIX(8);
281
0
    MIX(9);
282
0
    MIX(10);
283
284
    /* step 7. Perform 1 mashing round. */
285
0
    MASH;
286
287
    /* step 8. Perform 5 mixing rounds. */
288
289
0
    MIX(11);
290
0
    MIX(12);
291
0
    MIX(13);
292
0
    MIX(14);
293
0
    MIX(15);
294
295
    /* output results */
296
0
    output->s[0] = R0;
297
0
    output->s[1] = R1;
298
0
    output->s[2] = R2;
299
0
    output->s[3] = R3;
300
0
}
301
302
0
#define ROR(x, k) (x >> k | x << (16 - k))
303
#define R_MIX(j)                                         \
304
0
    R3 = ROR(R3, 5);                                     \
305
0
    R3 = R3 - cx->K[4 * j + 3] - (R2 & R1) - (~R2 & R0); \
306
0
    R2 = ROR(R2, 3);                                     \
307
0
    R2 = R2 - cx->K[4 * j + 2] - (R1 & R0) - (~R1 & R3); \
308
0
    R1 = ROR(R1, 2);                                     \
309
0
    R1 = R1 - cx->K[4 * j + 1] - (R0 & R3) - (~R0 & R2); \
310
0
    R0 = ROR(R0, 1);                                     \
311
0
    R0 = R0 - cx->K[4 * j + 0] - (R3 & R2) - (~R3 & R1)
312
#define R_MASH                \
313
0
    R3 = R3 - cx->K[R2 & 63]; \
314
0
    R2 = R2 - cx->K[R1 & 63]; \
315
0
    R1 = R1 - cx->K[R0 & 63]; \
316
0
    R0 = R0 - cx->K[R3 & 63]
317
318
/* Encrypt one block */
319
static void
320
rc2_Decrypt1Block(RC2Context *cx, RC2Block *output, RC2Block *input)
321
0
{
322
0
    register PRUint16 R0, R1, R2, R3;
323
324
    /* step 1. Initialize input. */
325
0
    R0 = input->s[0];
326
0
    R1 = input->s[1];
327
0
    R2 = input->s[2];
328
0
    R3 = input->s[3];
329
330
    /* step 2.  Expand Key (already done, in context) */
331
    /* step 3.  j = 63 */
332
    /* step 4.  Perform 5 r_mixing rounds. */
333
0
    R_MIX(15);
334
0
    R_MIX(14);
335
0
    R_MIX(13);
336
0
    R_MIX(12);
337
0
    R_MIX(11);
338
339
    /* step 5.  Perform 1 r_mashing round. */
340
0
    R_MASH;
341
342
    /* step 6.  Perform 6 r_mixing rounds. */
343
0
    R_MIX(10);
344
0
    R_MIX(9);
345
0
    R_MIX(8);
346
0
    R_MIX(7);
347
0
    R_MIX(6);
348
0
    R_MIX(5);
349
350
    /* step 7.  Perform 1 r_mashing round. */
351
0
    R_MASH;
352
353
    /* step 8.  Perform 5 r_mixing rounds. */
354
0
    R_MIX(4);
355
0
    R_MIX(3);
356
0
    R_MIX(2);
357
0
    R_MIX(1);
358
0
    R_MIX(0);
359
360
    /* output results */
361
0
    output->s[0] = R0;
362
0
    output->s[1] = R1;
363
0
    output->s[2] = R2;
364
0
    output->s[3] = R3;
365
0
}
366
367
static SECStatus NO_SANITIZE_ALIGNMENT
368
rc2_EncryptECB(RC2Context *cx, unsigned char *output,
369
               const unsigned char *input, unsigned int inputLen)
370
0
{
371
0
    RC2Block iBlock;
372
373
0
    while (inputLen > 0) {
374
0
        LOAD(iBlock.s)
375
0
        rc2_Encrypt1Block(cx, &iBlock, &iBlock);
376
0
        STORE(iBlock.s)
377
0
        output += RC2_BLOCK_SIZE;
378
0
        input += RC2_BLOCK_SIZE;
379
0
        inputLen -= RC2_BLOCK_SIZE;
380
0
    }
381
0
    return SECSuccess;
382
0
}
383
384
static SECStatus NO_SANITIZE_ALIGNMENT
385
rc2_DecryptECB(RC2Context *cx, unsigned char *output,
386
               const unsigned char *input, unsigned int inputLen)
387
0
{
388
0
    RC2Block iBlock;
389
390
0
    while (inputLen > 0) {
391
0
        LOAD(iBlock.s)
392
0
        rc2_Decrypt1Block(cx, &iBlock, &iBlock);
393
0
        STORE(iBlock.s)
394
0
        output += RC2_BLOCK_SIZE;
395
0
        input += RC2_BLOCK_SIZE;
396
0
        inputLen -= RC2_BLOCK_SIZE;
397
0
    }
398
0
    return SECSuccess;
399
0
}
400
401
static SECStatus NO_SANITIZE_ALIGNMENT
402
rc2_EncryptCBC(RC2Context *cx, unsigned char *output,
403
               const unsigned char *input, unsigned int inputLen)
404
0
{
405
0
    RC2Block iBlock;
406
407
0
    while (inputLen > 0) {
408
409
0
        LOAD(iBlock.s)
410
0
        iBlock.l[0] ^= cx->iv.l[0];
411
0
        iBlock.l[1] ^= cx->iv.l[1];
412
0
        rc2_Encrypt1Block(cx, &iBlock, &iBlock);
413
0
        cx->iv = iBlock;
414
0
        STORE(iBlock.s)
415
0
        output += RC2_BLOCK_SIZE;
416
0
        input += RC2_BLOCK_SIZE;
417
0
        inputLen -= RC2_BLOCK_SIZE;
418
0
    }
419
0
    return SECSuccess;
420
0
}
421
422
static SECStatus NO_SANITIZE_ALIGNMENT
423
rc2_DecryptCBC(RC2Context *cx, unsigned char *output,
424
               const unsigned char *input, unsigned int inputLen)
425
0
{
426
0
    RC2Block iBlock;
427
0
    RC2Block oBlock;
428
429
0
    while (inputLen > 0) {
430
0
        LOAD(iBlock.s)
431
0
        rc2_Decrypt1Block(cx, &oBlock, &iBlock);
432
0
        oBlock.l[0] ^= cx->iv.l[0];
433
0
        oBlock.l[1] ^= cx->iv.l[1];
434
0
        cx->iv = iBlock;
435
0
        STORE(oBlock.s)
436
0
        output += RC2_BLOCK_SIZE;
437
0
        input += RC2_BLOCK_SIZE;
438
0
        inputLen -= RC2_BLOCK_SIZE;
439
0
    }
440
0
    return SECSuccess;
441
0
}
442
443
/*
444
** Perform RC2 encryption.
445
**  "cx" the context
446
**  "output" the output buffer to store the encrypted data.
447
**  "outputLen" how much data is stored in "output". Set by the routine
448
**     after some data is stored in output.
449
**  "maxOutputLen" the maximum amount of data that can ever be
450
**     stored in "output"
451
**  "input" the input data
452
**  "inputLen" the amount of input data
453
*/
454
SECStatus
455
RC2_Encrypt(RC2Context *cx, unsigned char *output,
456
            unsigned int *outputLen, unsigned int maxOutputLen,
457
            const unsigned char *input, unsigned int inputLen)
458
0
{
459
0
    SECStatus rv = SECSuccess;
460
0
    if (inputLen) {
461
0
        if (inputLen % RC2_BLOCK_SIZE) {
462
0
            PORT_SetError(SEC_ERROR_INPUT_LEN);
463
0
            return SECFailure;
464
0
        }
465
0
        if (maxOutputLen < inputLen) {
466
0
            PORT_SetError(SEC_ERROR_OUTPUT_LEN);
467
0
            return SECFailure;
468
0
        }
469
0
        rv = (*cx->enc)(cx, output, input, inputLen);
470
0
    }
471
0
    if (rv == SECSuccess) {
472
0
        *outputLen = inputLen;
473
0
    }
474
0
    return rv;
475
0
}
476
477
/*
478
** Perform RC2 decryption.
479
**  "cx" the context
480
**  "output" the output buffer to store the decrypted data.
481
**  "outputLen" how much data is stored in "output". Set by the routine
482
**     after some data is stored in output.
483
**  "maxOutputLen" the maximum amount of data that can ever be
484
**     stored in "output"
485
**  "input" the input data
486
**  "inputLen" the amount of input data
487
*/
488
SECStatus
489
RC2_Decrypt(RC2Context *cx, unsigned char *output,
490
            unsigned int *outputLen, unsigned int maxOutputLen,
491
            const unsigned char *input, unsigned int inputLen)
492
0
{
493
0
    SECStatus rv = SECSuccess;
494
0
    if (inputLen) {
495
0
        if (inputLen % RC2_BLOCK_SIZE) {
496
0
            PORT_SetError(SEC_ERROR_INPUT_LEN);
497
0
            return SECFailure;
498
0
        }
499
0
        if (maxOutputLen < inputLen) {
500
0
            PORT_SetError(SEC_ERROR_OUTPUT_LEN);
501
0
            return SECFailure;
502
0
        }
503
0
        rv = (*cx->dec)(cx, output, input, inputLen);
504
0
    }
505
0
    if (rv == SECSuccess) {
506
0
        *outputLen = inputLen;
507
0
    }
508
0
    return rv;
509
0
}