Coverage Report

Created: 2025-04-22 06:14

/src/nss/lib/ssl/sslencode.c
Line
Count
Source (jump to first uncovered line)
1
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2
/*
3
 * This file is PRIVATE to SSL.
4
 *
5
 * This Source Code Form is subject to the terms of the Mozilla Public
6
 * License, v. 2.0. If a copy of the MPL was not distributed with this
7
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8
9
#include "nss.h"
10
#include "prnetdb.h"
11
#include "ssl.h"
12
#include "sslimpl.h"
13
#include "sslproto.h"
14
15
/* Helper function to encode an unsigned integer into a buffer. */
16
static void
17
ssl_EncodeUintX(PRUint8 *to, PRUint64 value, unsigned int bytes)
18
2.67M
{
19
2.67M
    PRUint64 encoded;
20
21
2.67M
    PORT_Assert(bytes > 0 && bytes <= sizeof(encoded));
22
23
2.67M
    encoded = PR_htonll(value);
24
2.67M
    PORT_Memcpy(to, ((unsigned char *)(&encoded)) + (sizeof(encoded) - bytes),
25
2.67M
                bytes);
26
2.67M
}
27
28
/* Grow a buffer to hold newLen bytes of data.  When used for recv/xmit buffers,
29
 * the caller must hold xmitBufLock or recvBufLock, as appropriate. */
30
SECStatus
31
sslBuffer_Grow(sslBuffer *b, unsigned int newLen)
32
3.16M
{
33
3.16M
    PORT_Assert(b);
34
3.16M
    if (b->fixed) {
35
274k
        PORT_Assert(newLen <= b->space);
36
274k
        if (newLen > b->space) {
37
0
            PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
38
0
            return SECFailure;
39
0
        }
40
274k
        return SECSuccess;
41
274k
    }
42
43
    /* If buf is non-NULL, space must be non-zero;
44
     * if buf is NULL, space must be zero. */
45
2.89M
    PORT_Assert((b->buf && b->space) || (!b->buf && !b->space));
46
2.89M
    if (newLen > b->space) {
47
361k
        newLen = PR_MAX(newLen, b->space + 2048);
48
361k
        unsigned char *newBuf;
49
361k
        if (b->buf) {
50
2.24k
            newBuf = (unsigned char *)PORT_Realloc(b->buf, newLen);
51
358k
        } else {
52
358k
            newBuf = (unsigned char *)PORT_Alloc(newLen);
53
358k
        }
54
361k
        if (!newBuf) {
55
0
            return SECFailure;
56
0
        }
57
361k
        b->buf = newBuf;
58
361k
        b->space = newLen;
59
361k
    }
60
2.89M
    return SECSuccess;
61
2.89M
}
62
63
/* Appends len copies of c to b */
64
SECStatus
65
sslBuffer_Fill(sslBuffer *b, PRUint8 c, size_t len)
66
4.54k
{
67
4.54k
    PORT_Assert(b);
68
4.54k
    SECStatus rv = sslBuffer_Grow(b, b->len + len);
69
4.54k
    if (rv != SECSuccess) {
70
0
        return SECFailure;
71
0
    }
72
4.54k
    if (len > 0) {
73
4.54k
        memset(SSL_BUFFER_NEXT(b), c, len);
74
4.54k
    }
75
4.54k
    b->len += len;
76
4.54k
    return SECSuccess;
77
4.54k
}
78
79
SECStatus
80
sslBuffer_Append(sslBuffer *b, const void *data, unsigned int len)
81
174k
{
82
174k
    SECStatus rv = sslBuffer_Grow(b, b->len + len);
83
174k
    if (rv != SECSuccess) {
84
0
        return SECFailure; /* Code already set. */
85
0
    }
86
174k
    if (len > 0) {
87
168k
        PORT_Assert(data);
88
168k
        PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len);
89
168k
    }
90
174k
    b->len += len;
91
174k
    return SECSuccess;
92
174k
}
93
94
SECStatus
95
sslBuffer_AppendNumber(sslBuffer *b, PRUint64 v, unsigned int size)
96
2.37M
{
97
2.37M
    SECStatus rv = sslBuffer_Grow(b, b->len + size);
98
2.37M
    if (rv != SECSuccess) {
99
0
        return SECFailure;
100
0
    }
101
2.37M
    ssl_EncodeUintX(SSL_BUFFER_NEXT(b), v, size);
102
2.37M
    b->len += size;
103
2.37M
    return SECSuccess;
104
2.37M
}
105
106
SECStatus
107
sslBuffer_AppendVariable(sslBuffer *b, const PRUint8 *data, unsigned int len,
108
                         unsigned int size)
109
165k
{
110
165k
    PORT_Assert(size <= 4 && size > 0);
111
165k
    PORT_Assert(b);
112
165k
    if (len >= (1ULL << (8 * size))) {
113
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
114
0
        return SECFailure;
115
0
    }
116
117
165k
    if (sslBuffer_Grow(b, b->len + len + size) != SECSuccess) {
118
0
        return SECFailure;
119
0
    }
120
121
165k
    ssl_EncodeUintX(SSL_BUFFER_NEXT(b), len, size);
122
165k
    b->len += size;
123
165k
    if (len != 0) {
124
141k
        PORT_Assert(data);
125
        /* We sometimes pass NULL, 0 and memcpy() doesn't want NULL. */
126
141k
        PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len);
127
141k
    }
128
165k
    b->len += len;
129
165k
    return SECSuccess;
130
165k
}
131
132
SECStatus
133
sslBuffer_AppendBuffer(sslBuffer *b, const sslBuffer *append)
134
18.5k
{
135
18.5k
    return sslBuffer_Append(b, append->buf, append->len);
136
18.5k
}
137
138
SECStatus
139
sslBuffer_AppendBufferVariable(sslBuffer *b, const sslBuffer *append,
140
                               unsigned int size)
141
19.8k
{
142
19.8k
    return sslBuffer_AppendVariable(b, append->buf, append->len, size);
143
19.8k
}
144
145
SECStatus
146
sslBuffer_Skip(sslBuffer *b, unsigned int size, unsigned int *savedOffset)
147
130k
{
148
130k
    if (sslBuffer_Grow(b, b->len + size) != SECSuccess) {
149
0
        return SECFailure;
150
0
    }
151
152
130k
    if (savedOffset) {
153
92.8k
        *savedOffset = b->len;
154
92.8k
    }
155
130k
    b->len += size;
156
130k
    return SECSuccess;
157
130k
}
158
159
/* A common problem is that a buffer is used to construct a variable length
160
 * structure of unknown length.  The length field for that structure is then
161
 * populated afterwards.  This function makes this process a little easier.
162
 *
163
 * To use this, before encoding the variable length structure, skip the spot
164
 * where the length would be using sslBuffer_Skip().  After encoding the
165
 * structure, and before encoding anything else, call this function passing the
166
 * value returned from sslBuffer_Skip() as |at| to have the length inserted.
167
 */
168
SECStatus
169
sslBuffer_InsertLength(sslBuffer *b, unsigned int at, unsigned int size)
170
81.9k
{
171
81.9k
    unsigned int len;
172
173
81.9k
    PORT_Assert(b->len >= at + size);
174
81.9k
    PORT_Assert(b->space >= at + size);
175
81.9k
    len = b->len - (at + size);
176
177
81.9k
    PORT_Assert(size <= 4 && size > 0);
178
81.9k
    if (len >= (1ULL << (8 * size))) {
179
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
180
0
        return SECFailure;
181
0
    }
182
183
81.9k
    ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, len, size);
184
81.9k
    return SECSuccess;
185
81.9k
}
186
187
SECStatus
188
sslBuffer_InsertNumber(sslBuffer *b, unsigned int at,
189
                       PRUint64 v, unsigned int size)
190
42.1k
{
191
42.1k
    PORT_Assert(b->len >= at + size);
192
42.1k
    PORT_Assert(b->space >= at + size);
193
194
42.1k
    PORT_Assert(size <= 4 && size > 0);
195
42.1k
    if (v >= (1ULL << (8 * size))) {
196
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
197
0
        return SECFailure;
198
0
    }
199
200
42.1k
    ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, v, size);
201
42.1k
    return SECSuccess;
202
42.1k
}
203
204
void
205
sslBuffer_Clear(sslBuffer *b)
206
537k
{
207
537k
    if (!b->fixed) {
208
537k
        if (b->buf) {
209
335k
            PORT_Free(b->buf);
210
335k
            b->buf = NULL;
211
335k
        }
212
537k
        b->space = 0;
213
537k
    }
214
537k
    b->len = 0;
215
537k
}
216
217
SECStatus
218
sslRead_Read(sslReader *reader, unsigned int count, sslReadBuffer *out)
219
172k
{
220
172k
    if (!reader || !out) {
221
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
222
0
        return SECFailure;
223
0
    }
224
172k
    if (reader->buf.len < reader->offset ||
225
172k
        count > SSL_READER_REMAINING(reader)) {
226
14
        PORT_SetError(SEC_ERROR_BAD_DATA);
227
14
        return SECFailure;
228
14
    }
229
230
172k
    out->buf = SSL_READER_CURRENT(reader);
231
172k
    out->len = count;
232
172k
    reader->offset += count;
233
234
172k
    return SECSuccess;
235
172k
}
236
237
SECStatus
238
sslRead_ReadVariable(sslReader *reader, unsigned int sizeLen, sslReadBuffer *out)
239
186k
{
240
186k
    PRUint64 variableLen = 0;
241
186k
    SECStatus rv = sslRead_ReadNumber(reader, sizeLen, &variableLen);
242
186k
    if (rv != SECSuccess) {
243
9
        PORT_SetError(SEC_ERROR_BAD_DATA);
244
9
        return SECFailure;
245
9
    }
246
186k
    if (!variableLen) {
247
        // It is ok to have an empty variable.
248
31.1k
        out->len = variableLen;
249
31.1k
        return SECSuccess;
250
31.1k
    }
251
155k
    return sslRead_Read(reader, variableLen, out);
252
186k
}
253
254
SECStatus
255
sslRead_ReadNumber(sslReader *reader, unsigned int bytes, PRUint64 *num)
256
371k
{
257
371k
    if (!reader || !num) {
258
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
259
0
        return SECFailure;
260
0
    }
261
371k
    if (reader->buf.len < reader->offset ||
262
371k
        bytes > SSL_READER_REMAINING(reader) ||
263
371k
        bytes > 8) {
264
11
        PORT_SetError(SEC_ERROR_BAD_DATA);
265
11
        return SECFailure;
266
11
    }
267
371k
    unsigned int i;
268
371k
    PRUint64 number = 0;
269
1.08M
    for (i = 0; i < bytes; i++) {
270
714k
        number = (number << 8) + reader->buf.buf[i + reader->offset];
271
714k
    }
272
273
371k
    reader->offset = reader->offset + bytes;
274
371k
    *num = number;
275
371k
    return SECSuccess;
276
371k
}
277
278
/**************************************************************************
279
 * Append Handshake functions.
280
 * All these functions set appropriate error codes.
281
 * Most rely on ssl3_AppendHandshake to set the error code.
282
 **************************************************************************/
283
41.6k
#define MAX_SEND_BUF_LENGTH 32000 /* watch for 16-bit integer overflow */
284
#define MIN_SEND_BUF_LENGTH 4000
285
286
static SECStatus
287
ssl3_AppendHandshakeInternal(sslSocket *ss, const void *void_src, unsigned int bytes, PRBool suppressHash)
288
20.8k
{
289
20.8k
    unsigned char *src = (unsigned char *)void_src;
290
20.8k
    int room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
291
20.8k
    SECStatus rv;
292
293
20.8k
    PORT_Assert(ss->opt.noLocks || ssl_HaveSSL3HandshakeLock(ss)); /* protects sendBuf. */
294
295
20.8k
    if (!bytes)
296
0
        return SECSuccess;
297
20.8k
    if (ss->sec.ci.sendBuf.space < MAX_SEND_BUF_LENGTH && room < bytes) {
298
9.65k
        rv = sslBuffer_Grow(&ss->sec.ci.sendBuf, PR_MAX(MIN_SEND_BUF_LENGTH,
299
9.65k
                                                        PR_MIN(MAX_SEND_BUF_LENGTH, ss->sec.ci.sendBuf.len + bytes)));
300
9.65k
        if (rv != SECSuccess)
301
0
            return SECFailure; /* sslBuffer_Grow sets a memory error code. */
302
9.65k
        room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
303
9.65k
    }
304
305
20.8k
    PRINT_BUF(60, (ss, "Append to Handshake", (unsigned char *)void_src, bytes));
306
    // TODO: Move firstHsDone and version check into callers as a suppression.
307
20.8k
    if (!suppressHash && (!ss->firstHsDone || ss->version < SSL_LIBRARY_VERSION_TLS_1_3)) {
308
16.2k
        rv = ssl3_UpdateHandshakeHashes(ss, src, bytes);
309
16.2k
        if (rv != SECSuccess)
310
0
            return SECFailure; /* error code set by ssl3_UpdateHandshakeHashes */
311
16.2k
    }
312
313
20.8k
    while (bytes > room) {
314
0
        if (room > 0)
315
0
            PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src,
316
0
                        room);
317
0
        ss->sec.ci.sendBuf.len += room;
318
0
        rv = ssl3_FlushHandshake(ss, ssl_SEND_FLAG_FORCE_INTO_BUFFER);
319
0
        if (rv != SECSuccess) {
320
0
            return SECFailure; /* error code set by ssl3_FlushHandshake */
321
0
        }
322
0
        bytes -= room;
323
0
        src += room;
324
0
        room = ss->sec.ci.sendBuf.space;
325
0
        PORT_Assert(ss->sec.ci.sendBuf.len == 0);
326
0
    }
327
20.8k
    PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src, bytes);
328
20.8k
    ss->sec.ci.sendBuf.len += bytes;
329
20.8k
    return SECSuccess;
330
20.8k
}
331
332
SECStatus
333
ssl3_AppendHandshakeSuppressHash(sslSocket *ss, const void *void_src, unsigned int bytes)
334
4.54k
{
335
4.54k
    return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_TRUE);
336
4.54k
}
337
338
SECStatus
339
ssl3_AppendHandshake(sslSocket *ss, const void *void_src, unsigned int bytes)
340
9.28k
{
341
9.28k
    return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_FALSE);
342
9.28k
}
343
344
SECStatus
345
ssl3_AppendHandshakeNumberSuppressHash(sslSocket *ss, PRUint64 num, unsigned int lenSize, PRBool suppressHash)
346
7.01k
{
347
7.01k
    if ((lenSize > 8) || ((lenSize < 8) && (num >= (1ULL << (8 * lenSize))))) {
348
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
349
0
        return SECFailure;
350
0
    }
351
352
7.01k
    PRUint8 b[sizeof(num)];
353
7.01k
    SSL_TRC(60, ("%d: number:", SSL_GETPID()));
354
7.01k
    ssl_EncodeUintX(b, num, lenSize);
355
7.01k
    return ssl3_AppendHandshakeInternal(ss, b, lenSize, suppressHash);
356
7.01k
}
357
358
SECStatus
359
ssl3_AppendHandshakeNumber(sslSocket *ss, PRUint64 num, unsigned int lenSize)
360
7.01k
{
361
7.01k
    return ssl3_AppendHandshakeNumberSuppressHash(ss, num, lenSize, PR_FALSE);
362
7.01k
}
363
364
SECStatus
365
ssl3_AppendHandshakeVariable(sslSocket *ss, const PRUint8 *src,
366
                             unsigned int bytes, unsigned int lenSize)
367
1.38k
{
368
1.38k
    SECStatus rv;
369
370
1.38k
    PORT_Assert((bytes < (1 << 8) && lenSize == 1) ||
371
1.38k
                (bytes < (1L << 16) && lenSize == 2) ||
372
1.38k
                (bytes < (1L << 24) && lenSize == 3));
373
374
1.38k
    SSL_TRC(60, ("%d: append variable:", SSL_GETPID()));
375
1.38k
    rv = ssl3_AppendHandshakeNumber(ss, bytes, lenSize);
376
1.38k
    if (rv != SECSuccess) {
377
0
        return SECFailure; /* error code set by AppendHandshake. */
378
0
    }
379
1.38k
    SSL_TRC(60, ("data:"));
380
1.38k
    return ssl3_AppendHandshake(ss, src, bytes);
381
1.38k
}
382
383
SECStatus
384
ssl3_AppendBufferToHandshake(sslSocket *ss, sslBuffer *buf)
385
8
{
386
8
    return ssl3_AppendHandshake(ss, buf->buf, buf->len);
387
8
}
388
389
SECStatus
390
ssl3_AppendBufferToHandshakeVariable(sslSocket *ss, sslBuffer *buf,
391
                                     unsigned int lenSize)
392
0
{
393
0
    return ssl3_AppendHandshakeVariable(ss, buf->buf, buf->len, lenSize);
394
0
}
395
396
SECStatus
397
ssl3_MaybeUpdateHashWithSavedRecord(sslSocket *ss)
398
9.53k
{
399
9.53k
    SECStatus rv;
400
    /* dtls13ClientMessageBuffer is not empty if ClientHello has sent DTLS1.3 */
401
9.53k
    if (ss->ssl3.hs.dtls13ClientMessageBuffer.len == 0) {
402
9.53k
        return SECSuccess;
403
9.53k
    }
404
405
0
    size_t offset = 0;
406
407
    /* the first clause checks the version that was received in ServerHello:
408
     * only if it's DTLS1.3, we remove the necessary fields.
409
     * the second clause checks if we send 0rtt (see TestTls13ZeroRttDowngrade).
410
     */
411
0
    if ((ss->version == ss->ssl3.cwSpec->version || ss->ssl3.hs.zeroRttState == ssl_0rtt_sent)) {
412
0
        if (ss->ssl3.hs.dtls13ClientMessageBuffer.len < 12) {
413
0
            PORT_SetError(SEC_ERROR_INVALID_ARGS);
414
0
            return SECFailure;
415
0
        }
416
417
0
        rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf, 4);
418
0
        if (rv != SECSuccess) {
419
0
            return SECFailure;
420
0
        }
421
0
        offset = 12;
422
0
    }
423
424
0
    PORT_Assert(offset < ss->ssl3.hs.dtls13ClientMessageBuffer.len);
425
0
    rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf + offset,
426
0
                                    ss->ssl3.hs.dtls13ClientMessageBuffer.len - offset);
427
0
    if (rv != SECSuccess) {
428
0
        return SECFailure;
429
0
    }
430
431
0
    sslBuffer_Clear(&ss->ssl3.hs.dtls13ClientMessageBuffer);
432
0
    ss->ssl3.hs.dtls13ClientMessageBuffer.len = 0;
433
0
    return SECSuccess;
434
0
}
435
436
SECStatus
437
ssl3_CopyToSECItem(sslBuffer *buf, SECItem *i)
438
0
{
439
0
    return SECITEM_MakeItem(NULL, i, buf->buf, buf->len);
440
0
}