Coverage Report

Created: 2026-11-15 06:55

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/nss/lib/ssl/sslencode.c
Line
Count
Source
1
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2
/*
3
 * This file is PRIVATE to SSL.
4
 *
5
 * This Source Code Form is subject to the terms of the Mozilla Public
6
 * License, v. 2.0. If a copy of the MPL was not distributed with this
7
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8
9
#include "nss.h"
10
#include "prnetdb.h"
11
#include "ssl.h"
12
#include "sslimpl.h"
13
#include "sslproto.h"
14
15
/* Helper function to encode an unsigned integer into a buffer. */
16
static void
17
ssl_EncodeUintX(PRUint8 *to, PRUint64 value, unsigned int bytes)
18
23.8M
{
19
23.8M
    PRUint64 encoded;
20
21
23.8M
    PORT_Assert(bytes > 0 && bytes <= sizeof(encoded));
22
23
23.8M
    encoded = PR_htonll(value);
24
23.8M
    PORT_Memcpy(to, ((unsigned char *)(&encoded)) + (sizeof(encoded) - bytes),
25
23.8M
                bytes);
26
23.8M
}
27
28
/* Grow a buffer to hold newLen bytes of data.  When used for recv/xmit buffers,
29
 * the caller must hold xmitBufLock or recvBufLock, as appropriate. */
30
SECStatus
31
sslBuffer_Grow(sslBuffer *b, unsigned int newLen)
32
43.2M
{
33
43.2M
    PORT_Assert(b);
34
43.2M
    if (b->fixed) {
35
10.8M
        PORT_Assert(newLen <= b->space);
36
10.8M
        if (newLen > b->space) {
37
0
            PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
38
0
            return SECFailure;
39
0
        }
40
10.8M
        return SECSuccess;
41
10.8M
    }
42
43
    /* If buf is non-NULL, space must be non-zero;
44
     * if buf is NULL, space must be zero. */
45
32.4M
    PORT_Assert((b->buf && b->space) || (!b->buf && !b->space));
46
32.4M
    if (newLen > b->space) {
47
5.08M
        newLen = PR_MAX(newLen, b->space + 2048);
48
5.08M
        unsigned char *newBuf;
49
5.08M
        if (b->buf) {
50
13.4k
            newBuf = (unsigned char *)PORT_Realloc(b->buf, newLen);
51
5.06M
        } else {
52
5.06M
            newBuf = (unsigned char *)PORT_Alloc(newLen);
53
5.06M
        }
54
5.08M
        if (!newBuf) {
55
0
            return SECFailure;
56
0
        }
57
5.08M
        b->buf = newBuf;
58
5.08M
        b->space = newLen;
59
5.08M
    }
60
32.4M
    return SECSuccess;
61
32.4M
}
62
63
/* Appends len copies of c to b */
64
SECStatus
65
sslBuffer_Fill(sslBuffer *b, PRUint8 c, size_t len)
66
3.84k
{
67
3.84k
    PORT_Assert(b);
68
3.84k
    SECStatus rv = sslBuffer_Grow(b, b->len + len);
69
3.84k
    if (rv != SECSuccess) {
70
0
        return SECFailure;
71
0
    }
72
3.84k
    if (len > 0) {
73
3.84k
        memset(SSL_BUFFER_NEXT(b), c, len);
74
3.84k
    }
75
3.84k
    b->len += len;
76
3.84k
    return SECSuccess;
77
3.84k
}
78
79
SECStatus
80
sslBuffer_Append(sslBuffer *b, const void *data, unsigned int len)
81
15.5M
{
82
15.5M
    SECStatus rv = sslBuffer_Grow(b, b->len + len);
83
15.5M
    if (rv != SECSuccess) {
84
0
        return SECFailure; /* Code already set. */
85
0
    }
86
15.5M
    if (len > 0) {
87
7.12M
        PORT_Assert(data);
88
7.12M
        PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len);
89
7.12M
    }
90
15.5M
    b->len += len;
91
15.5M
    return SECSuccess;
92
15.5M
}
93
94
SECStatus
95
sslBuffer_AppendNumber(sslBuffer *b, PRUint64 v, unsigned int size)
96
17.4M
{
97
17.4M
    SECStatus rv = sslBuffer_Grow(b, b->len + size);
98
17.4M
    if (rv != SECSuccess) {
99
0
        return SECFailure;
100
0
    }
101
17.4M
    ssl_EncodeUintX(SSL_BUFFER_NEXT(b), v, size);
102
17.4M
    b->len += size;
103
17.4M
    return SECSuccess;
104
17.4M
}
105
106
SECStatus
107
sslBuffer_AppendVariable(sslBuffer *b, const PRUint8 *data, unsigned int len,
108
                         unsigned int size)
109
2.37M
{
110
2.37M
    PORT_Assert(size <= 4 && size > 0);
111
2.37M
    PORT_Assert(b);
112
2.37M
    if (len >= (1ULL << (8 * size))) {
113
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
114
0
        return SECFailure;
115
0
    }
116
117
2.37M
    if (sslBuffer_Grow(b, b->len + len + size) != SECSuccess) {
118
0
        return SECFailure;
119
0
    }
120
121
2.37M
    ssl_EncodeUintX(SSL_BUFFER_NEXT(b), len, size);
122
2.37M
    b->len += size;
123
2.37M
    if (len != 0) {
124
380k
        PORT_Assert(data);
125
        /* We sometimes pass NULL, 0 and memcpy() doesn't want NULL. */
126
380k
        PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len);
127
380k
    }
128
2.37M
    b->len += len;
129
2.37M
    return SECSuccess;
130
2.37M
}
131
132
SECStatus
133
sslBuffer_AppendBuffer(sslBuffer *b, const sslBuffer *append)
134
75.5k
{
135
75.5k
    return sslBuffer_Append(b, append->buf, append->len);
136
75.5k
}
137
138
SECStatus
139
sslBuffer_AppendBufferVariable(sslBuffer *b, const sslBuffer *append,
140
                               unsigned int size)
141
41.4k
{
142
41.4k
    return sslBuffer_AppendVariable(b, append->buf, append->len, size);
143
41.4k
}
144
145
SECStatus
146
sslBuffer_Skip(sslBuffer *b, unsigned int size, unsigned int *savedOffset)
147
3.33M
{
148
3.33M
    if (sslBuffer_Grow(b, b->len + size) != SECSuccess) {
149
0
        return SECFailure;
150
0
    }
151
152
3.33M
    if (savedOffset) {
153
1.75M
        *savedOffset = b->len;
154
1.75M
    }
155
3.33M
    b->len += size;
156
3.33M
    return SECSuccess;
157
3.33M
}
158
159
/* A common problem is that a buffer is used to construct a variable length
160
 * structure of unknown length.  The length field for that structure is then
161
 * populated afterwards.  This function makes this process a little easier.
162
 *
163
 * To use this, before encoding the variable length structure, skip the spot
164
 * where the length would be using sslBuffer_Skip().  After encoding the
165
 * structure, and before encoding anything else, call this function passing the
166
 * value returned from sslBuffer_Skip() as |at| to have the length inserted.
167
 */
168
SECStatus
169
sslBuffer_InsertLength(sslBuffer *b, unsigned int at, unsigned int size)
170
1.74M
{
171
1.74M
    unsigned int len;
172
173
1.74M
    PORT_Assert(b->len >= at + size);
174
1.74M
    PORT_Assert(b->space >= at + size);
175
1.74M
    len = b->len - (at + size);
176
177
1.74M
    PORT_Assert(size <= 4 && size > 0);
178
1.74M
    if (len >= (1ULL << (8 * size))) {
179
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
180
0
        return SECFailure;
181
0
    }
182
183
1.74M
    ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, len, size);
184
1.74M
    return SECSuccess;
185
1.74M
}
186
187
SECStatus
188
sslBuffer_InsertNumber(sslBuffer *b, unsigned int at,
189
                       PRUint64 v, unsigned int size)
190
197k
{
191
197k
    PORT_Assert(b->len >= at + size);
192
197k
    PORT_Assert(b->space >= at + size);
193
194
197k
    PORT_Assert(size <= 4 && size > 0);
195
197k
    if (v >= (1ULL << (8 * size))) {
196
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
197
0
        return SECFailure;
198
0
    }
199
200
197k
    ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, v, size);
201
197k
    return SECSuccess;
202
197k
}
203
204
void
205
sslBuffer_Clear(sslBuffer *b)
206
14.2M
{
207
14.2M
    if (!b->fixed) {
208
14.2M
        if (b->buf) {
209
4.87M
            PORT_Free(b->buf);
210
4.87M
            b->buf = NULL;
211
4.87M
        }
212
14.2M
        b->space = 0;
213
14.2M
    }
214
14.2M
    b->len = 0;
215
14.2M
}
216
217
void
218
sslBuffer_ZeroClear(sslBuffer *b)
219
0
{
220
0
    if (b->buf) {
221
0
        PORT_SafeZero(b->buf, b->space);
222
0
    }
223
0
    sslBuffer_Clear(b);
224
0
}
225
226
SECStatus
227
sslRead_Read(sslReader *reader, unsigned int count, sslReadBuffer *out)
228
197k
{
229
197k
    if (!reader || !out) {
230
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
231
0
        return SECFailure;
232
0
    }
233
197k
    if (reader->buf.len < reader->offset ||
234
197k
        count > SSL_READER_REMAINING(reader)) {
235
603
        PORT_SetError(SEC_ERROR_BAD_DATA);
236
603
        return SECFailure;
237
603
    }
238
239
196k
    out->buf = SSL_READER_CURRENT(reader);
240
196k
    out->len = count;
241
196k
    reader->offset += count;
242
243
196k
    return SECSuccess;
244
197k
}
245
246
SECStatus
247
sslRead_ReadVariable(sslReader *reader, unsigned int sizeLen, sslReadBuffer *out)
248
221k
{
249
221k
    PRUint64 variableLen = 0;
250
221k
    SECStatus rv = sslRead_ReadNumber(reader, sizeLen, &variableLen);
251
221k
    if (rv != SECSuccess) {
252
194
        PORT_SetError(SEC_ERROR_BAD_DATA);
253
194
        return SECFailure;
254
194
    }
255
221k
    if (!variableLen) {
256
        // It is ok to have an empty variable.
257
57.1k
        out->len = variableLen;
258
57.1k
        return SECSuccess;
259
57.1k
    }
260
164k
    return sslRead_Read(reader, variableLen, out);
261
221k
}
262
263
SECStatus
264
sslRead_ReadNumber(sslReader *reader, unsigned int bytes, PRUint64 *num)
265
8.07M
{
266
8.07M
    if (!reader || !num) {
267
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
268
0
        return SECFailure;
269
0
    }
270
8.07M
    if (reader->buf.len < reader->offset ||
271
8.07M
        bytes > SSL_READER_REMAINING(reader) ||
272
8.07M
        bytes > 8) {
273
369
        PORT_SetError(SEC_ERROR_BAD_DATA);
274
369
        return SECFailure;
275
369
    }
276
8.07M
    unsigned int i;
277
8.07M
    PRUint64 number = 0;
278
54.1M
    for (i = 0; i < bytes; i++) {
279
46.1M
        number = (number << 8) + reader->buf.buf[i + reader->offset];
280
46.1M
    }
281
282
8.07M
    reader->offset = reader->offset + bytes;
283
8.07M
    *num = number;
284
8.07M
    return SECSuccess;
285
8.07M
}
286
287
/**************************************************************************
288
 * Append Handshake functions.
289
 * All these functions set appropriate error codes.
290
 * Most rely on ssl3_AppendHandshake to set the error code.
291
 **************************************************************************/
292
4.29M
#define MAX_SEND_BUF_LENGTH 32000 /* watch for 16-bit integer overflow */
293
#define MIN_SEND_BUF_LENGTH 4000
294
295
static SECStatus
296
ssl3_AppendHandshakeInternal(sslSocket *ss, const void *void_src, unsigned int bytes, PRBool suppressHash)
297
2.53M
{
298
2.53M
    unsigned char *src = (unsigned char *)void_src;
299
2.53M
    int room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
300
2.53M
    SECStatus rv;
301
302
2.53M
    PORT_Assert(ss->opt.noLocks || ssl_HaveSSL3HandshakeLock(ss)); /* protects sendBuf. */
303
304
2.53M
    if (!bytes)
305
1.25k
        return SECSuccess;
306
2.53M
    if (IS_DTLS(ss)) {
307
        /* Each flush of sendBuf is queued as a separate DTLSQueuedMessage, and
308
         * the fragmentation code assumes every queued message is a complete
309
         * handshake message beginning with a handshake header.  A DTLS
310
         * handshake message must therefore stay whole in sendBuf and never be
311
         * flushed partway through, so grow to hold it rather than capping at
312
         * MAX_SEND_BUF_LENGTH. */
313
391k
        if (room < bytes) {
314
25.4k
            rv = sslBuffer_Grow(&ss->sec.ci.sendBuf,
315
25.4k
                                ss->sec.ci.sendBuf.len + bytes);
316
25.4k
            if (rv != SECSuccess)
317
0
                return SECFailure; /* sslBuffer_Grow sets a memory error code. */
318
25.4k
            room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
319
25.4k
        }
320
2.14M
    } else if (ss->sec.ci.sendBuf.space < MAX_SEND_BUF_LENGTH && room < bytes) {
321
28.3k
        rv = sslBuffer_Grow(&ss->sec.ci.sendBuf, PR_MAX(MIN_SEND_BUF_LENGTH,
322
28.3k
                                                        PR_MIN(MAX_SEND_BUF_LENGTH, ss->sec.ci.sendBuf.len + bytes)));
323
28.3k
        if (rv != SECSuccess)
324
0
            return SECFailure; /* sslBuffer_Grow sets a memory error code. */
325
28.3k
        room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
326
28.3k
    }
327
328
2.53M
    PRINT_BUF(60, (ss, "Append to Handshake", (unsigned char *)void_src, bytes));
329
    // TODO: Move firstHsDone and version check into callers as a suppression.
330
2.53M
    if (!suppressHash && (!ss->firstHsDone || ss->version < SSL_LIBRARY_VERSION_TLS_1_3)) {
331
1.75M
        rv = ssl3_UpdateHandshakeHashes(ss, src, bytes);
332
1.75M
        if (rv != SECSuccess)
333
0
            return SECFailure; /* error code set by ssl3_UpdateHandshakeHashes */
334
1.75M
    }
335
336
2.53M
    while (bytes > room) {
337
0
        if (room > 0)
338
0
            PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src,
339
0
                        room);
340
0
        ss->sec.ci.sendBuf.len += room;
341
0
        rv = ssl3_FlushHandshake(ss, ssl_SEND_FLAG_FORCE_INTO_BUFFER);
342
0
        if (rv != SECSuccess) {
343
0
            return SECFailure; /* error code set by ssl3_FlushHandshake */
344
0
        }
345
0
        bytes -= room;
346
0
        src += room;
347
0
        room = ss->sec.ci.sendBuf.space;
348
0
        PORT_Assert(ss->sec.ci.sendBuf.len == 0);
349
0
    }
350
2.53M
    PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src, bytes);
351
2.53M
    ss->sec.ci.sendBuf.len += bytes;
352
2.53M
    return SECSuccess;
353
2.53M
}
354
355
SECStatus
356
ssl3_AppendHandshakeSuppressHash(sslSocket *ss, const void *void_src, unsigned int bytes)
357
12.9k
{
358
12.9k
    return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_TRUE);
359
12.9k
}
360
361
SECStatus
362
ssl3_AppendHandshake(sslSocket *ss, const void *void_src, unsigned int bytes)
363
480k
{
364
480k
    return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_FALSE);
365
480k
}
366
367
SECStatus
368
ssl3_AppendHandshakeNumberSuppressHash(sslSocket *ss, PRUint64 num, unsigned int lenSize, PRBool suppressHash)
369
2.04M
{
370
2.04M
    if ((lenSize > 8) || ((lenSize < 8) && (num >= (1ULL << (8 * lenSize))))) {
371
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
372
0
        return SECFailure;
373
0
    }
374
375
2.04M
    PRUint8 b[sizeof(num)];
376
2.04M
    SSL_TRC(60, ("%d: number:", SSL_GETPID()));
377
2.04M
    ssl_EncodeUintX(b, num, lenSize);
378
2.04M
    return ssl3_AppendHandshakeInternal(ss, b, lenSize, suppressHash);
379
2.04M
}
380
381
SECStatus
382
ssl3_AppendHandshakeNumber(sslSocket *ss, PRUint64 num, unsigned int lenSize)
383
1.88M
{
384
1.88M
    return ssl3_AppendHandshakeNumberSuppressHash(ss, num, lenSize, PR_FALSE);
385
1.88M
}
386
387
SECStatus
388
ssl3_AppendHandshakeVariable(sslSocket *ss, const PRUint8 *src,
389
                             unsigned int bytes, unsigned int lenSize)
390
203k
{
391
203k
    SECStatus rv;
392
393
203k
    PORT_Assert((bytes < (1 << 8) && lenSize == 1) ||
394
203k
                (bytes < (1L << 16) && lenSize == 2) ||
395
203k
                (bytes < (1L << 24) && lenSize == 3));
396
397
203k
    SSL_TRC(60, ("%d: append variable:", SSL_GETPID()));
398
203k
    rv = ssl3_AppendHandshakeNumber(ss, bytes, lenSize);
399
203k
    if (rv != SECSuccess) {
400
0
        return SECFailure; /* error code set by AppendHandshake. */
401
0
    }
402
203k
    SSL_TRC(60, ("data:"));
403
203k
    return ssl3_AppendHandshake(ss, src, bytes);
404
203k
}
405
406
SECStatus
407
ssl3_AppendBufferToHandshake(sslSocket *ss, sslBuffer *buf)
408
7.59k
{
409
7.59k
    return ssl3_AppendHandshake(ss, buf->buf, buf->len);
410
7.59k
}
411
412
SECStatus
413
ssl3_AppendBufferToHandshakeVariable(sslSocket *ss, sslBuffer *buf,
414
                                     unsigned int lenSize)
415
3.96k
{
416
3.96k
    return ssl3_AppendHandshakeVariable(ss, buf->buf, buf->len, lenSize);
417
3.96k
}
418
419
SECStatus
420
ssl3_MaybeUpdateHashWithSavedRecord(sslSocket *ss)
421
65.3k
{
422
65.3k
    SECStatus rv;
423
    /* dtls13ClientMessageBuffer is not empty if ClientHello has sent DTLS1.3 */
424
65.3k
    if (ss->ssl3.hs.dtls13ClientMessageBuffer.len == 0) {
425
59.0k
        return SECSuccess;
426
59.0k
    }
427
428
6.29k
    size_t offset = 0;
429
430
    /* the first clause checks the version that was received in ServerHello:
431
     * only if it's DTLS1.3, we remove the necessary fields.
432
     * the second clause checks if we send 0rtt (see TestTls13ZeroRttDowngrade).
433
     */
434
6.29k
    if ((ss->version == ss->ssl3.cwSpec->version || ss->ssl3.hs.zeroRttState == ssl_0rtt_sent)) {
435
3.05k
        if (ss->ssl3.hs.dtls13ClientMessageBuffer.len < 12) {
436
0
            PORT_SetError(SEC_ERROR_INVALID_ARGS);
437
0
            return SECFailure;
438
0
        }
439
440
3.05k
        rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf, 4);
441
3.05k
        if (rv != SECSuccess) {
442
0
            return SECFailure;
443
0
        }
444
3.05k
        offset = 12;
445
3.05k
    }
446
447
6.29k
    PORT_Assert(offset < ss->ssl3.hs.dtls13ClientMessageBuffer.len);
448
6.29k
    rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf + offset,
449
6.29k
                                    ss->ssl3.hs.dtls13ClientMessageBuffer.len - offset);
450
6.29k
    if (rv != SECSuccess) {
451
0
        return SECFailure;
452
0
    }
453
454
6.29k
    sslBuffer_Clear(&ss->ssl3.hs.dtls13ClientMessageBuffer);
455
6.29k
    ss->ssl3.hs.dtls13ClientMessageBuffer.len = 0;
456
6.29k
    return SECSuccess;
457
6.29k
}
458
459
SECStatus
460
ssl3_CopyToSECItem(sslBuffer *buf, SECItem *i)
461
5
{
462
    return SECITEM_MakeItem(NULL, i, buf->buf, buf->len);
463
5
}