Coverage Report

Created: 2026-01-22 06:19

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/nss/lib/ssl/sslencode.c
Line
Count
Source
1
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2
/*
3
 * This file is PRIVATE to SSL.
4
 *
5
 * This Source Code Form is subject to the terms of the Mozilla Public
6
 * License, v. 2.0. If a copy of the MPL was not distributed with this
7
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
8
9
#include "nss.h"
10
#include "prnetdb.h"
11
#include "ssl.h"
12
#include "sslimpl.h"
13
#include "sslproto.h"
14
15
/* Helper function to encode an unsigned integer into a buffer. */
16
static void
17
ssl_EncodeUintX(PRUint8 *to, PRUint64 value, unsigned int bytes)
18
17.8M
{
19
17.8M
    PRUint64 encoded;
20
21
17.8M
    PORT_Assert(bytes > 0 && bytes <= sizeof(encoded));
22
23
17.8M
    encoded = PR_htonll(value);
24
17.8M
    PORT_Memcpy(to, ((unsigned char *)(&encoded)) + (sizeof(encoded) - bytes),
25
17.8M
                bytes);
26
17.8M
}
27
28
/* Grow a buffer to hold newLen bytes of data.  When used for recv/xmit buffers,
29
 * the caller must hold xmitBufLock or recvBufLock, as appropriate. */
30
SECStatus
31
sslBuffer_Grow(sslBuffer *b, unsigned int newLen)
32
31.5M
{
33
31.5M
    PORT_Assert(b);
34
31.5M
    if (b->fixed) {
35
4.75M
        PORT_Assert(newLen <= b->space);
36
4.75M
        if (newLen > b->space) {
37
0
            PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
38
0
            return SECFailure;
39
0
        }
40
4.75M
        return SECSuccess;
41
4.75M
    }
42
43
    /* If buf is non-NULL, space must be non-zero;
44
     * if buf is NULL, space must be zero. */
45
26.7M
    PORT_Assert((b->buf && b->space) || (!b->buf && !b->space));
46
26.7M
    if (newLen > b->space) {
47
4.53M
        newLen = PR_MAX(newLen, b->space + 2048);
48
4.53M
        unsigned char *newBuf;
49
4.53M
        if (b->buf) {
50
13.0k
            newBuf = (unsigned char *)PORT_Realloc(b->buf, newLen);
51
4.51M
        } else {
52
4.51M
            newBuf = (unsigned char *)PORT_Alloc(newLen);
53
4.51M
        }
54
4.53M
        if (!newBuf) {
55
0
            return SECFailure;
56
0
        }
57
4.53M
        b->buf = newBuf;
58
4.53M
        b->space = newLen;
59
4.53M
    }
60
26.7M
    return SECSuccess;
61
26.7M
}
62
63
/* Appends len copies of c to b */
64
SECStatus
65
sslBuffer_Fill(sslBuffer *b, PRUint8 c, size_t len)
66
3.62k
{
67
3.62k
    PORT_Assert(b);
68
3.62k
    SECStatus rv = sslBuffer_Grow(b, b->len + len);
69
3.62k
    if (rv != SECSuccess) {
70
0
        return SECFailure;
71
0
    }
72
3.62k
    if (len > 0) {
73
3.62k
        memset(SSL_BUFFER_NEXT(b), c, len);
74
3.62k
    }
75
3.62k
    b->len += len;
76
3.62k
    return SECSuccess;
77
3.62k
}
78
79
SECStatus
80
sslBuffer_Append(sslBuffer *b, const void *data, unsigned int len)
81
10.5M
{
82
10.5M
    SECStatus rv = sslBuffer_Grow(b, b->len + len);
83
10.5M
    if (rv != SECSuccess) {
84
0
        return SECFailure; /* Code already set. */
85
0
    }
86
10.5M
    if (len > 0) {
87
4.59M
        PORT_Assert(data);
88
4.59M
        PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len);
89
4.59M
    }
90
10.5M
    b->len += len;
91
10.5M
    return SECSuccess;
92
10.5M
}
93
94
SECStatus
95
sslBuffer_AppendNumber(sslBuffer *b, PRUint64 v, unsigned int size)
96
13.8M
{
97
13.8M
    SECStatus rv = sslBuffer_Grow(b, b->len + size);
98
13.8M
    if (rv != SECSuccess) {
99
0
        return SECFailure;
100
0
    }
101
13.8M
    ssl_EncodeUintX(SSL_BUFFER_NEXT(b), v, size);
102
13.8M
    b->len += size;
103
13.8M
    return SECSuccess;
104
13.8M
}
105
106
SECStatus
107
sslBuffer_AppendVariable(sslBuffer *b, const PRUint8 *data, unsigned int len,
108
                         unsigned int size)
109
1.07M
{
110
1.07M
    PORT_Assert(size <= 4 && size > 0);
111
1.07M
    PORT_Assert(b);
112
1.07M
    if (len >= (1ULL << (8 * size))) {
113
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
114
0
        return SECFailure;
115
0
    }
116
117
1.07M
    if (sslBuffer_Grow(b, b->len + len + size) != SECSuccess) {
118
0
        return SECFailure;
119
0
    }
120
121
1.07M
    ssl_EncodeUintX(SSL_BUFFER_NEXT(b), len, size);
122
1.07M
    b->len += size;
123
1.07M
    if (len != 0) {
124
371k
        PORT_Assert(data);
125
        /* We sometimes pass NULL, 0 and memcpy() doesn't want NULL. */
126
371k
        PORT_Memcpy(SSL_BUFFER_NEXT(b), data, len);
127
371k
    }
128
1.07M
    b->len += len;
129
1.07M
    return SECSuccess;
130
1.07M
}
131
132
SECStatus
133
sslBuffer_AppendBuffer(sslBuffer *b, const sslBuffer *append)
134
77.6k
{
135
77.6k
    return sslBuffer_Append(b, append->buf, append->len);
136
77.6k
}
137
138
SECStatus
139
sslBuffer_AppendBufferVariable(sslBuffer *b, const sslBuffer *append,
140
                               unsigned int size)
141
39.3k
{
142
39.3k
    return sslBuffer_AppendVariable(b, append->buf, append->len, size);
143
39.3k
}
144
145
SECStatus
146
sslBuffer_Skip(sslBuffer *b, unsigned int size, unsigned int *savedOffset)
147
2.13M
{
148
2.13M
    if (sslBuffer_Grow(b, b->len + size) != SECSuccess) {
149
0
        return SECFailure;
150
0
    }
151
152
2.13M
    if (savedOffset) {
153
1.15M
        *savedOffset = b->len;
154
1.15M
    }
155
2.13M
    b->len += size;
156
2.13M
    return SECSuccess;
157
2.13M
}
158
159
/* A common problem is that a buffer is used to construct a variable length
160
 * structure of unknown length.  The length field for that structure is then
161
 * populated afterwards.  This function makes this process a little easier.
162
 *
163
 * To use this, before encoding the variable length structure, skip the spot
164
 * where the length would be using sslBuffer_Skip().  After encoding the
165
 * structure, and before encoding anything else, call this function passing the
166
 * value returned from sslBuffer_Skip() as |at| to have the length inserted.
167
 */
168
SECStatus
169
sslBuffer_InsertLength(sslBuffer *b, unsigned int at, unsigned int size)
170
1.15M
{
171
1.15M
    unsigned int len;
172
173
1.15M
    PORT_Assert(b->len >= at + size);
174
1.15M
    PORT_Assert(b->space >= at + size);
175
1.15M
    len = b->len - (at + size);
176
177
1.15M
    PORT_Assert(size <= 4 && size > 0);
178
1.15M
    if (len >= (1ULL << (8 * size))) {
179
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
180
0
        return SECFailure;
181
0
    }
182
183
1.15M
    ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, len, size);
184
1.15M
    return SECSuccess;
185
1.15M
}
186
187
SECStatus
188
sslBuffer_InsertNumber(sslBuffer *b, unsigned int at,
189
                       PRUint64 v, unsigned int size)
190
196k
{
191
196k
    PORT_Assert(b->len >= at + size);
192
196k
    PORT_Assert(b->space >= at + size);
193
194
196k
    PORT_Assert(size <= 4 && size > 0);
195
196k
    if (v >= (1ULL << (8 * size))) {
196
0
        PORT_SetError(SEC_ERROR_LIBRARY_FAILURE);
197
0
        return SECFailure;
198
0
    }
199
200
196k
    ssl_EncodeUintX(SSL_BUFFER_BASE(b) + at, v, size);
201
196k
    return SECSuccess;
202
196k
}
203
204
void
205
sslBuffer_Clear(sslBuffer *b)
206
11.6M
{
207
11.6M
    if (!b->fixed) {
208
11.6M
        if (b->buf) {
209
4.32M
            PORT_Free(b->buf);
210
4.32M
            b->buf = NULL;
211
4.32M
        }
212
11.6M
        b->space = 0;
213
11.6M
    }
214
11.6M
    b->len = 0;
215
11.6M
}
216
217
SECStatus
218
sslRead_Read(sslReader *reader, unsigned int count, sslReadBuffer *out)
219
185k
{
220
185k
    if (!reader || !out) {
221
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
222
0
        return SECFailure;
223
0
    }
224
185k
    if (reader->buf.len < reader->offset ||
225
185k
        count > SSL_READER_REMAINING(reader)) {
226
604
        PORT_SetError(SEC_ERROR_BAD_DATA);
227
604
        return SECFailure;
228
604
    }
229
230
185k
    out->buf = SSL_READER_CURRENT(reader);
231
185k
    out->len = count;
232
185k
    reader->offset += count;
233
234
185k
    return SECSuccess;
235
185k
}
236
237
SECStatus
238
sslRead_ReadVariable(sslReader *reader, unsigned int sizeLen, sslReadBuffer *out)
239
203k
{
240
203k
    PRUint64 variableLen = 0;
241
203k
    SECStatus rv = sslRead_ReadNumber(reader, sizeLen, &variableLen);
242
203k
    if (rv != SECSuccess) {
243
209
        PORT_SetError(SEC_ERROR_BAD_DATA);
244
209
        return SECFailure;
245
209
    }
246
203k
    if (!variableLen) {
247
        // It is ok to have an empty variable.
248
48.7k
        out->len = variableLen;
249
48.7k
        return SECSuccess;
250
48.7k
    }
251
154k
    return sslRead_Read(reader, variableLen, out);
252
203k
}
253
254
SECStatus
255
sslRead_ReadNumber(sslReader *reader, unsigned int bytes, PRUint64 *num)
256
5.99M
{
257
5.99M
    if (!reader || !num) {
258
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
259
0
        return SECFailure;
260
0
    }
261
5.99M
    if (reader->buf.len < reader->offset ||
262
5.99M
        bytes > SSL_READER_REMAINING(reader) ||
263
5.99M
        bytes > 8) {
264
366
        PORT_SetError(SEC_ERROR_BAD_DATA);
265
366
        return SECFailure;
266
366
    }
267
5.99M
    unsigned int i;
268
5.99M
    PRUint64 number = 0;
269
39.8M
    for (i = 0; i < bytes; i++) {
270
33.8M
        number = (number << 8) + reader->buf.buf[i + reader->offset];
271
33.8M
    }
272
273
5.99M
    reader->offset = reader->offset + bytes;
274
5.99M
    *num = number;
275
5.99M
    return SECSuccess;
276
5.99M
}
277
278
/**************************************************************************
279
 * Append Handshake functions.
280
 * All these functions set appropriate error codes.
281
 * Most rely on ssl3_AppendHandshake to set the error code.
282
 **************************************************************************/
283
4.16M
#define MAX_SEND_BUF_LENGTH 32000 /* watch for 16-bit integer overflow */
284
#define MIN_SEND_BUF_LENGTH 4000
285
286
static SECStatus
287
ssl3_AppendHandshakeInternal(sslSocket *ss, const void *void_src, unsigned int bytes, PRBool suppressHash)
288
2.08M
{
289
2.08M
    unsigned char *src = (unsigned char *)void_src;
290
2.08M
    int room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
291
2.08M
    SECStatus rv;
292
293
2.08M
    PORT_Assert(ss->opt.noLocks || ssl_HaveSSL3HandshakeLock(ss)); /* protects sendBuf. */
294
295
2.08M
    if (!bytes)
296
1.13k
        return SECSuccess;
297
2.08M
    if (ss->sec.ci.sendBuf.space < MAX_SEND_BUF_LENGTH && room < bytes) {
298
52.6k
        rv = sslBuffer_Grow(&ss->sec.ci.sendBuf, PR_MAX(MIN_SEND_BUF_LENGTH,
299
52.6k
                                                        PR_MIN(MAX_SEND_BUF_LENGTH, ss->sec.ci.sendBuf.len + bytes)));
300
52.6k
        if (rv != SECSuccess)
301
0
            return SECFailure; /* sslBuffer_Grow sets a memory error code. */
302
52.6k
        room = ss->sec.ci.sendBuf.space - ss->sec.ci.sendBuf.len;
303
52.6k
    }
304
305
2.08M
    PRINT_BUF(60, (ss, "Append to Handshake", (unsigned char *)void_src, bytes));
306
    // TODO: Move firstHsDone and version check into callers as a suppression.
307
2.08M
    if (!suppressHash && (!ss->firstHsDone || ss->version < SSL_LIBRARY_VERSION_TLS_1_3)) {
308
1.79M
        rv = ssl3_UpdateHandshakeHashes(ss, src, bytes);
309
1.79M
        if (rv != SECSuccess)
310
0
            return SECFailure; /* error code set by ssl3_UpdateHandshakeHashes */
311
1.79M
    }
312
313
2.08M
    while (bytes > room) {
314
0
        if (room > 0)
315
0
            PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src,
316
0
                        room);
317
0
        ss->sec.ci.sendBuf.len += room;
318
0
        rv = ssl3_FlushHandshake(ss, ssl_SEND_FLAG_FORCE_INTO_BUFFER);
319
0
        if (rv != SECSuccess) {
320
0
            return SECFailure; /* error code set by ssl3_FlushHandshake */
321
0
        }
322
0
        bytes -= room;
323
0
        src += room;
324
0
        room = ss->sec.ci.sendBuf.space;
325
0
        PORT_Assert(ss->sec.ci.sendBuf.len == 0);
326
0
    }
327
2.08M
    PORT_Memcpy(ss->sec.ci.sendBuf.buf + ss->sec.ci.sendBuf.len, src, bytes);
328
2.08M
    ss->sec.ci.sendBuf.len += bytes;
329
2.08M
    return SECSuccess;
330
2.08M
}
331
332
SECStatus
333
ssl3_AppendHandshakeSuppressHash(sslSocket *ss, const void *void_src, unsigned int bytes)
334
11.3k
{
335
11.3k
    return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_TRUE);
336
11.3k
}
337
338
SECStatus
339
ssl3_AppendHandshake(sslSocket *ss, const void *void_src, unsigned int bytes)
340
499k
{
341
499k
    return ssl3_AppendHandshakeInternal(ss, void_src, bytes, PR_FALSE);
342
499k
}
343
344
SECStatus
345
ssl3_AppendHandshakeNumberSuppressHash(sslSocket *ss, PRUint64 num, unsigned int lenSize, PRBool suppressHash)
346
1.57M
{
347
1.57M
    if ((lenSize > 8) || ((lenSize < 8) && (num >= (1ULL << (8 * lenSize))))) {
348
0
        PORT_SetError(SEC_ERROR_INVALID_ARGS);
349
0
        return SECFailure;
350
0
    }
351
352
1.57M
    PRUint8 b[sizeof(num)];
353
1.57M
    SSL_TRC(60, ("%d: number:", SSL_GETPID()));
354
1.57M
    ssl_EncodeUintX(b, num, lenSize);
355
1.57M
    return ssl3_AppendHandshakeInternal(ss, b, lenSize, suppressHash);
356
1.57M
}
357
358
SECStatus
359
ssl3_AppendHandshakeNumber(sslSocket *ss, PRUint64 num, unsigned int lenSize)
360
1.42M
{
361
1.42M
    return ssl3_AppendHandshakeNumberSuppressHash(ss, num, lenSize, PR_FALSE);
362
1.42M
}
363
364
SECStatus
365
ssl3_AppendHandshakeVariable(sslSocket *ss, const PRUint8 *src,
366
                             unsigned int bytes, unsigned int lenSize)
367
210k
{
368
210k
    SECStatus rv;
369
370
210k
    PORT_Assert((bytes < (1 << 8) && lenSize == 1) ||
371
210k
                (bytes < (1L << 16) && lenSize == 2) ||
372
210k
                (bytes < (1L << 24) && lenSize == 3));
373
374
210k
    SSL_TRC(60, ("%d: append variable:", SSL_GETPID()));
375
210k
    rv = ssl3_AppendHandshakeNumber(ss, bytes, lenSize);
376
210k
    if (rv != SECSuccess) {
377
0
        return SECFailure; /* error code set by AppendHandshake. */
378
0
    }
379
210k
    SSL_TRC(60, ("data:"));
380
210k
    return ssl3_AppendHandshake(ss, src, bytes);
381
210k
}
382
383
SECStatus
384
ssl3_AppendBufferToHandshake(sslSocket *ss, sslBuffer *buf)
385
7.78k
{
386
7.78k
    return ssl3_AppendHandshake(ss, buf->buf, buf->len);
387
7.78k
}
388
389
SECStatus
390
ssl3_AppendBufferToHandshakeVariable(sslSocket *ss, sslBuffer *buf,
391
                                     unsigned int lenSize)
392
3.53k
{
393
3.53k
    return ssl3_AppendHandshakeVariable(ss, buf->buf, buf->len, lenSize);
394
3.53k
}
395
396
SECStatus
397
ssl3_MaybeUpdateHashWithSavedRecord(sslSocket *ss)
398
68.5k
{
399
68.5k
    SECStatus rv;
400
    /* dtls13ClientMessageBuffer is not empty if ClientHello has sent DTLS1.3 */
401
68.5k
    if (ss->ssl3.hs.dtls13ClientMessageBuffer.len == 0) {
402
62.7k
        return SECSuccess;
403
62.7k
    }
404
405
5.81k
    size_t offset = 0;
406
407
    /* the first clause checks the version that was received in ServerHello:
408
     * only if it's DTLS1.3, we remove the necessary fields.
409
     * the second clause checks if we send 0rtt (see TestTls13ZeroRttDowngrade).
410
     */
411
5.81k
    if ((ss->version == ss->ssl3.cwSpec->version || ss->ssl3.hs.zeroRttState == ssl_0rtt_sent)) {
412
2.53k
        if (ss->ssl3.hs.dtls13ClientMessageBuffer.len < 12) {
413
0
            PORT_SetError(SEC_ERROR_INVALID_ARGS);
414
0
            return SECFailure;
415
0
        }
416
417
2.53k
        rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf, 4);
418
2.53k
        if (rv != SECSuccess) {
419
0
            return SECFailure;
420
0
        }
421
2.53k
        offset = 12;
422
2.53k
    }
423
424
5.81k
    PORT_Assert(offset < ss->ssl3.hs.dtls13ClientMessageBuffer.len);
425
5.81k
    rv = ssl3_UpdateHandshakeHashes(ss, ss->ssl3.hs.dtls13ClientMessageBuffer.buf + offset,
426
5.81k
                                    ss->ssl3.hs.dtls13ClientMessageBuffer.len - offset);
427
5.81k
    if (rv != SECSuccess) {
428
0
        return SECFailure;
429
0
    }
430
431
5.81k
    sslBuffer_Clear(&ss->ssl3.hs.dtls13ClientMessageBuffer);
432
5.81k
    ss->ssl3.hs.dtls13ClientMessageBuffer.len = 0;
433
5.81k
    return SECSuccess;
434
5.81k
}
435
436
SECStatus
437
ssl3_CopyToSECItem(sslBuffer *buf, SECItem *i)
438
5
{
439
    return SECITEM_MakeItem(NULL, i, buf->buf, buf->len);
440
5
}