Coverage Report

Created: 2026-08-13 07:17

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/h2o/deps/quicly/lib/quicly.c
Line
Count
Source
1
/*
2
 * Copyright (c) 2017 Fastly, Kazuho Oku
3
 *
4
 * Permission is hereby granted, free of charge, to any person obtaining a copy
5
 * of this software and associated documentation files (the "Software"), to
6
 * deal in the Software without restriction, including without limitation the
7
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8
 * sell copies of the Software, and to permit persons to whom the Software is
9
 * furnished to do so, subject to the following conditions:
10
 *
11
 * The above copyright notice and this permission notice shall be included in
12
 * all copies or substantial portions of the Software.
13
 *
14
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20
 * IN THE SOFTWARE.
21
 */
22
#include <assert.h>
23
#include <inttypes.h>
24
#include <arpa/inet.h>
25
#include <sys/types.h>
26
#include <netinet/in.h>
27
#include <netinet/ip.h>
28
#include <pthread.h>
29
#include <stdarg.h>
30
#include <stdio.h>
31
#include <stdlib.h>
32
#include <sys/socket.h>
33
#include <sys/time.h>
34
#include "khash.h"
35
#include "quicly.h"
36
#include "quicly/defaults.h"
37
#include "quicly/sentmap.h"
38
#include "quicly/pacer.h"
39
#include "quicly/frame.h"
40
#include "quicly/streambuf.h"
41
#include "quicly/cc.h"
42
#if QUICLY_USE_DTRACE
43
#include "quicly-probes.h"
44
#endif
45
46
0
#define QUICLY_TLS_EXTENSION_TYPE_TRANSPORT_PARAMETERS_FINAL 0x39
47
0
#define QUICLY_TLS_EXTENSION_TYPE_TRANSPORT_PARAMETERS_DRAFT 0xffa5
48
#define QUICLY_TRANSPORT_PARAMETER_ID_ORIGINAL_CONNECTION_ID 0
49
#define QUICLY_TRANSPORT_PARAMETER_ID_MAX_IDLE_TIMEOUT 1
50
#define QUICLY_TRANSPORT_PARAMETER_ID_STATELESS_RESET_TOKEN 2
51
#define QUICLY_TRANSPORT_PARAMETER_ID_MAX_UDP_PAYLOAD_SIZE 3
52
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_DATA 4
53
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL 5
54
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE 6
55
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_UNI 7
56
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_BIDI 8
57
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_UNI 9
58
#define QUICLY_TRANSPORT_PARAMETER_ID_ACK_DELAY_EXPONENT 10
59
#define QUICLY_TRANSPORT_PARAMETER_ID_MAX_ACK_DELAY 11
60
#define QUICLY_TRANSPORT_PARAMETER_ID_DISABLE_ACTIVE_MIGRATION 12
61
#define QUICLY_TRANSPORT_PARAMETER_ID_PREFERRED_ADDRESS 13
62
#define QUICLY_TRANSPORT_PARAMETER_ID_ACTIVE_CONNECTION_ID_LIMIT 14
63
#define QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_SOURCE_CONNECTION_ID 15
64
#define QUICLY_TRANSPORT_PARAMETER_ID_RETRY_SOURCE_CONNECTION_ID 16
65
#define QUICLY_TRANSPORT_PARAMETER_ID_MAX_DATAGRAM_FRAME_SIZE 0x20
66
#define QUICLY_TRANSPORT_PARAMETER_ID_MIN_ACK_DELAY 0xff04de1b
67
68
/**
69
 * maximum size of token that quicly accepts
70
 */
71
0
#define QUICLY_MAX_TOKEN_LEN 512
72
/**
73
 * Sends ACK bundled with PING, when number of gaps in the ack queue reaches or exceeds this threshold. This value should be much
74
 * smaller than QUICLY_MAX_RANGES.
75
 */
76
0
#define QUICLY_NUM_ACK_BLOCKS_TO_INDUCE_ACKACK 8
77
/**
78
 * maximum number of undecryptable packets to buffer
79
 */
80
0
#define QUICLY_MAX_DELAYED_PACKETS 10
81
82
KHASH_MAP_INIT_INT64(quicly_stream_t, quicly_stream_t *)
83
84
#if QUICLY_USE_TRACER
85
0
#define QUICLY_TRACER(label, conn, ...) QUICLY_TRACER_##label(conn, __VA_ARGS__)
86
#else
87
#define QUICLY_TRACER(...)
88
#endif
89
90
#if QUICLY_USE_DTRACE
91
#define QUICLY_PROBE(label, conn, ...)                                                                                             \
92
    do {                                                                                                                           \
93
        quicly_conn_t *_conn = (conn);                                                                                             \
94
        if (PTLS_UNLIKELY(QUICLY_##label##_ENABLED()))                                                                             \
95
            QUICLY_##label(_conn, __VA_ARGS__);                                                                                    \
96
        QUICLY_TRACER(label, _conn, __VA_ARGS__);                                                                                  \
97
    } while (0)
98
#define QUICLY_PROBE_ENABLED(label) QUICLY_##label##_ENABLED()
99
#else
100
0
#define QUICLY_PROBE(label, conn, ...) QUICLY_TRACER(label, conn, __VA_ARGS__)
101
0
#define QUICLY_PROBE_ENABLED(label) 0
102
#endif
103
#define QUICLY_PROBE_HEXDUMP(s, l)                                                                                                 \
104
    ({                                                                                                                             \
105
        size_t _l = (l);                                                                                                           \
106
        ptls_hexdump(alloca(_l * 2 + 1), (s), _l);                                                                                 \
107
    })
108
#define QUICLY_PROBE_ESCAPE_UNSAFE_STRING(s, l)                                                                                    \
109
    ({                                                                                                                             \
110
        size_t _l = (l);                                                                                                           \
111
        quicly_escape_unsafe_string(alloca(_l * 4 + 1), (s), _l);                                                                  \
112
    })
113
114
struct st_quicly_cipher_context_t {
115
    ptls_aead_context_t *aead;
116
    ptls_cipher_context_t *header_protection;
117
};
118
119
struct st_quicly_pn_space_t {
120
    /**
121
     * acks to be sent to remote peer
122
     */
123
    quicly_ranges_t ack_queue;
124
    /**
125
     * time at when the largest pn in the ack_queue has been received (or INT64_MAX if none)
126
     */
127
    int64_t largest_pn_received_at;
128
    /**
129
     *
130
     */
131
    uint64_t next_expected_packet_number;
132
    /**
133
     * number of ACK-eliciting packets that have not been ACKed yet
134
     */
135
    uint32_t unacked_count;
136
    /**
137
     * The previously received packet's ecn value
138
     */
139
    uint8_t prior_ecn : 2;
140
    /**
141
     * ECN in the order of ECT(0), ECT(1), CE
142
     */
143
    uint64_t ecn_counts[3];
144
    /**
145
     * maximum number of ACK-eliciting packets to be queued before sending an ACK
146
     */
147
    uint32_t packet_tolerance;
148
    /**
149
     * Maximum packet reordering before eliciting an immediate ACK. Zero disables immediate ACKS on out of order packets.
150
     */
151
    uint32_t reordering_threshold;
152
    /**
153
     * max(acked packet number, unacked ack-eliciting packet number).
154
     */
155
    uint64_t largest_acked_unacked;
156
    /**
157
     * smallest missing packet number within the packet reordering window.
158
     */
159
    uint64_t smallest_unreported_missing;
160
};
161
162
struct st_quicly_handshake_space_t {
163
    struct st_quicly_pn_space_t super;
164
    struct {
165
        struct st_quicly_cipher_context_t ingress;
166
        struct st_quicly_cipher_context_t egress;
167
    } cipher;
168
    uint16_t largest_ingress_udp_payload_size;
169
};
170
171
struct st_quicly_application_space_t {
172
    struct st_quicly_pn_space_t super;
173
    struct {
174
        struct {
175
            struct {
176
                ptls_cipher_context_t *zero_rtt, *one_rtt;
177
            } header_protection;
178
            ptls_aead_context_t *aead[2]; /* 0-RTT uses aead[1], 1-RTT uses aead[key_phase] */
179
            uint8_t secret[PTLS_MAX_DIGEST_SIZE];
180
            struct {
181
                uint64_t prepared;
182
                uint64_t decrypted;
183
            } key_phase;
184
        } ingress;
185
        struct {
186
            struct st_quicly_cipher_context_t key;
187
            uint8_t secret[PTLS_MAX_DIGEST_SIZE];
188
            uint64_t key_phase;
189
            struct {
190
                /**
191
                 * PN at which key update was initiated. Set to UINT64_MAX once key update is acked.
192
                 */
193
                uint64_t last;
194
                /**
195
                 * PN at which key update should be initiated. Set to UINT64_MAX when key update cannot be initiated.
196
                 */
197
                uint64_t next;
198
            } key_update_pn;
199
        } egress;
200
    } cipher;
201
    int one_rtt_writable;
202
};
203
204
struct st_quicly_conn_path_t {
205
    struct {
206
        /**
207
         * remote address (must not be AF_UNSPEC)
208
         */
209
        quicly_address_t remote;
210
        /**
211
         * local address (may be AF_UNSPEC)
212
         */
213
        quicly_address_t local;
214
    } address;
215
    /**
216
     * DCID being used for the path indicated by the sequence number; or UINT64_MAX if yet to be assigned. Multile paths will share
217
     * the same value of zero if peer CID is zero-length.
218
     */
219
    uint64_t dcid;
220
    /**
221
     * Maximum number of packets being received by the connection when a packet was last received on this path. This value is used
222
     * to determine the least-recently-used path which will be recycled.
223
     */
224
    uint64_t packet_last_received;
225
    /**
226
     * `send_at` indicates when a PATH_CHALLENGE frame carrying `data` should be sent, or if the value is INT64_MAX the path is
227
     * validated
228
     */
229
    struct {
230
        int64_t send_at;
231
        uint64_t num_sent;
232
        uint8_t data[QUICLY_PATH_CHALLENGE_DATA_LEN];
233
    } path_challenge;
234
    /**
235
     * path response to be sent, if `send_` is set
236
     */
237
    struct {
238
        uint8_t send_;
239
        uint8_t data[QUICLY_PATH_CHALLENGE_DATA_LEN];
240
    } path_response;
241
    /**
242
     * if this path is the initial path (i.e., the one on which handshake is done)
243
     */
244
    uint8_t initial : 1;
245
    /**
246
     * if only probe packets have been received (and hence have been sent) on the path
247
     */
248
    uint8_t probe_only : 1;
249
    /**
250
     * number of packets being sent / received on the path
251
     */
252
    struct {
253
        uint64_t sent;
254
        uint64_t received;
255
    } num_packets;
256
};
257
258
struct st_quicly_delayed_packet_t {
259
    struct st_quicly_delayed_packet_t *next;
260
    int64_t at;
261
    quicly_decoded_packet_t packet;
262
    uint8_t bytes[1];
263
};
264
265
struct st_quicly_conn_t {
266
    struct _st_quicly_conn_public_t super;
267
    /**
268
     * `paths[0]` is the non-probing path that is guaranteed to exist, others are backups that may be NULL
269
     */
270
    struct st_quicly_conn_path_t *paths[QUICLY_LOCAL_ACTIVE_CONNECTION_ID_LIMIT];
271
    /**
272
     * the initial context
273
     */
274
    struct st_quicly_handshake_space_t *initial;
275
    /**
276
     * the handshake context
277
     */
278
    struct st_quicly_handshake_space_t *handshake;
279
    /**
280
     * 0-RTT and 1-RTT context
281
     */
282
    struct st_quicly_application_space_t *application;
283
    /**
284
     * hashtable of streams
285
     */
286
    khash_t(quicly_stream_t) * streams;
287
    /**
288
     *
289
     */
290
    struct {
291
        /**
292
         *
293
         */
294
        struct {
295
            /**
296
             * sum of max(offset + len) for all the streams; used for checking if the peer stays in its flow control limit
297
             */
298
            uint64_t bytes_consumed;
299
            /**
300
             * sum of bytes shifted (read out) from the receive buffers, by calling `quicly_stream_sync_recvbuf`; as bytes are
301
             * shifted, additional connection-level flow control credits are provided to the peer
302
             */
303
            uint64_t bytes_shifted;
304
            quicly_maxsender_t sender;
305
        } max_data;
306
        /**
307
         *
308
         */
309
        struct {
310
            quicly_maxsender_t uni, bidi;
311
        } max_streams;
312
        /**
313
         *
314
         */
315
        struct {
316
            uint64_t next_sequence;
317
        } ack_frequency;
318
    } ingress;
319
    /**
320
     *
321
     */
322
    struct {
323
        /**
324
         * loss recovery
325
         */
326
        quicly_loss_t loss;
327
        /**
328
         * next or the currently encoding packet number
329
         */
330
        uint64_t packet_number;
331
        /**
332
         * next PN to be skipped
333
         */
334
        uint64_t next_pn_to_skip;
335
        /**
336
         *
337
         */
338
        uint16_t max_udp_payload_size;
339
        /**
340
         *
341
         */
342
        struct {
343
            uint64_t permitted;
344
            uint64_t sent;
345
        } max_data;
346
        /**
347
         *
348
         */
349
        struct {
350
            struct st_quicly_max_streams_t {
351
                uint64_t count;
352
                quicly_maxsender_t blocked_sender;
353
            } uni, bidi;
354
        } max_streams;
355
        /**
356
         *
357
         */
358
        struct {
359
            uint64_t generation;
360
            uint64_t max_acked;
361
            uint32_t num_inflight;
362
        } new_token;
363
        /**
364
         *
365
         */
366
        struct {
367
            int64_t update_at;
368
            uint64_t sequence;
369
        } ack_frequency;
370
        /**
371
         *
372
         */
373
        int64_t last_retransmittable_sent_at;
374
        /**
375
         * when to send an ACK, connection close frames or to destroy the connection
376
         */
377
        int64_t send_ack_at;
378
        /**
379
         * when a PATH_CHALLENGE or PATH_RESPONSE frame is to be sent on any path
380
         */
381
        int64_t send_probe_at;
382
        /**
383
         * congestion control
384
         */
385
        quicly_cc_t cc;
386
        /**
387
         * Next PN to be used when the path is initialized or promoted. As loss recovery / CC is reset upon path promotion, ACKs for
388
         * packets with PN below this property are ignored.
389
         */
390
        uint64_t pn_path_start;
391
        /**
392
         * pacer
393
         */
394
        quicly_pacer_t *pacer;
395
        /**
396
         * ECN
397
         */
398
        struct {
399
            enum en_quicly_ecn_state { QUICLY_ECN_OFF, QUICLY_ECN_ON, QUICLY_ECN_PROBING } state;
400
            uint64_t counts[QUICLY_NUM_EPOCHS][3];
401
        } ecn;
402
        /**
403
         * things to be sent at the stream-level, that are not governed by the stream scheduler
404
         */
405
        struct {
406
            /**
407
             * list of blocked streams (sorted in ascending order of stream_ids)
408
             */
409
            struct {
410
                quicly_linklist_t uni;
411
                quicly_linklist_t bidi;
412
            } blocked;
413
            /**
414
             * list of streams with pending control data (e.g., RESET_STREAM)
415
             */
416
            quicly_linklist_t control;
417
        } pending_streams;
418
        /**
419
         * send state for DATA_BLOCKED frame that corresponds to the current value of `conn->egress.max_data.permitted`
420
         */
421
        quicly_sender_state_t data_blocked;
422
        /**
423
         * bit vector indicating if there's any pending crypto data (the insignificant 4 bits), or other non-stream data
424
         */
425
        uint8_t pending_flows;
426
/* The flags below indicate if the respective frames have to be sent or not. There are no false positives. */
427
0
#define QUICLY_PENDING_FLOW_NEW_TOKEN_BIT (1 << 4)
428
0
#define QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT (1 << 5)
429
/* Indicates that MAX_STREAMS, MAX_DATA, DATA_BLOCKED, STREAMS_BLOCKED, NEW_CONNECTION_ID _might_ have to be sent. There could be
430
 * false positives; logic for sending each of these frames have the capability of detecting such false positives. The purpose of
431
 * this bit is to consolidate information as an optimization. */
432
0
#define QUICLY_PENDING_FLOW_OTHERS_BIT (1 << 6)
433
        /**
434
         *
435
         */
436
        uint8_t try_jumpstart : 1;
437
        /**
438
         * payload of DATAGRAM frames to be sent
439
         */
440
        struct {
441
            ptls_iovec_t payloads[10];
442
            size_t count;
443
        } datagram_frame_payloads;
444
        /**
445
         * delivery rate estimator
446
         */
447
        quicly_ratemeter_t ratemeter;
448
    } egress;
449
    /**
450
     * close reason
451
     */
452
    struct {
453
        quicly_error_t err;
454
        uint64_t frame_type; /* UINT64_MAX if application close */
455
        char *reason_phrase;
456
        uint8_t is_remote : 1;
457
        unsigned long num_packets_received;
458
    } connection_close;
459
    /**
460
     * crypto data
461
     */
462
    struct {
463
        ptls_t *tls;
464
        ptls_handshake_properties_t handshake_properties;
465
        struct {
466
            ptls_raw_extension_t ext[2];
467
            ptls_buffer_t buf;
468
        } transport_params;
469
        unsigned async_in_progress : 1;
470
    } crypto;
471
    /**
472
     * token (if the token is a Retry token can be determined by consulting the length of retry_scid)
473
     */
474
    ptls_iovec_t token;
475
    /**
476
     * len=UINT8_MAX if Retry was not used, use client_received_retry() to check
477
     */
478
    quicly_cid_t retry_scid;
479
    /**
480
     *
481
     */
482
    struct {
483
        /**
484
         * The moment when the idle timeout fires (including the additional 3 PTO). The value is set to INT64_MAX while the
485
         * handshake is in progress.
486
         */
487
        int64_t at;
488
        /**
489
         * idle timeout
490
         */
491
        uint8_t should_rearm_on_send : 1;
492
    } idle_timeout;
493
    /**
494
     * records the time when this connection was created
495
     */
496
    int64_t created_at;
497
    /**
498
     *
499
     */
500
    struct {
501
        union {
502
            struct {
503
                struct st_quicly_delayed_packet_linklist_t {
504
                    struct st_quicly_delayed_packet_t *head, **tail;
505
                } zero_rtt, handshake, one_rtt;
506
            };
507
            struct st_quicly_delayed_packet_linklist_t as_array[3];
508
        };
509
        size_t num_packets;
510
        unsigned slots_newly_processible;
511
    } delayed_packets;
512
    /**
513
     * structure to hold various data used internally
514
     */
515
    struct {
516
        /**
517
         * This value holds current time that remains constant while quicly functions that deal with time are running. Only
518
         * available when the lock is held using `lock_now`.
519
         */
520
        int64_t now;
521
        /**
522
         *
523
         */
524
        uint8_t lock_count;
525
        struct {
526
            /**
527
             * This cache is used to concatenate acked ranges of streams before processing them, reducing the frequency of function
528
             * calls to `quicly_sendstate_t` and to the application-level send window management callbacks. This approach works,
529
             * because in most cases acks will contain contiguous ranges of a single stream.
530
             */
531
            struct {
532
                /**
533
                 * set to INT64_MIN when the cache is invalid
534
                 */
535
                quicly_stream_id_t stream_id;
536
                quicly_sendstate_sent_t args;
537
            } active_acked_cache;
538
        } on_ack_stream;
539
    } stash;
540
};
541
542
#if QUICLY_USE_TRACER
543
#include "quicly-tracer.h"
544
#endif
545
546
struct st_quicly_handle_payload_state_t {
547
    const uint8_t *src, *const end;
548
    size_t epoch;
549
    size_t path_index;
550
    uint64_t frame_type;
551
};
552
553
static void crypto_stream_receive(quicly_stream_t *stream, size_t off, const void *src, size_t len);
554
555
static const quicly_stream_callbacks_t crypto_stream_callbacks = {quicly_streambuf_destroy, quicly_streambuf_egress_shift,
556
                                                                  quicly_streambuf_egress_emit, NULL, crypto_stream_receive};
557
558
static int update_traffic_key_cb(ptls_update_traffic_key_t *self, ptls_t *tls, int is_enc, size_t epoch, const void *secret);
559
static quicly_error_t initiate_close(quicly_conn_t *conn, quicly_error_t err, uint64_t frame_type, const char *reason_phrase);
560
static quicly_error_t handle_close(quicly_conn_t *conn, quicly_error_t err, uint64_t frame_type, ptls_iovec_t reason_phrase);
561
static quicly_error_t discard_sentmap_by_epoch(quicly_conn_t *conn, unsigned ack_epochs);
562
563
quicly_cid_plaintext_t quicly_cid_plaintext_invalid = {.node_id = UINT64_MAX, .thread_id = 0xffffff};
564
565
static const quicly_transport_parameters_t default_transport_params = {.max_udp_payload_size = QUICLY_DEFAULT_MAX_UDP_PAYLOAD_SIZE,
566
                                                                       .ack_delay_exponent = QUICLY_DEFAULT_ACK_DELAY_EXPONENT,
567
                                                                       .max_ack_delay = QUICLY_DEFAULT_MAX_ACK_DELAY,
568
                                                                       .min_ack_delay_usec = UINT64_MAX,
569
                                                                       .active_connection_id_limit =
570
                                                                           QUICLY_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT};
571
572
const quicly_salt_t *quicly_get_salt(uint32_t protocol_version)
573
0
{
574
0
    static const quicly_salt_t
575
0
        v1 = {.initial = {0x38, 0x76, 0x2c, 0xf7, 0xf5, 0x59, 0x34, 0xb3, 0x4d, 0x17,
576
0
                          0x9a, 0xe6, 0xa4, 0xc8, 0x0c, 0xad, 0xcc, 0xbb, 0x7f, 0x0a},
577
0
              .retry = {.key = {0xbe, 0x0c, 0x69, 0x0b, 0x9f, 0x66, 0x57, 0x5a, 0x1d, 0x76, 0x6b, 0x54, 0xe3, 0x68, 0xc8, 0x4e},
578
0
                        .iv = {0x46, 0x15, 0x99, 0xd3, 0x5d, 0x63, 0x2b, 0xf2, 0x23, 0x98, 0x25, 0xbb}}},
579
0
        draft29 = {.initial = {0xaf, 0xbf, 0xec, 0x28, 0x99, 0x93, 0xd2, 0x4c, 0x9e, 0x97,
580
0
                               0x86, 0xf1, 0x9c, 0x61, 0x11, 0xe0, 0x43, 0x90, 0xa8, 0x99},
581
0
                   .retry = {.key = {0xcc, 0xce, 0x18, 0x7e, 0xd0, 0x9a, 0x09, 0xd0, 0x57, 0x28, 0x15, 0x5a, 0x6c, 0xb9, 0x6b,
582
0
                                     0xe1},
583
0
                             .iv = {0xe5, 0x49, 0x30, 0xf9, 0x7f, 0x21, 0x36, 0xf0, 0x53, 0x0a, 0x8c, 0x1c}}},
584
0
        draft27 = {
585
0
            .initial = {0xc3, 0xee, 0xf7, 0x12, 0xc7, 0x2e, 0xbb, 0x5a, 0x11, 0xa7,
586
0
                        0xd2, 0x43, 0x2b, 0xb4, 0x63, 0x65, 0xbe, 0xf9, 0xf5, 0x02},
587
0
            .retry = {.key = {0x4d, 0x32, 0xec, 0xdb, 0x2a, 0x21, 0x33, 0xc8, 0x41, 0xe4, 0x04, 0x3d, 0xf2, 0x7d, 0x44, 0x30},
588
0
                      .iv = {0x4d, 0x16, 0x11, 0xd0, 0x55, 0x13, 0xa5, 0x52, 0xc5, 0x87, 0xd5, 0x75}}};
589
590
0
    switch (protocol_version) {
591
0
    case QUICLY_PROTOCOL_VERSION_1:
592
0
        return &v1;
593
0
    case QUICLY_PROTOCOL_VERSION_DRAFT29:
594
0
        return &draft29;
595
0
    case QUICLY_PROTOCOL_VERSION_DRAFT27:
596
0
        return &draft27;
597
0
        break;
598
0
    default:
599
0
        return NULL;
600
0
    }
601
0
}
602
603
static int enable_with_ratio255(uint8_t ratio, void (*random_bytes)(void *, size_t))
604
0
{
605
0
    if (ratio == 0)
606
0
        return 0;
607
0
    if (ratio == 255)
608
0
        return 1;
609
610
    /* approximate using 255*257=256*256-1 */
611
0
    uint16_t r;
612
0
    random_bytes(&r, sizeof(r));
613
0
    return r < ratio * 257u;
614
0
}
615
616
static void lock_now(quicly_conn_t *conn, int is_reentrant)
617
0
{
618
0
    if (conn->stash.now == 0) {
619
0
        assert(conn->stash.lock_count == 0);
620
0
        conn->stash.now = conn->super.ctx->now->cb(conn->super.ctx->now);
621
0
    } else {
622
0
        assert(is_reentrant && "caller must be reentrant");
623
0
        assert(conn->stash.lock_count != 0);
624
0
    }
625
626
0
    ++conn->stash.lock_count;
627
0
}
628
629
static void unlock_now(quicly_conn_t *conn)
630
0
{
631
0
    assert(conn->stash.now != 0);
632
633
0
    if (--conn->stash.lock_count == 0)
634
0
        conn->stash.now = 0;
635
0
}
636
637
static void set_address(quicly_address_t *addr, struct sockaddr *sa)
638
0
{
639
0
    if (sa == NULL) {
640
0
        addr->sa.sa_family = AF_UNSPEC;
641
0
        return;
642
0
    }
643
644
0
    switch (sa->sa_family) {
645
0
    case AF_UNSPEC:
646
0
        addr->sa.sa_family = AF_UNSPEC;
647
0
        break;
648
0
    case AF_INET:
649
0
        addr->sin = *(struct sockaddr_in *)sa;
650
0
        break;
651
0
    case AF_INET6:
652
0
        addr->sin6 = *(struct sockaddr_in6 *)sa;
653
0
        break;
654
0
    default:
655
0
        memset(addr, 0xff, sizeof(*addr));
656
0
        assert(!"unexpected address type");
657
0
        break;
658
0
    }
659
0
}
660
661
static ptls_cipher_suite_t *get_aes128gcmsha256(quicly_context_t *ctx)
662
0
{
663
0
    ptls_cipher_suite_t **cs;
664
665
0
    for (cs = ctx->tls->cipher_suites;; ++cs) {
666
0
        assert(cs != NULL);
667
0
        if ((*cs)->id == PTLS_CIPHER_SUITE_AES_128_GCM_SHA256)
668
0
            break;
669
0
    }
670
0
    return *cs;
671
0
}
672
673
static inline uint8_t get_epoch(uint8_t first_byte)
674
0
{
675
0
    if (!QUICLY_PACKET_IS_LONG_HEADER(first_byte))
676
0
        return QUICLY_EPOCH_1RTT;
677
678
0
    switch (first_byte & QUICLY_PACKET_TYPE_BITMASK) {
679
0
    case QUICLY_PACKET_TYPE_INITIAL:
680
0
        return QUICLY_EPOCH_INITIAL;
681
0
    case QUICLY_PACKET_TYPE_HANDSHAKE:
682
0
        return QUICLY_EPOCH_HANDSHAKE;
683
0
    case QUICLY_PACKET_TYPE_0RTT:
684
0
        return QUICLY_EPOCH_0RTT;
685
0
    default:
686
0
        assert(!"FIXME");
687
0
    }
688
0
}
689
690
static ptls_aead_context_t *create_retry_aead(quicly_context_t *ctx, uint32_t protocol_version, int is_enc)
691
0
{
692
0
    const quicly_salt_t *salt = quicly_get_salt(protocol_version);
693
0
    assert(salt != NULL);
694
695
0
    ptls_cipher_suite_t *algo = get_aes128gcmsha256(ctx);
696
0
    ptls_aead_context_t *aead = ptls_aead_new_direct(algo->aead, is_enc, salt->retry.key, salt->retry.iv);
697
0
    assert(aead != NULL);
698
0
    return aead;
699
0
}
700
701
static void dispose_cipher(struct st_quicly_cipher_context_t *ctx)
702
0
{
703
0
    ptls_aead_free(ctx->aead);
704
0
    ptls_cipher_free(ctx->header_protection);
705
0
}
706
707
static void clear_datagram_frame_payloads(quicly_conn_t *conn)
708
0
{
709
0
    for (size_t i = 0; i != conn->egress.datagram_frame_payloads.count; ++i) {
710
0
        free(conn->egress.datagram_frame_payloads.payloads[i].base);
711
0
        conn->egress.datagram_frame_payloads.payloads[i] = ptls_iovec_init(NULL, 0);
712
0
    }
713
0
    conn->egress.datagram_frame_payloads.count = 0;
714
0
}
715
716
/**
717
 * changes the raw bytes being referred to by `packet` to `octets`
718
 */
719
static void adjust_pointers_of_decoded_packet(quicly_decoded_packet_t *packet, uint8_t *octets)
720
0
{
721
0
    uint8_t *orig = packet->octets.base;
722
0
    uintptr_t diff = (uintptr_t)octets - (uintptr_t)packet->octets.base;
723
724
0
#define ADJUST(memb, nullable)                                                                                                     \
725
0
    do {                                                                                                                           \
726
0
        if (!(nullable && packet->memb == NULL)) {                                                                                 \
727
0
            assert(orig <= packet->memb && packet->memb <= orig + packet->octets.len);                                             \
728
0
            packet->memb = (void *)((uintptr_t)packet->memb + diff);                                                               \
729
0
        }                                                                                                                          \
730
0
    } while (0)
731
0
    ADJUST(octets.base, 0);
732
0
    ADJUST(cid.dest.encrypted.base, 1);
733
0
    ADJUST(cid.src.base, 1);
734
0
    ADJUST(token.base, 1);
735
0
#undef ADJUST
736
0
}
737
738
static int is_retry(quicly_conn_t *conn)
739
0
{
740
0
    return conn->retry_scid.len != UINT8_MAX;
741
0
}
742
743
static int needs_cid_auth(quicly_conn_t *conn)
744
0
{
745
0
    switch (conn->super.version) {
746
0
    case QUICLY_PROTOCOL_VERSION_1:
747
0
    case QUICLY_PROTOCOL_VERSION_DRAFT29:
748
0
        return 1;
749
0
    default:
750
0
        return 0;
751
0
    }
752
0
}
753
754
static int64_t get_sentmap_expiration_time(quicly_conn_t *conn)
755
0
{
756
0
    return quicly_loss_get_sentmap_expiration_time(&conn->egress.loss, conn->super.remote.transport_params.max_ack_delay);
757
0
}
758
759
/**
760
 * converts ECN bits to index in the order of ACK-ECN field (i.e., ECT(0) -> 0, ECT(1) -> 1, CE -> 2)
761
 */
762
static size_t get_ecn_index_from_bits(uint8_t bits)
763
0
{
764
0
    assert(1 <= bits && bits <= 3);
765
0
    return (18 >> bits) & 3;
766
0
}
767
768
static void update_ecn_state(quicly_conn_t *conn, enum en_quicly_ecn_state new_state)
769
0
{
770
0
    assert(new_state == QUICLY_ECN_ON || new_state == QUICLY_ECN_OFF);
771
772
0
    conn->egress.ecn.state = new_state;
773
0
    if (new_state == QUICLY_ECN_ON) {
774
0
        ++conn->super.stats.num_paths.ecn_validated;
775
0
    } else {
776
0
        ++conn->super.stats.num_paths.ecn_failed;
777
0
    }
778
779
0
    QUICLY_PROBE(ECN_VALIDATION, conn, conn->stash.now, (int)new_state);
780
0
    QUICLY_LOG_CONN(ecn_validation, conn, { PTLS_LOG_ELEMENT_SIGNED(state, (int)new_state); });
781
0
}
782
783
static void ack_frequency_set_next_update_at(quicly_conn_t *conn)
784
0
{
785
0
    if (conn->super.remote.transport_params.min_ack_delay_usec != UINT64_MAX)
786
0
        conn->egress.ack_frequency.update_at = conn->stash.now + get_sentmap_expiration_time(conn);
787
0
}
788
789
size_t quicly_decode_packet(quicly_context_t *ctx, quicly_decoded_packet_t *packet, const uint8_t *datagram, size_t datagram_size,
790
                            size_t *off)
791
0
{
792
0
    const uint8_t *src = datagram, *src_end = datagram + datagram_size;
793
794
0
    assert(*off <= datagram_size);
795
796
0
    packet->octets = ptls_iovec_init(src + *off, datagram_size - *off);
797
0
    if (packet->octets.len < 2)
798
0
        goto Error;
799
0
    packet->datagram_size = datagram_size;
800
0
    packet->first_packet = *off == 0;
801
0
    packet->token = ptls_iovec_init(NULL, 0);
802
0
    packet->decrypted.pn = UINT64_MAX;
803
0
    packet->ecn = 0; /* non-ECT */
804
805
    /* move the cursor to the second byte */
806
0
    src += *off + 1;
807
808
0
    if (QUICLY_PACKET_IS_LONG_HEADER(packet->octets.base[0])) {
809
        /* long header */
810
0
        uint64_t rest_length;
811
0
        if (src_end - src < 5)
812
0
            goto Error;
813
0
        packet->version = quicly_decode32(&src);
814
0
        packet->cid.dest.encrypted.len = *src++;
815
0
        if (src_end - src < packet->cid.dest.encrypted.len + 1)
816
0
            goto Error;
817
0
        packet->cid.dest.encrypted.base = (uint8_t *)src;
818
0
        src += packet->cid.dest.encrypted.len;
819
0
        packet->cid.src.len = *src++;
820
0
        if (src_end - src < packet->cid.src.len)
821
0
            goto Error;
822
0
        packet->cid.src.base = (uint8_t *)src;
823
0
        src += packet->cid.src.len;
824
0
        switch (packet->octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) {
825
0
        case QUICLY_PACKET_TYPE_INITIAL:
826
0
        case QUICLY_PACKET_TYPE_0RTT:
827
0
            if (ctx->cid_encryptor == NULL || packet->cid.dest.encrypted.len == 0 ||
828
0
                ctx->cid_encryptor->decrypt_cid(ctx->cid_encryptor, &packet->cid.dest.plaintext, packet->cid.dest.encrypted.base,
829
0
                                                packet->cid.dest.encrypted.len) == SIZE_MAX)
830
0
                packet->cid.dest.plaintext = quicly_cid_plaintext_invalid;
831
0
            packet->cid.dest.might_be_client_generated = 1;
832
0
            break;
833
0
        default:
834
0
            if (ctx->cid_encryptor != NULL) {
835
0
                if (packet->cid.dest.encrypted.len == 0)
836
0
                    goto Error;
837
0
                if (ctx->cid_encryptor->decrypt_cid(ctx->cid_encryptor, &packet->cid.dest.plaintext,
838
0
                                                    packet->cid.dest.encrypted.base, packet->cid.dest.encrypted.len) == SIZE_MAX)
839
0
                    goto Error;
840
0
            } else {
841
0
                packet->cid.dest.plaintext = quicly_cid_plaintext_invalid;
842
0
            }
843
0
            packet->cid.dest.might_be_client_generated = 0;
844
0
            break;
845
0
        }
846
0
        switch (packet->version) {
847
0
        case QUICLY_PROTOCOL_VERSION_1:
848
0
        case QUICLY_PROTOCOL_VERSION_DRAFT29:
849
0
        case QUICLY_PROTOCOL_VERSION_DRAFT27:
850
            /* these are the recognized versions, and they share the same packet header format */
851
0
            if (packet->cid.dest.encrypted.len > QUICLY_MAX_CID_LEN_V1 || packet->cid.src.len > QUICLY_MAX_CID_LEN_V1)
852
0
                goto Error;
853
0
            if ((packet->octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) == QUICLY_PACKET_TYPE_RETRY) {
854
                /* retry */
855
0
                if (src_end - src <= PTLS_AESGCM_TAG_SIZE)
856
0
                    goto Error;
857
0
                packet->token = ptls_iovec_init(src, src_end - src - PTLS_AESGCM_TAG_SIZE);
858
0
                src += packet->token.len;
859
0
                packet->encrypted_off = src - packet->octets.base;
860
0
            } else {
861
                /* coalescible long header packet */
862
0
                if ((packet->octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) == QUICLY_PACKET_TYPE_INITIAL) {
863
                    /* initial has a token */
864
0
                    uint64_t token_len;
865
0
                    if ((token_len = quicly_decodev(&src, src_end)) == UINT64_MAX)
866
0
                        goto Error;
867
0
                    if (src_end - src < token_len)
868
0
                        goto Error;
869
0
                    packet->token = ptls_iovec_init(src, token_len);
870
0
                    src += token_len;
871
0
                }
872
0
                if ((rest_length = quicly_decodev(&src, src_end)) == UINT64_MAX)
873
0
                    goto Error;
874
0
                if (rest_length < 1)
875
0
                    goto Error;
876
0
                if (src_end - src < rest_length)
877
0
                    goto Error;
878
0
                packet->encrypted_off = src - packet->octets.base;
879
0
                packet->octets.len = packet->encrypted_off + rest_length;
880
0
            }
881
0
            break;
882
0
        default:
883
            /* VN packet or packets of unknown version cannot be parsed. `encrypted_off` is set to the first byte after SCID. */
884
0
            packet->encrypted_off = src - packet->octets.base;
885
0
        }
886
0
        packet->_is_stateless_reset_cached = QUICLY__DECODED_PACKET_CACHED_NOT_STATELESS_RESET;
887
0
    } else {
888
        /* short header */
889
0
        if (ctx->cid_encryptor != NULL) {
890
0
            if (src_end - src < QUICLY_MAX_CID_LEN_V1)
891
0
                goto Error;
892
0
            size_t local_cidl = ctx->cid_encryptor->decrypt_cid(ctx->cid_encryptor, &packet->cid.dest.plaintext, src, 0);
893
0
            if (local_cidl == SIZE_MAX)
894
0
                goto Error;
895
0
            packet->cid.dest.encrypted = ptls_iovec_init(src, local_cidl);
896
0
            src += local_cidl;
897
0
        } else {
898
0
            packet->cid.dest.encrypted = ptls_iovec_init(NULL, 0);
899
0
            packet->cid.dest.plaintext = quicly_cid_plaintext_invalid;
900
0
        }
901
0
        packet->cid.dest.might_be_client_generated = 0;
902
0
        packet->cid.src = ptls_iovec_init(NULL, 0);
903
0
        packet->version = 0;
904
0
        packet->encrypted_off = src - packet->octets.base;
905
0
        packet->_is_stateless_reset_cached = QUICLY__DECODED_PACKET_CACHED_MAYBE_STATELESS_RESET;
906
0
    }
907
908
0
    *off += packet->octets.len;
909
0
    return packet->octets.len;
910
911
0
Error:
912
0
    return SIZE_MAX;
913
0
}
914
915
uint64_t quicly_determine_packet_number(uint32_t truncated, size_t num_bits, uint64_t expected)
916
0
{
917
0
    uint64_t win = (uint64_t)1 << num_bits, candidate = (expected & ~(win - 1)) | truncated;
918
919
0
    if (candidate + win / 2 <= expected)
920
0
        return candidate + win;
921
0
    if (candidate > expected + win / 2 && candidate >= win)
922
0
        return candidate - win;
923
0
    return candidate;
924
0
}
925
926
static void assert_consistency(quicly_conn_t *conn, int timer_must_be_in_future)
927
0
{
928
0
    if (conn->super.state >= QUICLY_STATE_CLOSING) {
929
0
        assert(!timer_must_be_in_future || conn->stash.now < conn->egress.send_ack_at);
930
0
        return;
931
0
    }
932
933
0
    if (conn->egress.loss.sentmap.bytes_in_flight != 0 || conn->super.remote.address_validation.send_probe) {
934
0
        assert(conn->egress.loss.alarm_at != INT64_MAX);
935
0
    } else {
936
0
        assert(conn->egress.loss.loss_time == INT64_MAX);
937
0
    }
938
    /* Allow timers not in the future when the remote peer is not yet validated, since we may not be able to send packets even when
939
     * timers fire. */
940
0
    if (timer_must_be_in_future && conn->super.remote.address_validation.validated)
941
0
        assert(conn->stash.now < conn->egress.loss.alarm_at);
942
0
}
943
944
static quicly_error_t on_invalid_ack(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
945
0
{
946
0
    if (acked)
947
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
948
0
    return 0;
949
0
}
950
951
static uint64_t calc_next_pn_to_skip(ptls_context_t *tlsctx, uint64_t next_pn, uint32_t cwnd, uint64_t mtu)
952
0
{
953
0
    static __thread struct {
954
0
        uint32_t values[8];
955
0
        size_t off;
956
0
    } cached_rand;
957
958
0
    if (cached_rand.off == 0) {
959
0
        tlsctx->random_bytes(cached_rand.values, sizeof(cached_rand.values));
960
0
        cached_rand.off = PTLS_ELEMENTSOF(cached_rand.values);
961
0
    }
962
963
    /* on average, skip one PN per every min(256 packets, 8 * CWND) */
964
0
    uint32_t packet_cwnd = cwnd / mtu;
965
0
    if (packet_cwnd < 32)
966
0
        packet_cwnd = 32;
967
0
    uint64_t skip_after = cached_rand.values[--cached_rand.off] % (16 * packet_cwnd);
968
0
    return next_pn + 1 + skip_after;
969
0
}
970
971
static void init_max_streams(struct st_quicly_max_streams_t *m)
972
0
{
973
0
    m->count = 0;
974
0
    quicly_maxsender_init(&m->blocked_sender, -1);
975
0
}
976
977
static quicly_error_t update_max_streams(struct st_quicly_max_streams_t *m, uint64_t count)
978
0
{
979
0
    if (count > (uint64_t)1 << 60)
980
0
        return QUICLY_TRANSPORT_ERROR_STREAM_LIMIT;
981
982
0
    if (m->count < count) {
983
0
        m->count = count;
984
0
        if (m->blocked_sender.max_acked < count)
985
0
            m->blocked_sender.max_acked = count;
986
0
    }
987
988
0
    return 0;
989
0
}
990
991
int quicly_connection_is_ready(quicly_conn_t *conn)
992
0
{
993
0
    return conn->application != NULL;
994
0
}
995
996
quicly_error_t quicly_get_close_reason(quicly_conn_t *conn, uint64_t *offending_frame_type, const char **reason_phrase,
997
                                       int *is_remote)
998
0
{
999
0
    assert(conn->super.state >= QUICLY_STATE_CLOSING);
1000
0
    if (offending_frame_type != NULL)
1001
0
        *offending_frame_type = conn->connection_close.frame_type;
1002
0
    if (reason_phrase != NULL)
1003
0
        *reason_phrase = conn->connection_close.reason_phrase;
1004
0
    if (is_remote != NULL)
1005
0
        *is_remote = conn->connection_close.is_remote;
1006
0
    return conn->connection_close.err;
1007
0
}
1008
1009
static int stream_is_destroyable(quicly_stream_t *stream)
1010
0
{
1011
0
    if (!quicly_recvstate_transfer_complete(&stream->recvstate))
1012
0
        return 0;
1013
0
    if (!quicly_sendstate_transfer_complete(&stream->sendstate))
1014
0
        return 0;
1015
0
    switch (stream->_send_aux.reset_stream.sender_state) {
1016
0
    case QUICLY_SENDER_STATE_NONE:
1017
0
    case QUICLY_SENDER_STATE_ACKED:
1018
0
        break;
1019
0
    default:
1020
0
        return 0;
1021
0
    }
1022
0
    return 1;
1023
0
}
1024
1025
static void sched_stream_control(quicly_stream_t *stream)
1026
0
{
1027
0
    assert(stream->stream_id >= 0);
1028
1029
0
    if (!quicly_linklist_is_linked(&stream->_send_aux.pending_link.control))
1030
0
        quicly_linklist_insert(stream->conn->egress.pending_streams.control.prev, &stream->_send_aux.pending_link.control);
1031
0
}
1032
1033
static void resched_stream_data(quicly_stream_t *stream)
1034
0
{
1035
0
    if (stream->stream_id < 0) {
1036
0
        assert(-4 <= stream->stream_id);
1037
0
        uint8_t mask = 1 << -(1 + stream->stream_id);
1038
0
        assert((mask & (QUICLY_PENDING_FLOW_NEW_TOKEN_BIT | QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT |
1039
0
                        QUICLY_PENDING_FLOW_OTHERS_BIT)) == 0);
1040
0
        if (stream->sendstate.pending.num_ranges != 0) {
1041
0
            stream->conn->egress.pending_flows |= mask;
1042
0
        } else {
1043
0
            stream->conn->egress.pending_flows &= ~mask;
1044
0
        }
1045
0
        return;
1046
0
    }
1047
1048
    /* do nothing if blocked */
1049
0
    if (stream->streams_blocked)
1050
0
        return;
1051
1052
0
    quicly_stream_scheduler_t *scheduler = stream->conn->super.ctx->stream_scheduler;
1053
0
    scheduler->update_state(scheduler, stream);
1054
0
}
1055
1056
static int should_send_max_data(quicly_conn_t *conn)
1057
0
{
1058
0
    return quicly_maxsender_should_send_max(&conn->ingress.max_data.sender, conn->ingress.max_data.bytes_shifted,
1059
0
                                            (uint32_t)conn->super.ctx->transport_params.max_data, 512);
1060
0
}
1061
1062
static int should_send_max_stream_data(quicly_stream_t *stream)
1063
0
{
1064
0
    if (stream->recvstate.eos != UINT64_MAX)
1065
0
        return 0;
1066
0
    return quicly_maxsender_should_send_max(&stream->_send_aux.max_stream_data_sender, stream->recvstate.data_off,
1067
0
                                            stream->_recv_aux.window, 512);
1068
0
}
1069
1070
int quicly_stream_sync_sendbuf(quicly_stream_t *stream, int activate)
1071
22.7k
{
1072
22.7k
    int ret;
1073
1074
22.7k
    if (activate) {
1075
22.7k
        if ((ret = quicly_sendstate_activate(&stream->sendstate)) != 0)
1076
0
            return ret;
1077
22.7k
    }
1078
1079
22.7k
    resched_stream_data(stream);
1080
22.7k
    return 0;
1081
22.7k
}
1082
1083
void quicly_stream_sync_recvbuf(quicly_stream_t *stream, size_t shift_amount)
1084
{
1085
    stream->recvstate.data_off += shift_amount;
1086
1087
    /* handle flow control unless the given stream is a CRYPTO stream, which are exempt from flow control */
1088
    if (stream->stream_id >= 0) {
1089
        if (should_send_max_stream_data(stream))
1090
            sched_stream_control(stream);
1091
        quicly_conn_t *conn = stream->conn;
1092
        conn->ingress.max_data.bytes_shifted += shift_amount;
1093
        if (should_send_max_data(conn))
1094
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
1095
    }
1096
}
1097
1098
/**
1099
 * calculate how many CIDs we provide to the remote peer
1100
 */
1101
static size_t local_cid_size(const quicly_conn_t *conn)
1102
0
{
1103
0
    PTLS_BUILD_ASSERT(QUICLY_LOCAL_ACTIVE_CONNECTION_ID_LIMIT < SIZE_MAX / sizeof(uint64_t));
1104
1105
    /* if we don't have an encryptor, the only CID we issue is the one we send during handshake */
1106
0
    if (conn->super.ctx->cid_encryptor == NULL)
1107
0
        return 1;
1108
1109
0
    uint64_t capacity = conn->super.remote.transport_params.active_connection_id_limit;
1110
0
    if (capacity > QUICLY_LOCAL_ACTIVE_CONNECTION_ID_LIMIT)
1111
0
        capacity = QUICLY_LOCAL_ACTIVE_CONNECTION_ID_LIMIT;
1112
0
    return capacity;
1113
0
}
1114
1115
/**
1116
 * Resets CIDs associated to paths if they are being retired. To maximize the chance of having enough number of CIDs to run all
1117
 * paths when new CIDs are provided through multiple NCID frames possibly scattered over multiple packets, CIDs are reassigned to
1118
 * the paths lazily.
1119
 */
1120
static void dissociate_cid(quicly_conn_t *conn, uint64_t sequence)
1121
0
{
1122
0
    for (size_t i = 0; i < PTLS_ELEMENTSOF(conn->paths); ++i) {
1123
0
        struct st_quicly_conn_path_t *path = conn->paths[i];
1124
0
        if (path != NULL && path->dcid == sequence)
1125
0
            path->dcid = UINT64_MAX;
1126
0
    }
1127
0
}
1128
1129
static int write_crypto_data(quicly_conn_t *conn, ptls_buffer_t *tlsbuf, size_t epoch_offsets[5])
1130
0
{
1131
0
    size_t epoch;
1132
0
    int ret;
1133
1134
0
    if (tlsbuf->off == 0)
1135
0
        return 0;
1136
1137
0
    for (epoch = 0; epoch < 4; ++epoch) {
1138
0
        size_t len = epoch_offsets[epoch + 1] - epoch_offsets[epoch];
1139
0
        if (len == 0)
1140
0
            continue;
1141
0
        quicly_stream_t *stream = quicly_get_stream(conn, -(quicly_stream_id_t)(1 + epoch));
1142
0
        assert(stream != NULL);
1143
0
        if ((ret = quicly_streambuf_egress_write(stream, tlsbuf->base + epoch_offsets[epoch], len)) != 0)
1144
0
            return ret;
1145
0
    }
1146
1147
0
    return 0;
1148
0
}
1149
1150
/**
1151
 * compresses a quicly error code into an int, converting QUIC transport error codes into negative ints
1152
 */
1153
static int compress_handshake_result(quicly_error_t quicly_err)
1154
0
{
1155
0
    if (QUICLY_ERROR_IS_QUIC_TRANSPORT(quicly_err)) {
1156
0
        assert(QUICLY_ERROR_GET_ERROR_CODE(quicly_err) <= INT32_MAX);
1157
0
        return (int)-QUICLY_ERROR_GET_ERROR_CODE(quicly_err);
1158
0
    } else {
1159
0
        assert(0 <= quicly_err && quicly_err < INT_MAX);
1160
0
        return (int)quicly_err;
1161
0
    }
1162
0
}
1163
1164
static quicly_error_t expand_handshake_result(int compressed_err)
1165
0
{
1166
0
    if (compressed_err < 0) {
1167
0
        return QUICLY_ERROR_FROM_TRANSPORT_ERROR_CODE(-compressed_err);
1168
0
    } else {
1169
0
        return compressed_err;
1170
0
    }
1171
0
}
1172
1173
static void crypto_handshake(quicly_conn_t *conn, size_t in_epoch, ptls_iovec_t input)
1174
0
{
1175
0
    ptls_buffer_t output;
1176
0
    size_t epoch_offsets[5] = {0};
1177
1178
0
    assert(!conn->crypto.async_in_progress);
1179
1180
0
    ptls_buffer_init(&output, "", 0);
1181
1182
0
    quicly_error_t handshake_result = expand_handshake_result(ptls_handle_message(
1183
0
        conn->crypto.tls, &output, epoch_offsets, in_epoch, input.base, input.len, &conn->crypto.handshake_properties));
1184
0
    QUICLY_PROBE(CRYPTO_HANDSHAKE, conn, conn->stash.now, handshake_result);
1185
0
    QUICLY_LOG_CONN(crypto_handshake, conn, { PTLS_LOG_ELEMENT_SIGNED(ret, handshake_result); });
1186
0
    switch (handshake_result) {
1187
0
    case 0:
1188
0
    case PTLS_ERROR_IN_PROGRESS:
1189
0
        break;
1190
0
    case PTLS_ERROR_ASYNC_OPERATION:
1191
0
        assert(conn->super.ctx->async_handshake != NULL &&
1192
0
               "async handshake is used but the quicly_context_t::async_handshake is NULL");
1193
0
        conn->crypto.async_in_progress = 1;
1194
0
        conn->super.ctx->async_handshake->cb(conn->super.ctx->async_handshake, conn->crypto.tls);
1195
0
        break;
1196
0
    default:
1197
0
        initiate_close(conn,
1198
0
                       QUICLY_ERROR_IS_QUIC_TRANSPORT(handshake_result) ||
1199
0
                               PTLS_ERROR_GET_CLASS(handshake_result) == PTLS_ERROR_CLASS_SELF_ALERT
1200
0
                           ? handshake_result
1201
0
                           : QUICLY_TRANSPORT_ERROR_INTERNAL,
1202
0
                       QUICLY_FRAME_TYPE_CRYPTO, NULL);
1203
0
        goto Exit;
1204
0
    }
1205
    /* drop 0-RTT write key if 0-RTT is rejected by remote peer */
1206
0
    if (conn->application != NULL && !conn->application->one_rtt_writable && conn->application->cipher.egress.key.aead != NULL) {
1207
0
        assert(quicly_is_client(conn));
1208
0
        if (conn->crypto.handshake_properties.client.early_data_acceptance == PTLS_EARLY_DATA_REJECTED) {
1209
0
            dispose_cipher(&conn->application->cipher.egress.key);
1210
0
            conn->application->cipher.egress.key = (struct st_quicly_cipher_context_t){NULL};
1211
            /* retire all packets with ack_epoch == 3; they are all 0-RTT packets */
1212
0
            quicly_error_t ret;
1213
0
            if ((ret = discard_sentmap_by_epoch(conn, 1u << QUICLY_EPOCH_1RTT)) != 0) {
1214
0
                initiate_close(conn, ret, QUICLY_FRAME_TYPE_CRYPTO, NULL);
1215
0
                goto Exit;
1216
0
            }
1217
0
        }
1218
0
    }
1219
1220
0
    write_crypto_data(conn, &output, epoch_offsets);
1221
1222
0
Exit:
1223
0
    ptls_buffer_dispose(&output);
1224
0
}
1225
1226
void crypto_stream_receive(quicly_stream_t *stream, size_t off, const void *src, size_t len)
1227
0
{
1228
0
    quicly_conn_t *conn = stream->conn;
1229
0
    ptls_iovec_t input;
1230
1231
    /* store input */
1232
0
    if (quicly_streambuf_ingress_receive(stream, off, src, len) != 0)
1233
0
        return;
1234
1235
    /* While the server generates the handshake signature asynchronously, clients would not send additional messages. They cannot
1236
     * generate Finished. They would not send Certificate / CertificateVerify before authenticating the server identity. */
1237
0
    if (conn->crypto.async_in_progress) {
1238
0
        initiate_close(conn, PTLS_ALERT_UNEXPECTED_MESSAGE, QUICLY_FRAME_TYPE_CRYPTO, NULL);
1239
0
        return;
1240
0
    }
1241
1242
    /* feed the input into TLS, send result */
1243
0
    if ((input = quicly_streambuf_ingress_get(stream)).len != 0) {
1244
0
        size_t in_epoch = -(1 + stream->stream_id);
1245
0
        crypto_handshake(conn, in_epoch, input);
1246
0
        quicly_streambuf_ingress_shift(stream, input.len);
1247
0
    }
1248
0
}
1249
1250
quicly_conn_t *quicly_resume_handshake(ptls_t *tls)
1251
0
{
1252
0
    quicly_conn_t *conn;
1253
1254
0
    if ((conn = *ptls_get_data_ptr(tls)) == NULL) {
1255
        /* QUIC connection has been closed while TLS async operation was inflight. */
1256
0
        ptls_free(tls);
1257
0
        return NULL;
1258
0
    }
1259
1260
0
    assert(conn->crypto.async_in_progress);
1261
0
    conn->crypto.async_in_progress = 0;
1262
1263
0
    if (conn->super.state >= QUICLY_STATE_CLOSING)
1264
0
        return conn;
1265
1266
0
    crypto_handshake(conn, 0, ptls_iovec_init(NULL, 0));
1267
0
    return conn;
1268
0
}
1269
1270
static void init_stream_properties(quicly_stream_t *stream, uint32_t initial_max_stream_data_local,
1271
                                   uint64_t initial_max_stream_data_remote)
1272
0
{
1273
0
    int is_client = quicly_is_client(stream->conn);
1274
1275
0
    if (quicly_stream_has_send_side(is_client, stream->stream_id)) {
1276
0
        quicly_sendstate_init(&stream->sendstate);
1277
0
    } else {
1278
0
        quicly_sendstate_init_closed(&stream->sendstate);
1279
0
    }
1280
0
    if (quicly_stream_has_receive_side(is_client, stream->stream_id)) {
1281
0
        quicly_recvstate_init(&stream->recvstate);
1282
0
    } else {
1283
0
        quicly_recvstate_init_closed(&stream->recvstate);
1284
0
    }
1285
0
    stream->streams_blocked = 0;
1286
1287
0
    stream->_send_aux.max_stream_data = initial_max_stream_data_remote;
1288
0
    stream->_send_aux.stop_sending.sender_state = QUICLY_SENDER_STATE_NONE;
1289
0
    stream->_send_aux.stop_sending.error_code = 0;
1290
0
    stream->_send_aux.reset_stream.sender_state = QUICLY_SENDER_STATE_NONE;
1291
0
    stream->_send_aux.reset_stream.error_code = 0;
1292
0
    quicly_maxsender_init(&stream->_send_aux.max_stream_data_sender, initial_max_stream_data_local);
1293
0
    stream->_send_aux.blocked = QUICLY_SENDER_STATE_NONE;
1294
0
    quicly_linklist_init(&stream->_send_aux.pending_link.control);
1295
0
    quicly_linklist_init(&stream->_send_aux.pending_link.default_scheduler);
1296
1297
0
    stream->_recv_aux.window = initial_max_stream_data_local;
1298
1299
    /* Set the number of max ranges to be capable of handling following case:
1300
     * * every one of the two packets being sent are lost
1301
     * * average size of a STREAM frame found in a packet is >= ~512 bytes, or small STREAM frame is sent for every other stream
1302
     *   being opened (e.g., sending QPACK encoder/decoder stream frame for each HTTP/3 request)
1303
     * See also: the doc-comment on `_recv_aux.max_ranges`.
1304
     */
1305
0
    uint32_t fragments_minmax = (uint32_t)(stream->conn->super.ctx->transport_params.max_streams_uni +
1306
0
                                           stream->conn->super.ctx->transport_params.max_streams_bidi);
1307
0
    if (fragments_minmax < 63)
1308
0
        fragments_minmax = 63;
1309
0
    if ((stream->_recv_aux.max_ranges = initial_max_stream_data_local / 1024) < fragments_minmax)
1310
0
        stream->_recv_aux.max_ranges = fragments_minmax;
1311
0
}
1312
1313
static void dispose_stream_properties(quicly_stream_t *stream)
1314
0
{
1315
0
    quicly_sendstate_dispose(&stream->sendstate);
1316
0
    quicly_recvstate_dispose(&stream->recvstate);
1317
0
    quicly_maxsender_dispose(&stream->_send_aux.max_stream_data_sender);
1318
0
    quicly_linklist_unlink(&stream->_send_aux.pending_link.control);
1319
0
    quicly_linklist_unlink(&stream->_send_aux.pending_link.default_scheduler);
1320
0
}
1321
1322
static quicly_stream_t *open_stream(quicly_conn_t *conn, uint64_t stream_id, uint32_t initial_max_stream_data_local,
1323
                                    uint64_t initial_max_stream_data_remote)
1324
0
{
1325
0
    quicly_stream_t *stream;
1326
1327
0
    if ((stream = malloc(sizeof(*stream))) == NULL)
1328
0
        return NULL;
1329
0
    stream->conn = conn;
1330
0
    stream->stream_id = stream_id;
1331
0
    stream->callbacks = NULL;
1332
0
    stream->data = NULL;
1333
1334
0
    int r;
1335
0
    khiter_t iter = kh_put(quicly_stream_t, conn->streams, stream_id, &r);
1336
0
    assert(iter != kh_end(conn->streams));
1337
0
    kh_val(conn->streams, iter) = stream;
1338
1339
0
    init_stream_properties(stream, initial_max_stream_data_local, initial_max_stream_data_remote);
1340
1341
0
    return stream;
1342
0
}
1343
1344
static struct st_quicly_conn_streamgroup_state_t *get_streamgroup_state(quicly_conn_t *conn, quicly_stream_id_t stream_id)
1345
0
{
1346
0
    if (quicly_is_client(conn) == quicly_stream_is_client_initiated(stream_id)) {
1347
0
        return quicly_stream_is_unidirectional(stream_id) ? &conn->super.local.uni : &conn->super.local.bidi;
1348
0
    } else {
1349
0
        return quicly_stream_is_unidirectional(stream_id) ? &conn->super.remote.uni : &conn->super.remote.bidi;
1350
0
    }
1351
0
}
1352
1353
static int should_send_max_streams(quicly_conn_t *conn, int uni)
1354
0
{
1355
0
    uint64_t concurrency;
1356
0
    quicly_maxsender_t *maxsender;
1357
0
    struct st_quicly_conn_streamgroup_state_t *group;
1358
1359
0
#define INIT_VARS(type)                                                                                                            \
1360
0
    do {                                                                                                                           \
1361
0
        concurrency = conn->super.ctx->transport_params.max_streams_##type;                                                        \
1362
0
        maxsender = &conn->ingress.max_streams.type;                                                                               \
1363
0
        group = &conn->super.remote.type;                                                                                          \
1364
0
    } while (0)
1365
0
    if (uni) {
1366
0
        INIT_VARS(uni);
1367
0
    } else {
1368
0
        INIT_VARS(bidi);
1369
0
    }
1370
0
#undef INIT_VARS
1371
1372
0
    if (concurrency == 0)
1373
0
        return 0;
1374
1375
0
    if (!quicly_maxsender_should_send_max(maxsender, group->next_stream_id / 4, group->num_streams, 768))
1376
0
        return 0;
1377
1378
0
    return 1;
1379
0
}
1380
1381
static void destroy_stream(quicly_stream_t *stream, quicly_error_t err)
1382
0
{
1383
0
    quicly_conn_t *conn = stream->conn;
1384
1385
0
    QUICLY_PROBE(STREAM_ON_DESTROY, conn, conn->stash.now, stream, err);
1386
0
    QUICLY_LOG_CONN(stream_on_destroy, conn, {
1387
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
1388
0
        PTLS_LOG_ELEMENT_SIGNED(err, err);
1389
0
    });
1390
1391
0
    if (stream->callbacks != NULL)
1392
0
        stream->callbacks->on_destroy(stream, err);
1393
1394
0
    khiter_t iter = kh_get(quicly_stream_t, conn->streams, stream->stream_id);
1395
0
    assert(iter != kh_end(conn->streams));
1396
0
    kh_del(quicly_stream_t, conn->streams, iter);
1397
1398
0
    if (stream->stream_id < 0) {
1399
0
        size_t epoch = -(1 + stream->stream_id);
1400
0
        stream->conn->egress.pending_flows &= ~(uint8_t)(1 << epoch);
1401
0
    } else {
1402
0
        struct st_quicly_conn_streamgroup_state_t *group = get_streamgroup_state(conn, stream->stream_id);
1403
0
        --group->num_streams;
1404
0
    }
1405
1406
0
    dispose_stream_properties(stream);
1407
1408
0
    if (conn->application != NULL && should_send_max_streams(conn, quicly_stream_is_unidirectional(stream->stream_id)))
1409
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
1410
1411
0
    free(stream);
1412
0
}
1413
1414
static void destroy_all_streams(quicly_conn_t *conn, quicly_error_t err, int including_crypto_streams)
1415
0
{
1416
0
    quicly_stream_t *stream;
1417
0
    kh_foreach_value(conn->streams, stream, {
1418
        /* TODO do we need to send reset signals to open streams? */
1419
0
        if (including_crypto_streams || stream->stream_id >= 0)
1420
0
            destroy_stream(stream, err);
1421
0
    });
1422
0
    assert(quicly_num_streams(conn) == 0);
1423
0
}
1424
1425
int64_t quicly_foreach_stream(quicly_conn_t *conn, void *thunk, int64_t (*cb)(void *thunk, quicly_stream_t *stream))
1426
0
{
1427
0
    quicly_stream_t *stream;
1428
0
    kh_foreach_value(conn->streams, stream, {
1429
0
        if (stream->stream_id >= 0) {
1430
0
            int64_t ret = cb(thunk, stream);
1431
0
            if (ret != 0)
1432
0
                return ret;
1433
0
        }
1434
0
    });
1435
0
    return 0;
1436
0
}
1437
1438
quicly_stream_t *quicly_get_stream(quicly_conn_t *conn, quicly_stream_id_t stream_id)
1439
0
{
1440
0
    khiter_t iter = kh_get(quicly_stream_t, conn->streams, stream_id);
1441
0
    if (iter != kh_end(conn->streams))
1442
0
        return kh_val(conn->streams, iter);
1443
0
    return NULL;
1444
0
}
1445
1446
ptls_t *quicly_get_tls(quicly_conn_t *conn)
1447
409
{
1448
409
    return conn->crypto.tls;
1449
409
}
1450
1451
uint32_t quicly_num_streams_by_group(quicly_conn_t *conn, int uni, int locally_initiated)
1452
6.28k
{
1453
6.28k
    int server_initiated = quicly_is_client(conn) != locally_initiated;
1454
6.28k
    struct st_quicly_conn_streamgroup_state_t *state = get_streamgroup_state(conn, uni * 2 + server_initiated);
1455
6.28k
    return state->num_streams;
1456
6.28k
}
1457
1458
struct sockaddr *quicly_get_sockname(quicly_conn_t *conn)
1459
0
{
1460
0
    return &conn->paths[0]->address.local.sa;
1461
0
}
1462
1463
struct sockaddr *quicly_get_peername(quicly_conn_t *conn)
1464
409
{
1465
409
    return &conn->paths[0]->address.remote.sa;
1466
409
}
1467
1468
quicly_error_t quicly_get_stats(quicly_conn_t *conn, quicly_stats_t *stats)
1469
{
1470
    /* copy the pre-built stats fields */
1471
    memcpy(stats, &conn->super.stats, sizeof(conn->super.stats));
1472
1473
    /* set or generate the non-pre-built stats fields here */
1474
    stats->rtt = conn->egress.loss.rtt;
1475
    stats->loss_thresholds = conn->egress.loss.thresholds;
1476
    stats->cc = conn->egress.cc;
1477
    /* convert `exit_slow_start_at` to time spent since the connection was created */
1478
    if (stats->cc.exit_slow_start_at != INT64_MAX) {
1479
        assert(stats->cc.exit_slow_start_at >= conn->created_at);
1480
        stats->cc.exit_slow_start_at -= conn->created_at;
1481
    }
1482
    quicly_ratemeter_report(&conn->egress.ratemeter, &stats->delivery_rate);
1483
    stats->num_sentmap_packets_largest = conn->egress.loss.sentmap.num_packets_largest;
1484
1485
    return 0;
1486
}
1487
1488
quicly_error_t quicly_get_delivery_rate(quicly_conn_t *conn, quicly_rate_t *delivery_rate)
1489
0
{
1490
0
    quicly_ratemeter_report(&conn->egress.ratemeter, delivery_rate);
1491
0
    return 0;
1492
0
}
1493
1494
quicly_stream_id_t quicly_get_ingress_max_streams(quicly_conn_t *conn, int uni)
1495
0
{
1496
0
    quicly_maxsender_t *maxsender = uni ? &conn->ingress.max_streams.uni : &conn->ingress.max_streams.bidi;
1497
0
    return maxsender->max_committed;
1498
0
}
1499
1500
void quicly_get_max_data(quicly_conn_t *conn, uint64_t *send_permitted, uint64_t *sent, uint64_t *consumed, uint64_t *shifted)
1501
8.53k
{
1502
8.53k
    if (send_permitted != NULL)
1503
0
        *send_permitted = conn->egress.max_data.permitted;
1504
8.53k
    if (sent != NULL)
1505
8.53k
        *sent = conn->egress.max_data.sent;
1506
8.53k
    if (consumed != NULL)
1507
0
        *consumed = conn->ingress.max_data.bytes_consumed;
1508
8.53k
    if (shifted != NULL)
1509
0
        *shifted = conn->ingress.max_data.bytes_shifted;
1510
8.53k
}
1511
1512
static void update_idle_timeout(quicly_conn_t *conn, int is_in_receive)
1513
0
{
1514
0
    if (!is_in_receive && !conn->idle_timeout.should_rearm_on_send)
1515
0
        return;
1516
1517
    /* calculate the minimum of the two max_idle_timeout */
1518
0
    int64_t idle_msec = INT64_MAX;
1519
0
    if (conn->initial == NULL && conn->handshake == NULL && conn->super.remote.transport_params.max_idle_timeout != 0)
1520
0
        idle_msec = conn->super.remote.transport_params.max_idle_timeout;
1521
0
    if (conn->super.ctx->transport_params.max_idle_timeout != 0 && conn->super.ctx->transport_params.max_idle_timeout < idle_msec)
1522
0
        idle_msec = conn->super.ctx->transport_params.max_idle_timeout;
1523
1524
0
    if (idle_msec == INT64_MAX)
1525
0
        return;
1526
1527
0
    uint32_t three_pto = 3 * quicly_rtt_get_pto(&conn->egress.loss.rtt, conn->super.remote.transport_params.max_ack_delay,
1528
0
                                                conn->egress.loss.conf->min_pto);
1529
0
    conn->idle_timeout.at = conn->stash.now + (idle_msec > three_pto ? idle_msec : three_pto);
1530
0
    conn->idle_timeout.should_rearm_on_send = is_in_receive;
1531
0
}
1532
1533
static int scheduler_can_send(quicly_conn_t *conn)
1534
0
{
1535
    /* invoke the scheduler only when we are able to send stream data; skipping STATE_ACCEPTING is important as the application
1536
     * would not have setup data pointer. */
1537
0
    switch (conn->super.state) {
1538
0
    case QUICLY_STATE_FIRSTFLIGHT:
1539
0
    case QUICLY_STATE_CONNECTED:
1540
0
        break;
1541
0
    default:
1542
0
        return 0;
1543
0
    }
1544
1545
    /* scheduler would never have data to send, until application keys become available */
1546
0
    if (conn->application == NULL || conn->application->cipher.egress.key.aead == NULL)
1547
0
        return 0;
1548
1549
0
    int conn_is_saturated = !(conn->egress.max_data.sent < conn->egress.max_data.permitted);
1550
0
    return conn->super.ctx->stream_scheduler->can_send(conn->super.ctx->stream_scheduler, conn, conn_is_saturated);
1551
0
}
1552
1553
static void update_send_alarm(quicly_conn_t *conn, int can_send_stream_data, int is_after_send)
1554
0
{
1555
0
    int has_outstanding = conn->egress.loss.sentmap.bytes_in_flight != 0 || conn->super.remote.address_validation.send_probe,
1556
0
        handshake_is_in_progress = conn->initial != NULL || conn->handshake != NULL;
1557
0
    quicly_loss_update_alarm(&conn->egress.loss, conn->stash.now, conn->egress.last_retransmittable_sent_at, has_outstanding,
1558
0
                             can_send_stream_data, handshake_is_in_progress, conn->egress.max_data.sent, is_after_send);
1559
0
}
1560
1561
static void update_ratemeter(quicly_conn_t *conn, int is_cc_limited)
1562
0
{
1563
0
    if (quicly_ratemeter_is_cc_limited(&conn->egress.ratemeter) != is_cc_limited) {
1564
0
        if (is_cc_limited) {
1565
0
            quicly_ratemeter_enter_cc_limited(&conn->egress.ratemeter, conn->egress.packet_number);
1566
0
            QUICLY_PROBE(ENTER_CC_LIMITED, conn, conn->stash.now, conn->egress.packet_number);
1567
0
            QUICLY_LOG_CONN(enter_cc_limited, conn, { PTLS_LOG_ELEMENT_UNSIGNED(pn, conn->egress.packet_number); });
1568
0
        } else {
1569
0
            quicly_ratemeter_exit_cc_limited(&conn->egress.ratemeter, conn->egress.packet_number);
1570
0
            QUICLY_PROBE(EXIT_CC_LIMITED, conn, conn->stash.now, conn->egress.packet_number);
1571
0
            QUICLY_LOG_CONN(exit_cc_limited, conn, { PTLS_LOG_ELEMENT_UNSIGNED(pn, conn->egress.packet_number); });
1572
0
        }
1573
0
    }
1574
0
}
1575
1576
/**
1577
 * Updates the send alarm and adjusts the delivery rate estimator. This function is called from the receive path. From the sendp
1578
 * path, `update_send_alarm` is called directly.
1579
 */
1580
static void setup_next_send(quicly_conn_t *conn)
1581
0
{
1582
0
    int can_send_stream_data = scheduler_can_send(conn);
1583
1584
0
    update_send_alarm(conn, can_send_stream_data, 0);
1585
1586
    /* When the flow becomes application-limited due to receiving some information, stop collecting delivery rate samples. */
1587
0
    if (!can_send_stream_data)
1588
0
        update_ratemeter(conn, 0);
1589
0
}
1590
1591
static int create_handshake_flow(quicly_conn_t *conn, size_t epoch)
1592
0
{
1593
0
    quicly_stream_t *stream;
1594
0
    int ret;
1595
1596
0
    if ((stream = open_stream(conn, -(quicly_stream_id_t)(1 + epoch), conn->super.ctx->max_crypto_bytes,
1597
0
                              16777216 /* CRYPTO streams are not flow controlled; set the remote threshold to a huge value that we'd
1598
0
                                        * never hit */)) == NULL)
1599
0
        return PTLS_ERROR_NO_MEMORY;
1600
0
    if ((ret = quicly_streambuf_create(stream, sizeof(quicly_streambuf_t))) != 0) {
1601
0
        destroy_stream(stream, ret);
1602
0
        return ret;
1603
0
    }
1604
0
    stream->callbacks = &crypto_stream_callbacks;
1605
1606
0
    return 0;
1607
0
}
1608
1609
static void destroy_handshake_flow(quicly_conn_t *conn, size_t epoch)
1610
0
{
1611
0
    quicly_stream_t *stream = quicly_get_stream(conn, -(quicly_stream_id_t)(1 + epoch));
1612
0
    if (stream != NULL)
1613
0
        destroy_stream(stream, 0);
1614
0
}
1615
1616
static struct st_quicly_pn_space_t *alloc_pn_space(size_t sz, uint32_t packet_tolerance)
1617
0
{
1618
0
    struct st_quicly_pn_space_t *space;
1619
1620
0
    if ((space = malloc(sz)) == NULL)
1621
0
        return NULL;
1622
1623
0
    quicly_ranges_init(&space->ack_queue);
1624
0
    space->largest_pn_received_at = INT64_MAX;
1625
0
    space->next_expected_packet_number = 0;
1626
0
    space->unacked_count = 0;
1627
0
    space->prior_ecn = 0;
1628
0
    for (size_t i = 0; i < PTLS_ELEMENTSOF(space->ecn_counts); ++i)
1629
0
        space->ecn_counts[i] = 0;
1630
0
    space->packet_tolerance = packet_tolerance;
1631
0
    space->reordering_threshold = 1;
1632
0
    space->largest_acked_unacked = 0;
1633
0
    space->smallest_unreported_missing = 0;
1634
0
    if (sz != sizeof(*space))
1635
0
        memset((uint8_t *)space + sizeof(*space), 0, sz - sizeof(*space));
1636
1637
0
    return space;
1638
0
}
1639
1640
static void do_free_pn_space(struct st_quicly_pn_space_t *space)
1641
0
{
1642
0
    quicly_ranges_clear(&space->ack_queue);
1643
0
    free(space);
1644
0
}
1645
1646
static void update_smallest_unreported_missing_on_send_ack(quicly_ranges_t *ranges, uint64_t *largest_acked_unacked,
1647
                                                           uint64_t *smallest_unreported_missing, uint32_t reordering_threshold)
1648
0
{
1649
0
    assert(ranges->num_ranges != 0 && "on_send_ack is never called until the first packet is received");
1650
1651
0
    uint64_t largest_acked = ranges->ranges[ranges->num_ranges - 1].end - 1;
1652
0
    if (largest_acked <= *largest_acked_unacked)
1653
0
        return;
1654
0
    *largest_acked_unacked = largest_acked;
1655
1656
0
    if (reordering_threshold <= 1) {
1657
        /* For these cases simply set the smallest_unreported missing to the next expected PN. When reordering_threshold is 0,
1658
         * smallest_unreported_missing isn't used, but it's convenient to keep its state consistent if the threshold changes. */
1659
0
        *smallest_unreported_missing = largest_acked + 1;
1660
0
    } else {
1661
0
        uint64_t largest_pn_outside_reorder_window = largest_acked - (uint64_t)reordering_threshold;
1662
0
        if (largest_pn_outside_reorder_window >= *smallest_unreported_missing)
1663
0
            *smallest_unreported_missing = quicly_ranges_next_missing(ranges, largest_pn_outside_reorder_window + 1, NULL);
1664
0
    }
1665
0
}
1666
1667
static int change_outside_reorder_window(quicly_ranges_t *ranges, uint64_t largest_acked_unacked,
1668
                                         uint64_t *smallest_unreported_missing, uint64_t received_pn, uint32_t reordering_threshold)
1669
0
{
1670
    /* as this function is called after `record_pn`, `received_pn` will be registered */
1671
0
    assert(ranges->num_ranges != 0);
1672
0
    if (reordering_threshold == 0) {
1673
        /* We don't use this when the reordering_threshold is 0, but by
1674
         * maintaining it, we avoid having to do extra work if the
1675
         * reordering_threshold changes. */
1676
0
        *smallest_unreported_missing = largest_acked_unacked + 1;
1677
0
        return 0;
1678
0
    }
1679
1680
0
    uint64_t prev_smallest_unreported_missing = *smallest_unreported_missing;
1681
0
    size_t slots_traversed_for_next_missing = 0;
1682
1683
0
    if (received_pn == prev_smallest_unreported_missing) {
1684
0
        if (received_pn == largest_acked_unacked) {
1685
            // fast path. We received the packets in order.
1686
0
            *smallest_unreported_missing = largest_acked_unacked + 1;
1687
0
        } else {
1688
0
            *smallest_unreported_missing = quicly_ranges_next_missing(ranges, received_pn + 1, &slots_traversed_for_next_missing);
1689
0
        }
1690
0
    }
1691
1692
0
    if (largest_acked_unacked < reordering_threshold)
1693
0
        return 0;
1694
1695
0
    uint64_t largest_pn_outside_reorder_window = largest_acked_unacked - (uint64_t)reordering_threshold;
1696
1697
0
    if (*smallest_unreported_missing <= largest_pn_outside_reorder_window)
1698
0
        *smallest_unreported_missing =
1699
0
            quicly_ranges_next_missing(ranges, largest_pn_outside_reorder_window + 1, &slots_traversed_for_next_missing);
1700
1701
0
    return (prev_smallest_unreported_missing <= largest_pn_outside_reorder_window) ||
1702
0
           received_pn <= largest_pn_outside_reorder_window ||
1703
           // Send an ack if the next smallest unreported missing is past 1/4 of
1704
           // our max ranges to make sure all ack ranges get reported to the
1705
           // peer.
1706
0
           slots_traversed_for_next_missing > QUICLY_MAX_ACK_BLOCKS / 4;
1707
0
}
1708
1709
static quicly_error_t record_pn(quicly_ranges_t *ranges, uint64_t pn, int *is_out_of_order)
1710
0
{
1711
0
    quicly_error_t ret;
1712
1713
0
    *is_out_of_order = 0;
1714
1715
0
    if (ranges->num_ranges != 0) {
1716
        /* fast path that is taken when we receive a packet in-order */
1717
0
        if (ranges->ranges[ranges->num_ranges - 1].end == pn) {
1718
0
            ranges->ranges[ranges->num_ranges - 1].end = pn + 1;
1719
0
            return 0;
1720
0
        }
1721
0
        *is_out_of_order = 1;
1722
0
    }
1723
1724
    /* slow path; we add, then remove the oldest ranges when the number of ranges exceed the maximum */
1725
0
    if ((ret = quicly_ranges_add(ranges, pn, pn + 1)) != 0)
1726
0
        return ret;
1727
0
    if (ranges->num_ranges > QUICLY_MAX_ACK_BLOCKS)
1728
0
        quicly_ranges_drop_by_range_indices(ranges, ranges->num_ranges - QUICLY_MAX_ACK_BLOCKS, ranges->num_ranges);
1729
1730
0
    return 0;
1731
0
}
1732
1733
static quicly_error_t record_receipt(struct st_quicly_pn_space_t *space, uint64_t pn, uint8_t ecn, int is_ack_only,
1734
                                     int64_t received_at, int64_t *send_ack_at, uint64_t *received_out_of_order)
1735
0
{
1736
0
    int ack_now, is_out_of_order;
1737
0
    quicly_error_t ret;
1738
1739
0
    if ((ret = record_pn(&space->ack_queue, pn, &is_out_of_order)) != 0)
1740
0
        goto Exit;
1741
0
    if (is_out_of_order)
1742
0
        *received_out_of_order += 1;
1743
0
    if (!is_ack_only && space->largest_acked_unacked < pn)
1744
0
        space->largest_acked_unacked = pn;
1745
1746
0
    if (space->reordering_threshold == 1) {
1747
        // Keep previous code paths when using RFC 9000 reordering_threshold.
1748
0
        ack_now = !is_ack_only && (is_out_of_order || ecn == IPTOS_ECN_CE);
1749
0
        space->smallest_unreported_missing = space->largest_acked_unacked + 1;
1750
0
    } else {
1751
0
        ack_now = change_outside_reorder_window(&space->ack_queue, space->largest_acked_unacked,
1752
0
                                                &space->smallest_unreported_missing, pn, space->reordering_threshold);
1753
        /* Only ack a change outside the reordering window if the packet is
1754
         * ack-eliciting.
1755
         *
1756
         * Note that we must still call `change_outside_reorder_window` to maintain
1757
         * the correct `smallest_unreported_missing` value. */
1758
0
        ack_now = !is_ack_only && ack_now;
1759
        // https://datatracker.ietf.org/doc/html/draft-ietf-quic-ack-frequency-11#section-6.4-1
1760
0
        ack_now = ack_now || (ecn == IPTOS_ECN_CE && space->prior_ecn != IPTOS_ECN_CE);
1761
0
    }
1762
1763
    /* update largest_pn_received_at (TODO implement deduplication at an earlier moment?) */
1764
0
    if (space->ack_queue.ranges[space->ack_queue.num_ranges - 1].end == pn + 1)
1765
0
        space->largest_pn_received_at = received_at;
1766
1767
    /* increment ecn counters */
1768
0
    if (ecn != 0)
1769
0
        space->ecn_counts[get_ecn_index_from_bits(ecn)] += 1;
1770
1771
    /* if the received packet is ack-eliciting, update / schedule transmission of ACK */
1772
0
    if (!is_ack_only) {
1773
0
        space->unacked_count++;
1774
0
        if (space->unacked_count >= space->packet_tolerance)
1775
0
            ack_now = 1;
1776
0
    }
1777
1778
0
    if (ack_now) {
1779
0
        *send_ack_at = received_at;
1780
0
    } else if (*send_ack_at == INT64_MAX && space->unacked_count != 0) {
1781
0
        *send_ack_at = received_at + QUICLY_DELAYED_ACK_TIMEOUT;
1782
0
    }
1783
1784
0
    space->prior_ecn = ecn;
1785
0
    ret = 0;
1786
0
Exit:
1787
0
    return ret;
1788
0
}
1789
1790
static void free_handshake_space(struct st_quicly_handshake_space_t **space)
1791
0
{
1792
0
    if (*space != NULL) {
1793
0
        if ((*space)->cipher.ingress.aead != NULL)
1794
0
            dispose_cipher(&(*space)->cipher.ingress);
1795
0
        if ((*space)->cipher.egress.aead != NULL)
1796
0
            dispose_cipher(&(*space)->cipher.egress);
1797
0
        do_free_pn_space(&(*space)->super);
1798
0
        *space = NULL;
1799
0
    }
1800
0
}
1801
1802
static int setup_cipher(quicly_conn_t *conn, size_t epoch, int is_enc, ptls_cipher_context_t **hp_ctx,
1803
                        ptls_aead_context_t **aead_ctx, ptls_aead_algorithm_t *aead, ptls_hash_algorithm_t *hash,
1804
                        const void *secret)
1805
0
{
1806
    /* quicly_accept builds cipher before instantiating a connection. In such case, we use the default crypto engine */
1807
0
    quicly_crypto_engine_t *engine = conn != NULL ? conn->super.ctx->crypto_engine : &quicly_default_crypto_engine;
1808
1809
0
    return engine->setup_cipher(engine, conn, epoch, is_enc, hp_ctx, aead_ctx, aead, hash, secret);
1810
0
}
1811
1812
static int setup_handshake_space_and_flow(quicly_conn_t *conn, size_t epoch)
1813
0
{
1814
0
    struct st_quicly_handshake_space_t **space = epoch == QUICLY_EPOCH_INITIAL ? &conn->initial : &conn->handshake;
1815
0
    if ((*space = (void *)alloc_pn_space(sizeof(struct st_quicly_handshake_space_t), 1)) == NULL)
1816
0
        return PTLS_ERROR_NO_MEMORY;
1817
0
    return create_handshake_flow(conn, epoch);
1818
0
}
1819
1820
static void free_application_space(struct st_quicly_application_space_t **space)
1821
0
{
1822
0
    if (*space != NULL) {
1823
0
#define DISPOSE_INGRESS(label, func)                                                                                               \
1824
0
    if ((*space)->cipher.ingress.label != NULL)                                                                                    \
1825
0
    func((*space)->cipher.ingress.label)
1826
0
        DISPOSE_INGRESS(header_protection.zero_rtt, ptls_cipher_free);
1827
0
        DISPOSE_INGRESS(header_protection.one_rtt, ptls_cipher_free);
1828
0
        DISPOSE_INGRESS(aead[0], ptls_aead_free);
1829
0
        DISPOSE_INGRESS(aead[1], ptls_aead_free);
1830
0
#undef DISPOSE_INGRESS
1831
0
        if ((*space)->cipher.egress.key.aead != NULL)
1832
0
            dispose_cipher(&(*space)->cipher.egress.key);
1833
0
        ptls_clear_memory((*space)->cipher.egress.secret, sizeof((*space)->cipher.egress.secret));
1834
0
        do_free_pn_space(&(*space)->super);
1835
0
        *space = NULL;
1836
0
    }
1837
0
}
1838
1839
static int setup_application_space(quicly_conn_t *conn)
1840
0
{
1841
0
    if ((conn->application =
1842
0
             (void *)alloc_pn_space(sizeof(struct st_quicly_application_space_t), QUICLY_DEFAULT_PACKET_TOLERANCE)) == NULL)
1843
0
        return PTLS_ERROR_NO_MEMORY;
1844
1845
    /* prohibit key-update until receiving an ACK for an 1-RTT packet */
1846
0
    conn->application->cipher.egress.key_update_pn.last = 0;
1847
0
    conn->application->cipher.egress.key_update_pn.next = UINT64_MAX;
1848
1849
0
    return create_handshake_flow(conn, QUICLY_EPOCH_1RTT);
1850
0
}
1851
1852
static quicly_error_t discard_handshake_context(quicly_conn_t *conn, size_t epoch)
1853
0
{
1854
0
    quicly_error_t ret;
1855
1856
0
    assert(epoch == QUICLY_EPOCH_INITIAL || epoch == QUICLY_EPOCH_HANDSHAKE);
1857
1858
0
    if ((ret = discard_sentmap_by_epoch(conn, 1u << epoch)) != 0)
1859
0
        return ret;
1860
0
    destroy_handshake_flow(conn, epoch);
1861
0
    if (epoch == QUICLY_EPOCH_HANDSHAKE) {
1862
0
        assert(conn->stash.now != 0);
1863
0
        conn->super.stats.handshake_confirmed_msec = conn->stash.now - conn->created_at;
1864
0
    }
1865
0
    free_handshake_space(epoch == QUICLY_EPOCH_INITIAL ? &conn->initial : &conn->handshake);
1866
1867
0
    return 0;
1868
0
}
1869
1870
static quicly_error_t apply_remote_transport_params(quicly_conn_t *conn)
1871
0
{
1872
0
    quicly_error_t ret;
1873
1874
0
    conn->egress.max_data.permitted = conn->super.remote.transport_params.max_data;
1875
0
    if ((ret = update_max_streams(&conn->egress.max_streams.uni, conn->super.remote.transport_params.max_streams_uni)) != 0)
1876
0
        return ret;
1877
0
    if ((ret = update_max_streams(&conn->egress.max_streams.bidi, conn->super.remote.transport_params.max_streams_bidi)) != 0)
1878
0
        return ret;
1879
1880
0
    return 0;
1881
0
}
1882
1883
static int update_1rtt_key(quicly_conn_t *conn, ptls_cipher_suite_t *cipher, int is_enc, ptls_aead_context_t **aead,
1884
                           uint8_t *secret)
1885
0
{
1886
0
    uint8_t new_secret[PTLS_MAX_DIGEST_SIZE];
1887
0
    ptls_aead_context_t *new_aead = NULL;
1888
0
    int ret;
1889
1890
    /* generate next AEAD key */
1891
0
    if ((ret = ptls_hkdf_expand_label(cipher->hash, new_secret, cipher->hash->digest_size,
1892
0
                                      ptls_iovec_init(secret, cipher->hash->digest_size), "quic ku", ptls_iovec_init(NULL, 0),
1893
0
                                      NULL)) != 0)
1894
0
        goto Exit;
1895
0
    if ((ret = setup_cipher(conn, QUICLY_EPOCH_1RTT, is_enc, NULL, &new_aead, cipher->aead, cipher->hash, new_secret)) != 0)
1896
0
        goto Exit;
1897
1898
    /* success! update AEAD and secret */
1899
0
    if (*aead != NULL)
1900
0
        ptls_aead_free(*aead);
1901
0
    *aead = new_aead;
1902
0
    new_aead = NULL;
1903
0
    memcpy(secret, new_secret, cipher->hash->digest_size);
1904
1905
0
    ret = 0;
1906
0
Exit:
1907
0
    if (new_aead != NULL)
1908
0
        ptls_aead_free(new_aead);
1909
0
    ptls_clear_memory(new_secret, cipher->hash->digest_size);
1910
0
    return ret;
1911
0
}
1912
1913
static int update_1rtt_egress_key(quicly_conn_t *conn)
1914
0
{
1915
0
    struct st_quicly_application_space_t *space = conn->application;
1916
0
    ptls_cipher_suite_t *cipher = ptls_get_cipher(conn->crypto.tls);
1917
0
    int ret;
1918
1919
    /* generate next AEAD key, and increment key phase if it succeeds */
1920
0
    if ((ret = update_1rtt_key(conn, cipher, 1, &space->cipher.egress.key.aead, space->cipher.egress.secret)) != 0)
1921
0
        return ret;
1922
0
    ++space->cipher.egress.key_phase;
1923
1924
    /* signal that we are waiting for an ACK */
1925
0
    space->cipher.egress.key_update_pn.last = conn->egress.packet_number;
1926
0
    space->cipher.egress.key_update_pn.next = UINT64_MAX;
1927
1928
0
    QUICLY_PROBE(CRYPTO_SEND_KEY_UPDATE, conn, conn->stash.now, space->cipher.egress.key_phase,
1929
0
                 QUICLY_PROBE_HEXDUMP(space->cipher.egress.secret, cipher->hash->digest_size));
1930
0
    QUICLY_LOG_CONN(crypto_send_key_update, conn, {
1931
0
        PTLS_LOG_ELEMENT_UNSIGNED(phase, space->cipher.egress.key_phase);
1932
0
        PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(secret, space->cipher.egress.secret, cipher->hash->digest_size);
1933
0
    });
1934
1935
0
    return 0;
1936
0
}
1937
1938
static int received_key_update(quicly_conn_t *conn, uint64_t newly_decrypted_key_phase)
1939
0
{
1940
0
    struct st_quicly_application_space_t *space = conn->application;
1941
1942
0
    assert(space->cipher.ingress.key_phase.decrypted < newly_decrypted_key_phase);
1943
0
    assert(newly_decrypted_key_phase <= space->cipher.ingress.key_phase.prepared);
1944
1945
0
    space->cipher.ingress.key_phase.decrypted = newly_decrypted_key_phase;
1946
1947
0
    QUICLY_PROBE(CRYPTO_RECEIVE_KEY_UPDATE, conn, conn->stash.now, space->cipher.ingress.key_phase.decrypted,
1948
0
                 QUICLY_PROBE_HEXDUMP(space->cipher.ingress.secret, ptls_get_cipher(conn->crypto.tls)->hash->digest_size));
1949
0
    QUICLY_LOG_CONN(crypto_receive_key_update, conn, {
1950
0
        PTLS_LOG_ELEMENT_UNSIGNED(phase, space->cipher.ingress.key_phase.decrypted);
1951
0
        PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(secret, space->cipher.ingress.secret,
1952
0
                                         ptls_get_cipher(conn->crypto.tls)->hash->digest_size);
1953
0
    });
1954
1955
0
    if (space->cipher.egress.key_phase < space->cipher.ingress.key_phase.decrypted) {
1956
0
        return update_1rtt_egress_key(conn);
1957
0
    } else {
1958
0
        return 0;
1959
0
    }
1960
0
}
1961
1962
static void calc_resume_sendrate(quicly_conn_t *conn, uint64_t *rate, uint32_t *rtt)
1963
0
{
1964
0
    quicly_rate_t reported;
1965
1966
0
    quicly_ratemeter_report(&conn->egress.ratemeter, &reported);
1967
1968
0
    if (reported.smoothed != 0 || reported.latest != 0) {
1969
0
        *rate = reported.smoothed > reported.latest ? reported.smoothed : reported.latest;
1970
0
        *rtt = conn->egress.loss.rtt.minimum;
1971
0
    } else {
1972
0
        *rate = 0;
1973
0
        *rtt = 0;
1974
0
    }
1975
0
}
1976
1977
static inline void update_open_count(quicly_context_t *ctx, ssize_t delta)
1978
0
{
1979
0
    if (ctx->update_open_count != NULL)
1980
0
        ctx->update_open_count->cb(ctx->update_open_count, delta);
1981
0
}
1982
1983
0
#define LONGEST_ADDRESS_STR "[0000:1111:2222:3333:4444:5555:6666:7777]:12345"
1984
static void stringify_address(char *buf, struct sockaddr *sa)
1985
0
{
1986
0
    char *p = buf;
1987
0
    uint16_t port = 0;
1988
1989
0
    p = buf;
1990
0
    switch (sa->sa_family) {
1991
0
    case AF_INET:
1992
0
        inet_ntop(AF_INET, &((struct sockaddr_in *)sa)->sin_addr, p, sizeof(LONGEST_ADDRESS_STR));
1993
0
        p += strlen(p);
1994
0
        port = ntohs(((struct sockaddr_in *)sa)->sin_port);
1995
0
        break;
1996
0
    case AF_INET6:
1997
0
        *p++ = '[';
1998
0
        inet_ntop(AF_INET6, &((struct sockaddr_in6 *)sa)->sin6_addr, p, sizeof(LONGEST_ADDRESS_STR));
1999
0
        *p++ = ']';
2000
0
        port = ntohs(((struct sockaddr_in *)sa)->sin_port);
2001
0
        break;
2002
0
    default:
2003
0
        assert("unexpected address family");
2004
0
        break;
2005
0
    }
2006
2007
0
    *p++ = ':';
2008
0
    sprintf(p, "%" PRIu16, port);
2009
0
}
2010
2011
static int new_path(quicly_conn_t *conn, size_t path_index, struct sockaddr *remote_addr, struct sockaddr *local_addr)
2012
0
{
2013
0
    struct st_quicly_conn_path_t *path;
2014
2015
0
    assert(conn->paths[path_index] == NULL);
2016
2017
0
    if ((path = malloc(sizeof(*conn->paths[path_index]))) == NULL)
2018
0
        return PTLS_ERROR_NO_MEMORY;
2019
2020
0
    if (path_index == 0) {
2021
        /* default path used for handshake */
2022
0
        *path = (struct st_quicly_conn_path_t){
2023
0
            .dcid = 0,
2024
0
            .path_challenge.send_at = INT64_MAX,
2025
0
            .initial = 1,
2026
0
            .probe_only = 0,
2027
0
        };
2028
0
    } else {
2029
0
        *path = (struct st_quicly_conn_path_t){
2030
0
            .dcid = UINT64_MAX,
2031
0
            .path_challenge.send_at = 0,
2032
0
            .probe_only = 1,
2033
0
        };
2034
0
        conn->super.ctx->tls->random_bytes(path->path_challenge.data, sizeof(path->path_challenge.data));
2035
0
        conn->super.stats.num_paths.created += 1;
2036
0
    }
2037
0
    set_address(&path->address.remote, remote_addr);
2038
0
    set_address(&path->address.local, local_addr);
2039
2040
0
    conn->paths[path_index] = path;
2041
2042
0
    PTLS_LOG_DEFINE_POINT(quicly, new_path, new_path_logpoint);
2043
0
    if (QUICLY_PROBE_ENABLED(NEW_PATH) ||
2044
0
        (ptls_log_point_maybe_active(&new_path_logpoint) &
2045
0
         ptls_log_conn_maybe_active(ptls_get_log_state(conn->crypto.tls), ptls_log_getsni_ptls(conn->crypto.tls))) != 0) {
2046
0
        char remote[sizeof(LONGEST_ADDRESS_STR)];
2047
0
        stringify_address(remote, &path->address.remote.sa);
2048
0
        QUICLY_PROBE(NEW_PATH, conn, conn->stash.now, path_index, remote);
2049
0
        QUICLY_LOG_CONN(new_path, conn, {
2050
0
            PTLS_LOG_ELEMENT_UNSIGNED(path_index, path_index);
2051
0
            PTLS_LOG_ELEMENT_SAFESTR(remote, remote);
2052
0
        });
2053
0
    }
2054
2055
0
    return 0;
2056
0
}
2057
2058
static int do_delete_path(quicly_conn_t *conn, struct st_quicly_conn_path_t *path)
2059
0
{
2060
0
    int ret = 0;
2061
2062
0
    if (path->dcid != UINT64_MAX && conn->super.remote.cid_set.cids[0].cid.len != 0) {
2063
0
        uint64_t cid = path->dcid;
2064
0
        dissociate_cid(conn, cid);
2065
0
        ret = quicly_remote_cid_unregister(&conn->super.remote.cid_set, cid);
2066
0
        assert(conn->super.remote.cid_set.retired.count != 0);
2067
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
2068
0
    }
2069
2070
0
    free(path);
2071
2072
0
    return ret;
2073
0
}
2074
2075
static int delete_path(quicly_conn_t *conn, size_t path_index)
2076
0
{
2077
0
    QUICLY_PROBE(DELETE_PATH, conn, conn->stash.now, path_index);
2078
0
    QUICLY_LOG_CONN(delete_path, conn, { PTLS_LOG_ELEMENT_UNSIGNED(path_index, path_index); });
2079
2080
0
    struct st_quicly_conn_path_t *path = conn->paths[path_index];
2081
0
    conn->paths[path_index] = NULL;
2082
0
    if (path->path_challenge.send_at != INT64_MAX)
2083
0
        conn->super.stats.num_paths.validation_failed += 1;
2084
2085
0
    return do_delete_path(conn, path);
2086
0
}
2087
2088
/**
2089
 * paths[0] (the default path) is freed and the path specified by `path_index` is promoted
2090
 */
2091
static quicly_error_t promote_path(quicly_conn_t *conn, size_t path_index)
2092
0
{
2093
0
    quicly_error_t ret;
2094
2095
0
    QUICLY_PROBE(PROMOTE_PATH, conn, conn->stash.now, path_index);
2096
0
    QUICLY_LOG_CONN(promote_path, conn, { PTLS_LOG_ELEMENT_UNSIGNED(path_index, path_index); });
2097
2098
0
    { /* mark all packets as lost, as it is unlikely that packets sent on the old path would be acknowledged */
2099
0
        quicly_sentmap_iter_t iter;
2100
0
        if ((ret = quicly_loss_init_sentmap_iter(&conn->egress.loss, &iter, conn->stash.now,
2101
0
                                                 conn->super.remote.transport_params.max_ack_delay, 0)) != 0)
2102
0
            return ret;
2103
0
        const quicly_sent_packet_t *sent;
2104
0
        while ((sent = quicly_sentmap_get(&iter))->packet_number != UINT64_MAX) {
2105
0
            if ((ret = quicly_sentmap_update(&conn->egress.loss.sentmap, &iter, QUICLY_SENTMAP_EVENT_PTO)) != 0)
2106
0
                return ret;
2107
0
        }
2108
0
    }
2109
2110
    /* reset CC (FIXME flush sentmap and reset loss recovery) */
2111
0
    conn->egress.cc.type->cc_init->cb(
2112
0
        conn->egress.cc.type->cc_init, &conn->egress.cc,
2113
0
        quicly_cc_calc_initial_cwnd(conn->super.ctx->initcwnd_packets, conn->egress.max_udp_payload_size), conn->stash.now);
2114
0
    if (conn->super.stats.num_rapid_start != 0 && conn->egress.cc.type->enable_rapid_start != NULL)
2115
0
        conn->egress.cc.type->enable_rapid_start(&conn->egress.cc, conn->stash.now);
2116
2117
    /* set jumpstart target */
2118
0
    calc_resume_sendrate(conn, &conn->super.stats.jumpstart.prev_rate, &conn->super.stats.jumpstart.prev_rtt);
2119
2120
    /* reset RTT estimate, adopting SRTT of the original path as initial RTT (TODO calculate RTT based on path challenge RT) */
2121
0
    quicly_rtt_init(&conn->egress.loss.rtt, &conn->super.ctx->loss,
2122
0
                    conn->egress.loss.rtt.smoothed < conn->super.ctx->loss.default_initial_rtt
2123
0
                        ? conn->egress.loss.rtt.smoothed
2124
0
                        : conn->super.ctx->loss.default_initial_rtt);
2125
2126
    /* reset ratemeter */
2127
0
    quicly_ratemeter_init(&conn->egress.ratemeter);
2128
2129
    /* remember PN when the path was promoted */
2130
0
    conn->egress.pn_path_start = conn->egress.packet_number;
2131
2132
    /* update path mapping */
2133
0
    struct st_quicly_conn_path_t *path = conn->paths[0];
2134
0
    conn->paths[0] = conn->paths[path_index];
2135
0
    conn->paths[path_index] = NULL;
2136
0
    conn->super.stats.num_paths.promoted += 1;
2137
2138
0
    ret = do_delete_path(conn, path);
2139
2140
    /* rearm the loss timer, now that the RTT estimate has been changed */
2141
0
    setup_next_send(conn);
2142
2143
0
    return ret;
2144
0
}
2145
2146
static int open_path(quicly_conn_t *conn, size_t *path_index, struct sockaddr *remote_addr, struct sockaddr *local_addr)
2147
0
{
2148
0
    int ret;
2149
2150
    /* choose a slot that in unused or the least-recently-used one that has completed validation */
2151
0
    *path_index = SIZE_MAX;
2152
0
    for (size_t i = 1; i < PTLS_ELEMENTSOF(conn->paths); ++i) {
2153
0
        struct st_quicly_conn_path_t *p = conn->paths[i];
2154
0
        if (p == NULL) {
2155
0
            *path_index = i;
2156
0
            break;
2157
0
        }
2158
0
        if (p->path_challenge.send_at != INT64_MAX)
2159
0
            continue;
2160
0
        if (*path_index == SIZE_MAX || p->packet_last_received < conn->paths[*path_index]->packet_last_received)
2161
0
            *path_index = i;
2162
0
    }
2163
0
    if (*path_index == SIZE_MAX)
2164
0
        return QUICLY_ERROR_PACKET_IGNORED;
2165
2166
    /* free existing path info */
2167
0
    if (conn->paths[*path_index] != NULL && (ret = delete_path(conn, *path_index)) != 0)
2168
0
        return ret;
2169
2170
    /* initialize new path info */
2171
0
    if ((ret = new_path(conn, *path_index, remote_addr, local_addr)) != 0)
2172
0
        return ret;
2173
2174
    /* schedule emission of PATH_CHALLENGE */
2175
0
    conn->egress.send_probe_at = 0;
2176
2177
0
    return 0;
2178
0
}
2179
2180
static void recalc_send_probe_at(quicly_conn_t *conn)
2181
0
{
2182
0
    conn->egress.send_probe_at = INT64_MAX;
2183
2184
0
    for (size_t i = 0; i < PTLS_ELEMENTSOF(conn->paths); ++i) {
2185
0
        if (conn->paths[i] == NULL)
2186
0
            continue;
2187
0
        if (conn->egress.send_probe_at > conn->paths[i]->path_challenge.send_at)
2188
0
            conn->egress.send_probe_at = conn->paths[i]->path_challenge.send_at;
2189
0
        if (conn->paths[i]->path_response.send_) {
2190
0
            conn->egress.send_probe_at = 0;
2191
0
            break;
2192
0
        }
2193
0
    }
2194
0
}
2195
2196
void quicly_free(quicly_conn_t *conn)
2197
0
{
2198
0
    lock_now(conn, 0);
2199
2200
0
    QUICLY_PROBE(FREE, conn, conn->stash.now);
2201
0
    QUICLY_LOG_CONN(free, conn, {});
2202
2203
0
    PTLS_LOG_DEFINE_POINT(quicly, conn_stats, conn_stats_logpoint);
2204
0
    if (QUICLY_PROBE_ENABLED(CONN_STATS) ||
2205
0
        (ptls_log_point_maybe_active(&conn_stats_logpoint) &
2206
0
         ptls_log_conn_maybe_active(ptls_get_log_state(conn->crypto.tls), ptls_log_getsni_ptls(conn->crypto.tls))) != 0) {
2207
0
        quicly_stats_t stats;
2208
0
        if (quicly_get_stats(conn, &stats) == 0) {
2209
0
            QUICLY_PROBE(CONN_STATS, conn, conn->stash.now, &stats, sizeof(stats));
2210
0
#define EMIT_FIELD(fld, lit) PTLS_LOG__DO_ELEMENT_UNSIGNED(lit, stats.fld);
2211
0
            QUICLY_LOG_CONN(conn_stats, conn, { QUICLY_STATS_FOREACH(EMIT_FIELD); });
2212
0
#undef EMIT_FIELD
2213
0
        }
2214
0
    }
2215
2216
0
    destroy_all_streams(conn, 0, 1);
2217
0
    update_open_count(conn->super.ctx, -1);
2218
0
    clear_datagram_frame_payloads(conn);
2219
2220
0
    for (size_t i = 0; i != PTLS_ELEMENTSOF(conn->delayed_packets.as_array); ++i) {
2221
0
        while (conn->delayed_packets.as_array[i].head != NULL) {
2222
0
            struct st_quicly_delayed_packet_t *delayed = conn->delayed_packets.as_array[i].head;
2223
0
            conn->delayed_packets.as_array[i].head = delayed->next;
2224
0
            free(delayed);
2225
0
        }
2226
0
    }
2227
2228
0
    quicly_maxsender_dispose(&conn->ingress.max_data.sender);
2229
0
    quicly_maxsender_dispose(&conn->ingress.max_streams.uni);
2230
0
    quicly_maxsender_dispose(&conn->ingress.max_streams.bidi);
2231
0
    quicly_loss_dispose(&conn->egress.loss);
2232
2233
0
    kh_destroy(quicly_stream_t, conn->streams);
2234
2235
0
    assert(!quicly_linklist_is_linked(&conn->egress.pending_streams.blocked.uni));
2236
0
    assert(!quicly_linklist_is_linked(&conn->egress.pending_streams.blocked.bidi));
2237
0
    assert(!quicly_linklist_is_linked(&conn->egress.pending_streams.control));
2238
0
    assert(!quicly_linklist_is_linked(&conn->super._default_scheduler.active));
2239
0
    assert(!quicly_linklist_is_linked(&conn->super._default_scheduler.blocked));
2240
2241
0
    free_handshake_space(&conn->initial);
2242
0
    free_handshake_space(&conn->handshake);
2243
0
    free_application_space(&conn->application);
2244
2245
0
    ptls_buffer_dispose(&conn->crypto.transport_params.buf);
2246
2247
0
    for (size_t i = 0; i < PTLS_ELEMENTSOF(conn->paths); ++i) {
2248
0
        if (conn->paths[i] != NULL)
2249
0
            delete_path(conn, i);
2250
0
    }
2251
2252
    /* `crytpo.tls` is disposed late, because logging relies on `ptls_skip_tracing` */
2253
0
    if (conn->crypto.async_in_progress) {
2254
        /* When async signature generation is inflight, `ptls_free` will be called from `quicly_resume_handshake` laterwards. */
2255
0
        *ptls_get_data_ptr(conn->crypto.tls) = NULL;
2256
0
    } else {
2257
0
        ptls_free(conn->crypto.tls);
2258
0
    }
2259
2260
0
    unlock_now(conn);
2261
2262
0
    if (conn->egress.pacer != NULL)
2263
0
        free(conn->egress.pacer);
2264
0
    if (conn->connection_close.reason_phrase != NULL)
2265
0
        free(conn->connection_close.reason_phrase);
2266
0
    free(conn->token.base);
2267
0
    free(conn);
2268
0
}
2269
2270
static int calc_initial_key(ptls_cipher_suite_t *cs, uint8_t *traffic_secret, const void *master_secret, const char *label)
2271
0
{
2272
0
    return ptls_hkdf_expand_label(cs->hash, traffic_secret, cs->hash->digest_size,
2273
0
                                  ptls_iovec_init(master_secret, cs->hash->digest_size), label, ptls_iovec_init(NULL, 0), NULL);
2274
0
}
2275
2276
int quicly_calc_initial_keys(ptls_cipher_suite_t *cs, uint8_t *ingress, uint8_t *egress, ptls_iovec_t cid, int is_client,
2277
                             ptls_iovec_t salt)
2278
0
{
2279
0
    static const char *labels[2] = {"client in", "server in"};
2280
0
    uint8_t master_secret[PTLS_MAX_DIGEST_SIZE];
2281
0
    int ret;
2282
2283
    /* extract master secret */
2284
0
    if ((ret = ptls_hkdf_extract(cs->hash, master_secret, salt, cid)) != 0)
2285
0
        goto Exit;
2286
2287
    /* calc secrets */
2288
0
    if (ingress != NULL && (ret = calc_initial_key(cs, ingress, master_secret, labels[is_client])) != 0)
2289
0
        goto Exit;
2290
0
    if (egress != NULL && (ret = calc_initial_key(cs, egress, master_secret, labels[!is_client])) != 0)
2291
0
        goto Exit;
2292
2293
0
Exit:
2294
0
    ptls_clear_memory(master_secret, sizeof(master_secret));
2295
0
    return ret;
2296
0
}
2297
2298
/**
2299
 * @param conn maybe NULL when called by quicly_accept
2300
 */
2301
static int setup_initial_encryption(ptls_cipher_suite_t *cs, struct st_quicly_cipher_context_t *ingress,
2302
                                    struct st_quicly_cipher_context_t *egress, ptls_iovec_t cid, int is_client, ptls_iovec_t salt,
2303
                                    quicly_conn_t *conn)
2304
0
{
2305
0
    struct {
2306
0
        uint8_t ingress[PTLS_MAX_DIGEST_SIZE];
2307
0
        uint8_t egress[PTLS_MAX_DIGEST_SIZE];
2308
0
    } secrets;
2309
0
    int ret;
2310
2311
0
    if ((ret = quicly_calc_initial_keys(cs, ingress != NULL ? secrets.ingress : NULL, egress != NULL ? secrets.egress : NULL, cid,
2312
0
                                        is_client, salt)) != 0)
2313
0
        goto Exit;
2314
2315
0
    if (ingress != NULL && (ret = setup_cipher(conn, QUICLY_EPOCH_INITIAL, 0, &ingress->header_protection, &ingress->aead, cs->aead,
2316
0
                                               cs->hash, secrets.ingress)) != 0)
2317
0
        goto Exit;
2318
0
    if (egress != NULL && (ret = setup_cipher(conn, QUICLY_EPOCH_INITIAL, 1, &egress->header_protection, &egress->aead, cs->aead,
2319
0
                                              cs->hash, secrets.egress)) != 0)
2320
0
        goto Exit;
2321
2322
0
Exit:
2323
0
    ptls_clear_memory(&secrets, sizeof(secrets));
2324
0
    return ret;
2325
0
}
2326
2327
static quicly_error_t reinstall_initial_encryption(quicly_conn_t *conn, quicly_error_t err_code_if_unknown_version)
2328
0
{
2329
0
    const quicly_salt_t *salt;
2330
2331
    /* get salt */
2332
0
    if ((salt = quicly_get_salt(conn->super.version)) == NULL)
2333
0
        return err_code_if_unknown_version;
2334
2335
    /* dispose existing context */
2336
0
    dispose_cipher(&conn->initial->cipher.ingress);
2337
0
    dispose_cipher(&conn->initial->cipher.egress);
2338
2339
    /* setup encryption context */
2340
0
    return setup_initial_encryption(
2341
0
        get_aes128gcmsha256(conn->super.ctx), &conn->initial->cipher.ingress, &conn->initial->cipher.egress,
2342
0
        ptls_iovec_init(conn->super.remote.cid_set.cids[0].cid.cid, conn->super.remote.cid_set.cids[0].cid.len), 1,
2343
0
        ptls_iovec_init(salt->initial, sizeof(salt->initial)), NULL);
2344
0
}
2345
2346
static quicly_error_t apply_stream_frame(quicly_stream_t *stream, quicly_stream_frame_t *frame)
2347
0
{
2348
0
    quicly_error_t ret;
2349
2350
0
    QUICLY_PROBE(STREAM_RECEIVE, stream->conn, stream->conn->stash.now, stream, frame->offset, frame->data.base, frame->data.len,
2351
0
                 (int)frame->is_fin);
2352
0
    QUICLY_LOG_CONN(stream_receive, stream->conn, {
2353
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
2354
0
        PTLS_LOG_ELEMENT_UNSIGNED(off, frame->offset);
2355
0
        PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(data, frame->data.base, frame->data.len);
2356
0
        PTLS_LOG_ELEMENT_BOOL(is_fin, frame->is_fin);
2357
0
    });
2358
2359
0
    if (quicly_recvstate_transfer_complete(&stream->recvstate))
2360
0
        return 0;
2361
2362
    /* flow control */
2363
0
    if (stream->stream_id >= 0) {
2364
        /* STREAMs */
2365
0
        uint64_t max_stream_data = frame->offset + frame->data.len;
2366
0
        if ((int64_t)stream->_recv_aux.window < (int64_t)max_stream_data - (int64_t)stream->recvstate.data_off)
2367
0
            return QUICLY_TRANSPORT_ERROR_FLOW_CONTROL;
2368
0
        if (stream->recvstate.received.ranges[stream->recvstate.received.num_ranges - 1].end < max_stream_data) {
2369
0
            uint64_t newly_received =
2370
0
                max_stream_data - stream->recvstate.received.ranges[stream->recvstate.received.num_ranges - 1].end;
2371
0
            if (stream->conn->ingress.max_data.bytes_consumed + newly_received >
2372
0
                stream->conn->ingress.max_data.sender.max_committed)
2373
0
                return QUICLY_TRANSPORT_ERROR_FLOW_CONTROL;
2374
0
            stream->conn->ingress.max_data.bytes_consumed += newly_received;
2375
            /* FIXME send MAX_DATA if necessary */
2376
0
        }
2377
0
    } else {
2378
        /* CRYPTO streams; maybe add different limit for 1-RTT CRYPTO? */
2379
0
        if (frame->offset + frame->data.len > stream->conn->super.ctx->max_crypto_bytes)
2380
0
            return QUICLY_TRANSPORT_ERROR_CRYPTO_BUFFER_EXCEEDED;
2381
0
    }
2382
2383
    /* update recvbuf */
2384
0
    size_t apply_len = frame->data.len;
2385
0
    if ((ret = quicly_recvstate_update(&stream->recvstate, frame->offset, &apply_len, frame->is_fin,
2386
0
                                       stream->_recv_aux.max_ranges)) != 0)
2387
0
        return ret;
2388
2389
0
    if (apply_len != 0 || quicly_recvstate_transfer_complete(&stream->recvstate)) {
2390
0
        uint64_t buf_offset = frame->offset + frame->data.len - apply_len - stream->recvstate.data_off;
2391
0
        size_t apply_off = frame->data.len - apply_len;
2392
0
        QUICLY_PROBE(STREAM_ON_RECEIVE, stream->conn, stream->conn->stash.now, stream, (size_t)buf_offset, apply_off, apply_len);
2393
0
        QUICLY_LOG_CONN(stream_on_receive, stream->conn, {
2394
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
2395
0
            PTLS_LOG_ELEMENT_UNSIGNED(buf_off, buf_offset);
2396
0
            PTLS_LOG_ELEMENT_UNSIGNED(apply_off, apply_off);
2397
0
            PTLS_LOG_ELEMENT_UNSIGNED(apply_len, apply_len);
2398
0
        });
2399
0
        stream->callbacks->on_receive(stream, (size_t)buf_offset, frame->data.base + apply_off, apply_len);
2400
0
        if (stream->conn->super.state >= QUICLY_STATE_CLOSING)
2401
0
            return QUICLY_ERROR_IS_CLOSING;
2402
0
    }
2403
2404
0
    if (stream->stream_id >= 0 && should_send_max_stream_data(stream))
2405
0
        sched_stream_control(stream);
2406
2407
0
    if (stream_is_destroyable(stream))
2408
0
        destroy_stream(stream, 0);
2409
2410
0
    return 0;
2411
0
}
2412
2413
int quicly_encode_transport_parameter_list(ptls_buffer_t *buf, const quicly_transport_parameters_t *params,
2414
                                           const quicly_cid_t *original_dcid, const quicly_cid_t *initial_scid,
2415
                                           const quicly_cid_t *retry_scid, const void *stateless_reset_token, size_t expand_by)
2416
0
{
2417
0
    int ret;
2418
2419
0
#define PUSH_TP(buf, id, block)                                                                                                    \
2420
0
    do {                                                                                                                           \
2421
0
        ptls_buffer_push_quicint((buf), (id));                                                                                     \
2422
0
        ptls_buffer_push_block((buf), -1, block);                                                                                  \
2423
0
    } while (0)
2424
2425
0
    PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_MAX_UDP_PAYLOAD_SIZE,
2426
0
            { ptls_buffer_push_quicint(buf, params->max_udp_payload_size); });
2427
0
    if (params->max_stream_data.bidi_local != 0)
2428
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL,
2429
0
                { ptls_buffer_push_quicint(buf, params->max_stream_data.bidi_local); });
2430
0
    if (params->max_stream_data.bidi_remote != 0)
2431
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE,
2432
0
                { ptls_buffer_push_quicint(buf, params->max_stream_data.bidi_remote); });
2433
0
    if (params->max_stream_data.uni != 0)
2434
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_UNI,
2435
0
                { ptls_buffer_push_quicint(buf, params->max_stream_data.uni); });
2436
0
    if (params->max_data != 0)
2437
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_DATA, { ptls_buffer_push_quicint(buf, params->max_data); });
2438
0
    if (params->max_idle_timeout != 0)
2439
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_MAX_IDLE_TIMEOUT, { ptls_buffer_push_quicint(buf, params->max_idle_timeout); });
2440
0
    if (original_dcid != NULL)
2441
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_ORIGINAL_CONNECTION_ID,
2442
0
                { ptls_buffer_pushv(buf, original_dcid->cid, original_dcid->len); });
2443
0
    if (initial_scid != NULL)
2444
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_SOURCE_CONNECTION_ID,
2445
0
                { ptls_buffer_pushv(buf, initial_scid->cid, initial_scid->len); });
2446
0
    if (retry_scid != NULL)
2447
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_RETRY_SOURCE_CONNECTION_ID,
2448
0
                { ptls_buffer_pushv(buf, retry_scid->cid, retry_scid->len); });
2449
0
    if (stateless_reset_token != NULL)
2450
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_STATELESS_RESET_TOKEN,
2451
0
                { ptls_buffer_pushv(buf, stateless_reset_token, QUICLY_STATELESS_RESET_TOKEN_LEN); });
2452
0
    if (params->max_streams_bidi != 0)
2453
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_BIDI,
2454
0
                { ptls_buffer_push_quicint(buf, params->max_streams_bidi); });
2455
0
    if (params->max_streams_uni != 0)
2456
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_UNI,
2457
0
                { ptls_buffer_push_quicint(buf, params->max_streams_uni); });
2458
0
    if (QUICLY_LOCAL_ACK_DELAY_EXPONENT != QUICLY_DEFAULT_ACK_DELAY_EXPONENT)
2459
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_ACK_DELAY_EXPONENT,
2460
0
                { ptls_buffer_push_quicint(buf, QUICLY_LOCAL_ACK_DELAY_EXPONENT); });
2461
0
    if (QUICLY_LOCAL_MAX_ACK_DELAY != QUICLY_DEFAULT_MAX_ACK_DELAY)
2462
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_MAX_ACK_DELAY, { ptls_buffer_push_quicint(buf, QUICLY_LOCAL_MAX_ACK_DELAY); });
2463
0
    if (params->min_ack_delay_usec != UINT64_MAX) {
2464
        /* TODO consider the value we should advertise. */
2465
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_MIN_ACK_DELAY,
2466
0
                { ptls_buffer_push_quicint(buf, QUICLY_LOCAL_MAX_ACK_DELAY * 1000 /* in microseconds */); });
2467
0
    }
2468
0
    if (params->disable_active_migration)
2469
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_DISABLE_ACTIVE_MIGRATION, {});
2470
0
    if (QUICLY_LOCAL_ACTIVE_CONNECTION_ID_LIMIT != QUICLY_DEFAULT_ACTIVE_CONNECTION_ID_LIMIT)
2471
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_ACTIVE_CONNECTION_ID_LIMIT,
2472
0
                { ptls_buffer_push_quicint(buf, QUICLY_LOCAL_ACTIVE_CONNECTION_ID_LIMIT); });
2473
0
    if (params->max_datagram_frame_size != 0)
2474
0
        PUSH_TP(buf, QUICLY_TRANSPORT_PARAMETER_ID_MAX_DATAGRAM_FRAME_SIZE,
2475
0
                { ptls_buffer_push_quicint(buf, params->max_datagram_frame_size); });
2476
    /* if requested, add a greasing TP of 1 MTU size so that CH spans across multiple packets */
2477
0
    if (expand_by != 0) {
2478
0
        PUSH_TP(buf, 31 * 100 + 27, {
2479
0
            if ((ret = ptls_buffer_reserve(buf, expand_by)) != 0)
2480
0
                goto Exit;
2481
0
            memset(buf->base + buf->off, 0, expand_by);
2482
0
            buf->off += expand_by;
2483
0
        });
2484
0
    }
2485
2486
0
#undef PUSH_TP
2487
2488
0
    ret = 0;
2489
0
Exit:
2490
0
    return ret;
2491
0
}
2492
2493
/**
2494
 * sentinel used for indicating that the corresponding TP should be ignored
2495
 */
2496
static const quicly_cid_t _tp_cid_ignore;
2497
0
#define tp_cid_ignore (*(quicly_cid_t *)&_tp_cid_ignore)
2498
2499
quicly_error_t quicly_decode_transport_parameter_list(quicly_transport_parameters_t *params, quicly_cid_t *original_dcid,
2500
                                                      quicly_cid_t *initial_scid, quicly_cid_t *retry_scid,
2501
                                                      void *stateless_reset_token, const uint8_t *src, const uint8_t *end)
2502
0
{
2503
/* When non-negative, tp_index contains the literal position within the list of transport parameters recognized by this function.
2504
 * That index is being used to find duplicates using a 64-bit bitmap (found_bits). When the transport parameter is being processed,
2505
 * tp_index is set to -1. */
2506
0
#define DECODE_TP(_id, block)                                                                                                      \
2507
0
    do {                                                                                                                           \
2508
0
        if (tp_index >= 0) {                                                                                                       \
2509
0
            if (id == (_id)) {                                                                                                     \
2510
0
                if ((found_bits & ((uint64_t)1 << tp_index)) != 0) {                                                               \
2511
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;                                                              \
2512
0
                    goto Exit;                                                                                                     \
2513
0
                }                                                                                                                  \
2514
0
                found_bits |= (uint64_t)1 << tp_index;                                                                             \
2515
0
                {block} tp_index = -1;                                                                                             \
2516
0
            } else {                                                                                                               \
2517
0
                ++tp_index;                                                                                                        \
2518
0
            }                                                                                                                      \
2519
0
        }                                                                                                                          \
2520
0
    } while (0)
2521
0
#define DECODE_CID_TP(_id, dest)                                                                                                   \
2522
0
    DECODE_TP(_id, {                                                                                                               \
2523
0
        size_t cidl = end - src;                                                                                                   \
2524
0
        if (cidl > QUICLY_MAX_CID_LEN_V1) {                                                                                        \
2525
0
            ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;                                                                      \
2526
0
            goto Exit;                                                                                                             \
2527
0
        }                                                                                                                          \
2528
0
        if (dest == NULL) {                                                                                                        \
2529
0
            ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;                                                                      \
2530
0
            goto Exit;                                                                                                             \
2531
0
        } else if (dest != &tp_cid_ignore) {                                                                                       \
2532
0
            quicly_set_cid(dest, ptls_iovec_init(src, cidl));                                                                      \
2533
0
        }                                                                                                                          \
2534
0
        src = end;                                                                                                                 \
2535
0
    });
2536
2537
0
    uint64_t found_bits = 0;
2538
0
    quicly_error_t ret;
2539
2540
    /* set parameters to their default values */
2541
0
    *params = default_transport_params;
2542
2543
    /* Set optional parameters to UINT8_MAX. It is used to as a sentinel for detecting missing TPs. */
2544
0
    if (original_dcid != NULL && original_dcid != &tp_cid_ignore)
2545
0
        original_dcid->len = UINT8_MAX;
2546
0
    if (initial_scid != NULL && initial_scid != &tp_cid_ignore)
2547
0
        initial_scid->len = UINT8_MAX;
2548
0
    if (retry_scid != NULL && retry_scid != &tp_cid_ignore)
2549
0
        retry_scid->len = UINT8_MAX;
2550
2551
    /* decode the parameters block */
2552
0
    while (src != end) {
2553
0
        uint64_t id;
2554
0
        if ((id = quicly_decodev(&src, end)) == UINT64_MAX) {
2555
0
            ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2556
0
            goto Exit;
2557
0
        }
2558
0
        int tp_index = 0;
2559
0
        ptls_decode_open_block(src, end, -1, {
2560
0
            DECODE_CID_TP(QUICLY_TRANSPORT_PARAMETER_ID_ORIGINAL_CONNECTION_ID, original_dcid);
2561
0
            DECODE_CID_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_SOURCE_CONNECTION_ID, initial_scid);
2562
0
            DECODE_CID_TP(QUICLY_TRANSPORT_PARAMETER_ID_RETRY_SOURCE_CONNECTION_ID, retry_scid);
2563
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_MAX_UDP_PAYLOAD_SIZE, {
2564
0
                uint64_t v;
2565
0
                if ((v = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2566
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2567
0
                    goto Exit;
2568
0
                }
2569
0
                if (v < 1200) {
2570
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2571
0
                    goto Exit;
2572
0
                }
2573
0
                if (v > UINT16_MAX)
2574
0
                    v = UINT16_MAX;
2575
0
                params->max_udp_payload_size = (uint16_t)v;
2576
0
            });
2577
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL, {
2578
0
                if ((params->max_stream_data.bidi_local = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2579
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2580
0
                    goto Exit;
2581
0
                }
2582
0
            });
2583
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE, {
2584
0
                if ((params->max_stream_data.bidi_remote = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2585
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2586
0
                    goto Exit;
2587
0
                }
2588
0
            });
2589
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_UNI, {
2590
0
                if ((params->max_stream_data.uni = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2591
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2592
0
                    goto Exit;
2593
0
                }
2594
0
            });
2595
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_DATA, {
2596
0
                if ((params->max_data = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2597
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2598
0
                    goto Exit;
2599
0
                }
2600
0
            });
2601
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_STATELESS_RESET_TOKEN, {
2602
0
                if (!(stateless_reset_token != NULL && end - src == QUICLY_STATELESS_RESET_TOKEN_LEN)) {
2603
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2604
0
                    goto Exit;
2605
0
                }
2606
0
                memcpy(stateless_reset_token, src, QUICLY_STATELESS_RESET_TOKEN_LEN);
2607
0
                src = end;
2608
0
            });
2609
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_MAX_IDLE_TIMEOUT, {
2610
0
                if ((params->max_idle_timeout = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2611
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2612
0
                    goto Exit;
2613
0
                }
2614
0
            });
2615
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_BIDI, {
2616
0
                if ((params->max_streams_bidi = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2617
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2618
0
                    goto Exit;
2619
0
                }
2620
0
            });
2621
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_UNI, {
2622
0
                if ((params->max_streams_uni = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2623
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2624
0
                    goto Exit;
2625
0
                }
2626
0
            });
2627
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_ACK_DELAY_EXPONENT, {
2628
0
                uint64_t v;
2629
0
                if ((v = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2630
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2631
0
                    goto Exit;
2632
0
                }
2633
0
                if (v > 20) {
2634
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2635
0
                    goto Exit;
2636
0
                }
2637
0
                params->ack_delay_exponent = (uint8_t)v;
2638
0
            });
2639
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_MAX_ACK_DELAY, {
2640
0
                uint64_t v;
2641
0
                if ((v = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2642
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2643
0
                    goto Exit;
2644
0
                }
2645
0
                if (v >= 16384) { /* "values of 2^14 or greater are invalid" */
2646
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2647
0
                    goto Exit;
2648
0
                }
2649
0
                params->max_ack_delay = (uint16_t)v;
2650
0
            });
2651
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_MIN_ACK_DELAY, {
2652
0
                if ((params->min_ack_delay_usec = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2653
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2654
0
                    goto Exit;
2655
0
                }
2656
0
            });
2657
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_ACTIVE_CONNECTION_ID_LIMIT, {
2658
0
                uint64_t v;
2659
0
                if ((v = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2660
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2661
0
                    goto Exit;
2662
0
                }
2663
0
                if (v < QUICLY_MIN_ACTIVE_CONNECTION_ID_LIMIT) {
2664
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2665
0
                    goto Exit;
2666
0
                }
2667
0
                params->active_connection_id_limit = v;
2668
0
            });
2669
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_DISABLE_ACTIVE_MIGRATION, { params->disable_active_migration = 1; });
2670
0
            DECODE_TP(QUICLY_TRANSPORT_PARAMETER_ID_MAX_DATAGRAM_FRAME_SIZE, {
2671
0
                uint64_t v;
2672
0
                if ((v = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
2673
0
                    ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2674
0
                    goto Exit;
2675
0
                }
2676
0
                if (v > UINT16_MAX)
2677
0
                    v = UINT16_MAX;
2678
0
                params->max_datagram_frame_size = (uint16_t)v;
2679
0
            });
2680
            /* skip unknown extension */
2681
0
            if (tp_index >= 0)
2682
0
                src = end;
2683
0
        });
2684
0
    }
2685
2686
    /* check consistency between the transport parameters */
2687
0
    if (params->min_ack_delay_usec != UINT64_MAX) {
2688
0
        if (params->min_ack_delay_usec > params->max_ack_delay * 1000) {
2689
0
            ret = QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
2690
0
            goto Exit;
2691
0
        }
2692
0
    }
2693
2694
    /* check the absence of CIDs */
2695
0
    if ((original_dcid != NULL && original_dcid->len == UINT8_MAX) || (initial_scid != NULL && initial_scid->len == UINT8_MAX) ||
2696
0
        (retry_scid != NULL && retry_scid->len == UINT8_MAX)) {
2697
0
        ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2698
0
        goto Exit;
2699
0
    }
2700
2701
0
    ret = 0;
2702
0
Exit:
2703
0
    if (ret == PTLS_ALERT_DECODE_ERROR)
2704
0
        ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2705
0
    return ret;
2706
2707
0
#undef DECODE_TP
2708
0
#undef DECODE_CID_TP
2709
0
}
2710
2711
static uint16_t get_transport_parameters_extension_id(uint32_t quic_version)
2712
0
{
2713
0
    switch (quic_version) {
2714
0
    case QUICLY_PROTOCOL_VERSION_DRAFT27:
2715
0
    case QUICLY_PROTOCOL_VERSION_DRAFT29:
2716
0
        return QUICLY_TLS_EXTENSION_TYPE_TRANSPORT_PARAMETERS_DRAFT;
2717
0
    default:
2718
0
        return QUICLY_TLS_EXTENSION_TYPE_TRANSPORT_PARAMETERS_FINAL;
2719
0
    }
2720
0
}
2721
2722
static int collect_transport_parameters(ptls_t *tls, struct st_ptls_handshake_properties_t *properties, uint16_t type)
2723
0
{
2724
0
    quicly_conn_t *conn = (void *)((char *)properties - offsetof(quicly_conn_t, crypto.handshake_properties));
2725
0
    return type == get_transport_parameters_extension_id(conn->super.version);
2726
0
}
2727
2728
static quicly_conn_t *create_connection(quicly_context_t *ctx, uint32_t protocol_version, const char *server_name,
2729
                                        struct sockaddr *remote_addr, struct sockaddr *local_addr, ptls_iovec_t *remote_cid,
2730
                                        const quicly_cid_plaintext_t *local_cid, ptls_handshake_properties_t *handshake_properties,
2731
                                        void *appdata, uint32_t initcwnd)
2732
0
{
2733
0
    ptls_log_conn_state_t log_state_override;
2734
0
    ptls_t *tls;
2735
0
    quicly_conn_t *conn;
2736
0
    quicly_pacer_t *pacer = NULL;
2737
2738
    /* consistency checks */
2739
0
    assert(remote_addr != NULL && remote_addr->sa_family != AF_UNSPEC);
2740
0
    if (ctx->transport_params.max_datagram_frame_size != 0)
2741
0
        assert(ctx->receive_datagram_frame != NULL);
2742
2743
    /* build log state and override, unless already set by the caller */
2744
0
    if (ptls_log_conn_state_override == NULL) {
2745
0
        ptls_log_init_conn_state(&log_state_override, ctx->tls->random_bytes, 0, remote_addr);
2746
0
        ptls_log_conn_state_override = &log_state_override;
2747
0
    }
2748
2749
    /* create TLS context */
2750
0
    tls = ptls_new(ctx->tls, server_name == NULL);
2751
2752
    /* clear the override if we had set our own */
2753
0
    if (ptls_log_conn_state_override == &log_state_override)
2754
0
        ptls_log_conn_state_override = NULL;
2755
2756
0
    if (tls == NULL)
2757
0
        return NULL;
2758
0
    if (server_name != NULL && ptls_set_server_name(tls, server_name, strlen(server_name)) != 0) {
2759
0
        ptls_free(tls);
2760
0
        return NULL;
2761
0
    }
2762
2763
    /* allocate memory and start creating QUIC context */
2764
0
    if ((conn = malloc(sizeof(*conn))) == NULL) {
2765
0
        ptls_free(tls);
2766
0
        return NULL;
2767
0
    }
2768
0
    if (enable_with_ratio255(ctx->enable_ratio.pacing, ctx->tls->random_bytes) && (pacer = malloc(sizeof(*pacer))) == NULL) {
2769
0
        ptls_free(tls);
2770
0
        free(conn);
2771
0
        return NULL;
2772
0
    }
2773
0
    memset(conn, 0, sizeof(*conn));
2774
0
    conn->super.ctx = ctx;
2775
0
    conn->super.data = appdata;
2776
0
    lock_now(conn, 0);
2777
0
    conn->created_at = conn->stash.now;
2778
0
    conn->super.stats.handshake_confirmed_msec = UINT64_MAX;
2779
0
    conn->super.stats.num_paced = pacer != NULL;
2780
0
    conn->super.stats.num_respected_app_limited =
2781
0
        enable_with_ratio255(conn->super.ctx->enable_ratio.respect_app_limited, ctx->tls->random_bytes);
2782
0
    conn->crypto.tls = tls;
2783
0
    if (new_path(conn, 0, remote_addr, local_addr) != 0) {
2784
0
        unlock_now(conn);
2785
0
        if (pacer != NULL)
2786
0
            free(pacer);
2787
0
        ptls_free(tls);
2788
0
        free(conn);
2789
0
        return NULL;
2790
0
    }
2791
0
    quicly_local_cid_init_set(&conn->super.local.cid_set, ctx->cid_encryptor, local_cid);
2792
0
    conn->super.local.long_header_src_cid = conn->super.local.cid_set.cids[0].cid;
2793
0
    quicly_remote_cid_init_set(&conn->super.remote.cid_set, remote_cid, ctx->tls->random_bytes);
2794
0
    assert(conn->paths[0]->dcid == 0 && conn->super.remote.cid_set.cids[0].sequence == 0 &&
2795
0
           conn->super.remote.cid_set.cids[0].state == QUICLY_REMOTE_CID_IN_USE && "paths[0].dcid uses cids[0]");
2796
0
    conn->super.state = QUICLY_STATE_FIRSTFLIGHT;
2797
0
    if (server_name != NULL) {
2798
0
        conn->super.local.bidi.next_stream_id = 0;
2799
0
        conn->super.local.uni.next_stream_id = 2;
2800
0
        conn->super.remote.bidi.next_stream_id = 1;
2801
0
        conn->super.remote.uni.next_stream_id = 3;
2802
0
    } else {
2803
0
        conn->super.local.bidi.next_stream_id = 1;
2804
0
        conn->super.local.uni.next_stream_id = 3;
2805
0
        conn->super.remote.bidi.next_stream_id = 0;
2806
0
        conn->super.remote.uni.next_stream_id = 2;
2807
0
    }
2808
0
    conn->super.remote.transport_params = default_transport_params;
2809
0
    conn->super.version = protocol_version;
2810
0
    quicly_linklist_init(&conn->super._default_scheduler.active);
2811
0
    quicly_linklist_init(&conn->super._default_scheduler.blocked);
2812
0
    conn->streams = kh_init(quicly_stream_t);
2813
0
    quicly_maxsender_init(&conn->ingress.max_data.sender, conn->super.ctx->transport_params.max_data);
2814
0
    quicly_maxsender_init(&conn->ingress.max_streams.uni, conn->super.ctx->transport_params.max_streams_uni);
2815
0
    quicly_maxsender_init(&conn->ingress.max_streams.bidi, conn->super.ctx->transport_params.max_streams_bidi);
2816
0
    quicly_loss_init(&conn->egress.loss, &conn->super.ctx->loss,
2817
0
                     conn->super.ctx->loss.default_initial_rtt /* FIXME remember initial_rtt in session ticket */,
2818
0
                     &conn->super.remote.transport_params.max_ack_delay, &conn->super.remote.transport_params.ack_delay_exponent);
2819
0
    conn->egress.max_udp_payload_size = conn->super.ctx->initial_egress_max_udp_payload_size;
2820
0
    init_max_streams(&conn->egress.max_streams.uni);
2821
0
    init_max_streams(&conn->egress.max_streams.bidi);
2822
0
    conn->egress.ack_frequency.update_at = INT64_MAX;
2823
0
    conn->egress.send_ack_at = INT64_MAX;
2824
0
    conn->egress.send_probe_at = INT64_MAX;
2825
0
    conn->super.ctx->init_cc->cb(conn->super.ctx->init_cc, &conn->egress.cc, initcwnd, conn->stash.now);
2826
0
    if (conn->egress.cc.type->enable_rapid_start != NULL &&
2827
0
        enable_with_ratio255(conn->super.ctx->enable_ratio.rapid_start, conn->super.ctx->tls->random_bytes)) {
2828
0
        conn->egress.cc.type->enable_rapid_start(&conn->egress.cc, conn->stash.now);
2829
0
        conn->super.stats.num_rapid_start = 1;
2830
0
    }
2831
0
    if (pacer != NULL) {
2832
0
        conn->egress.pacer = pacer;
2833
0
        quicly_pacer_reset(conn->egress.pacer);
2834
0
    }
2835
0
    conn->egress.ecn.state = enable_with_ratio255(conn->super.ctx->enable_ratio.ecn, conn->super.ctx->tls->random_bytes)
2836
0
                                 ? QUICLY_ECN_PROBING
2837
0
                                 : QUICLY_ECN_OFF;
2838
0
    quicly_linklist_init(&conn->egress.pending_streams.blocked.uni);
2839
0
    quicly_linklist_init(&conn->egress.pending_streams.blocked.bidi);
2840
0
    quicly_linklist_init(&conn->egress.pending_streams.control);
2841
0
    quicly_ratemeter_init(&conn->egress.ratemeter);
2842
0
    conn->egress.try_jumpstart = 1;
2843
0
    if (handshake_properties != NULL) {
2844
0
        assert(handshake_properties->additional_extensions == NULL);
2845
0
        assert(handshake_properties->collect_extension == NULL);
2846
0
        assert(handshake_properties->collected_extensions == NULL);
2847
0
        conn->crypto.handshake_properties = *handshake_properties;
2848
0
    } else {
2849
0
        conn->crypto.handshake_properties = (ptls_handshake_properties_t){{{{NULL}}}};
2850
0
    }
2851
0
    conn->crypto.handshake_properties.collect_extension = collect_transport_parameters;
2852
0
    conn->retry_scid.len = UINT8_MAX;
2853
0
    conn->idle_timeout.at = INT64_MAX;
2854
0
    conn->idle_timeout.should_rearm_on_send = 1;
2855
0
    for (size_t i = 0; i != PTLS_ELEMENTSOF(conn->delayed_packets.as_array); ++i)
2856
0
        conn->delayed_packets.as_array[i].tail = &conn->delayed_packets.as_array[i].head;
2857
0
    conn->stash.on_ack_stream.active_acked_cache.stream_id = INT64_MIN;
2858
2859
0
    *ptls_get_data_ptr(tls) = conn;
2860
2861
0
    update_open_count(conn->super.ctx, 1);
2862
2863
0
    return conn;
2864
0
}
2865
2866
static int client_collected_extensions(ptls_t *tls, ptls_handshake_properties_t *properties, ptls_raw_extension_t *slots)
2867
0
{
2868
0
    quicly_conn_t *conn = (void *)((char *)properties - offsetof(quicly_conn_t, crypto.handshake_properties));
2869
0
    quicly_error_t ret;
2870
2871
0
    assert(properties->client.early_data_acceptance != PTLS_EARLY_DATA_ACCEPTANCE_UNKNOWN);
2872
2873
0
    if (slots[0].type == UINT16_MAX) {
2874
0
        ret = PTLS_ALERT_MISSING_EXTENSION;
2875
0
        goto Exit;
2876
0
    }
2877
0
    assert(slots[0].type == get_transport_parameters_extension_id(conn->super.version));
2878
0
    assert(slots[1].type == UINT16_MAX);
2879
2880
0
    const uint8_t *src = slots[0].data.base, *end = src + slots[0].data.len;
2881
0
    quicly_transport_parameters_t params;
2882
0
    quicly_cid_t original_dcid, initial_scid, retry_scid = {};
2883
2884
    /* obtain pointer to initial CID of the peer. It is guaranteed to exist in the first slot, as TP is received before any frame
2885
     * that updates the CID set. */
2886
0
    quicly_remote_cid_t *remote_cid = &conn->super.remote.cid_set.cids[0];
2887
0
    assert(remote_cid->sequence == 0);
2888
2889
    /* decode */
2890
0
    if ((ret = quicly_decode_transport_parameter_list(&params, needs_cid_auth(conn) || is_retry(conn) ? &original_dcid : NULL,
2891
0
                                                      needs_cid_auth(conn) ? &initial_scid : &tp_cid_ignore,
2892
0
                                                      needs_cid_auth(conn) ? is_retry(conn) ? &retry_scid : NULL : &tp_cid_ignore,
2893
0
                                                      remote_cid->stateless_reset_token, src, end)) != 0)
2894
0
        goto Exit;
2895
2896
    /* validate CIDs */
2897
0
    if (needs_cid_auth(conn) || is_retry(conn)) {
2898
0
        if (!quicly_cid_is_equal(&conn->super.original_dcid, ptls_iovec_init(original_dcid.cid, original_dcid.len))) {
2899
0
            ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2900
0
            goto Exit;
2901
0
        }
2902
0
    }
2903
0
    if (needs_cid_auth(conn)) {
2904
0
        if (!quicly_cid_is_equal(&remote_cid->cid, ptls_iovec_init(initial_scid.cid, initial_scid.len))) {
2905
0
            ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2906
0
            goto Exit;
2907
0
        }
2908
0
        if (is_retry(conn)) {
2909
0
            if (!quicly_cid_is_equal(&conn->retry_scid, ptls_iovec_init(retry_scid.cid, retry_scid.len))) {
2910
0
                ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;
2911
0
                goto Exit;
2912
0
            }
2913
0
        }
2914
0
    }
2915
2916
0
    if (properties->client.early_data_acceptance == PTLS_EARLY_DATA_ACCEPTED) {
2917
0
#define ZERORTT_VALIDATE(x)                                                                                                        \
2918
0
    if (params.x < conn->super.remote.transport_params.x) {                                                                        \
2919
0
        ret = QUICLY_TRANSPORT_ERROR_TRANSPORT_PARAMETER;                                                                          \
2920
0
        goto Exit;                                                                                                                 \
2921
0
    }
2922
0
        ZERORTT_VALIDATE(max_data);
2923
0
        ZERORTT_VALIDATE(max_stream_data.bidi_local);
2924
0
        ZERORTT_VALIDATE(max_stream_data.bidi_remote);
2925
0
        ZERORTT_VALIDATE(max_stream_data.uni);
2926
0
        ZERORTT_VALIDATE(max_streams_bidi);
2927
0
        ZERORTT_VALIDATE(max_streams_uni);
2928
0
#undef ZERORTT_VALIDATE
2929
0
    }
2930
2931
    /* store the results */
2932
0
    conn->super.remote.transport_params = params;
2933
0
    ack_frequency_set_next_update_at(conn);
2934
2935
0
Exit:
2936
0
    return compress_handshake_result(ret);
2937
0
}
2938
2939
quicly_error_t quicly_connect(quicly_conn_t **_conn, quicly_context_t *ctx, const char *server_name, struct sockaddr *dest_addr,
2940
                              struct sockaddr *src_addr, const quicly_cid_plaintext_t *new_cid, ptls_iovec_t address_token,
2941
                              ptls_handshake_properties_t *handshake_properties,
2942
                              const quicly_transport_parameters_t *resumed_transport_params, void *appdata)
2943
0
{
2944
0
    const quicly_salt_t *salt;
2945
0
    quicly_conn_t *conn = NULL;
2946
0
    const quicly_cid_t *server_cid;
2947
0
    ptls_buffer_t buf;
2948
0
    size_t epoch_offsets[5] = {0};
2949
0
    size_t max_early_data_size = 0;
2950
0
    quicly_error_t ret;
2951
2952
0
    if ((salt = quicly_get_salt(ctx->initial_version)) == NULL) {
2953
0
        if ((ctx->initial_version & 0x0f0f0f0f) == 0x0a0a0a0a) {
2954
            /* greasing version, use our own greasing salt */
2955
0
            static const quicly_salt_t grease_salt = {.initial = {0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad,
2956
0
                                                                  0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef, 0xde, 0xad, 0xbe, 0xef}};
2957
0
            salt = &grease_salt;
2958
0
        } else {
2959
0
            ret = QUICLY_ERROR_INVALID_INITIAL_VERSION;
2960
0
            goto Exit;
2961
0
        }
2962
0
    }
2963
2964
0
    if ((conn = create_connection(
2965
0
             ctx, ctx->initial_version, server_name, dest_addr, src_addr, NULL, new_cid, handshake_properties, appdata,
2966
0
             quicly_cc_calc_initial_cwnd(ctx->initcwnd_packets, ctx->transport_params.max_udp_payload_size))) == NULL) {
2967
0
        ret = PTLS_ERROR_NO_MEMORY;
2968
0
        goto Exit;
2969
0
    }
2970
0
    conn->super.remote.address_validation.validated = 1;
2971
0
    conn->super.remote.address_validation.send_probe = 1;
2972
0
    if (address_token.len != 0) {
2973
0
        if ((conn->token.base = malloc(address_token.len)) == NULL) {
2974
0
            ret = PTLS_ERROR_NO_MEMORY;
2975
0
            goto Exit;
2976
0
        }
2977
0
        memcpy(conn->token.base, address_token.base, address_token.len);
2978
0
        conn->token.len = address_token.len;
2979
0
    }
2980
0
    server_cid = quicly_get_remote_cid(conn);
2981
0
    conn->super.original_dcid = *server_cid;
2982
2983
0
    QUICLY_PROBE(CONNECT, conn, conn->stash.now, conn->super.version);
2984
0
    QUICLY_LOG_CONN(connect, conn, { PTLS_LOG_ELEMENT_UNSIGNED(version, conn->super.version); });
2985
2986
0
    if ((ret = setup_handshake_space_and_flow(conn, QUICLY_EPOCH_INITIAL)) != 0)
2987
0
        goto Exit;
2988
0
    if ((ret = setup_initial_encryption(get_aes128gcmsha256(ctx), &conn->initial->cipher.ingress, &conn->initial->cipher.egress,
2989
0
                                        ptls_iovec_init(server_cid->cid, server_cid->len), 1,
2990
0
                                        ptls_iovec_init(salt->initial, sizeof(salt->initial)), conn)) != 0)
2991
0
        goto Exit;
2992
2993
    /* handshake (we always encode authentication CIDs, as we do not (yet) regenerate ClientHello when receiving Retry) */
2994
0
    ptls_buffer_init(&conn->crypto.transport_params.buf, "", 0);
2995
0
    if ((ret = quicly_encode_transport_parameter_list(
2996
0
             &conn->crypto.transport_params.buf, &conn->super.ctx->transport_params, NULL, &conn->super.local.cid_set.cids[0].cid,
2997
0
             NULL, NULL, conn->super.ctx->expand_client_hello ? conn->super.ctx->initial_egress_max_udp_payload_size : 0)) != 0)
2998
0
        goto Exit;
2999
0
    conn->crypto.transport_params.ext[0] =
3000
0
        (ptls_raw_extension_t){get_transport_parameters_extension_id(conn->super.version),
3001
0
                               {conn->crypto.transport_params.buf.base, conn->crypto.transport_params.buf.off}};
3002
0
    conn->crypto.transport_params.ext[1] = (ptls_raw_extension_t){UINT16_MAX};
3003
0
    conn->crypto.handshake_properties.additional_extensions = conn->crypto.transport_params.ext;
3004
0
    conn->crypto.handshake_properties.collected_extensions = client_collected_extensions;
3005
3006
0
    ptls_buffer_init(&buf, "", 0);
3007
0
    if (resumed_transport_params != NULL)
3008
0
        conn->crypto.handshake_properties.client.max_early_data_size = &max_early_data_size;
3009
0
    ret = expand_handshake_result(
3010
0
        ptls_handle_message(conn->crypto.tls, &buf, epoch_offsets, 0, NULL, 0, &conn->crypto.handshake_properties));
3011
0
    conn->crypto.handshake_properties.client.max_early_data_size = NULL;
3012
0
    if (ret != PTLS_ERROR_IN_PROGRESS) {
3013
0
        assert(ret > 0); /* no QUIC errors */
3014
0
        goto Exit;
3015
0
    }
3016
0
    write_crypto_data(conn, &buf, epoch_offsets);
3017
0
    ptls_buffer_dispose(&buf);
3018
3019
0
    if (max_early_data_size != 0) {
3020
        /* when attempting 0-RTT, apply the remembered transport parameters */
3021
0
#define APPLY(n) conn->super.remote.transport_params.n = resumed_transport_params->n
3022
0
        APPLY(active_connection_id_limit);
3023
0
        APPLY(max_data);
3024
0
        APPLY(max_stream_data.bidi_local);
3025
0
        APPLY(max_stream_data.bidi_remote);
3026
0
        APPLY(max_stream_data.uni);
3027
0
        APPLY(max_streams_bidi);
3028
0
        APPLY(max_streams_uni);
3029
0
#undef APPLY
3030
0
        if ((ret = apply_remote_transport_params(conn)) != 0)
3031
0
            goto Exit;
3032
0
    }
3033
3034
0
    *_conn = conn;
3035
0
    ret = 0;
3036
3037
0
Exit:
3038
0
    if (conn != NULL)
3039
0
        unlock_now(conn);
3040
0
    if (ret != 0) {
3041
0
        if (conn != NULL)
3042
0
            quicly_free(conn);
3043
0
    }
3044
0
    return ret;
3045
0
}
3046
3047
static int server_collected_extensions(ptls_t *tls, ptls_handshake_properties_t *properties, ptls_raw_extension_t *slots)
3048
0
{
3049
0
    quicly_conn_t *conn = (void *)((char *)properties - offsetof(quicly_conn_t, crypto.handshake_properties));
3050
0
    quicly_cid_t initial_scid;
3051
0
    quicly_error_t ret;
3052
3053
0
    if (slots[0].type == UINT16_MAX) {
3054
0
        ret = PTLS_ALERT_MISSING_EXTENSION;
3055
0
        goto Exit;
3056
0
    }
3057
0
    assert(slots[0].type == get_transport_parameters_extension_id(conn->super.version));
3058
0
    assert(slots[1].type == UINT16_MAX);
3059
3060
0
    { /* decode transport_parameters extension */
3061
0
        const uint8_t *src = slots[0].data.base, *end = src + slots[0].data.len;
3062
0
        if ((ret = quicly_decode_transport_parameter_list(&conn->super.remote.transport_params,
3063
0
                                                          needs_cid_auth(conn) ? NULL : &tp_cid_ignore,
3064
0
                                                          needs_cid_auth(conn) ? &initial_scid : &tp_cid_ignore,
3065
0
                                                          needs_cid_auth(conn) ? NULL : &tp_cid_ignore, NULL, src, end)) != 0)
3066
0
            goto Exit;
3067
0
        if (needs_cid_auth(conn) &&
3068
0
            !quicly_cid_is_equal(&conn->super.remote.cid_set.cids[0].cid, ptls_iovec_init(initial_scid.cid, initial_scid.len))) {
3069
0
            ret = QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
3070
0
            goto Exit;
3071
0
        }
3072
0
    }
3073
3074
    /* setup ack frequency */
3075
0
    ack_frequency_set_next_update_at(conn);
3076
3077
    /* update UDP max payload size, as described in the doc-comment of
3078
    `quicly_context_t::initial_egress_max_udp_payload_size` */
3079
0
    assert(conn->initial != NULL);
3080
0
    if (conn->egress.max_udp_payload_size < conn->initial->largest_ingress_udp_payload_size) {
3081
0
        uint16_t size = conn->initial->largest_ingress_udp_payload_size;
3082
0
        if (size > conn->super.ctx->transport_params.max_udp_payload_size)
3083
0
            size = conn->super.ctx->transport_params.max_udp_payload_size;
3084
0
        conn->egress.max_udp_payload_size = size;
3085
0
    }
3086
0
    if (conn->egress.max_udp_payload_size > conn->super.remote.transport_params.max_udp_payload_size)
3087
0
        conn->egress.max_udp_payload_size = conn->super.remote.transport_params.max_udp_payload_size;
3088
3089
    /* set transport_parameters extension to be sent in EE */
3090
0
    assert(properties->additional_extensions == NULL);
3091
0
    ptls_buffer_init(&conn->crypto.transport_params.buf, "", 0);
3092
0
    assert(conn->super.local.cid_set.cids[0].sequence == 0 && "make sure that local_cid is in expected state before sending SRT");
3093
0
    if ((ret = quicly_encode_transport_parameter_list(
3094
0
             &conn->crypto.transport_params.buf, &conn->super.ctx->transport_params,
3095
0
             needs_cid_auth(conn) || is_retry(conn) ? &conn->super.original_dcid : NULL,
3096
0
             needs_cid_auth(conn) ? &conn->super.local.cid_set.cids[0].cid : NULL,
3097
0
             needs_cid_auth(conn) && is_retry(conn) ? &conn->retry_scid : NULL,
3098
0
             conn->super.ctx->cid_encryptor != NULL ? conn->super.local.cid_set.cids[0].stateless_reset_token : NULL, 0)) != 0)
3099
0
        goto Exit;
3100
0
    properties->additional_extensions = conn->crypto.transport_params.ext;
3101
0
    conn->crypto.transport_params.ext[0] =
3102
0
        (ptls_raw_extension_t){get_transport_parameters_extension_id(conn->super.version),
3103
0
                               {conn->crypto.transport_params.buf.base, conn->crypto.transport_params.buf.off}};
3104
0
    conn->crypto.transport_params.ext[1] = (ptls_raw_extension_t){UINT16_MAX};
3105
0
    conn->crypto.handshake_properties.additional_extensions = conn->crypto.transport_params.ext;
3106
3107
0
    ret = 0;
3108
3109
0
Exit:
3110
0
    return compress_handshake_result(ret);
3111
0
}
3112
3113
static size_t aead_decrypt_core(ptls_aead_context_t *aead, uint64_t pn, quicly_decoded_packet_t *packet, size_t aead_off)
3114
0
{
3115
0
    return ptls_aead_decrypt(aead, packet->octets.base + aead_off, packet->octets.base + aead_off, packet->octets.len - aead_off,
3116
0
                             pn, packet->octets.base, aead_off);
3117
0
}
3118
3119
static int aead_decrypt_fixed_key(void *ctx, uint64_t pn, quicly_decoded_packet_t *packet, size_t aead_off, size_t *ptlen)
3120
0
{
3121
0
    ptls_aead_context_t *aead = ctx;
3122
3123
0
    if ((*ptlen = aead_decrypt_core(aead, pn, packet, aead_off)) == SIZE_MAX)
3124
0
        return QUICLY_ERROR_PACKET_IGNORED;
3125
0
    return 0;
3126
0
}
3127
3128
static int aead_decrypt_1rtt(void *ctx, uint64_t pn, quicly_decoded_packet_t *packet, size_t aead_off, size_t *ptlen)
3129
0
{
3130
0
    quicly_conn_t *conn = ctx;
3131
0
    struct st_quicly_application_space_t *space = conn->application;
3132
0
    size_t aead_index = (packet->octets.base[0] & QUICLY_KEY_PHASE_BIT) != 0;
3133
0
    int ret;
3134
3135
    /* prepare key, when not available (yet) */
3136
0
    if (space->cipher.ingress.aead[aead_index] == NULL) {
3137
0
    Retry_1RTT: {
3138
        /* Replace the AEAD key at the alternative slot (note: decryption key slots are shared by 0-RTT and 1-RTT), at the same time
3139
         * dropping 0-RTT header protection key. */
3140
0
        if (conn->application->cipher.ingress.header_protection.zero_rtt != NULL) {
3141
0
            ptls_cipher_free(conn->application->cipher.ingress.header_protection.zero_rtt);
3142
0
            conn->application->cipher.ingress.header_protection.zero_rtt = NULL;
3143
0
        }
3144
0
        ptls_cipher_suite_t *cipher = ptls_get_cipher(conn->crypto.tls);
3145
0
        if ((ret = update_1rtt_key(conn, cipher, 0, &space->cipher.ingress.aead[aead_index], space->cipher.ingress.secret)) != 0)
3146
0
            return ret;
3147
0
        ++space->cipher.ingress.key_phase.prepared;
3148
0
        QUICLY_PROBE(CRYPTO_RECEIVE_KEY_UPDATE_PREPARE, conn, conn->stash.now, space->cipher.ingress.key_phase.prepared,
3149
0
                     QUICLY_PROBE_HEXDUMP(space->cipher.ingress.secret, cipher->hash->digest_size));
3150
0
        QUICLY_LOG_CONN(crypto_receive_key_update_prepare, conn, {
3151
0
            PTLS_LOG_ELEMENT_UNSIGNED(phase, space->cipher.ingress.key_phase.prepared);
3152
0
            PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(secret, space->cipher.ingress.secret, cipher->hash->digest_size);
3153
0
        });
3154
0
    }
3155
0
    }
3156
3157
    /* decrypt */
3158
0
    ptls_aead_context_t *aead = space->cipher.ingress.aead[aead_index];
3159
0
    if ((*ptlen = aead_decrypt_core(aead, pn, packet, aead_off)) == SIZE_MAX) {
3160
        /* retry with a new key, if possible */
3161
0
        if (space->cipher.ingress.key_phase.decrypted == space->cipher.ingress.key_phase.prepared &&
3162
0
            space->cipher.ingress.key_phase.decrypted % 2 != aead_index) {
3163
            /* reapply AEAD to revert payload to the encrypted form. This assumes that the cipher used in AEAD is CTR. */
3164
0
            aead_decrypt_core(aead, pn, packet, aead_off);
3165
0
            goto Retry_1RTT;
3166
0
        }
3167
        /* otherwise return failure */
3168
0
        return QUICLY_ERROR_PACKET_IGNORED;
3169
0
    }
3170
3171
    /* update the confirmed key phase and also the egress key phase, if necessary */
3172
0
    if (space->cipher.ingress.key_phase.prepared != space->cipher.ingress.key_phase.decrypted &&
3173
0
        space->cipher.ingress.key_phase.prepared % 2 == aead_index) {
3174
0
        if ((ret = received_key_update(conn, space->cipher.ingress.key_phase.prepared)) != 0)
3175
0
            return ret;
3176
0
    }
3177
3178
0
    return 0;
3179
0
}
3180
3181
static quicly_error_t do_decrypt_packet(ptls_cipher_context_t *header_protection,
3182
                                        int (*aead_cb)(void *, uint64_t, quicly_decoded_packet_t *, size_t, size_t *),
3183
                                        void *aead_ctx, uint64_t *next_expected_pn, quicly_decoded_packet_t *packet, uint64_t *pn,
3184
                                        ptls_iovec_t *payload)
3185
0
{
3186
0
    size_t encrypted_len = packet->octets.len - packet->encrypted_off;
3187
0
    uint8_t hpmask[5] = {0};
3188
0
    uint32_t pnbits = 0;
3189
0
    size_t pnlen, ptlen, i;
3190
3191
    /* decipher the header protection, as well as obtaining pnbits, pnlen */
3192
0
    if (encrypted_len < header_protection->algo->iv_size + QUICLY_MAX_PN_SIZE) {
3193
0
        *pn = UINT64_MAX;
3194
0
        return QUICLY_ERROR_PACKET_IGNORED;
3195
0
    }
3196
0
    ptls_cipher_init(header_protection, packet->octets.base + packet->encrypted_off + QUICLY_MAX_PN_SIZE);
3197
0
    ptls_cipher_encrypt(header_protection, hpmask, hpmask, sizeof(hpmask));
3198
0
    packet->octets.base[0] ^= hpmask[0] & (QUICLY_PACKET_IS_LONG_HEADER(packet->octets.base[0]) ? 0xf : 0x1f);
3199
0
    pnlen = (packet->octets.base[0] & 0x3) + 1;
3200
0
    for (i = 0; i != pnlen; ++i) {
3201
0
        packet->octets.base[packet->encrypted_off + i] ^= hpmask[i + 1];
3202
0
        pnbits = (pnbits << 8) | packet->octets.base[packet->encrypted_off + i];
3203
0
    }
3204
3205
0
    size_t aead_off = packet->encrypted_off + pnlen;
3206
0
    *pn = quicly_determine_packet_number(pnbits, pnlen * 8, *next_expected_pn);
3207
3208
    /* AEAD decryption */
3209
0
    int ret;
3210
0
    if ((ret = (*aead_cb)(aead_ctx, *pn, packet, aead_off, &ptlen)) != 0) {
3211
0
        return ret;
3212
0
    }
3213
0
    if (*next_expected_pn <= *pn)
3214
0
        *next_expected_pn = *pn + 1;
3215
3216
0
    *payload = ptls_iovec_init(packet->octets.base + aead_off, ptlen);
3217
0
    return 0;
3218
0
}
3219
3220
static quicly_error_t decrypt_packet(ptls_cipher_context_t *header_protection,
3221
                                     int (*aead_cb)(void *, uint64_t, quicly_decoded_packet_t *, size_t, size_t *), void *aead_ctx,
3222
                                     uint64_t *next_expected_pn, quicly_decoded_packet_t *packet, uint64_t *pn,
3223
                                     ptls_iovec_t *payload)
3224
0
{
3225
0
    quicly_error_t ret;
3226
3227
    /* decrypt ourselves, or use the pre-decrypted input */
3228
0
    if (packet->decrypted.pn == UINT64_MAX) {
3229
0
        if ((ret = do_decrypt_packet(header_protection, aead_cb, aead_ctx, next_expected_pn, packet, pn, payload)) != 0)
3230
0
            return ret;
3231
0
    } else {
3232
0
        *payload = ptls_iovec_init(packet->octets.base + packet->encrypted_off, packet->octets.len - packet->encrypted_off);
3233
0
        *pn = packet->decrypted.pn;
3234
0
        if (aead_cb == aead_decrypt_1rtt) {
3235
0
            quicly_conn_t *conn = aead_ctx;
3236
0
            if (conn->application->cipher.ingress.key_phase.decrypted < packet->decrypted.key_phase) {
3237
0
                if ((ret = received_key_update(conn, packet->decrypted.key_phase)) != 0)
3238
0
                    return ret;
3239
0
            }
3240
0
        }
3241
0
        if (*next_expected_pn < *pn)
3242
0
            *next_expected_pn = *pn + 1;
3243
0
    }
3244
3245
    /* check reserved bits after AEAD decryption */
3246
0
    if ((packet->octets.base[0] & (QUICLY_PACKET_IS_LONG_HEADER(packet->octets.base[0]) ? QUICLY_LONG_HEADER_RESERVED_BITS
3247
0
                                                                                        : QUICLY_SHORT_HEADER_RESERVED_BITS)) !=
3248
0
        0) {
3249
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
3250
0
    }
3251
0
    if (payload->len == 0) {
3252
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
3253
0
    }
3254
3255
0
    return 0;
3256
0
}
3257
3258
static quicly_error_t do_on_ack_ack(quicly_conn_t *conn, const quicly_sent_packet_t *packet, uint64_t start, uint64_t start_length,
3259
                                    struct st_quicly_sent_ack_additional_t *additional, size_t additional_capacity)
3260
0
{
3261
    /* find the pn space */
3262
0
    struct st_quicly_pn_space_t *space;
3263
0
    switch (packet->ack_epoch) {
3264
0
    case QUICLY_EPOCH_INITIAL:
3265
0
        space = &conn->initial->super;
3266
0
        break;
3267
0
    case QUICLY_EPOCH_HANDSHAKE:
3268
0
        space = &conn->handshake->super;
3269
0
        break;
3270
0
    case QUICLY_EPOCH_1RTT:
3271
0
        space = &conn->application->super;
3272
0
        break;
3273
0
    default:
3274
0
        assert(!"FIXME");
3275
0
        return QUICLY_TRANSPORT_ERROR_INTERNAL;
3276
0
    }
3277
3278
    /* subtract given ACK ranges */
3279
0
    int ret;
3280
0
    uint64_t end = start + start_length;
3281
0
    if ((ret = quicly_ranges_subtract(&space->ack_queue, start, end)) != 0)
3282
0
        return ret;
3283
0
    for (size_t i = 0; i < additional_capacity && additional[i].gap != 0; ++i) {
3284
0
        start = end + additional[i].gap;
3285
0
        end = start + additional[i].length;
3286
0
        if ((ret = quicly_ranges_subtract(&space->ack_queue, start, end)) != 0)
3287
0
            return ret;
3288
0
    }
3289
3290
    /* make adjustments */
3291
0
    if (space->ack_queue.num_ranges == 0) {
3292
0
        space->largest_pn_received_at = INT64_MAX;
3293
0
        space->unacked_count = 0;
3294
0
    } else if (space->ack_queue.num_ranges > QUICLY_MAX_ACK_BLOCKS) {
3295
0
        quicly_ranges_drop_by_range_indices(&space->ack_queue, space->ack_queue.num_ranges - QUICLY_MAX_ACK_BLOCKS,
3296
0
                                            space->ack_queue.num_ranges);
3297
0
    }
3298
3299
0
    return 0;
3300
0
}
3301
3302
static quicly_error_t on_ack_ack_ranges64(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3303
0
{
3304
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3305
3306
    /* TODO log */
3307
3308
0
    return acked ? do_on_ack_ack(conn, packet, sent->data.ack.start, sent->data.ack.ranges64.start_length,
3309
0
                                 sent->data.ack.ranges64.additional, PTLS_ELEMENTSOF(sent->data.ack.ranges64.additional))
3310
0
                 : 0;
3311
0
}
3312
3313
static quicly_error_t on_ack_ack_ranges8(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3314
0
{
3315
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3316
3317
    /* TODO log */
3318
3319
0
    return acked ? do_on_ack_ack(conn, packet, sent->data.ack.start, sent->data.ack.ranges8.start_length,
3320
0
                                 sent->data.ack.ranges8.additional, PTLS_ELEMENTSOF(sent->data.ack.ranges8.additional))
3321
0
                 : 0;
3322
0
}
3323
3324
static quicly_error_t on_ack_stream_ack_one(quicly_conn_t *conn, quicly_stream_id_t stream_id, quicly_sendstate_sent_t *sent)
3325
0
{
3326
0
    quicly_stream_t *stream;
3327
3328
0
    if ((stream = quicly_get_stream(conn, stream_id)) == NULL)
3329
0
        return 0;
3330
3331
0
    size_t bytes_to_shift;
3332
0
    int ret;
3333
0
    if ((ret = quicly_sendstate_acked(&stream->sendstate, sent, &bytes_to_shift)) != 0)
3334
0
        return ret;
3335
0
    if (bytes_to_shift != 0) {
3336
0
        QUICLY_PROBE(STREAM_ON_SEND_SHIFT, stream->conn, stream->conn->stash.now, stream, bytes_to_shift);
3337
0
        stream->callbacks->on_send_shift(stream, bytes_to_shift);
3338
0
        QUICLY_LOG_CONN(stream_on_send_shift, stream->conn, {
3339
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
3340
0
            PTLS_LOG_ELEMENT_UNSIGNED(delta, bytes_to_shift);
3341
0
        });
3342
0
    }
3343
0
    if (stream_is_destroyable(stream)) {
3344
0
        destroy_stream(stream, 0);
3345
0
    } else if (stream->_send_aux.reset_stream.sender_state == QUICLY_SENDER_STATE_NONE) {
3346
0
        resched_stream_data(stream);
3347
0
    }
3348
3349
0
    return 0;
3350
0
}
3351
3352
static quicly_error_t on_ack_stream_ack_cached(quicly_conn_t *conn)
3353
0
{
3354
0
    if (conn->stash.on_ack_stream.active_acked_cache.stream_id == INT64_MIN)
3355
0
        return 0;
3356
0
    quicly_error_t ret = on_ack_stream_ack_one(conn, conn->stash.on_ack_stream.active_acked_cache.stream_id,
3357
0
                                               &conn->stash.on_ack_stream.active_acked_cache.args);
3358
0
    conn->stash.on_ack_stream.active_acked_cache.stream_id = INT64_MIN;
3359
0
    return ret;
3360
0
}
3361
3362
static quicly_error_t on_ack_stream(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3363
0
{
3364
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3365
0
    quicly_error_t ret;
3366
3367
0
    if (acked) {
3368
3369
0
        QUICLY_PROBE(STREAM_ACKED, conn, conn->stash.now, sent->data.stream.stream_id, sent->data.stream.args.start,
3370
0
                     sent->data.stream.args.end - sent->data.stream.args.start);
3371
0
        QUICLY_LOG_CONN(stream_acked, conn, {
3372
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, sent->data.stream.stream_id);
3373
0
            PTLS_LOG_ELEMENT_UNSIGNED(off, sent->data.stream.args.start);
3374
0
            PTLS_LOG_ELEMENT_UNSIGNED(len, sent->data.stream.args.end - sent->data.stream.args.start);
3375
0
        });
3376
3377
0
        if (packet->frames_in_flight && conn->stash.on_ack_stream.active_acked_cache.stream_id == sent->data.stream.stream_id &&
3378
0
            conn->stash.on_ack_stream.active_acked_cache.args.end == sent->data.stream.args.start) {
3379
            /* Fast path: append the newly supplied range to the existing cached range. */
3380
0
            conn->stash.on_ack_stream.active_acked_cache.args.end = sent->data.stream.args.end;
3381
0
        } else {
3382
            /* Slow path: submit the cached range, and if possible, cache the newly supplied range. Else submit the newly supplied
3383
             * range directly. */
3384
0
            if ((ret = on_ack_stream_ack_cached(conn)) != 0)
3385
0
                return ret;
3386
0
            if (packet->frames_in_flight) {
3387
0
                conn->stash.on_ack_stream.active_acked_cache.stream_id = sent->data.stream.stream_id;
3388
0
                conn->stash.on_ack_stream.active_acked_cache.args = sent->data.stream.args;
3389
0
            } else {
3390
0
                if ((ret = on_ack_stream_ack_one(conn, sent->data.stream.stream_id, &sent->data.stream.args)) != 0)
3391
0
                    return ret;
3392
0
            }
3393
0
        }
3394
3395
0
    } else {
3396
3397
0
        QUICLY_PROBE(STREAM_LOST, conn, conn->stash.now, sent->data.stream.stream_id, sent->data.stream.args.start,
3398
0
                     sent->data.stream.args.end - sent->data.stream.args.start);
3399
0
        QUICLY_LOG_CONN(stream_lost, conn, {
3400
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, sent->data.stream.stream_id);
3401
0
            PTLS_LOG_ELEMENT_UNSIGNED(off, sent->data.stream.args.start);
3402
0
            PTLS_LOG_ELEMENT_UNSIGNED(len, sent->data.stream.args.end - sent->data.stream.args.start);
3403
0
        });
3404
3405
0
        quicly_stream_t *stream;
3406
0
        if ((stream = quicly_get_stream(conn, sent->data.stream.stream_id)) == NULL)
3407
0
            return 0;
3408
        /* FIXME handle rto error */
3409
0
        if ((ret = quicly_sendstate_lost(&stream->sendstate, &sent->data.stream.args)) != 0)
3410
0
            return ret;
3411
0
        if (stream->_send_aux.reset_stream.sender_state == QUICLY_SENDER_STATE_NONE)
3412
0
            resched_stream_data(stream);
3413
0
    }
3414
3415
0
    return 0;
3416
0
}
3417
3418
static quicly_error_t on_ack_max_stream_data(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked,
3419
                                             quicly_sent_t *sent)
3420
0
{
3421
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3422
0
    quicly_stream_t *stream;
3423
3424
0
    if ((stream = quicly_get_stream(conn, sent->data.stream.stream_id)) != NULL) {
3425
0
        if (acked) {
3426
0
            quicly_maxsender_acked(&stream->_send_aux.max_stream_data_sender, &sent->data.max_stream_data.args);
3427
0
        } else {
3428
0
            quicly_maxsender_lost(&stream->_send_aux.max_stream_data_sender, &sent->data.max_stream_data.args);
3429
0
            if (should_send_max_stream_data(stream))
3430
0
                sched_stream_control(stream);
3431
0
        }
3432
0
    }
3433
3434
0
    return 0;
3435
0
}
3436
3437
static quicly_error_t on_ack_max_data(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3438
0
{
3439
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3440
3441
0
    if (acked) {
3442
0
        quicly_maxsender_acked(&conn->ingress.max_data.sender, &sent->data.max_data.args);
3443
0
    } else {
3444
0
        quicly_maxsender_lost(&conn->ingress.max_data.sender, &sent->data.max_data.args);
3445
0
    }
3446
3447
0
    return 0;
3448
0
}
3449
3450
static quicly_error_t on_ack_max_streams(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3451
0
{
3452
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3453
0
    quicly_maxsender_t *maxsender = sent->data.max_streams.uni ? &conn->ingress.max_streams.uni : &conn->ingress.max_streams.bidi;
3454
0
    assert(maxsender != NULL); /* we would only receive an ACK if we have sent the frame */
3455
3456
0
    if (acked) {
3457
0
        quicly_maxsender_acked(maxsender, &sent->data.max_streams.args);
3458
0
    } else {
3459
0
        quicly_maxsender_lost(maxsender, &sent->data.max_streams.args);
3460
0
    }
3461
3462
0
    return 0;
3463
0
}
3464
3465
static void on_ack_stream_state_sender(quicly_sender_state_t *sender_state, int acked)
3466
0
{
3467
0
    *sender_state = acked ? QUICLY_SENDER_STATE_ACKED : QUICLY_SENDER_STATE_SEND;
3468
0
}
3469
3470
static quicly_error_t on_ack_reset_stream(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3471
0
{
3472
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3473
0
    quicly_stream_t *stream;
3474
3475
0
    if ((stream = quicly_get_stream(conn, sent->data.stream_state_sender.stream_id)) != NULL) {
3476
0
        on_ack_stream_state_sender(&stream->_send_aux.reset_stream.sender_state, acked);
3477
0
        if (stream_is_destroyable(stream))
3478
0
            destroy_stream(stream, 0);
3479
0
    }
3480
3481
0
    return 0;
3482
0
}
3483
3484
static quicly_error_t on_ack_stop_sending(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3485
0
{
3486
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3487
0
    quicly_stream_t *stream;
3488
3489
0
    if ((stream = quicly_get_stream(conn, sent->data.stream_state_sender.stream_id)) != NULL) {
3490
0
        on_ack_stream_state_sender(&stream->_send_aux.stop_sending.sender_state, acked);
3491
0
        if (stream->_send_aux.stop_sending.sender_state != QUICLY_SENDER_STATE_ACKED)
3492
0
            sched_stream_control(stream);
3493
0
    }
3494
3495
0
    return 0;
3496
0
}
3497
3498
static quicly_error_t on_ack_streams_blocked(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked,
3499
                                             quicly_sent_t *sent)
3500
0
{
3501
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3502
0
    struct st_quicly_max_streams_t *m =
3503
0
        sent->data.streams_blocked.uni ? &conn->egress.max_streams.uni : &conn->egress.max_streams.bidi;
3504
3505
0
    if (acked) {
3506
0
        quicly_maxsender_acked(&m->blocked_sender, &sent->data.streams_blocked.args);
3507
0
    } else {
3508
0
        quicly_maxsender_lost(&m->blocked_sender, &sent->data.streams_blocked.args);
3509
0
    }
3510
3511
0
    return 0;
3512
0
}
3513
3514
static quicly_error_t on_ack_handshake_done(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked,
3515
                                            quicly_sent_t *sent)
3516
0
{
3517
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3518
3519
    /* When lost, reschedule for transmission. When acked, suppress retransmission if scheduled. */
3520
0
    if (acked) {
3521
0
        conn->egress.pending_flows &= ~QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT;
3522
0
    } else {
3523
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT;
3524
0
    }
3525
0
    return 0;
3526
0
}
3527
3528
static quicly_error_t on_ack_data_blocked(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3529
0
{
3530
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3531
3532
0
    if (conn->egress.max_data.permitted == sent->data.data_blocked.offset) {
3533
0
        if (acked) {
3534
0
            conn->egress.data_blocked = QUICLY_SENDER_STATE_ACKED;
3535
0
        } else if (packet->frames_in_flight && conn->egress.data_blocked == QUICLY_SENDER_STATE_UNACKED) {
3536
0
            conn->egress.data_blocked = QUICLY_SENDER_STATE_SEND;
3537
0
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
3538
0
        }
3539
0
    }
3540
3541
0
    return 0;
3542
0
}
3543
3544
static quicly_error_t on_ack_stream_data_blocked_frame(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked,
3545
                                                       quicly_sent_t *sent)
3546
0
{
3547
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3548
0
    quicly_stream_t *stream;
3549
3550
0
    if ((stream = quicly_get_stream(conn, sent->data.stream_data_blocked.stream_id)) == NULL)
3551
0
        return 0;
3552
3553
0
    if (stream->_send_aux.max_stream_data == sent->data.stream_data_blocked.offset) {
3554
0
        if (acked) {
3555
0
            stream->_send_aux.blocked = QUICLY_SENDER_STATE_ACKED;
3556
0
        } else if (packet->frames_in_flight && stream->_send_aux.blocked == QUICLY_SENDER_STATE_UNACKED) {
3557
0
            stream->_send_aux.blocked = QUICLY_SENDER_STATE_SEND;
3558
0
            sched_stream_control(stream);
3559
0
        }
3560
0
    }
3561
3562
0
    return 0;
3563
0
}
3564
3565
static quicly_error_t on_ack_new_token(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
3566
0
{
3567
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3568
3569
0
    if (sent->data.new_token.is_inflight) {
3570
0
        --conn->egress.new_token.num_inflight;
3571
0
        sent->data.new_token.is_inflight = 0;
3572
0
    }
3573
0
    if (acked) {
3574
0
        QUICLY_PROBE(NEW_TOKEN_ACKED, conn, conn->stash.now, sent->data.new_token.generation);
3575
0
        QUICLY_LOG_CONN(new_token_acked, conn, { PTLS_LOG_ELEMENT_UNSIGNED(generation, sent->data.new_token.generation); });
3576
0
        if (conn->egress.new_token.max_acked < sent->data.new_token.generation)
3577
0
            conn->egress.new_token.max_acked = sent->data.new_token.generation;
3578
0
    }
3579
3580
0
    if (conn->egress.new_token.num_inflight == 0 && conn->egress.new_token.max_acked < conn->egress.new_token.generation)
3581
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
3582
3583
0
    return 0;
3584
0
}
3585
3586
static quicly_error_t on_ack_new_connection_id(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked,
3587
                                               quicly_sent_t *sent)
3588
0
{
3589
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3590
0
    uint64_t sequence = sent->data.new_connection_id.sequence;
3591
3592
0
    if (acked) {
3593
0
        quicly_local_cid_on_acked(&conn->super.local.cid_set, sequence);
3594
0
    } else {
3595
0
        if (quicly_local_cid_on_lost(&conn->super.local.cid_set, sequence))
3596
0
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
3597
0
    }
3598
3599
0
    return 0;
3600
0
}
3601
3602
static quicly_error_t on_ack_retire_connection_id(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked,
3603
                                                  quicly_sent_t *sent)
3604
0
{
3605
0
    quicly_conn_t *conn = (quicly_conn_t *)((char *)map - offsetof(quicly_conn_t, egress.loss.sentmap));
3606
0
    uint64_t sequence = sent->data.retire_connection_id.sequence;
3607
0
    int ret;
3608
3609
0
    if (!acked) {
3610
0
        if ((ret = quicly_remote_cid_push_retired(&conn->super.remote.cid_set, sequence)) != 0)
3611
0
            return ret;
3612
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
3613
0
    }
3614
3615
0
    return 0;
3616
0
}
3617
3618
static uint32_t calc_pacer_send_rate(quicly_conn_t *conn)
3619
0
{
3620
0
    uint32_t multiplier;
3621
3622
0
    if (conn->egress.cc.num_loss_episodes == 0) {
3623
0
        if (quicly_cc_in_jumpstart(&conn->egress.cc)) {
3624
0
            multiplier = 1;
3625
0
        } else {
3626
0
            multiplier = quicly_cc_rapid_start_use_3x(&conn->egress.cc.rapid_start, &conn->egress.loss.rtt) ? 3 : 2;
3627
0
        }
3628
0
    } else {
3629
        /* We use of 2x during congestion avoidance, which is different from Linux using 1.25x. The rationale behind this choice is
3630
         * that 1.25x is not sufficiently aggressive immediately after a loss event. Following a loss event, the congestion window
3631
         * (CWND) is halved (i.e., beta), but the RTT remains high for one RTT and SRTT can remain high even loger, since it is a
3632
         * moving average adjusted with each ACK received. Consequently, if the multiplier is set to 1.25x, the calculated send rate
3633
         * could drop to as low as 1.25 * 1/2 = 0.625. By using a 2x multiplier, the send rate is guaranteed to become no less than
3634
         * that immediately before the loss event, which would have been the link throughput. */
3635
0
        multiplier = 2;
3636
0
    }
3637
3638
0
    return quicly_pacer_calc_send_rate(multiplier, conn->egress.cc.cwnd, conn->egress.loss.rtt.smoothed);
3639
0
}
3640
3641
static int should_send_datagram_frame(quicly_conn_t *conn)
3642
0
{
3643
0
    if (conn->egress.datagram_frame_payloads.count == 0)
3644
0
        return 0;
3645
0
    if (conn->application == NULL)
3646
0
        return 0;
3647
0
    if (conn->application->cipher.egress.key.aead == NULL)
3648
0
        return 0;
3649
0
    return 1;
3650
0
}
3651
3652
static inline uint64_t calc_amplification_limit_allowance(quicly_conn_t *conn)
3653
0
{
3654
0
    if (conn->super.remote.address_validation.validated)
3655
0
        return UINT64_MAX;
3656
0
    uint64_t budget = conn->super.stats.num_bytes.received * conn->super.ctx->pre_validation_amplification_limit;
3657
0
    if (budget <= conn->super.stats.num_bytes.sent)
3658
0
        return 0;
3659
0
    return budget - conn->super.stats.num_bytes.sent;
3660
0
}
3661
3662
/* Helper function to compute send window based on:
3663
 * * state of peer validation,
3664
 * * current cwnd,
3665
 * * minimum send requirements in |min_bytes_to_send|, and
3666
 * * if sending is to be restricted to the minimum, indicated in |restrict_sending|
3667
 */
3668
static size_t calc_send_window(quicly_conn_t *conn, size_t min_bytes_to_send, uint64_t amp_window, uint64_t pacer_window,
3669
                               int restrict_sending)
3670
0
{
3671
0
    uint64_t window = 0;
3672
0
    if (restrict_sending) {
3673
        /* Send min_bytes_to_send on PTO */
3674
0
        window = min_bytes_to_send;
3675
0
    } else {
3676
        /* Limit to cwnd */
3677
0
        if (conn->egress.cc.cwnd > conn->egress.loss.sentmap.bytes_in_flight) {
3678
0
            window = conn->egress.cc.cwnd - conn->egress.loss.sentmap.bytes_in_flight;
3679
0
            if (window > pacer_window)
3680
0
                window = pacer_window;
3681
0
        }
3682
        /* Allow at least one packet on time-threshold loss detection */
3683
0
        window = window > min_bytes_to_send ? window : min_bytes_to_send;
3684
0
    }
3685
    /* Cap the window by the amount allowed by address validation */
3686
0
    if (amp_window < window)
3687
0
        window = amp_window;
3688
3689
0
    return window;
3690
0
}
3691
3692
/**
3693
 * Checks if the server is waiting for ClientFinished. When that is the case, the loss timer is deactivated, to avoid repeatedly
3694
 * sending 1-RTT packets while the client spends time verifying the certificate chain at the same time buffering 1-RTT packets.
3695
 */
3696
static int is_point5rtt_with_no_handshake_data_to_send(quicly_conn_t *conn)
3697
0
{
3698
    /* bail out unless this is a server-side connection waiting for ClientFinished */
3699
0
    if (!(conn->handshake != NULL && conn->application != NULL && !quicly_is_client(conn)))
3700
0
        return 0;
3701
0
    quicly_stream_t *stream = quicly_get_stream(conn, (quicly_stream_id_t)-1 - QUICLY_EPOCH_HANDSHAKE);
3702
0
    assert(stream != NULL);
3703
0
    return stream->sendstate.pending.num_ranges == 0 && stream->sendstate.acked.ranges[0].end == stream->sendstate.size_inflight;
3704
0
}
3705
3706
static int64_t pacer_can_send_at(quicly_conn_t *conn)
3707
0
{
3708
0
    if (conn->egress.pacer == NULL)
3709
0
        return 0;
3710
3711
0
    uint32_t bytes_per_msec = calc_pacer_send_rate(conn);
3712
0
    return quicly_pacer_can_send_at(conn->egress.pacer, bytes_per_msec, conn->egress.max_udp_payload_size);
3713
0
}
3714
3715
int64_t quicly_get_first_timeout(quicly_conn_t *conn)
3716
0
{
3717
0
    if (conn->super.state >= QUICLY_STATE_CLOSING)
3718
0
        return conn->egress.send_ack_at;
3719
3720
0
    if (should_send_datagram_frame(conn))
3721
0
        return 0;
3722
3723
0
    uint64_t amp_window = calc_amplification_limit_allowance(conn);
3724
0
    int64_t at = conn->idle_timeout.at, pacer_at = pacer_can_send_at(conn);
3725
3726
    /* reduce at to the moment pacer provides credit, if we are not CC-limited and there's something to be sent over CC */
3727
0
    if (pacer_at < at && calc_send_window(conn, 0, amp_window, UINT64_MAX, 0) > 0) {
3728
0
        if (conn->egress.pending_flows != 0) {
3729
            /* crypto streams (as indicated by lower 4 bits) can be sent whenever CWND is available; other flows need application
3730
             * packet number space */
3731
0
            if ((conn->application != NULL && conn->application->cipher.egress.key.header_protection != NULL) ||
3732
0
                (conn->egress.pending_flows & 0xf) != 0)
3733
0
                at = pacer_at;
3734
0
        }
3735
0
        if (pacer_at < at && (quicly_linklist_is_linked(&conn->egress.pending_streams.control) || scheduler_can_send(conn)))
3736
0
            at = pacer_at;
3737
0
    }
3738
3739
    /* if something can be sent, return the earliest timeout. Otherwise return the idle timeout. */
3740
0
    if (amp_window > 0) {
3741
0
        if (conn->egress.loss.alarm_at < at && !is_point5rtt_with_no_handshake_data_to_send(conn))
3742
0
            at = conn->egress.loss.alarm_at;
3743
0
        if (conn->egress.send_ack_at < at)
3744
0
            at = conn->egress.send_ack_at;
3745
0
    }
3746
0
    if (at > conn->egress.send_probe_at)
3747
0
        at = conn->egress.send_probe_at;
3748
3749
0
    return at;
3750
0
}
3751
3752
uint64_t quicly_get_next_expected_packet_number(quicly_conn_t *conn)
3753
0
{
3754
0
    if (!conn->application)
3755
0
        return UINT64_MAX;
3756
3757
0
    return conn->application->super.next_expected_packet_number;
3758
0
}
3759
3760
static int setup_path_dcid(quicly_conn_t *conn, size_t path_index)
3761
0
{
3762
0
    struct st_quicly_conn_path_t *path = conn->paths[path_index];
3763
0
    quicly_remote_cid_set_t *set = &conn->super.remote.cid_set;
3764
0
    size_t found = SIZE_MAX;
3765
3766
0
    assert(path->dcid == UINT64_MAX);
3767
3768
0
    if (set->cids[0].cid.len == 0) {
3769
        /* if peer CID is zero-length, we can send packets to whatever address without the fear of corelation */
3770
0
        found = 0;
3771
0
    } else {
3772
        /* find the unused entry with a smallest sequence number */
3773
0
        for (size_t i = 0; i < PTLS_ELEMENTSOF(set->cids); ++i) {
3774
0
            if (set->cids[i].state == QUICLY_REMOTE_CID_AVAILABLE &&
3775
0
                (found == SIZE_MAX || set->cids[i].sequence < set->cids[found].sequence))
3776
0
                found = i;
3777
0
        }
3778
0
        if (found == SIZE_MAX)
3779
0
            return 0;
3780
0
    }
3781
3782
    /* associate */
3783
0
    set->cids[found].state = QUICLY_REMOTE_CID_IN_USE;
3784
0
    path->dcid = set->cids[found].sequence;
3785
3786
0
    return 1;
3787
0
}
3788
3789
static quicly_cid_t *get_dcid(quicly_conn_t *conn, size_t path_index)
3790
0
{
3791
0
    struct st_quicly_conn_path_t *path = conn->paths[path_index];
3792
3793
0
    assert(path->dcid != UINT64_MAX);
3794
3795
    /* lookup DCID and return */
3796
0
    for (size_t i = 0; i < PTLS_ELEMENTSOF(conn->super.remote.cid_set.cids); ++i) {
3797
0
        if (conn->super.remote.cid_set.cids[i].sequence == path->dcid)
3798
0
            return &conn->super.remote.cid_set.cids[i].cid;
3799
0
    }
3800
0
    assert(!"CID lookup failure");
3801
0
    return NULL;
3802
0
}
3803
3804
/**
3805
 * data structure that is used during one call through quicly_send()
3806
 */
3807
struct st_quicly_send_context_t {
3808
    /**
3809
     * current encryption context
3810
     */
3811
    struct {
3812
        struct st_quicly_cipher_context_t *cipher;
3813
        uint8_t first_byte;
3814
    } current;
3815
    /**
3816
     * packet under construction
3817
     */
3818
    struct {
3819
        struct st_quicly_cipher_context_t *cipher;
3820
        /**
3821
         * points to the first byte of the target QUIC packet. It will not point to packet->octets.base[0] when the datagram
3822
         * contains multiple QUIC packet.
3823
         */
3824
        uint8_t *first_byte_at;
3825
        /**
3826
         * if the target QUIC packet contains an ack-eliciting frame
3827
         */
3828
        uint8_t ack_eliciting : 1;
3829
        /**
3830
         * if the target datagram should be padded to full size
3831
         */
3832
        uint8_t full_size : 1;
3833
    } target;
3834
    /**
3835
     * output buffer into which list of datagrams is written
3836
     */
3837
    struct iovec *datagrams;
3838
    /**
3839
     * max number of datagrams that can be stored in |packets|
3840
     */
3841
    size_t max_datagrams;
3842
    /**
3843
     * number of datagrams currently stored in |packets|
3844
     */
3845
    size_t num_datagrams;
3846
    /**
3847
     * buffer in which packets are built
3848
     */
3849
    struct {
3850
        /**
3851
         * starting position of the current (or next) datagram
3852
         */
3853
        uint8_t *datagram;
3854
        /**
3855
         * end position of the payload buffer
3856
         */
3857
        uint8_t *end;
3858
    } payload_buf;
3859
    /**
3860
     * Currently available window for sending (in bytes); the value becomes negative when the sender uses more space than permitted.
3861
     * That happens because the sender operates at packet-level rather than byte-level.
3862
     */
3863
    ssize_t send_window;
3864
    /**
3865
     * location where next frame should be written
3866
     */
3867
    uint8_t *dst;
3868
    /**
3869
     * end of the payload area, beyond which frames cannot be written
3870
     */
3871
    uint8_t *dst_end;
3872
    /**
3873
     * address at which payload starts
3874
     */
3875
    uint8_t *dst_payload_from;
3876
    /**
3877
     * index of `conn->paths[]` to which we are sending
3878
     */
3879
    size_t path_index;
3880
    /**
3881
     * DCID to be used for the path
3882
     */
3883
    quicly_cid_t *dcid;
3884
    /**
3885
     * if `conn->egress.send_probe_at` should be recalculated
3886
     */
3887
    unsigned recalc_send_probe_at : 1;
3888
};
3889
3890
static quicly_error_t commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, int coalesced)
3891
0
{
3892
0
    size_t datagram_size, packet_bytes_in_flight;
3893
3894
0
    assert(s->target.cipher->aead != NULL);
3895
3896
0
    assert(s->dst != s->dst_payload_from);
3897
3898
    /* pad so that the pn + payload would be at least 4 bytes */
3899
0
    while (s->dst - s->dst_payload_from < QUICLY_MAX_PN_SIZE - QUICLY_SEND_PN_SIZE)
3900
0
        *s->dst++ = QUICLY_FRAME_TYPE_PADDING;
3901
3902
0
    if (!coalesced && s->target.full_size) {
3903
0
        assert(s->num_datagrams == 0 || s->datagrams[s->num_datagrams - 1].iov_len == conn->egress.max_udp_payload_size);
3904
0
        const size_t max_size = conn->egress.max_udp_payload_size - QUICLY_AEAD_TAG_SIZE;
3905
0
        assert(s->dst - s->payload_buf.datagram <= max_size);
3906
0
        memset(s->dst, QUICLY_FRAME_TYPE_PADDING, s->payload_buf.datagram + max_size - s->dst);
3907
0
        s->dst = s->payload_buf.datagram + max_size;
3908
0
    }
3909
3910
    /* encode packet size, packet number, key-phase */
3911
0
    if (QUICLY_PACKET_IS_LONG_HEADER(*s->target.first_byte_at)) {
3912
0
        uint16_t length = s->dst - s->dst_payload_from + s->target.cipher->aead->algo->tag_size + QUICLY_SEND_PN_SIZE;
3913
        /* length is always 2 bytes, see _do_prepare_packet */
3914
0
        length |= 0x4000;
3915
0
        quicly_encode16(s->dst_payload_from - QUICLY_SEND_PN_SIZE - 2, length);
3916
0
        switch (*s->target.first_byte_at & QUICLY_PACKET_TYPE_BITMASK) {
3917
0
        case QUICLY_PACKET_TYPE_INITIAL:
3918
0
            conn->super.stats.num_packets.initial_sent++;
3919
0
            break;
3920
0
        case QUICLY_PACKET_TYPE_0RTT:
3921
0
            conn->super.stats.num_packets.zero_rtt_sent++;
3922
0
            break;
3923
0
        case QUICLY_PACKET_TYPE_HANDSHAKE:
3924
0
            conn->super.stats.num_packets.handshake_sent++;
3925
0
            break;
3926
0
        }
3927
0
    } else {
3928
0
        if (conn->egress.packet_number >= conn->application->cipher.egress.key_update_pn.next) {
3929
0
            int ret;
3930
0
            if ((ret = update_1rtt_egress_key(conn)) != 0)
3931
0
                return ret;
3932
0
        }
3933
0
        if ((conn->application->cipher.egress.key_phase & 1) != 0)
3934
0
            *s->target.first_byte_at |= QUICLY_KEY_PHASE_BIT;
3935
0
    }
3936
0
    quicly_encode16(s->dst_payload_from - QUICLY_SEND_PN_SIZE, (uint16_t)conn->egress.packet_number);
3937
3938
    /* encrypt the packet */
3939
0
    s->dst += s->target.cipher->aead->algo->tag_size;
3940
0
    datagram_size = s->dst - s->payload_buf.datagram;
3941
0
    assert(datagram_size <= conn->egress.max_udp_payload_size);
3942
3943
0
    conn->super.ctx->crypto_engine->encrypt_packet(
3944
0
        conn->super.ctx->crypto_engine, conn, s->target.cipher->header_protection, s->target.cipher->aead,
3945
0
        ptls_iovec_init(s->payload_buf.datagram, datagram_size), s->target.first_byte_at - s->payload_buf.datagram,
3946
0
        s->dst_payload_from - s->payload_buf.datagram, conn->egress.packet_number, coalesced);
3947
3948
    /* update CC, commit sentmap */
3949
0
    int on_promoted_path = s->path_index == 0 && !conn->paths[0]->initial;
3950
0
    if (s->target.ack_eliciting) {
3951
0
        packet_bytes_in_flight = s->dst - s->target.first_byte_at;
3952
0
        s->send_window -= packet_bytes_in_flight;
3953
0
    } else {
3954
0
        packet_bytes_in_flight = 0;
3955
0
    }
3956
0
    if (quicly_sentmap_is_open(&conn->egress.loss.sentmap)) {
3957
0
        int cc_limited = conn->egress.loss.sentmap.bytes_in_flight + packet_bytes_in_flight >=
3958
0
                         conn->egress.cc.cwnd / 2; /* for the rationale behind this formula, see handle_ack_frame */
3959
0
        quicly_sentmap_commit(&conn->egress.loss.sentmap, (uint16_t)packet_bytes_in_flight, cc_limited, on_promoted_path);
3960
0
    }
3961
3962
0
    if (packet_bytes_in_flight != 0) {
3963
0
        assert(s->path_index == 0 && "CC governs path 0 and data is sent only on that path");
3964
0
        conn->egress.cc.type->cc_on_sent(&conn->egress.cc, &conn->egress.loss, (uint32_t)packet_bytes_in_flight, conn->stash.now);
3965
0
        if (conn->egress.pacer != NULL)
3966
0
            quicly_pacer_consume_window(conn->egress.pacer, packet_bytes_in_flight);
3967
0
    }
3968
3969
0
    QUICLY_PROBE(PACKET_SENT, conn, conn->stash.now, conn->egress.packet_number, s->dst - s->target.first_byte_at,
3970
0
                 get_epoch(*s->target.first_byte_at), !s->target.ack_eliciting);
3971
0
    QUICLY_LOG_CONN(packet_sent, conn, {
3972
0
        PTLS_LOG_ELEMENT_UNSIGNED(pn, conn->egress.packet_number);
3973
0
        PTLS_LOG_ELEMENT_UNSIGNED(len, s->dst - s->target.first_byte_at);
3974
0
        PTLS_LOG_ELEMENT_UNSIGNED(packet_type, get_epoch(*s->target.first_byte_at));
3975
0
        PTLS_LOG_ELEMENT_BOOL(ack_only, !s->target.ack_eliciting);
3976
0
    });
3977
3978
0
    ++conn->egress.packet_number;
3979
0
    ++conn->super.stats.num_packets.sent;
3980
0
    ++conn->paths[s->path_index]->num_packets.sent;
3981
0
    if (on_promoted_path)
3982
0
        ++conn->super.stats.num_packets.sent_promoted_paths;
3983
3984
0
    if (!coalesced) {
3985
0
        conn->super.stats.num_bytes.sent += datagram_size;
3986
0
        s->datagrams[s->num_datagrams++] = (struct iovec){.iov_base = s->payload_buf.datagram, .iov_len = datagram_size};
3987
0
        s->payload_buf.datagram += datagram_size;
3988
0
        s->target.cipher = NULL;
3989
0
        s->target.first_byte_at = NULL;
3990
0
    }
3991
3992
    /* insert PN gap if necessary, registering the PN to the ack queue so that we'd close the connection in the event of receiving
3993
     * an ACK for that gap. */
3994
0
    if (conn->egress.packet_number >= conn->egress.next_pn_to_skip && !QUICLY_PACKET_IS_LONG_HEADER(s->current.first_byte) &&
3995
0
        conn->super.state < QUICLY_STATE_CLOSING) {
3996
0
        quicly_error_t ret;
3997
0
        if ((ret = quicly_sentmap_prepare(&conn->egress.loss.sentmap, conn->egress.packet_number, conn->stash.now,
3998
0
                                          QUICLY_EPOCH_1RTT)) != 0)
3999
0
            return ret;
4000
0
        if (quicly_sentmap_allocate(&conn->egress.loss.sentmap, on_invalid_ack) == NULL)
4001
0
            return PTLS_ERROR_NO_MEMORY;
4002
0
        quicly_sentmap_commit(&conn->egress.loss.sentmap, 0, 0, 0);
4003
0
        ++conn->egress.packet_number;
4004
0
        conn->egress.next_pn_to_skip = calc_next_pn_to_skip(conn->super.ctx->tls, conn->egress.packet_number, conn->egress.cc.cwnd,
4005
0
                                                            conn->egress.max_udp_payload_size);
4006
0
    }
4007
4008
0
    return 0;
4009
0
}
4010
4011
static inline uint8_t *emit_cid(uint8_t *dst, const quicly_cid_t *cid)
4012
0
{
4013
0
    if (cid->len != 0) {
4014
0
        memcpy(dst, cid->cid, cid->len);
4015
0
        dst += cid->len;
4016
0
    }
4017
0
    return dst;
4018
0
}
4019
4020
enum allocate_frame_type {
4021
    ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING,
4022
    ALLOCATE_FRAME_TYPE_ACK_ELICITING,
4023
    ALLOCATE_FRAME_TYPE_ACK_ELICITING_NO_CC,
4024
};
4025
4026
static quicly_error_t do_allocate_frame(quicly_conn_t *conn, quicly_send_context_t *s, size_t min_space,
4027
                                        enum allocate_frame_type frame_type)
4028
0
{
4029
0
    int coalescible;
4030
0
    quicly_error_t ret;
4031
4032
0
    assert((s->current.first_byte & QUICLY_QUIC_BIT) != 0);
4033
4034
    /* allocate and setup the new packet if necessary */
4035
0
    if (s->dst_end - s->dst < min_space || s->target.first_byte_at == NULL) {
4036
0
        coalescible = 0;
4037
0
    } else if (((*s->target.first_byte_at ^ s->current.first_byte) & QUICLY_PACKET_TYPE_BITMASK) != 0) {
4038
0
        coalescible = QUICLY_PACKET_IS_LONG_HEADER(*s->target.first_byte_at);
4039
0
    } else if (s->dst_end - s->dst < min_space) {
4040
0
        coalescible = 0;
4041
0
    } else {
4042
        /* use the existing packet */
4043
0
        goto TargetReady;
4044
0
    }
4045
4046
    /* commit at the same time determining if we will coalesce the packets */
4047
0
    if (s->target.first_byte_at != NULL) {
4048
0
        if (coalescible) {
4049
0
            size_t overhead = 1 /* type */ + s->dcid->len + QUICLY_SEND_PN_SIZE + s->current.cipher->aead->algo->tag_size;
4050
0
            if (QUICLY_PACKET_IS_LONG_HEADER(s->current.first_byte))
4051
0
                overhead += 4 /* version */ + 1 /* cidl */ + s->dcid->len + conn->super.local.long_header_src_cid.len +
4052
0
                            (s->current.first_byte == QUICLY_PACKET_TYPE_INITIAL) /* token_length == 0 */ + 2 /* length */;
4053
0
            size_t packet_min_space = QUICLY_MAX_PN_SIZE - QUICLY_SEND_PN_SIZE;
4054
0
            if (packet_min_space < min_space)
4055
0
                packet_min_space = min_space;
4056
0
            if (overhead + packet_min_space > s->dst_end - s->dst)
4057
0
                coalescible = 0;
4058
0
        }
4059
        /* Close the packet under construction. Datagrams being returned by `quicly_send` are padded to full-size (except for the
4060
         * last one datagram) so that they can be sent at once using GSO. */
4061
0
        if (!coalescible)
4062
0
            s->target.full_size = 1;
4063
0
        if ((ret = commit_send_packet(conn, s, coalescible)) != 0)
4064
0
            return ret;
4065
0
    } else {
4066
0
        coalescible = 0;
4067
0
    }
4068
4069
    /* allocate packet */
4070
0
    if (coalescible) {
4071
0
        s->dst_end += s->target.cipher->aead->algo->tag_size; /* restore the AEAD tag size (tag size can differ bet. epochs) */
4072
0
        s->target.cipher = s->current.cipher;
4073
0
    } else {
4074
0
        if (s->num_datagrams >= s->max_datagrams)
4075
0
            return QUICLY_ERROR_SENDBUF_FULL;
4076
        /* note: send_window (ssize_t) can become negative; see doc-comment */
4077
0
        if (frame_type == ALLOCATE_FRAME_TYPE_ACK_ELICITING && s->send_window <= 0)
4078
0
            return QUICLY_ERROR_SENDBUF_FULL;
4079
0
        if (s->payload_buf.end - s->payload_buf.datagram < conn->egress.max_udp_payload_size)
4080
0
            return QUICLY_ERROR_SENDBUF_FULL;
4081
0
        s->target.cipher = s->current.cipher;
4082
0
        s->target.full_size = 0;
4083
0
        s->dst = s->payload_buf.datagram;
4084
0
        s->dst_end = s->dst + conn->egress.max_udp_payload_size;
4085
0
    }
4086
0
    s->target.ack_eliciting = 0;
4087
4088
0
    QUICLY_PROBE(PACKET_PREPARE, conn, conn->stash.now, s->current.first_byte, QUICLY_PROBE_HEXDUMP(s->dcid->cid, s->dcid->len));
4089
0
    QUICLY_LOG_CONN(packet_prepare, conn, {
4090
0
        PTLS_LOG_ELEMENT_UNSIGNED(first_octet, s->current.first_byte);
4091
0
        PTLS_LOG_ELEMENT_HEXDUMP(dcid, s->dcid->cid, s->dcid->len);
4092
0
    });
4093
4094
    /* emit header */
4095
0
    s->target.first_byte_at = s->dst;
4096
0
    *s->dst++ = s->current.first_byte | 0x1 /* pnlen == 2 */;
4097
0
    if (QUICLY_PACKET_IS_LONG_HEADER(s->current.first_byte)) {
4098
0
        s->dst = quicly_encode32(s->dst, conn->super.version);
4099
0
        *s->dst++ = s->dcid->len;
4100
0
        s->dst = emit_cid(s->dst, s->dcid);
4101
0
        *s->dst++ = conn->super.local.long_header_src_cid.len;
4102
0
        s->dst = emit_cid(s->dst, &conn->super.local.long_header_src_cid);
4103
        /* token */
4104
0
        if (s->current.first_byte == QUICLY_PACKET_TYPE_INITIAL) {
4105
0
            s->dst = quicly_encodev(s->dst, conn->token.len);
4106
0
            if (conn->token.len != 0) {
4107
0
                assert(s->dst_end - s->dst > conn->token.len);
4108
0
                memcpy(s->dst, conn->token.base, conn->token.len);
4109
0
                s->dst += conn->token.len;
4110
0
            }
4111
0
        }
4112
        /* payload length is filled laterwards (see commit_send_packet) */
4113
0
        *s->dst++ = 0;
4114
0
        *s->dst++ = 0;
4115
0
    } else {
4116
0
        s->dst = emit_cid(s->dst, s->dcid);
4117
0
    }
4118
0
    s->dst += QUICLY_SEND_PN_SIZE; /* space for PN bits, filled in at commit time */
4119
0
    s->dst_payload_from = s->dst;
4120
0
    assert(s->target.cipher->aead != NULL);
4121
0
    s->dst_end -= s->target.cipher->aead->algo->tag_size;
4122
0
    assert(s->dst_end - s->dst >= QUICLY_MAX_PN_SIZE - QUICLY_SEND_PN_SIZE);
4123
4124
0
    if (conn->super.state < QUICLY_STATE_CLOSING) {
4125
        /* register to sentmap */
4126
0
        uint8_t ack_epoch = get_epoch(s->current.first_byte);
4127
0
        if (ack_epoch == QUICLY_EPOCH_0RTT)
4128
0
            ack_epoch = QUICLY_EPOCH_1RTT;
4129
0
        if ((ret = quicly_sentmap_prepare(&conn->egress.loss.sentmap, conn->egress.packet_number, conn->stash.now, ack_epoch)) != 0)
4130
0
            return ret;
4131
        /* adjust ack-frequency */
4132
0
        if (frame_type == ALLOCATE_FRAME_TYPE_ACK_ELICITING && conn->stash.now >= conn->egress.ack_frequency.update_at &&
4133
0
            s->dst_end - s->dst >= QUICLY_ACK_FREQUENCY_FRAME_CAPACITY + min_space) {
4134
0
            assert(conn->super.remote.transport_params.min_ack_delay_usec != UINT64_MAX);
4135
0
            if (conn->egress.cc.num_loss_episodes >= QUICLY_FIRST_ACK_FREQUENCY_LOSS_EPISODE && conn->initial == NULL &&
4136
0
                conn->handshake == NULL) {
4137
0
                uint32_t fraction_of_cwnd = (uint32_t)((uint64_t)conn->egress.cc.cwnd * conn->super.ctx->ack_frequency / 1024);
4138
0
                if (fraction_of_cwnd >= conn->egress.max_udp_payload_size * 3) {
4139
0
                    uint32_t packet_tolerance = fraction_of_cwnd / conn->egress.max_udp_payload_size;
4140
0
                    if (packet_tolerance > QUICLY_MAX_PACKET_TOLERANCE)
4141
0
                        packet_tolerance = QUICLY_MAX_PACKET_TOLERANCE;
4142
                    /* TODO: Discuss (and possibly test) the strategy for choosing max_ack_delay; note the chosen value should be
4143
                     * passed to quicly_loss_detect_loss too. */
4144
0
                    uint64_t max_ack_delay = conn->super.remote.transport_params.max_ack_delay * 1000;
4145
0
                    uint64_t reordering_threshold =
4146
0
                        conn->egress.loss.thresholds.use_packet_based ? QUICLY_LOSS_DEFAULT_PACKET_THRESHOLD : 0;
4147
                    /* TODO: Adjust the max_ack_delay we use for loss recovery to be consistent with this value */
4148
0
                    s->dst = quicly_encode_ack_frequency_frame(s->dst, conn->egress.ack_frequency.sequence++, packet_tolerance,
4149
0
                                                               max_ack_delay, reordering_threshold);
4150
0
                    ++conn->super.stats.num_frames_sent.ack_frequency;
4151
0
                }
4152
0
            }
4153
0
            ack_frequency_set_next_update_at(conn);
4154
0
        }
4155
0
    }
4156
4157
0
TargetReady:
4158
0
    if (frame_type != ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING) {
4159
0
        s->target.ack_eliciting = 1;
4160
0
        conn->egress.last_retransmittable_sent_at = conn->stash.now;
4161
0
    }
4162
0
    return 0;
4163
0
}
4164
4165
static quicly_error_t allocate_ack_eliciting_frame(quicly_conn_t *conn, quicly_send_context_t *s, size_t min_space,
4166
                                                   quicly_sent_t **sent, quicly_sent_acked_cb acked)
4167
0
{
4168
0
    quicly_error_t ret;
4169
4170
0
    if ((ret = do_allocate_frame(conn, s, min_space, ALLOCATE_FRAME_TYPE_ACK_ELICITING)) != 0)
4171
0
        return ret;
4172
0
    if ((*sent = quicly_sentmap_allocate(&conn->egress.loss.sentmap, acked)) == NULL)
4173
0
        return PTLS_ERROR_NO_MEMORY;
4174
4175
0
    return ret;
4176
0
}
4177
4178
static quicly_error_t send_ack(quicly_conn_t *conn, struct st_quicly_pn_space_t *space, quicly_send_context_t *s)
4179
0
{
4180
0
    uint64_t ack_delay;
4181
0
    quicly_error_t ret;
4182
4183
0
    if (space->ack_queue.num_ranges == 0)
4184
0
        return 0;
4185
4186
    /* calc ack_delay */
4187
0
    if (space->largest_pn_received_at < conn->stash.now) {
4188
        /* We underreport ack_delay up to 1 milliseconds assuming that QUICLY_LOCAL_ACK_DELAY_EXPONENT is 10. It's considered a
4189
         * non-issue because our time measurement is at millisecond granularity anyways. */
4190
0
        ack_delay = ((conn->stash.now - space->largest_pn_received_at) * 1000) >> QUICLY_LOCAL_ACK_DELAY_EXPONENT;
4191
0
    } else {
4192
0
        ack_delay = 0;
4193
0
    }
4194
4195
0
Emit: /* emit an ACK frame */
4196
0
    if ((ret = do_allocate_frame(conn, s, QUICLY_ACK_FRAME_CAPACITY, ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING)) != 0)
4197
0
        return ret;
4198
0
    uint8_t *dst = s->dst;
4199
0
    dst = quicly_encode_ack_frame(dst, s->dst_end, &space->ack_queue, space->ecn_counts, ack_delay);
4200
4201
    /* when there's no space, retry with a new MTU-sized packet */
4202
0
    if (dst == NULL) {
4203
        /* [rare case] A coalesced packet might not have enough space to hold only an ACK. If so, pad it, as that's easier than
4204
         * rolling back. */
4205
0
        if (s->dst == s->dst_payload_from) {
4206
0
            assert(s->target.first_byte_at != s->payload_buf.datagram);
4207
0
            *s->dst++ = QUICLY_FRAME_TYPE_PADDING;
4208
0
        }
4209
0
        s->target.full_size = 1;
4210
0
        if ((ret = commit_send_packet(conn, s, 0)) != 0)
4211
0
            return ret;
4212
0
        goto Emit;
4213
0
    }
4214
4215
0
    ++conn->super.stats.num_frames_sent.ack;
4216
0
    QUICLY_PROBE(ACK_SEND, conn, conn->stash.now, space->ack_queue.ranges[space->ack_queue.num_ranges - 1].end - 1, ack_delay);
4217
0
    QUICLY_LOG_CONN(ack_send, conn, {
4218
0
        PTLS_LOG_ELEMENT_UNSIGNED(largest_acked, space->ack_queue.ranges[space->ack_queue.num_ranges - 1].end - 1);
4219
0
        PTLS_LOG_ELEMENT_UNSIGNED(ack_delay, ack_delay);
4220
0
    });
4221
4222
    /* when there are no less than QUICLY_NUM_ACK_BLOCKS_TO_INDUCE_ACKACK (8) gaps, bundle PING once every 4 packets being sent */
4223
0
    if (space->ack_queue.num_ranges >= QUICLY_NUM_ACK_BLOCKS_TO_INDUCE_ACKACK && conn->egress.packet_number % 4 == 0 &&
4224
0
        dst < s->dst_end) {
4225
0
        *dst++ = QUICLY_FRAME_TYPE_PING;
4226
0
        ++conn->super.stats.num_frames_sent.ping;
4227
0
        QUICLY_PROBE(PING_SEND, conn, conn->stash.now);
4228
0
        QUICLY_LOG_CONN(ping_send, conn, {});
4229
0
    }
4230
4231
0
    s->dst = dst;
4232
4233
0
    { /* save what's inflight */
4234
0
        size_t range_index = 0;
4235
0
        while (range_index < space->ack_queue.num_ranges) {
4236
0
            quicly_sent_t *sent;
4237
0
            struct st_quicly_sent_ack_additional_t *additional, *additional_end;
4238
            /* allocate */
4239
0
            if ((sent = quicly_sentmap_allocate(&conn->egress.loss.sentmap, on_ack_ack_ranges8)) == NULL)
4240
0
                return PTLS_ERROR_NO_MEMORY;
4241
            /* store the first range, as well as preparing references to the additional slots */
4242
0
            sent->data.ack.start = space->ack_queue.ranges[range_index].start;
4243
0
            uint64_t length = space->ack_queue.ranges[range_index].end - space->ack_queue.ranges[range_index].start;
4244
0
            if (length <= UINT8_MAX) {
4245
0
                sent->data.ack.ranges8.start_length = length;
4246
0
                additional = sent->data.ack.ranges8.additional;
4247
0
                additional_end = additional + PTLS_ELEMENTSOF(sent->data.ack.ranges8.additional);
4248
0
            } else {
4249
0
                sent->acked = on_ack_ack_ranges64;
4250
0
                sent->data.ack.ranges64.start_length = length;
4251
0
                additional = sent->data.ack.ranges64.additional;
4252
0
                additional_end = additional + PTLS_ELEMENTSOF(sent->data.ack.ranges64.additional);
4253
0
            }
4254
            /* store additional ranges, if possible */
4255
0
            for (++range_index; range_index < space->ack_queue.num_ranges && additional < additional_end;
4256
0
                 ++range_index, ++additional) {
4257
0
                uint64_t gap = space->ack_queue.ranges[range_index].start - space->ack_queue.ranges[range_index - 1].end;
4258
0
                uint64_t length = space->ack_queue.ranges[range_index].end - space->ack_queue.ranges[range_index].start;
4259
0
                if (gap > UINT8_MAX || length > UINT8_MAX)
4260
0
                    break;
4261
0
                additional->gap = gap;
4262
0
                additional->length = length;
4263
0
            }
4264
            /* additional list is zero-terminated, if not full */
4265
0
            if (additional < additional_end)
4266
0
                additional->gap = 0;
4267
0
        }
4268
0
    }
4269
4270
0
    space->unacked_count = 0;
4271
0
    update_smallest_unreported_missing_on_send_ack(&space->ack_queue, &space->largest_acked_unacked,
4272
0
                                                   &space->smallest_unreported_missing, space->reordering_threshold);
4273
0
    return ret;
4274
0
}
4275
4276
static quicly_error_t prepare_stream_state_sender(quicly_stream_t *stream, quicly_sender_state_t *sender, quicly_send_context_t *s,
4277
                                                  size_t min_space, quicly_sent_acked_cb ack_cb)
4278
0
{
4279
0
    quicly_sent_t *sent;
4280
0
    quicly_error_t ret;
4281
4282
0
    if ((ret = allocate_ack_eliciting_frame(stream->conn, s, min_space, &sent, ack_cb)) != 0)
4283
0
        return ret;
4284
0
    sent->data.stream_state_sender.stream_id = stream->stream_id;
4285
0
    *sender = QUICLY_SENDER_STATE_UNACKED;
4286
4287
0
    return 0;
4288
0
}
4289
4290
static quicly_error_t send_control_frames_of_stream(quicly_stream_t *stream, quicly_send_context_t *s)
4291
0
{
4292
0
    quicly_error_t ret;
4293
4294
    /* send STOP_SENDING if necessary */
4295
0
    if (stream->_send_aux.stop_sending.sender_state == QUICLY_SENDER_STATE_SEND) {
4296
        /* FIXME also send an empty STREAM frame */
4297
0
        if ((ret = prepare_stream_state_sender(stream, &stream->_send_aux.stop_sending.sender_state, s,
4298
0
                                               QUICLY_STOP_SENDING_FRAME_CAPACITY, on_ack_stop_sending)) != 0)
4299
0
            return ret;
4300
0
        s->dst = quicly_encode_stop_sending_frame(s->dst, stream->stream_id, stream->_send_aux.stop_sending.error_code);
4301
0
        ++stream->conn->super.stats.num_frames_sent.stop_sending;
4302
0
        QUICLY_PROBE(STOP_SENDING_SEND, stream->conn, stream->conn->stash.now, stream->stream_id,
4303
0
                     stream->_send_aux.stop_sending.error_code);
4304
0
        QUICLY_LOG_CONN(stop_sending_send, stream->conn, {
4305
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
4306
0
            PTLS_LOG_ELEMENT_UNSIGNED(error_code, stream->_send_aux.stop_sending.error_code);
4307
0
        });
4308
0
    }
4309
4310
    /* send MAX_STREAM_DATA if necessary */
4311
0
    if (should_send_max_stream_data(stream)) {
4312
0
        uint64_t new_value = stream->recvstate.data_off + stream->_recv_aux.window;
4313
0
        quicly_sent_t *sent;
4314
        /* prepare */
4315
0
        if ((ret = allocate_ack_eliciting_frame(stream->conn, s, QUICLY_MAX_STREAM_DATA_FRAME_CAPACITY, &sent,
4316
0
                                                on_ack_max_stream_data)) != 0)
4317
0
            return ret;
4318
        /* send */
4319
0
        s->dst = quicly_encode_max_stream_data_frame(s->dst, stream->stream_id, new_value);
4320
        /* register ack */
4321
0
        sent->data.max_stream_data.stream_id = stream->stream_id;
4322
0
        quicly_maxsender_record(&stream->_send_aux.max_stream_data_sender, new_value, &sent->data.max_stream_data.args);
4323
        /* update stats */
4324
0
        ++stream->conn->super.stats.num_frames_sent.max_stream_data;
4325
0
        QUICLY_PROBE(MAX_STREAM_DATA_SEND, stream->conn, stream->conn->stash.now, stream, new_value);
4326
0
        QUICLY_LOG_CONN(max_stream_data_send, stream->conn, {
4327
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
4328
0
            PTLS_LOG_ELEMENT_UNSIGNED(maximum, new_value);
4329
0
        });
4330
0
    }
4331
4332
    /* send RESET_STREAM if necessary */
4333
0
    if (stream->_send_aux.reset_stream.sender_state == QUICLY_SENDER_STATE_SEND) {
4334
0
        if ((ret = prepare_stream_state_sender(stream, &stream->_send_aux.reset_stream.sender_state, s, QUICLY_RST_FRAME_CAPACITY,
4335
0
                                               on_ack_reset_stream)) != 0)
4336
0
            return ret;
4337
0
        s->dst = quicly_encode_reset_stream_frame(s->dst, stream->stream_id, stream->_send_aux.reset_stream.error_code,
4338
0
                                                  stream->sendstate.size_inflight);
4339
0
        ++stream->conn->super.stats.num_frames_sent.reset_stream;
4340
0
        QUICLY_PROBE(RESET_STREAM_SEND, stream->conn, stream->conn->stash.now, stream->stream_id,
4341
0
                     stream->_send_aux.reset_stream.error_code, stream->sendstate.size_inflight);
4342
0
        QUICLY_LOG_CONN(reset_stream_send, stream->conn, {
4343
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
4344
0
            PTLS_LOG_ELEMENT_UNSIGNED(error_code, stream->_send_aux.reset_stream.error_code);
4345
0
            PTLS_LOG_ELEMENT_UNSIGNED(final_size, stream->sendstate.size_inflight);
4346
0
        });
4347
0
    }
4348
4349
    /* send STREAM_DATA_BLOCKED if necessary */
4350
0
    if (stream->_send_aux.blocked == QUICLY_SENDER_STATE_SEND) {
4351
0
        quicly_sent_t *sent;
4352
0
        if ((ret = allocate_ack_eliciting_frame(stream->conn, s, QUICLY_STREAM_DATA_BLOCKED_FRAME_CAPACITY, &sent,
4353
0
                                                on_ack_stream_data_blocked_frame)) != 0)
4354
0
            return ret;
4355
0
        uint64_t offset = stream->_send_aux.max_stream_data;
4356
0
        sent->data.stream_data_blocked.stream_id = stream->stream_id;
4357
0
        sent->data.stream_data_blocked.offset = offset;
4358
0
        s->dst = quicly_encode_stream_data_blocked_frame(s->dst, stream->stream_id, offset);
4359
0
        stream->_send_aux.blocked = QUICLY_SENDER_STATE_UNACKED;
4360
0
        ++stream->conn->super.stats.num_frames_sent.stream_data_blocked;
4361
0
        QUICLY_PROBE(STREAM_DATA_BLOCKED_SEND, stream->conn, stream->conn->stash.now, stream->stream_id, offset);
4362
0
        QUICLY_LOG_CONN(stream_data_blocked_send, stream->conn, {
4363
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
4364
0
            PTLS_LOG_ELEMENT_UNSIGNED(maximum, offset);
4365
0
        });
4366
0
    }
4367
4368
0
    return 0;
4369
0
}
4370
4371
static quicly_error_t send_stream_control_frames(quicly_conn_t *conn, quicly_send_context_t *s)
4372
0
{
4373
0
    quicly_error_t ret = 0;
4374
4375
0
    while (s->num_datagrams != s->max_datagrams && quicly_linklist_is_linked(&conn->egress.pending_streams.control)) {
4376
0
        quicly_stream_t *stream =
4377
0
            (void *)((char *)conn->egress.pending_streams.control.next - offsetof(quicly_stream_t, _send_aux.pending_link.control));
4378
0
        if ((ret = send_control_frames_of_stream(stream, s)) != 0)
4379
0
            goto Exit;
4380
0
        quicly_linklist_unlink(&stream->_send_aux.pending_link.control);
4381
0
    }
4382
4383
0
Exit:
4384
0
    return ret;
4385
0
}
4386
4387
int quicly_is_blocked(quicly_conn_t *conn)
4388
{
4389
    if (conn->egress.max_data.sent < conn->egress.max_data.permitted)
4390
        return 0;
4391
4392
    /* schedule the transmission of DATA_BLOCKED frame, if it's new information */
4393
    if (conn->egress.data_blocked == QUICLY_SENDER_STATE_NONE) {
4394
        conn->egress.data_blocked = QUICLY_SENDER_STATE_SEND;
4395
        conn->egress.pending_flows = QUICLY_PENDING_FLOW_OTHERS_BIT;
4396
    }
4397
4398
    return 1;
4399
}
4400
4401
int quicly_stream_can_send(quicly_stream_t *stream, int at_stream_level)
4402
0
{
4403
    /* return if there is nothing to be sent */
4404
0
    if (stream->sendstate.pending.num_ranges == 0)
4405
0
        return 0;
4406
4407
    /* return if flow is capped neither by MAX_STREAM_DATA nor (in case we are hitting connection-level flow control) by the number
4408
     * of bytes we've already sent */
4409
0
    uint64_t blocked_at = at_stream_level ? stream->_send_aux.max_stream_data : stream->sendstate.size_inflight;
4410
0
    if (stream->sendstate.pending.ranges[0].start < blocked_at)
4411
0
        return 1;
4412
    /* we can always send EOS, if that is the only thing to be sent */
4413
0
    if (stream->sendstate.pending.ranges[0].start >= stream->sendstate.final_size) {
4414
0
        assert(stream->sendstate.pending.ranges[0].start == stream->sendstate.final_size);
4415
0
        return 1;
4416
0
    }
4417
4418
    /* if known to be blocked at stream-level, schedule the emission of STREAM_DATA_BLOCKED frame */
4419
0
    if (at_stream_level && stream->_send_aux.blocked == QUICLY_SENDER_STATE_NONE) {
4420
0
        stream->_send_aux.blocked = QUICLY_SENDER_STATE_SEND;
4421
0
        sched_stream_control(stream);
4422
0
    }
4423
4424
0
    return 0;
4425
0
}
4426
4427
int quicly_can_send_data(quicly_conn_t *conn, quicly_send_context_t *s)
4428
{
4429
    return s->num_datagrams < s->max_datagrams;
4430
}
4431
4432
/**
4433
 * If necessary, changes the frame representation from one without length field to one that has if necessary. Or, as an alternative,
4434
 * prepends PADDING frames. Upon return, `dst` points to the end of the frame being built. `*len`, `*wrote_all`, `*frame_type_at`
4435
 * are also updated reflecting their values post-adjustment.
4436
 */
4437
static inline void adjust_stream_frame_layout(uint8_t **dst, uint8_t *const dst_end, size_t *len, int *wrote_all,
4438
                                              uint8_t **frame_at)
4439
0
{
4440
0
    size_t space_left = (dst_end - *dst) - *len, len_of_len = quicly_encodev_capacity(*len);
4441
4442
0
    if (**frame_at == QUICLY_FRAME_TYPE_CRYPTO) {
4443
        /* CRYPTO frame: adjust payload length to make space for the length field, if necessary. */
4444
0
        if (space_left < len_of_len) {
4445
0
            *len = dst_end - *dst - len_of_len;
4446
0
            *wrote_all = 0;
4447
0
        }
4448
0
    } else {
4449
        /* STREAM frame: insert length if space can be left for more frames. Otherwise, retain STREAM frame header omitting the
4450
         * length field, prepending PADDING if necessary. */
4451
0
        if (space_left <= len_of_len) {
4452
0
            if (space_left != 0) {
4453
0
                memmove(*frame_at + space_left, *frame_at, *dst + *len - *frame_at);
4454
0
                memset(*frame_at, QUICLY_FRAME_TYPE_PADDING, space_left);
4455
0
                *dst += space_left;
4456
0
                *frame_at += space_left;
4457
0
            }
4458
0
            *dst += *len;
4459
0
            return;
4460
0
        }
4461
0
        **frame_at |= QUICLY_FRAME_TYPE_STREAM_BIT_LEN;
4462
0
    }
4463
4464
    /* insert length before payload of `*len` bytes */
4465
0
    memmove(*dst + len_of_len, *dst, *len);
4466
0
    *dst = quicly_encodev(*dst, *len);
4467
0
    *dst += *len;
4468
0
}
4469
4470
quicly_error_t quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s)
4471
0
{
4472
0
    uint64_t off = stream->sendstate.pending.ranges[0].start;
4473
0
    quicly_sent_t *sent;
4474
0
    uint8_t *dst; /* this pointer points to the current write position within the frame being built, while `s->dst` points to the
4475
                   * beginning of the frame. */
4476
0
    size_t len;
4477
0
    int wrote_all, is_fin;
4478
0
    quicly_error_t ret;
4479
4480
    /* write frame type, stream_id and offset, calculate capacity (and store that in `len`) */
4481
0
    if (stream->stream_id < 0) {
4482
0
        if ((ret = allocate_ack_eliciting_frame(stream->conn, s,
4483
0
                                                1 + quicly_encodev_capacity(off) + 2 /* type + offset + len + 1-byte payload */,
4484
0
                                                &sent, on_ack_stream)) != 0)
4485
0
            return ret;
4486
0
        dst = s->dst;
4487
0
        *dst++ = QUICLY_FRAME_TYPE_CRYPTO;
4488
0
        dst = quicly_encodev(dst, off);
4489
0
        len = s->dst_end - dst;
4490
0
    } else {
4491
0
        uint8_t header[18], *hp = header + 1;
4492
0
        hp = quicly_encodev(hp, stream->stream_id);
4493
0
        if (off != 0) {
4494
0
            header[0] = QUICLY_FRAME_TYPE_STREAM_BASE | QUICLY_FRAME_TYPE_STREAM_BIT_OFF;
4495
0
            hp = quicly_encodev(hp, off);
4496
0
        } else {
4497
0
            header[0] = QUICLY_FRAME_TYPE_STREAM_BASE;
4498
0
        }
4499
0
        if (off == stream->sendstate.final_size) {
4500
0
            assert(!quicly_sendstate_is_open(&stream->sendstate));
4501
            /* special case for emitting FIN only */
4502
0
            header[0] |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN;
4503
0
            if ((ret = allocate_ack_eliciting_frame(stream->conn, s, hp - header, &sent, on_ack_stream)) != 0)
4504
0
                return ret;
4505
0
            if (hp - header != s->dst_end - s->dst) {
4506
0
                header[0] |= QUICLY_FRAME_TYPE_STREAM_BIT_LEN;
4507
0
                *hp++ = 0; /* empty length */
4508
0
            }
4509
0
            memcpy(s->dst, header, hp - header);
4510
0
            s->dst += hp - header;
4511
0
            len = 0;
4512
0
            wrote_all = 1;
4513
0
            is_fin = 1;
4514
0
            goto UpdateState;
4515
0
        }
4516
0
        if ((ret = allocate_ack_eliciting_frame(stream->conn, s, hp - header + 1, &sent, on_ack_stream)) != 0)
4517
0
            return ret;
4518
0
        dst = s->dst;
4519
0
        memcpy(dst, header, hp - header);
4520
0
        dst += hp - header;
4521
0
        len = s->dst_end - dst;
4522
        /* cap by max_stream_data */
4523
0
        if (off + len > stream->_send_aux.max_stream_data)
4524
0
            len = stream->_send_aux.max_stream_data - off;
4525
        /* cap by max_data */
4526
0
        if (off + len > stream->sendstate.size_inflight) {
4527
0
            uint64_t new_bytes = off + len - stream->sendstate.size_inflight;
4528
0
            if (new_bytes > stream->conn->egress.max_data.permitted - stream->conn->egress.max_data.sent) {
4529
0
                size_t max_stream_data =
4530
0
                    stream->sendstate.size_inflight + stream->conn->egress.max_data.permitted - stream->conn->egress.max_data.sent;
4531
0
                len = max_stream_data - off;
4532
0
            }
4533
0
        }
4534
0
    }
4535
0
    { /* cap len to the current range */
4536
0
        uint64_t range_capacity = stream->sendstate.pending.ranges[0].end - off;
4537
0
        if (off + range_capacity > stream->sendstate.final_size) {
4538
0
            assert(!quicly_sendstate_is_open(&stream->sendstate));
4539
0
            assert(range_capacity > 1); /* see the special case above */
4540
0
            range_capacity -= 1;
4541
0
        }
4542
0
        if (len > range_capacity)
4543
0
            len = range_capacity;
4544
0
    }
4545
4546
    /* Write payload, adjusting len to actual size. Note that `on_send_emit` might fail (e.g., when underlying pread(2) fails), in
4547
     * which case the application will either close the connection immediately or reset the stream. If that happens, we return
4548
     * immediately without updating state. */
4549
0
    assert(len != 0);
4550
0
    size_t emit_off = (size_t)(off - stream->sendstate.acked.ranges[0].end);
4551
0
    QUICLY_PROBE(STREAM_ON_SEND_EMIT, stream->conn, stream->conn->stash.now, stream, emit_off, len);
4552
0
    QUICLY_LOG_CONN(stream_on_send_emit, stream->conn, {
4553
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
4554
0
        PTLS_LOG_ELEMENT_UNSIGNED(off, off);
4555
0
        PTLS_LOG_ELEMENT_UNSIGNED(capacity, len);
4556
0
    });
4557
0
    stream->callbacks->on_send_emit(stream, emit_off, dst, &len, &wrote_all);
4558
0
    if (stream->conn->super.state >= QUICLY_STATE_CLOSING) {
4559
0
        return QUICLY_ERROR_IS_CLOSING;
4560
0
    } else if (stream->_send_aux.reset_stream.sender_state != QUICLY_SENDER_STATE_NONE) {
4561
0
        return 0;
4562
0
    }
4563
0
    assert(len != 0);
4564
4565
0
    adjust_stream_frame_layout(&dst, s->dst_end, &len, &wrote_all, &s->dst);
4566
4567
    /* determine if the frame incorporates FIN */
4568
0
    if (off + len == stream->sendstate.final_size) {
4569
0
        assert(!quicly_sendstate_is_open(&stream->sendstate));
4570
0
        assert(s->dst != NULL);
4571
0
        is_fin = 1;
4572
0
        *s->dst |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN;
4573
0
    } else {
4574
0
        is_fin = 0;
4575
0
    }
4576
4577
    /* update s->dst now that frame construction is complete */
4578
0
    s->dst = dst;
4579
4580
0
UpdateState:
4581
0
    if (stream->stream_id < 0) {
4582
0
        ++stream->conn->super.stats.num_frames_sent.crypto;
4583
0
    } else {
4584
0
        ++stream->conn->super.stats.num_frames_sent.stream;
4585
0
    }
4586
0
    stream->conn->super.stats.num_bytes.stream_data_sent += len;
4587
0
    if (off < stream->sendstate.size_inflight)
4588
0
        stream->conn->super.stats.num_bytes.stream_data_resent +=
4589
0
            (stream->sendstate.size_inflight < off + len ? stream->sendstate.size_inflight : off + len) - off;
4590
0
    QUICLY_PROBE(STREAM_SEND, stream->conn, stream->conn->stash.now, stream, off, s->dst - len, len, is_fin, wrote_all);
4591
0
    QUICLY_LOG_CONN(stream_send, stream->conn, {
4592
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
4593
0
        PTLS_LOG_ELEMENT_UNSIGNED(off, off);
4594
0
        PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(data, s->dst - len, len);
4595
0
        PTLS_LOG_ELEMENT_BOOL(is_fin, is_fin);
4596
0
        PTLS_LOG_ELEMENT_BOOL(wrote_all, wrote_all);
4597
0
    });
4598
4599
0
    QUICLY_PROBE(QUICTRACE_SEND_STREAM, stream->conn, stream->conn->stash.now, stream, off, len, is_fin);
4600
    /* update sendstate (and also MAX_DATA counter) */
4601
0
    if (stream->sendstate.size_inflight < off + len) {
4602
0
        if (stream->stream_id >= 0)
4603
0
            stream->conn->egress.max_data.sent += off + len - stream->sendstate.size_inflight;
4604
0
        stream->sendstate.size_inflight = off + len;
4605
0
    }
4606
0
    if ((ret = quicly_ranges_subtract(&stream->sendstate.pending, off, off + len + is_fin)) != 0)
4607
0
        return ret;
4608
0
    if (wrote_all) {
4609
0
        if ((ret = quicly_ranges_subtract(&stream->sendstate.pending, stream->sendstate.size_inflight, UINT64_MAX)) != 0)
4610
0
            return ret;
4611
0
    }
4612
4613
    /* setup sentmap */
4614
0
    sent->data.stream.stream_id = stream->stream_id;
4615
0
    sent->data.stream.args.start = off;
4616
0
    sent->data.stream.args.end = off + len + is_fin;
4617
4618
0
    return 0;
4619
0
}
4620
4621
static inline quicly_error_t init_acks_iter(quicly_conn_t *conn, quicly_sentmap_iter_t *iter)
4622
0
{
4623
0
    return quicly_loss_init_sentmap_iter(&conn->egress.loss, iter, conn->stash.now,
4624
0
                                         conn->super.remote.transport_params.max_ack_delay,
4625
0
                                         conn->super.state >= QUICLY_STATE_CLOSING);
4626
0
}
4627
4628
quicly_error_t discard_sentmap_by_epoch(quicly_conn_t *conn, unsigned ack_epochs)
4629
0
{
4630
0
    quicly_sentmap_iter_t iter;
4631
0
    const quicly_sent_packet_t *sent;
4632
0
    quicly_error_t ret;
4633
4634
0
    if ((ret = init_acks_iter(conn, &iter)) != 0)
4635
0
        return ret;
4636
4637
0
    while ((sent = quicly_sentmap_get(&iter))->packet_number != UINT64_MAX) {
4638
0
        if ((ack_epochs & (1u << sent->ack_epoch)) != 0) {
4639
0
            if ((ret = quicly_sentmap_update(&conn->egress.loss.sentmap, &iter, QUICLY_SENTMAP_EVENT_EXPIRED)) != 0)
4640
0
                return ret;
4641
0
        } else {
4642
0
            quicly_sentmap_skip(&iter);
4643
0
        }
4644
0
    }
4645
4646
0
    return ret;
4647
0
}
4648
4649
/**
4650
 * Mark frames of given epoch as pending, until `*bytes_to_mark` becomes zero.
4651
 */
4652
static quicly_error_t mark_frames_on_pto(quicly_conn_t *conn, uint8_t ack_epoch, size_t *bytes_to_mark)
4653
0
{
4654
0
    quicly_sentmap_iter_t iter;
4655
0
    const quicly_sent_packet_t *sent;
4656
0
    quicly_error_t ret;
4657
4658
0
    if ((ret = init_acks_iter(conn, &iter)) != 0)
4659
0
        return ret;
4660
4661
0
    while ((sent = quicly_sentmap_get(&iter))->packet_number != UINT64_MAX) {
4662
0
        if (sent->ack_epoch == ack_epoch && sent->frames_in_flight) {
4663
0
            *bytes_to_mark = *bytes_to_mark > sent->cc_bytes_in_flight ? *bytes_to_mark - sent->cc_bytes_in_flight : 0;
4664
0
            if ((ret = quicly_sentmap_update(&conn->egress.loss.sentmap, &iter, QUICLY_SENTMAP_EVENT_PTO)) != 0)
4665
0
                return ret;
4666
0
            assert(!sent->frames_in_flight);
4667
0
            if (*bytes_to_mark == 0)
4668
0
                break;
4669
0
        } else {
4670
0
            quicly_sentmap_skip(&iter);
4671
0
        }
4672
0
    }
4673
4674
0
    return 0;
4675
0
}
4676
4677
static void notify_congestion_to_cc(quicly_conn_t *conn, uint16_t lost_bytes, uint64_t lost_pn)
4678
0
{
4679
0
    if (conn->egress.pn_path_start <= lost_pn) {
4680
0
        conn->egress.cc.type->cc_on_lost(&conn->egress.cc, &conn->egress.loss, lost_bytes, lost_pn, conn->egress.packet_number,
4681
0
                                         conn->stash.now, conn->egress.max_udp_payload_size);
4682
0
        QUICLY_PROBE(CC_CONGESTION, conn, conn->stash.now, lost_pn + 1, conn->egress.loss.sentmap.bytes_in_flight,
4683
0
                     conn->egress.cc.cwnd);
4684
0
        QUICLY_LOG_CONN(cc_congestion, conn, {
4685
0
            PTLS_LOG_ELEMENT_UNSIGNED(max_lost_pn, lost_pn + 1);
4686
0
            PTLS_LOG_ELEMENT_UNSIGNED(flight, conn->egress.loss.sentmap.bytes_in_flight);
4687
0
            PTLS_LOG_ELEMENT_UNSIGNED(cwnd, conn->egress.cc.cwnd);
4688
0
        });
4689
0
    }
4690
0
}
4691
4692
static void on_loss_detected(quicly_loss_t *loss, const quicly_sent_packet_t *lost_packet, int is_time_threshold)
4693
0
{
4694
0
    quicly_conn_t *conn = (void *)((char *)loss - offsetof(quicly_conn_t, egress.loss));
4695
4696
0
    assert(lost_packet->cc_bytes_in_flight != 0);
4697
4698
0
    ++conn->super.stats.num_packets.lost;
4699
0
    if (is_time_threshold)
4700
0
        ++conn->super.stats.num_packets.lost_time_threshold;
4701
0
    conn->super.stats.num_bytes.lost += lost_packet->cc_bytes_in_flight;
4702
0
    QUICLY_PROBE(PACKET_LOST, conn, conn->stash.now, lost_packet->packet_number, lost_packet->ack_epoch);
4703
0
    QUICLY_LOG_CONN(packet_lost, conn, {
4704
0
        PTLS_LOG_ELEMENT_UNSIGNED(pn, lost_packet->packet_number);
4705
0
        PTLS_LOG_ELEMENT_UNSIGNED(packet_type, lost_packet->ack_epoch);
4706
0
    });
4707
0
    notify_congestion_to_cc(conn, lost_packet->cc_bytes_in_flight, lost_packet->packet_number);
4708
0
    QUICLY_PROBE(QUICTRACE_CC_LOST, conn, conn->stash.now, &conn->egress.loss.rtt, conn->egress.cc.cwnd,
4709
0
                 conn->egress.loss.sentmap.bytes_in_flight);
4710
0
}
4711
4712
static quicly_error_t send_max_streams(quicly_conn_t *conn, int uni, quicly_send_context_t *s)
4713
0
{
4714
0
    if (!should_send_max_streams(conn, uni))
4715
0
        return 0;
4716
4717
0
    quicly_maxsender_t *maxsender = uni ? &conn->ingress.max_streams.uni : &conn->ingress.max_streams.bidi;
4718
0
    struct st_quicly_conn_streamgroup_state_t *group = uni ? &conn->super.remote.uni : &conn->super.remote.bidi;
4719
0
    quicly_error_t ret;
4720
4721
0
    uint64_t new_count =
4722
0
        group->next_stream_id / 4 +
4723
0
        (uni ? conn->super.ctx->transport_params.max_streams_uni : conn->super.ctx->transport_params.max_streams_bidi) -
4724
0
        group->num_streams;
4725
4726
0
    quicly_sent_t *sent;
4727
0
    if ((ret = allocate_ack_eliciting_frame(conn, s, QUICLY_MAX_STREAMS_FRAME_CAPACITY, &sent, on_ack_max_streams)) != 0)
4728
0
        return ret;
4729
0
    s->dst = quicly_encode_max_streams_frame(s->dst, uni, new_count);
4730
0
    sent->data.max_streams.uni = uni;
4731
0
    quicly_maxsender_record(maxsender, new_count, &sent->data.max_streams.args);
4732
4733
0
    if (uni) {
4734
0
        ++conn->super.stats.num_frames_sent.max_streams_uni;
4735
0
    } else {
4736
0
        ++conn->super.stats.num_frames_sent.max_streams_bidi;
4737
0
    }
4738
0
    QUICLY_PROBE(MAX_STREAMS_SEND, conn, conn->stash.now, new_count, uni);
4739
0
    QUICLY_LOG_CONN(max_streams_send, conn, {
4740
0
        PTLS_LOG_ELEMENT_UNSIGNED(maximum, new_count);
4741
0
        PTLS_LOG_ELEMENT_BOOL(is_unidirectional, uni);
4742
0
    });
4743
4744
0
    return 0;
4745
0
}
4746
4747
static quicly_error_t send_streams_blocked(quicly_conn_t *conn, int uni, quicly_send_context_t *s)
4748
0
{
4749
0
    quicly_linklist_t *blocked_list = uni ? &conn->egress.pending_streams.blocked.uni : &conn->egress.pending_streams.blocked.bidi;
4750
0
    quicly_error_t ret;
4751
4752
0
    if (!quicly_linklist_is_linked(blocked_list))
4753
0
        return 0;
4754
4755
0
    struct st_quicly_max_streams_t *max_streams = uni ? &conn->egress.max_streams.uni : &conn->egress.max_streams.bidi;
4756
0
    quicly_stream_t *oldest_blocked_stream =
4757
0
        (void *)((char *)blocked_list->next - offsetof(quicly_stream_t, _send_aux.pending_link.control));
4758
0
    assert(max_streams->count == oldest_blocked_stream->stream_id / 4);
4759
4760
0
    if (!quicly_maxsender_should_send_blocked(&max_streams->blocked_sender, max_streams->count))
4761
0
        return 0;
4762
4763
0
    quicly_sent_t *sent;
4764
0
    if ((ret = allocate_ack_eliciting_frame(conn, s, QUICLY_STREAMS_BLOCKED_FRAME_CAPACITY, &sent, on_ack_streams_blocked)) != 0)
4765
0
        return ret;
4766
0
    s->dst = quicly_encode_streams_blocked_frame(s->dst, uni, max_streams->count);
4767
0
    sent->data.streams_blocked.uni = uni;
4768
0
    quicly_maxsender_record(&max_streams->blocked_sender, max_streams->count, &sent->data.streams_blocked.args);
4769
4770
0
    ++conn->super.stats.num_frames_sent.streams_blocked;
4771
0
    QUICLY_PROBE(STREAMS_BLOCKED_SEND, conn, conn->stash.now, max_streams->count, uni);
4772
0
    QUICLY_LOG_CONN(streams_blocked_send, conn, {
4773
0
        PTLS_LOG_ELEMENT_UNSIGNED(maximum, max_streams->count);
4774
0
        PTLS_LOG_ELEMENT_BOOL(is_unidirectional, uni);
4775
0
    });
4776
4777
0
    return 0;
4778
0
}
4779
4780
static void open_blocked_streams(quicly_conn_t *conn, int uni)
4781
0
{
4782
0
    uint64_t count;
4783
0
    quicly_linklist_t *anchor;
4784
4785
0
    if (uni) {
4786
0
        count = conn->egress.max_streams.uni.count;
4787
0
        anchor = &conn->egress.pending_streams.blocked.uni;
4788
0
    } else {
4789
0
        count = conn->egress.max_streams.bidi.count;
4790
0
        anchor = &conn->egress.pending_streams.blocked.bidi;
4791
0
    }
4792
4793
0
    while (quicly_linklist_is_linked(anchor)) {
4794
0
        quicly_stream_t *stream = (void *)((char *)anchor->next - offsetof(quicly_stream_t, _send_aux.pending_link.control));
4795
0
        if (stream->stream_id / 4 >= count)
4796
0
            break;
4797
0
        assert(stream->streams_blocked);
4798
0
        quicly_linklist_unlink(&stream->_send_aux.pending_link.control);
4799
0
        stream->streams_blocked = 0;
4800
0
        stream->_send_aux.max_stream_data = quicly_stream_is_unidirectional(stream->stream_id)
4801
0
                                                ? conn->super.remote.transport_params.max_stream_data.uni
4802
0
                                                : conn->super.remote.transport_params.max_stream_data.bidi_remote;
4803
        /* TODO retain separate flags for stream states so that we do not always need to sched for both control and data */
4804
0
        sched_stream_control(stream);
4805
0
        resched_stream_data(stream);
4806
0
    }
4807
0
}
4808
4809
static quicly_error_t send_handshake_done(quicly_conn_t *conn, quicly_send_context_t *s)
4810
0
{
4811
0
    quicly_sent_t *sent;
4812
0
    quicly_error_t ret;
4813
4814
0
    if ((ret = allocate_ack_eliciting_frame(conn, s, 1, &sent, on_ack_handshake_done)) != 0)
4815
0
        goto Exit;
4816
0
    *s->dst++ = QUICLY_FRAME_TYPE_HANDSHAKE_DONE;
4817
0
    conn->egress.pending_flows &= ~QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT;
4818
0
    ++conn->super.stats.num_frames_sent.handshake_done;
4819
0
    QUICLY_PROBE(HANDSHAKE_DONE_SEND, conn, conn->stash.now);
4820
0
    QUICLY_LOG_CONN(handshake_done_send, conn, {});
4821
4822
0
    ret = 0;
4823
0
Exit:
4824
0
    return ret;
4825
0
}
4826
4827
static quicly_error_t send_data_blocked(quicly_conn_t *conn, quicly_send_context_t *s)
4828
0
{
4829
0
    quicly_sent_t *sent;
4830
0
    quicly_error_t ret;
4831
4832
0
    uint64_t offset = conn->egress.max_data.permitted;
4833
0
    if ((ret = allocate_ack_eliciting_frame(conn, s, QUICLY_DATA_BLOCKED_FRAME_CAPACITY, &sent, on_ack_data_blocked)) != 0)
4834
0
        goto Exit;
4835
0
    sent->data.data_blocked.offset = offset;
4836
0
    s->dst = quicly_encode_data_blocked_frame(s->dst, offset);
4837
0
    conn->egress.data_blocked = QUICLY_SENDER_STATE_UNACKED;
4838
4839
0
    ++conn->super.stats.num_frames_sent.data_blocked;
4840
0
    QUICLY_PROBE(DATA_BLOCKED_SEND, conn, conn->stash.now, offset);
4841
0
    QUICLY_LOG_CONN(data_blocked_send, conn, { PTLS_LOG_ELEMENT_UNSIGNED(off, offset); });
4842
4843
0
    ret = 0;
4844
0
Exit:
4845
0
    return ret;
4846
0
}
4847
4848
#define QUICLY_RESUMPTION_ENTRY_TYPE_CAREFUL_RESUME 0
4849
4850
/**
4851
 * derives size of the new CWND given previous delivery rate and min RTTs of the previous and the new session
4852
 */
4853
static uint32_t derive_jumpstart_cwnd(quicly_context_t *ctx, uint32_t new_rtt, uint64_t prev_rate, uint32_t prev_rtt)
4854
0
{
4855
    /* convert previous rate to CWND size */
4856
0
    double cwnd = (double)prev_rate * prev_rtt / 1000;
4857
4858
    /* if new RTT is smaller, reduce new CWND so that the rate does not become greater than the previous session */
4859
0
    if (new_rtt < prev_rtt)
4860
0
        cwnd = cwnd * new_rtt / prev_rtt;
4861
4862
    /* cap to the configured value */
4863
0
    size_t jumpstart_cwnd =
4864
0
        quicly_cc_calc_initial_cwnd(ctx->max_jumpstart_cwnd_packets, ctx->transport_params.max_udp_payload_size);
4865
0
    if (cwnd > jumpstart_cwnd)
4866
0
        cwnd = jumpstart_cwnd;
4867
4868
0
    return (uint32_t)cwnd;
4869
0
}
4870
4871
static int decode_resumption_info(const uint8_t *src, size_t len, uint64_t *rate, uint32_t *min_rtt)
4872
0
{
4873
0
    const uint8_t *end = src + len;
4874
0
    int ret = 0;
4875
4876
0
    *rate = 0;
4877
4878
0
    while (src < end) {
4879
0
        uint64_t id;
4880
0
        if ((id = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
4881
0
            ret = PTLS_ALERT_DECODE_ERROR;
4882
0
            goto Exit;
4883
0
        }
4884
0
        ptls_decode_open_block(src, end, -1, {
4885
0
            switch (id) {
4886
0
            case QUICLY_RESUMPTION_ENTRY_TYPE_CAREFUL_RESUME: {
4887
0
                if ((*rate = ptls_decode_quicint(&src, end)) == UINT64_MAX) {
4888
0
                    ret = PTLS_ALERT_DECODE_ERROR;
4889
0
                    goto Exit;
4890
0
                }
4891
0
                uint64_t v;
4892
0
                if ((v = ptls_decode_quicint(&src, end)) > UINT32_MAX) {
4893
0
                    ret = PTLS_ALERT_DECODE_ERROR;
4894
0
                    goto Exit;
4895
0
                }
4896
0
                *min_rtt = (uint32_t)v;
4897
0
            } break;
4898
0
            default:
4899
                /* ignore unknown types */
4900
0
                src = end;
4901
0
                break;
4902
0
            }
4903
0
        });
4904
0
    }
4905
4906
0
Exit:
4907
0
    return ret;
4908
0
}
4909
4910
static size_t encode_resumption_info(quicly_conn_t *conn, uint8_t *dst, size_t capacity)
4911
0
{
4912
0
    ptls_buffer_t buf;
4913
0
    int ret;
4914
4915
0
    ptls_buffer_init(&buf, dst, capacity);
4916
4917
0
#define PUSH_ENTRY(id, block)                                                                                                      \
4918
0
    do {                                                                                                                           \
4919
0
        ptls_buffer_push_quicint(&buf, (id));                                                                                      \
4920
0
        ptls_buffer_push_block(&buf, -1, block);                                                                                   \
4921
0
    } while (0)
4922
4923
    /* emit delivery rate for Careful Resume */
4924
0
    if (conn->super.stats.token_sent.rate != 0 && conn->super.stats.token_sent.rtt != 0) {
4925
0
        PUSH_ENTRY(QUICLY_RESUMPTION_ENTRY_TYPE_CAREFUL_RESUME, {
4926
0
            ptls_buffer_push_quicint(&buf, conn->super.stats.token_sent.rate);
4927
0
            ptls_buffer_push_quicint(&buf, conn->super.stats.token_sent.rtt);
4928
0
        });
4929
0
    }
4930
4931
0
#undef PUSH_ENTRY
4932
4933
0
Exit:
4934
0
    assert(!buf.is_allocated);
4935
0
    return buf.off;
4936
0
}
4937
4938
static quicly_error_t send_resumption_token(quicly_conn_t *conn, quicly_send_context_t *s)
4939
0
{
4940
    /* fill conn->super.stats.token_sent the information we are sending now */
4941
0
    calc_resume_sendrate(conn, &conn->super.stats.token_sent.rate, &conn->super.stats.token_sent.rtt);
4942
4943
0
    quicly_address_token_plaintext_t token;
4944
0
    ptls_buffer_t tokenbuf;
4945
0
    uint8_t tokenbuf_small[128];
4946
0
    quicly_sent_t *sent;
4947
0
    quicly_error_t ret;
4948
4949
0
    ptls_buffer_init(&tokenbuf, tokenbuf_small, sizeof(tokenbuf_small));
4950
4951
    /* build token */
4952
0
    token =
4953
0
        (quicly_address_token_plaintext_t){QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION, conn->super.ctx->now->cb(conn->super.ctx->now)};
4954
0
    token.remote = conn->paths[0]->address.remote;
4955
0
    token.resumption.len = encode_resumption_info(conn, token.resumption.bytes, sizeof(token.resumption.bytes));
4956
4957
    /* encrypt */
4958
0
    if ((ret = conn->super.ctx->generate_resumption_token->cb(conn->super.ctx->generate_resumption_token, conn, &tokenbuf,
4959
0
                                                              &token)) != 0)
4960
0
        goto Exit;
4961
0
    assert(tokenbuf.off < QUICLY_MIN_CLIENT_INITIAL_SIZE / 2 && "this is a ballpark figure, but tokens ought to be small");
4962
4963
    /* emit frame */
4964
0
    if ((ret = allocate_ack_eliciting_frame(conn, s, quicly_new_token_frame_capacity(ptls_iovec_init(tokenbuf.base, tokenbuf.off)),
4965
0
                                            &sent, on_ack_new_token)) != 0)
4966
0
        goto Exit;
4967
0
    ++conn->egress.new_token.num_inflight;
4968
0
    sent->data.new_token.is_inflight = 1;
4969
0
    sent->data.new_token.generation = conn->egress.new_token.generation;
4970
0
    s->dst = quicly_encode_new_token_frame(s->dst, ptls_iovec_init(tokenbuf.base, tokenbuf.off));
4971
0
    conn->egress.pending_flows &= ~QUICLY_PENDING_FLOW_NEW_TOKEN_BIT;
4972
4973
0
    ++conn->super.stats.num_frames_sent.new_token;
4974
0
    QUICLY_PROBE(NEW_TOKEN_SEND, conn, conn->stash.now, tokenbuf.base, tokenbuf.off, sent->data.new_token.generation);
4975
0
    QUICLY_LOG_CONN(new_token_send, conn, {
4976
0
        PTLS_LOG_ELEMENT_HEXDUMP(token, tokenbuf.base, tokenbuf.off);
4977
0
        PTLS_LOG_ELEMENT_UNSIGNED(generation, sent->data.new_token.generation);
4978
0
    });
4979
0
    ret = 0;
4980
0
Exit:
4981
0
    ptls_buffer_dispose(&tokenbuf);
4982
0
    return ret;
4983
0
}
4984
4985
size_t quicly_send_version_negotiation(quicly_context_t *ctx, ptls_iovec_t dest_cid, ptls_iovec_t src_cid, const uint32_t *versions,
4986
                                       void *payload)
4987
0
{
4988
0
    uint8_t *dst = payload;
4989
4990
    /* type_flags */
4991
0
    ctx->tls->random_bytes(dst, 1);
4992
0
    *dst |= QUICLY_LONG_HEADER_BIT;
4993
0
    ++dst;
4994
    /* version */
4995
0
    dst = quicly_encode32(dst, 0);
4996
    /* connection-id */
4997
0
    *dst++ = dest_cid.len;
4998
0
    if (dest_cid.len != 0) {
4999
0
        memcpy(dst, dest_cid.base, dest_cid.len);
5000
0
        dst += dest_cid.len;
5001
0
    }
5002
0
    *dst++ = src_cid.len;
5003
0
    if (src_cid.len != 0) {
5004
0
        memcpy(dst, src_cid.base, src_cid.len);
5005
0
        dst += src_cid.len;
5006
0
    }
5007
    /* supported_versions */
5008
0
    for (const uint32_t *v = versions; *v != 0; ++v)
5009
0
        dst = quicly_encode32(dst, *v);
5010
    /* add a greasing version. This also covers the case where an empty list is specified by the caller to indicate rejection. */
5011
0
    uint32_t grease_version = 0;
5012
0
    if (src_cid.len >= sizeof(grease_version))
5013
0
        memcpy(&grease_version, src_cid.base, sizeof(grease_version));
5014
0
    grease_version = (grease_version & 0xf0f0f0f0) | 0x0a0a0a0a;
5015
0
    dst = quicly_encode32(dst, grease_version);
5016
5017
0
    return dst - (uint8_t *)payload;
5018
0
}
5019
5020
quicly_error_t quicly_retry_calc_cidpair_hash(ptls_hash_algorithm_t *sha256, ptls_iovec_t client_cid, ptls_iovec_t server_cid,
5021
                                              uint64_t *value)
5022
0
{
5023
0
    uint8_t digest[PTLS_SHA256_DIGEST_SIZE], buf[(QUICLY_MAX_CID_LEN_V1 + 1) * 2], *p = buf;
5024
0
    int ret;
5025
5026
0
    *p++ = (uint8_t)client_cid.len;
5027
0
    memcpy(p, client_cid.base, client_cid.len);
5028
0
    p += client_cid.len;
5029
0
    *p++ = (uint8_t)server_cid.len;
5030
0
    memcpy(p, server_cid.base, server_cid.len);
5031
0
    p += server_cid.len;
5032
5033
0
    if ((ret = ptls_calc_hash(sha256, digest, buf, p - buf)) != 0)
5034
0
        return ret;
5035
0
    p = digest;
5036
0
    *value = quicly_decode64((void *)&p);
5037
5038
0
    return 0;
5039
0
}
5040
5041
size_t quicly_send_retry(quicly_context_t *ctx, ptls_aead_context_t *token_encrypt_ctx, uint32_t protocol_version,
5042
                         struct sockaddr *dest_addr, ptls_iovec_t dest_cid, struct sockaddr *src_addr, ptls_iovec_t src_cid,
5043
                         ptls_iovec_t odcid, ptls_iovec_t token_prefix, ptls_iovec_t appdata,
5044
                         ptls_aead_context_t **retry_aead_cache, uint8_t *datagram)
5045
0
{
5046
0
    quicly_address_token_plaintext_t token;
5047
0
    ptls_buffer_t buf;
5048
0
    quicly_error_t ret;
5049
5050
0
    assert(!(src_cid.len == odcid.len && memcmp(src_cid.base, odcid.base, src_cid.len) == 0));
5051
5052
    /* build token as plaintext */
5053
0
    token = (quicly_address_token_plaintext_t){QUICLY_ADDRESS_TOKEN_TYPE_RETRY, ctx->now->cb(ctx->now)};
5054
0
    set_address(&token.remote, dest_addr);
5055
0
    set_address(&token.local, src_addr);
5056
5057
0
    quicly_set_cid(&token.retry.original_dcid, odcid);
5058
0
    quicly_set_cid(&token.retry.client_cid, dest_cid);
5059
0
    quicly_set_cid(&token.retry.server_cid, src_cid);
5060
0
    if (appdata.len != 0) {
5061
0
        assert(appdata.len <= sizeof(token.appdata.bytes));
5062
0
        memcpy(token.appdata.bytes, appdata.base, appdata.len);
5063
0
        token.appdata.len = appdata.len;
5064
0
    }
5065
5066
    /* start building the packet */
5067
0
    ptls_buffer_init(&buf, datagram, QUICLY_MIN_CLIENT_INITIAL_SIZE);
5068
5069
    /* first generate a pseudo packet */
5070
0
    ptls_buffer_push_block(&buf, 1, { ptls_buffer_pushv(&buf, odcid.base, odcid.len); });
5071
0
    ctx->tls->random_bytes(buf.base + buf.off, 1);
5072
0
    buf.base[buf.off] = QUICLY_PACKET_TYPE_RETRY | (buf.base[buf.off] & 0x0f);
5073
0
    ++buf.off;
5074
0
    ptls_buffer_push32(&buf, protocol_version);
5075
0
    ptls_buffer_push_block(&buf, 1, { ptls_buffer_pushv(&buf, dest_cid.base, dest_cid.len); });
5076
0
    ptls_buffer_push_block(&buf, 1, { ptls_buffer_pushv(&buf, src_cid.base, src_cid.len); });
5077
0
    if (token_prefix.len != 0) {
5078
0
        assert(token_prefix.len <= buf.capacity - buf.off);
5079
0
        memcpy(buf.base + buf.off, token_prefix.base, token_prefix.len);
5080
0
        buf.off += token_prefix.len;
5081
0
    }
5082
0
    if ((ret = quicly_encrypt_address_token(ctx->tls->random_bytes, token_encrypt_ctx, &buf, buf.off - token_prefix.len, &token)) !=
5083
0
        0)
5084
0
        goto Exit;
5085
5086
    /* append AEAD tag */
5087
0
    ret = ptls_buffer_reserve(&buf, PTLS_AESGCM_TAG_SIZE);
5088
0
    assert(ret == 0);
5089
0
    assert(!buf.is_allocated && "retry packet is too large");
5090
0
    {
5091
0
        ptls_aead_context_t *aead =
5092
0
            retry_aead_cache != NULL && *retry_aead_cache != NULL ? *retry_aead_cache : create_retry_aead(ctx, protocol_version, 1);
5093
0
        ptls_aead_encrypt(aead, buf.base + buf.off, "", 0, 0, buf.base, buf.off);
5094
0
        if (retry_aead_cache != NULL) {
5095
0
            *retry_aead_cache = aead;
5096
0
        } else {
5097
0
            ptls_aead_free(aead);
5098
0
        }
5099
0
    }
5100
0
    buf.off += PTLS_AESGCM_TAG_SIZE;
5101
5102
    /* convert the image to a Retry packet, by stripping the ODCID field */
5103
0
    memmove(buf.base, buf.base + odcid.len + 1, buf.off - (odcid.len + 1));
5104
0
    buf.off -= odcid.len + 1;
5105
5106
0
    ret = 0;
5107
5108
0
Exit:
5109
0
    return ret == 0 ? buf.off : SIZE_MAX;
5110
0
}
5111
5112
static struct st_quicly_pn_space_t *setup_send_space(quicly_conn_t *conn, size_t epoch, quicly_send_context_t *s)
5113
0
{
5114
0
    struct st_quicly_pn_space_t *space = NULL;
5115
5116
0
    switch (epoch) {
5117
0
    case QUICLY_EPOCH_INITIAL:
5118
0
        if (conn->initial == NULL || (s->current.cipher = &conn->initial->cipher.egress)->aead == NULL)
5119
0
            return NULL;
5120
0
        s->current.first_byte = QUICLY_PACKET_TYPE_INITIAL;
5121
0
        space = &conn->initial->super;
5122
0
        break;
5123
0
    case QUICLY_EPOCH_HANDSHAKE:
5124
0
        if (conn->handshake == NULL || (s->current.cipher = &conn->handshake->cipher.egress)->aead == NULL)
5125
0
            return NULL;
5126
0
        s->current.first_byte = QUICLY_PACKET_TYPE_HANDSHAKE;
5127
0
        space = &conn->handshake->super;
5128
0
        break;
5129
0
    case QUICLY_EPOCH_0RTT:
5130
0
    case QUICLY_EPOCH_1RTT:
5131
0
        if (conn->application == NULL || conn->application->cipher.egress.key.header_protection == NULL)
5132
0
            return NULL;
5133
0
        if ((epoch == QUICLY_EPOCH_0RTT) == conn->application->one_rtt_writable)
5134
0
            return NULL;
5135
0
        s->current.cipher = &conn->application->cipher.egress.key;
5136
0
        s->current.first_byte = epoch == QUICLY_EPOCH_0RTT ? QUICLY_PACKET_TYPE_0RTT : QUICLY_QUIC_BIT;
5137
0
        space = &conn->application->super;
5138
0
        break;
5139
0
    default:
5140
0
        assert(!"logic flaw");
5141
0
        break;
5142
0
    }
5143
5144
0
    return space;
5145
0
}
5146
5147
static quicly_error_t send_handshake_flow(quicly_conn_t *conn, size_t epoch, quicly_send_context_t *s, int ack_only, int send_probe)
5148
0
{
5149
0
    struct st_quicly_pn_space_t *space;
5150
0
    quicly_error_t ret = 0;
5151
5152
    /* setup send epoch, or return if it's impossible to send in this epoch */
5153
0
    if ((space = setup_send_space(conn, epoch, s)) == NULL)
5154
0
        return 0;
5155
5156
    /* send ACK */
5157
0
    if (space != NULL && (space->unacked_count != 0 || send_probe))
5158
0
        if ((ret = send_ack(conn, space, s)) != 0)
5159
0
            goto Exit;
5160
5161
0
    if (!ack_only) {
5162
        /* send data */
5163
0
        while ((conn->egress.pending_flows & (uint8_t)(1 << epoch)) != 0) {
5164
0
            quicly_stream_t *stream = quicly_get_stream(conn, -(quicly_stream_id_t)(1 + epoch));
5165
0
            assert(stream != NULL);
5166
0
            if ((ret = quicly_send_stream(stream, s)) != 0)
5167
0
                goto Exit;
5168
0
            resched_stream_data(stream);
5169
0
            send_probe = 0;
5170
0
        }
5171
5172
        /* send probe if requested */
5173
0
        if (send_probe) {
5174
0
            if ((ret = do_allocate_frame(conn, s, 1, ALLOCATE_FRAME_TYPE_ACK_ELICITING)) != 0)
5175
0
                goto Exit;
5176
0
            *s->dst++ = QUICLY_FRAME_TYPE_PING;
5177
0
            conn->egress.last_retransmittable_sent_at = conn->stash.now;
5178
0
            ++conn->super.stats.num_frames_sent.ping;
5179
0
            QUICLY_PROBE(PING_SEND, conn, conn->stash.now);
5180
0
            QUICLY_LOG_CONN(ping_send, conn, {});
5181
0
        }
5182
0
    }
5183
5184
0
Exit:
5185
0
    return ret;
5186
0
}
5187
5188
static quicly_error_t send_connection_close(quicly_conn_t *conn, size_t epoch, quicly_send_context_t *s)
5189
0
{
5190
0
    quicly_error_t ret;
5191
5192
0
    assert(!conn->connection_close.is_remote);
5193
5194
    /* setup send epoch, or return if it's impossible to send in this epoch */
5195
0
    if (setup_send_space(conn, epoch, s) == NULL)
5196
0
        return 0;
5197
5198
    /* determine the payload, masking the application error when sending the frame using an unauthenticated epoch */
5199
0
    uint64_t error_code, offending_frame_type = conn->connection_close.frame_type;
5200
0
    const char *reason_phrase = conn->connection_close.reason_phrase;
5201
0
    if (conn->connection_close.err == 0) {
5202
0
        error_code = 0;
5203
0
        offending_frame_type = QUICLY_FRAME_TYPE_PADDING;
5204
0
    } else if (conn->connection_close.err == QUICLY_ERROR_STATE_EXHAUSTION) {
5205
        /* State exhaution is an error induced by the peer, but as there is no specific error code, the generic error code
5206
         * (PROTOCOL_VIOLATION) is used. The exact cause is communicated using the reason phrase field because it is sometimes
5207
         * difficult for the peer to understand the problem without; e.g., when an ACK triggering the loss of a RETIRE_CONNECTION_ID
5208
         * frame leading to the overflow of `quicly_conn_t::egress.retire_cid`. */
5209
0
        assert(offending_frame_type != UINT64_MAX);
5210
0
        error_code = QUICLY_ERROR_GET_ERROR_CODE(QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION);
5211
0
        if (reason_phrase[0] == '\0')
5212
0
            reason_phrase = "state exhaustion";
5213
0
    } else if (QUICLY_ERROR_IS_QUIC_TRANSPORT(conn->connection_close.err)) {
5214
0
        assert(offending_frame_type != UINT64_MAX);
5215
0
        error_code = QUICLY_ERROR_GET_ERROR_CODE(conn->connection_close.err);
5216
0
    } else if (QUICLY_ERROR_IS_QUIC_APPLICATION(conn->connection_close.err)) {
5217
        /* conceal application errors unless sending in the application packet number space */
5218
0
        assert(offending_frame_type == UINT64_MAX);
5219
0
        switch (get_epoch(s->current.first_byte)) {
5220
0
        case QUICLY_EPOCH_INITIAL:
5221
0
        case QUICLY_EPOCH_HANDSHAKE:
5222
0
            error_code = QUICLY_ERROR_GET_ERROR_CODE(QUICLY_TRANSPORT_ERROR_APPLICATION);
5223
0
            offending_frame_type = QUICLY_FRAME_TYPE_PADDING;
5224
0
            reason_phrase = "";
5225
0
            break;
5226
0
        default:
5227
0
            error_code = QUICLY_ERROR_GET_ERROR_CODE(conn->connection_close.err);
5228
0
            break;
5229
0
        }
5230
0
    } else if (PTLS_ERROR_GET_CLASS(conn->connection_close.err) == PTLS_ERROR_CLASS_SELF_ALERT) {
5231
0
        assert(offending_frame_type != UINT64_MAX);
5232
0
        error_code = QUICLY_ERROR_GET_ERROR_CODE(QUICLY_TRANSPORT_ERROR_CRYPTO(PTLS_ERROR_TO_ALERT(conn->connection_close.err)));
5233
0
    } else {
5234
0
        assert(offending_frame_type != UINT64_MAX);
5235
0
        error_code = QUICLY_ERROR_GET_ERROR_CODE(QUICLY_TRANSPORT_ERROR_INTERNAL);
5236
0
    }
5237
5238
    /* write frame */
5239
0
    if ((ret = do_allocate_frame(conn, s, quicly_close_frame_capacity(error_code, offending_frame_type, reason_phrase),
5240
0
                                 ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING)) != 0)
5241
0
        return ret;
5242
0
    s->dst = quicly_encode_close_frame(s->dst, error_code, offending_frame_type, reason_phrase);
5243
5244
    /* update counter, probe */
5245
0
    if (offending_frame_type != UINT64_MAX) {
5246
0
        ++conn->super.stats.num_frames_sent.transport_close;
5247
0
        QUICLY_PROBE(TRANSPORT_CLOSE_SEND, conn, conn->stash.now, error_code, offending_frame_type, reason_phrase);
5248
0
        QUICLY_LOG_CONN(transport_close_send, conn, {
5249
0
            PTLS_LOG_ELEMENT_UNSIGNED(error_code, error_code);
5250
0
            PTLS_LOG_ELEMENT_UNSIGNED(frame_type, offending_frame_type);
5251
0
            PTLS_LOG_ELEMENT_UNSAFESTR(reason_phrase, reason_phrase, strlen(reason_phrase));
5252
0
        });
5253
0
    } else {
5254
0
        ++conn->super.stats.num_frames_sent.application_close;
5255
0
        QUICLY_PROBE(APPLICATION_CLOSE_SEND, conn, conn->stash.now, error_code, reason_phrase);
5256
0
        QUICLY_LOG_CONN(application_close_send, conn, {
5257
0
            PTLS_LOG_ELEMENT_UNSIGNED(error_code, error_code);
5258
0
            PTLS_LOG_ELEMENT_UNSAFESTR(reason_phrase, reason_phrase, strlen(reason_phrase));
5259
0
        });
5260
0
    }
5261
5262
0
    return 0;
5263
0
}
5264
5265
static quicly_error_t send_new_connection_id(quicly_conn_t *conn, quicly_send_context_t *s, struct st_quicly_local_cid_t *new_cid)
5266
0
{
5267
0
    quicly_sent_t *sent;
5268
0
    uint64_t retire_prior_to = 0; /* TODO */
5269
0
    quicly_error_t ret;
5270
5271
0
    if ((ret = allocate_ack_eliciting_frame(
5272
0
             conn, s, quicly_new_connection_id_frame_capacity(new_cid->sequence, retire_prior_to, new_cid->cid.len), &sent,
5273
0
             on_ack_new_connection_id)) != 0)
5274
0
        return ret;
5275
0
    sent->data.new_connection_id.sequence = new_cid->sequence;
5276
5277
0
    s->dst = quicly_encode_new_connection_id_frame(s->dst, new_cid->sequence, retire_prior_to, new_cid->cid.cid, new_cid->cid.len,
5278
0
                                                   new_cid->stateless_reset_token);
5279
5280
0
    ++conn->super.stats.num_frames_sent.new_connection_id;
5281
0
    QUICLY_PROBE(NEW_CONNECTION_ID_SEND, conn, conn->stash.now, new_cid->sequence, retire_prior_to,
5282
0
                 QUICLY_PROBE_HEXDUMP(new_cid->cid.cid, new_cid->cid.len),
5283
0
                 QUICLY_PROBE_HEXDUMP(new_cid->stateless_reset_token, QUICLY_STATELESS_RESET_TOKEN_LEN));
5284
0
    QUICLY_LOG_CONN(new_connection_id_send, conn, {
5285
0
        PTLS_LOG_ELEMENT_UNSIGNED(sequence, new_cid->sequence);
5286
0
        PTLS_LOG_ELEMENT_UNSIGNED(retire_prior_to, retire_prior_to);
5287
0
        PTLS_LOG_ELEMENT_HEXDUMP(cid, new_cid->cid.cid, new_cid->cid.len);
5288
0
        PTLS_LOG_ELEMENT_HEXDUMP(stateless_reset_token, new_cid->stateless_reset_token, QUICLY_STATELESS_RESET_TOKEN_LEN);
5289
0
    });
5290
5291
0
    return 0;
5292
0
}
5293
5294
static quicly_error_t send_retire_connection_id(quicly_conn_t *conn, quicly_send_context_t *s, uint64_t sequence)
5295
0
{
5296
0
    quicly_sent_t *sent;
5297
0
    quicly_error_t ret;
5298
5299
0
    if ((ret = allocate_ack_eliciting_frame(conn, s, quicly_retire_connection_id_frame_capacity(sequence), &sent,
5300
0
                                            on_ack_retire_connection_id)) != 0)
5301
0
        return ret;
5302
0
    sent->data.retire_connection_id.sequence = sequence;
5303
5304
0
    s->dst = quicly_encode_retire_connection_id_frame(s->dst, sequence);
5305
5306
0
    ++conn->super.stats.num_frames_sent.retire_connection_id;
5307
0
    QUICLY_PROBE(RETIRE_CONNECTION_ID_SEND, conn, conn->stash.now, sequence);
5308
0
    QUICLY_LOG_CONN(retire_connection_id_send, conn, { PTLS_LOG_ELEMENT_UNSIGNED(sequence, sequence); });
5309
5310
0
    return 0;
5311
0
}
5312
5313
static quicly_error_t send_path_challenge(quicly_conn_t *conn, quicly_send_context_t *s, int is_response, const uint8_t *data)
5314
0
{
5315
0
    quicly_error_t ret;
5316
5317
0
    if ((ret = do_allocate_frame(conn, s, QUICLY_PATH_CHALLENGE_FRAME_CAPACITY, ALLOCATE_FRAME_TYPE_NON_ACK_ELICITING)) != 0)
5318
0
        return ret;
5319
5320
0
    s->dst = quicly_encode_path_challenge_frame(s->dst, is_response, data);
5321
0
    s->target.full_size = 1; /* ensure that the path can transfer full-size packets */
5322
5323
0
    if (!is_response) {
5324
0
        ++conn->super.stats.num_frames_sent.path_challenge;
5325
0
        QUICLY_PROBE(PATH_CHALLENGE_SEND, conn, conn->stash.now, data, QUICLY_PATH_CHALLENGE_DATA_LEN);
5326
0
        QUICLY_LOG_CONN(path_challenge_send, conn, { PTLS_LOG_ELEMENT_HEXDUMP(data, data, QUICLY_PATH_CHALLENGE_DATA_LEN); });
5327
0
    } else {
5328
0
        ++conn->super.stats.num_frames_sent.path_response;
5329
0
        QUICLY_PROBE(PATH_RESPONSE_SEND, conn, conn->stash.now, data, QUICLY_PATH_CHALLENGE_DATA_LEN);
5330
0
        QUICLY_LOG_CONN(path_response_send, conn, { PTLS_LOG_ELEMENT_HEXDUMP(data, data, QUICLY_PATH_CHALLENGE_DATA_LEN); });
5331
0
    }
5332
5333
0
    return 0;
5334
0
}
5335
5336
static int update_traffic_key_cb(ptls_update_traffic_key_t *self, ptls_t *tls, int is_enc, size_t epoch, const void *secret)
5337
0
{
5338
0
    quicly_conn_t *conn = *ptls_get_data_ptr(tls);
5339
0
    ptls_context_t *tlsctx = ptls_get_context(tls);
5340
0
    ptls_cipher_suite_t *cipher = ptls_get_cipher(tls);
5341
0
    ptls_cipher_context_t **hp_slot;
5342
0
    ptls_aead_context_t **aead_slot;
5343
0
    int ret;
5344
0
    static const char *log_labels[2][4] = {
5345
0
        {NULL, "CLIENT_EARLY_TRAFFIC_SECRET", "CLIENT_HANDSHAKE_TRAFFIC_SECRET", "CLIENT_TRAFFIC_SECRET_0"},
5346
0
        {NULL, NULL, "SERVER_HANDSHAKE_TRAFFIC_SECRET", "SERVER_TRAFFIC_SECRET_0"}};
5347
0
    const char *log_label = log_labels[ptls_is_server(tls) == is_enc][epoch];
5348
5349
0
    QUICLY_PROBE(CRYPTO_UPDATE_SECRET, conn, conn->stash.now, is_enc, epoch, log_label,
5350
0
                 QUICLY_PROBE_HEXDUMP(secret, cipher->hash->digest_size));
5351
0
    QUICLY_LOG_CONN(crypto_update_secret, conn, {
5352
0
        PTLS_LOG_ELEMENT_BOOL(is_enc, is_enc);
5353
0
        PTLS_LOG_ELEMENT_UNSIGNED(epoch, epoch);
5354
0
        PTLS_LOG_ELEMENT_SAFESTR(label, log_label);
5355
0
        PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(secret, secret, cipher->hash->digest_size);
5356
0
    });
5357
5358
0
    if (tlsctx->log_event != NULL) {
5359
0
        char hexbuf[PTLS_MAX_DIGEST_SIZE * 2 + 1];
5360
0
        ptls_hexdump(hexbuf, secret, cipher->hash->digest_size);
5361
0
        tlsctx->log_event->cb(tlsctx->log_event, tls, log_label, "%s", hexbuf);
5362
0
    }
5363
5364
0
#define SELECT_CIPHER_CONTEXT(p)                                                                                                   \
5365
0
    do {                                                                                                                           \
5366
0
        hp_slot = &(p)->header_protection;                                                                                         \
5367
0
        aead_slot = &(p)->aead;                                                                                                    \
5368
0
    } while (0)
5369
5370
0
    switch (epoch) {
5371
0
    case QUICLY_EPOCH_0RTT:
5372
0
        assert(is_enc == quicly_is_client(conn));
5373
0
        if (conn->application == NULL && (ret = setup_application_space(conn)) != 0)
5374
0
            return ret;
5375
0
        if (is_enc) {
5376
0
            SELECT_CIPHER_CONTEXT(&conn->application->cipher.egress.key);
5377
0
        } else {
5378
0
            hp_slot = &conn->application->cipher.ingress.header_protection.zero_rtt;
5379
0
            aead_slot = &conn->application->cipher.ingress.aead[1];
5380
0
            conn->delayed_packets.slots_newly_processible |= 1
5381
0
                                                             << (&conn->delayed_packets.zero_rtt - conn->delayed_packets.as_array);
5382
0
        }
5383
0
        break;
5384
0
    case QUICLY_EPOCH_HANDSHAKE:
5385
0
        if (conn->handshake == NULL && (ret = setup_handshake_space_and_flow(conn, QUICLY_EPOCH_HANDSHAKE)) != 0)
5386
0
            return ret;
5387
0
        SELECT_CIPHER_CONTEXT(is_enc ? &conn->handshake->cipher.egress : &conn->handshake->cipher.ingress);
5388
0
        if (!is_enc)
5389
0
            conn->delayed_packets.slots_newly_processible |= 1
5390
0
                                                             << (&conn->delayed_packets.handshake - conn->delayed_packets.as_array);
5391
0
        break;
5392
0
    case QUICLY_EPOCH_1RTT: {
5393
0
        if (is_enc)
5394
0
            if ((ret = compress_handshake_result(apply_remote_transport_params(conn))) != 0)
5395
0
                return ret;
5396
0
        if (conn->application == NULL && (ret = setup_application_space(conn)) != 0)
5397
0
            return ret;
5398
0
        uint8_t *secret_store;
5399
0
        if (is_enc) {
5400
0
            if (conn->application->cipher.egress.key.aead != NULL)
5401
0
                dispose_cipher(&conn->application->cipher.egress.key);
5402
0
            SELECT_CIPHER_CONTEXT(&conn->application->cipher.egress.key);
5403
0
            secret_store = conn->application->cipher.egress.secret;
5404
0
        } else {
5405
0
            hp_slot = &conn->application->cipher.ingress.header_protection.one_rtt;
5406
0
            aead_slot = &conn->application->cipher.ingress.aead[0];
5407
0
            secret_store = conn->application->cipher.ingress.secret;
5408
0
            conn->delayed_packets.slots_newly_processible |= 1 << (&conn->delayed_packets.one_rtt - conn->delayed_packets.as_array);
5409
0
        }
5410
0
        memcpy(secret_store, secret, cipher->hash->digest_size);
5411
0
    } break;
5412
0
    default:
5413
0
        assert(!"logic flaw");
5414
0
        break;
5415
0
    }
5416
5417
0
#undef SELECT_CIPHER_CONTEXT
5418
5419
0
    if ((ret = setup_cipher(conn, epoch, is_enc, hp_slot, aead_slot, cipher->aead, cipher->hash, secret)) != 0)
5420
0
        return ret;
5421
5422
0
    if (epoch == QUICLY_EPOCH_1RTT && is_enc) {
5423
        /* update states now that we have 1-RTT write key */
5424
0
        conn->application->one_rtt_writable = 1;
5425
0
        open_blocked_streams(conn, 1);
5426
0
        open_blocked_streams(conn, 0);
5427
0
        if (quicly_linklist_is_linked(&conn->egress.pending_streams.blocked.bidi) ||
5428
0
            quicly_linklist_is_linked(&conn->egress.pending_streams.blocked.uni))
5429
0
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
5430
        /* send the first resumption token using the 0.5 RTT window */
5431
0
        if (!quicly_is_client(conn) && conn->super.ctx->generate_resumption_token != NULL) {
5432
0
            quicly_error_t ret64 = quicly_send_resumption_token(conn);
5433
0
            assert(ret64 == 0);
5434
0
        }
5435
5436
        /* schedule NEW_CONNECTION_IDs */
5437
0
        size_t size = local_cid_size(conn);
5438
0
        if (quicly_local_cid_set_size(&conn->super.local.cid_set, size))
5439
0
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
5440
0
    }
5441
5442
0
    return 0;
5443
0
}
5444
5445
static quicly_error_t send_other_control_frames(quicly_conn_t *conn, quicly_send_context_t *s)
5446
0
{
5447
0
    quicly_error_t ret;
5448
5449
    /* MAX_STREAMS */
5450
0
    if ((ret = send_max_streams(conn, 1, s)) != 0)
5451
0
        return ret;
5452
0
    if ((ret = send_max_streams(conn, 0, s)) != 0)
5453
0
        return ret;
5454
5455
    /* MAX_DATA */
5456
0
    if (should_send_max_data(conn)) {
5457
0
        quicly_sent_t *sent;
5458
0
        if ((ret = allocate_ack_eliciting_frame(conn, s, QUICLY_MAX_DATA_FRAME_CAPACITY, &sent, on_ack_max_data)) != 0)
5459
0
            return ret;
5460
0
        uint64_t new_value = conn->ingress.max_data.bytes_shifted + conn->super.ctx->transport_params.max_data;
5461
0
        s->dst = quicly_encode_max_data_frame(s->dst, new_value);
5462
0
        quicly_maxsender_record(&conn->ingress.max_data.sender, new_value, &sent->data.max_data.args);
5463
0
        ++conn->super.stats.num_frames_sent.max_data;
5464
0
        QUICLY_PROBE(MAX_DATA_SEND, conn, conn->stash.now, new_value);
5465
0
        QUICLY_LOG_CONN(max_data_send, conn, { PTLS_LOG_ELEMENT_UNSIGNED(maximum, new_value); });
5466
0
    }
5467
5468
    /* DATA_BLOCKED */
5469
0
    if (conn->egress.data_blocked == QUICLY_SENDER_STATE_SEND && (ret = send_data_blocked(conn, s)) != 0)
5470
0
        return ret;
5471
5472
    /* STREAMS_BLOCKED */
5473
0
    if ((ret = send_streams_blocked(conn, 1, s)) != 0)
5474
0
        return ret;
5475
0
    if ((ret = send_streams_blocked(conn, 0, s)) != 0)
5476
0
        return ret;
5477
5478
0
    { /* NEW_CONNECTION_ID */
5479
0
        size_t i, size = quicly_local_cid_get_size(&conn->super.local.cid_set);
5480
0
        for (i = 0; i < size; i++) {
5481
            /* PENDING CIDs are located at the front */
5482
0
            struct st_quicly_local_cid_t *c = &conn->super.local.cid_set.cids[i];
5483
0
            if (c->state != QUICLY_LOCAL_CID_STATE_PENDING)
5484
0
                break;
5485
0
            if ((ret = send_new_connection_id(conn, s, c)) != 0)
5486
0
                break;
5487
0
        }
5488
0
        quicly_local_cid_on_sent(&conn->super.local.cid_set, i);
5489
0
        if (ret != 0)
5490
0
            return ret;
5491
0
    }
5492
5493
0
    { /* RETIRE_CONNECTION_ID */
5494
0
        size_t i;
5495
0
        for (i = 0; i < conn->super.remote.cid_set.retired.count; ++i) {
5496
0
            uint64_t sequence = conn->super.remote.cid_set.retired.cids[i];
5497
0
            if ((ret = send_retire_connection_id(conn, s, sequence)) != 0)
5498
0
                break;
5499
0
        }
5500
0
        quicly_remote_cid_shift_retired(&conn->super.remote.cid_set, i);
5501
0
        if (ret != 0)
5502
0
            return ret;
5503
0
    }
5504
5505
0
    return 0;
5506
0
}
5507
5508
static quicly_error_t do_send(quicly_conn_t *conn, quicly_send_context_t *s)
5509
0
{
5510
0
    int restrict_sending = 0, ack_only = 0;
5511
0
    size_t min_packets_to_send = 0, orig_bytes_inflight = 0;
5512
0
    quicly_error_t ret = 0;
5513
5514
    /* handle timeouts */
5515
0
    if (conn->idle_timeout.at <= conn->stash.now) {
5516
0
        QUICLY_PROBE(IDLE_TIMEOUT, conn, conn->stash.now);
5517
0
        QUICLY_LOG_CONN(idle_timeout, conn, {});
5518
0
        goto CloseNow;
5519
0
    }
5520
    /* handle handshake timeouts */
5521
0
    if ((conn->initial != NULL || conn->handshake != NULL) &&
5522
0
        conn->created_at + (uint64_t)conn->super.ctx->handshake_timeout_rtt_multiplier * conn->egress.loss.rtt.smoothed <=
5523
0
            conn->stash.now) {
5524
0
        QUICLY_PROBE(HANDSHAKE_TIMEOUT, conn, conn->stash.now, conn->stash.now - conn->created_at, conn->egress.loss.rtt.smoothed);
5525
0
        QUICLY_LOG_CONN(handshake_timeout, conn, {
5526
0
            PTLS_LOG_ELEMENT_SIGNED(elapsed, conn->stash.now - conn->created_at);
5527
0
            PTLS_LOG_ELEMENT_UNSIGNED(rtt_smoothed, conn->egress.loss.rtt.smoothed);
5528
0
        });
5529
0
        conn->super.stats.num_handshake_timeouts++;
5530
0
        goto CloseNow;
5531
0
    }
5532
0
    uint64_t initial_handshake_sent = conn->super.stats.num_packets.initial_sent + conn->super.stats.num_packets.handshake_sent;
5533
0
    if (initial_handshake_sent > conn->super.ctx->max_initial_handshake_packets) {
5534
0
        QUICLY_PROBE(INITIAL_HANDSHAKE_PACKET_EXCEED, conn, conn->stash.now, initial_handshake_sent);
5535
0
        QUICLY_LOG_CONN(initial_handshake_packet_exceed, conn, { PTLS_LOG_ELEMENT_UNSIGNED(num_packets, initial_handshake_sent); });
5536
0
        conn->super.stats.num_initial_handshake_exceeded++;
5537
0
        goto CloseNow;
5538
0
    }
5539
0
    if (conn->egress.loss.alarm_at <= conn->stash.now) {
5540
0
        if ((ret = quicly_loss_on_alarm(&conn->egress.loss, conn->stash.now, conn->super.remote.transport_params.max_ack_delay,
5541
0
                                        conn->initial == NULL && conn->handshake == NULL, &min_packets_to_send, &restrict_sending,
5542
0
                                        on_loss_detected)) != 0)
5543
0
            goto Exit;
5544
0
        assert(min_packets_to_send > 0);
5545
0
        assert(min_packets_to_send <= s->max_datagrams);
5546
5547
0
        if (restrict_sending) {
5548
            /* PTO: when handshake is in progress, send from the very first unacknowledged byte so as to maximize the chance of
5549
             * making progress. When handshake is complete, transmit new data if any, else retransmit the oldest unacknowledged data
5550
             * that is considered inflight. */
5551
0
            QUICLY_PROBE(PTO, conn, conn->stash.now, conn->egress.loss.sentmap.bytes_in_flight, conn->egress.cc.cwnd,
5552
0
                         conn->egress.loss.pto_count);
5553
0
            QUICLY_LOG_CONN(pto, conn, {
5554
0
                PTLS_LOG_ELEMENT_SIGNED(inflight, conn->egress.loss.sentmap.bytes_in_flight);
5555
0
                PTLS_LOG_ELEMENT_UNSIGNED(cwnd, conn->egress.cc.cwnd);
5556
0
                PTLS_LOG_ELEMENT_SIGNED(pto_count, conn->egress.loss.pto_count);
5557
0
            });
5558
0
            ++conn->super.stats.num_ptos;
5559
0
            size_t bytes_to_mark = min_packets_to_send * conn->egress.max_udp_payload_size;
5560
0
            if (conn->initial != NULL && (ret = mark_frames_on_pto(conn, QUICLY_EPOCH_INITIAL, &bytes_to_mark)) != 0)
5561
0
                goto Exit;
5562
0
            if (bytes_to_mark != 0 && conn->handshake != NULL &&
5563
0
                (ret = mark_frames_on_pto(conn, QUICLY_EPOCH_HANDSHAKE, &bytes_to_mark)) != 0)
5564
0
                goto Exit;
5565
            /* Mark already sent 1-RTT data for PTO only if there's no new data, i.e., when scheduler_can_send() return false. */
5566
0
            if (bytes_to_mark != 0 && !scheduler_can_send(conn) &&
5567
0
                (ret = mark_frames_on_pto(conn, QUICLY_EPOCH_1RTT, &bytes_to_mark)) != 0)
5568
0
                goto Exit;
5569
0
        }
5570
0
    }
5571
5572
    /* disable ECN if zero packets where acked in the first 3 PTO of the connection during which all sent packets are ECT(0) */
5573
0
    if (conn->egress.ecn.state == QUICLY_ECN_PROBING && conn->created_at + conn->egress.loss.rtt.smoothed * 3 < conn->stash.now) {
5574
0
        update_ecn_state(conn, QUICLY_ECN_OFF);
5575
        /* TODO reset CC? */
5576
0
    }
5577
5578
0
    { /* calculate send window */
5579
0
        uint64_t pacer_window = SIZE_MAX;
5580
0
        if (conn->egress.pacer != NULL) {
5581
0
            uint32_t bytes_per_msec = calc_pacer_send_rate(conn);
5582
0
            pacer_window =
5583
0
                quicly_pacer_get_window(conn->egress.pacer, conn->stash.now, bytes_per_msec, conn->egress.max_udp_payload_size);
5584
0
        }
5585
0
        s->send_window = calc_send_window(conn, min_packets_to_send * conn->egress.max_udp_payload_size,
5586
0
                                          calc_amplification_limit_allowance(conn), pacer_window, restrict_sending);
5587
0
    }
5588
5589
0
    orig_bytes_inflight = conn->egress.loss.sentmap.bytes_in_flight;
5590
5591
0
    if (s->send_window == 0)
5592
0
        ack_only = 1;
5593
5594
0
    s->dcid = get_dcid(conn, s->path_index);
5595
5596
    /* send handshake flows; when PTO fires...
5597
     *  * quicly running as a client sends either a Handshake probe (or data) if the handshake keys are available, or else an
5598
     *    Initial probe (or data).
5599
     *  * quicly running as a server sends both Initial and Handshake probes (or data) if the corresponding keys are available. */
5600
0
    if (s->path_index == 0) {
5601
0
        if ((ret = send_handshake_flow(conn, QUICLY_EPOCH_INITIAL, s, ack_only,
5602
0
                                       min_packets_to_send != 0 && (!quicly_is_client(conn) || conn->handshake == NULL))) != 0)
5603
0
            goto Exit;
5604
0
        if ((ret = send_handshake_flow(conn, QUICLY_EPOCH_HANDSHAKE, s, ack_only, min_packets_to_send != 0)) != 0)
5605
0
            goto Exit;
5606
0
    }
5607
5608
    /* setup 0-RTT or 1-RTT send context (as the availability of the two epochs are mutually exclusive, we can try 1-RTT first as an
5609
     * optimization), then send application data if that succeeds */
5610
0
    if (setup_send_space(conn, QUICLY_EPOCH_1RTT, s) != NULL || setup_send_space(conn, QUICLY_EPOCH_0RTT, s) != NULL) {
5611
0
        { /* path_challenge / response */
5612
0
            struct st_quicly_conn_path_t *path = conn->paths[s->path_index];
5613
0
            assert(path != NULL);
5614
0
            if (path->path_challenge.send_at <= conn->stash.now) {
5615
                /* emit path challenge frame, doing exponential back off using PTO(initial_rtt) */
5616
0
                if ((ret = send_path_challenge(conn, s, 0, path->path_challenge.data)) != 0)
5617
0
                    goto Exit;
5618
0
                path->path_challenge.num_sent += 1;
5619
0
                path->path_challenge.send_at =
5620
0
                    conn->stash.now + ((3 * conn->super.ctx->loss.default_initial_rtt) << (path->path_challenge.num_sent - 1));
5621
0
                s->recalc_send_probe_at = 1;
5622
0
            }
5623
0
            if (path->path_response.send_) {
5624
0
                if ((ret = send_path_challenge(conn, s, 1, path->path_response.data)) != 0)
5625
0
                    goto Exit;
5626
0
                path->path_response.send_ = 0;
5627
0
                s->recalc_send_probe_at = 1;
5628
0
            }
5629
0
        }
5630
        /* non probing frames are sent only on path zero */
5631
0
        if (s->path_index == 0) {
5632
            /* acks */
5633
0
            if (conn->application->one_rtt_writable && conn->egress.send_ack_at <= conn->stash.now &&
5634
0
                conn->application->super.unacked_count != 0) {
5635
0
                if ((ret = send_ack(conn, &conn->application->super, s)) != 0)
5636
0
                    goto Exit;
5637
0
            }
5638
            /* DATAGRAM frame. Notes regarding current implementation:
5639
             * * Not limited by CC, nor the bytes counted by CC.
5640
             * * When given payload is too large and does not fit into a QUIC packet, a packet containing only PADDING frames is
5641
             *   sent. This is because we do not have a way to retract the generation of a QUIC packet.
5642
             * * Does not notify the application that the frame was dropped internally. */
5643
0
            if (should_send_datagram_frame(conn)) {
5644
0
                for (size_t i = 0; i != conn->egress.datagram_frame_payloads.count; ++i) {
5645
0
                    ptls_iovec_t *payload = conn->egress.datagram_frame_payloads.payloads + i;
5646
0
                    size_t required_space = quicly_datagram_frame_capacity(*payload);
5647
0
                    if ((ret = do_allocate_frame(conn, s, required_space, ALLOCATE_FRAME_TYPE_ACK_ELICITING_NO_CC)) != 0)
5648
0
                        goto Exit;
5649
0
                    if (s->dst_end - s->dst >= required_space) {
5650
0
                        s->dst = quicly_encode_datagram_frame(s->dst, *payload);
5651
0
                        QUICLY_PROBE(DATAGRAM_SEND, conn, conn->stash.now, payload->base, payload->len);
5652
0
                        QUICLY_LOG_CONN(datagram_send, conn,
5653
0
                                        { PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(payload, payload->base, payload->len); });
5654
0
                    } else {
5655
                        /* FIXME: At the moment, we add a padding because we do not have a way to reclaim allocated space, and
5656
                         * because it is forbidden to send an empty QUIC packet. */
5657
0
                        *s->dst++ = QUICLY_FRAME_TYPE_PADDING;
5658
0
                    }
5659
0
                }
5660
0
            }
5661
0
            if (!ack_only) {
5662
                /* PTO or loss detection timeout, always send PING. This is the easiest thing to do in terms of timer control. */
5663
0
                if (min_packets_to_send != 0) {
5664
0
                    if ((ret = do_allocate_frame(conn, s, 1, ALLOCATE_FRAME_TYPE_ACK_ELICITING)) != 0)
5665
0
                        goto Exit;
5666
0
                    if (get_epoch(s->current.first_byte) == QUICLY_EPOCH_1RTT &&
5667
0
                        conn->super.remote.transport_params.min_ack_delay_usec != UINT64_MAX) {
5668
0
                        *s->dst++ = QUICLY_FRAME_TYPE_IMMEDIATE_ACK;
5669
0
                        ++conn->super.stats.num_frames_sent.immediate_ack;
5670
0
                        QUICLY_PROBE(IMMEDIATE_ACK_SEND, conn, conn->stash.now);
5671
0
                        QUICLY_LOG_CONN(immediate_ack_send, conn, {});
5672
0
                    } else {
5673
0
                        *s->dst++ = QUICLY_FRAME_TYPE_PING;
5674
0
                        ++conn->super.stats.num_frames_sent.ping;
5675
0
                        QUICLY_PROBE(PING_SEND, conn, conn->stash.now);
5676
0
                        QUICLY_LOG_CONN(ping_send, conn, {});
5677
0
                    }
5678
0
                }
5679
                /* take actions only permitted for short header packets */
5680
0
                if (conn->application->one_rtt_writable) {
5681
                    /* send HANDSHAKE_DONE */
5682
0
                    if ((conn->egress.pending_flows & QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT) != 0 &&
5683
0
                        (ret = send_handshake_done(conn, s)) != 0)
5684
0
                        goto Exit;
5685
                    /* post-handshake messages */
5686
0
                    if ((conn->egress.pending_flows & (uint8_t)(1 << QUICLY_EPOCH_1RTT)) != 0) {
5687
0
                        quicly_stream_t *stream = quicly_get_stream(conn, -(1 + QUICLY_EPOCH_1RTT));
5688
0
                        assert(stream != NULL);
5689
0
                        if ((ret = quicly_send_stream(stream, s)) != 0)
5690
0
                            goto Exit;
5691
0
                        resched_stream_data(stream);
5692
0
                    }
5693
                    /* send other connection-level control frames, and iff we succeed in sending all of them, clear OTHERS_BIT to
5694
                     * disable `quicly_send` being called right again to send more control frames */
5695
0
                    if ((ret = send_other_control_frames(conn, s)) != 0)
5696
0
                        goto Exit;
5697
0
                    conn->egress.pending_flows &= ~QUICLY_PENDING_FLOW_OTHERS_BIT;
5698
                    /* send NEW_TOKEN */
5699
0
                    if ((conn->egress.pending_flows & QUICLY_PENDING_FLOW_NEW_TOKEN_BIT) != 0 &&
5700
0
                        (ret = send_resumption_token(conn, s)) != 0)
5701
0
                        goto Exit;
5702
0
                }
5703
                /* send stream-level control frames */
5704
0
                if ((ret = send_stream_control_frames(conn, s)) != 0)
5705
0
                    goto Exit;
5706
                /* send STREAM frames */
5707
0
                if ((ret = conn->super.ctx->stream_scheduler->do_send(conn->super.ctx->stream_scheduler, conn, s)) != 0)
5708
0
                    goto Exit;
5709
                /* once more, send control frames related to streams, as the state might have changed */
5710
0
                if ((ret = send_stream_control_frames(conn, s)) != 0)
5711
0
                    goto Exit;
5712
0
                if ((conn->egress.pending_flows & QUICLY_PENDING_FLOW_OTHERS_BIT) != 0) {
5713
0
                    if ((ret = send_other_control_frames(conn, s)) != 0)
5714
0
                        goto Exit;
5715
0
                    conn->egress.pending_flows &= ~QUICLY_PENDING_FLOW_OTHERS_BIT;
5716
0
                }
5717
0
            }
5718
            /* stream operations might have requested emission of NEW_TOKEN at the tail; if so, try to bundle it */
5719
0
            if ((conn->egress.pending_flows & QUICLY_PENDING_FLOW_NEW_TOKEN_BIT) != 0) {
5720
0
                assert(conn->application->one_rtt_writable);
5721
0
                if ((ret = send_resumption_token(conn, s)) != 0)
5722
0
                    goto Exit;
5723
0
            }
5724
0
        }
5725
0
    }
5726
5727
0
Exit:
5728
0
    if (ret == QUICLY_ERROR_SENDBUF_FULL) {
5729
0
        ret = 0;
5730
        /* when the buffer becomes full for the first time, try to use jumpstart; acting after the buffer becomes full does not
5731
         * delay switch to jump start, assuming that the buffer provided by the caller of quicly_send is no greater than the burst
5732
         * size of the pacer (10 packets) */
5733
0
        if (conn->egress.try_jumpstart && conn->egress.loss.rtt.minimum != UINT32_MAX) {
5734
0
            conn->egress.try_jumpstart = 0;
5735
0
            conn->super.stats.jumpstart.new_rtt = 0;
5736
0
            conn->super.stats.jumpstart.cwnd = 0;
5737
0
            if (conn->egress.pacer != NULL && conn->egress.cc.type->cc_jumpstart != NULL &&
5738
0
                (conn->super.ctx->default_jumpstart_cwnd_packets != 0 || conn->super.ctx->max_jumpstart_cwnd_packets != 0) &&
5739
0
                conn->egress.cc.num_loss_episodes == 0) {
5740
0
                conn->super.stats.jumpstart.new_rtt = conn->egress.loss.rtt.minimum;
5741
0
                if (conn->super.ctx->max_jumpstart_cwnd_packets != 0 && conn->super.stats.jumpstart.prev_rate != 0 &&
5742
0
                    conn->super.stats.jumpstart.prev_rtt != 0) {
5743
                    /* Careful Resume */
5744
0
                    conn->super.stats.jumpstart.cwnd =
5745
0
                        derive_jumpstart_cwnd(conn->super.ctx, conn->super.stats.jumpstart.new_rtt,
5746
0
                                              conn->super.stats.jumpstart.prev_rate, conn->super.stats.jumpstart.prev_rtt);
5747
0
                } else if (conn->super.ctx->default_jumpstart_cwnd_packets != 0) {
5748
                    /* jumpstart without previous information */
5749
0
                    conn->super.stats.jumpstart.cwnd = quicly_cc_calc_initial_cwnd(
5750
0
                        conn->super.ctx->default_jumpstart_cwnd_packets, conn->super.ctx->transport_params.max_udp_payload_size);
5751
0
                }
5752
                /* Jumpstart only if the amount that can be sent in 1 RTT would be higher than without. Comparison target is CWND +
5753
                 * inflight, as that is the amount that can be sent at most. Note the flow rate can become smaller due to packets
5754
                 * paced across the entire RTT during jumpstart. */
5755
0
                if (conn->super.stats.jumpstart.cwnd <= conn->egress.cc.cwnd + orig_bytes_inflight)
5756
0
                    conn->super.stats.jumpstart.cwnd = 0;
5757
0
            }
5758
            /* disable jumpstart probablistically based on the specified ratios; disablement is observable from the probes as
5759
             * `jumpstart.cwnd == 0` */
5760
0
            if (conn->super.stats.jumpstart.cwnd > 0) {
5761
0
                conn->super.stats.num_jumpstart_applicable = 1;
5762
0
                uint8_t ratio = conn->super.stats.jumpstart.prev_rate != 0 ? conn->super.ctx->enable_ratio.jumpstart.resume
5763
0
                                                                           : conn->super.ctx->enable_ratio.jumpstart.non_resume;
5764
0
                if (!enable_with_ratio255(ratio, conn->super.ctx->tls->random_bytes))
5765
0
                    conn->super.stats.jumpstart.cwnd = 0;
5766
0
                QUICLY_PROBE(ENTER_JUMPSTART, conn, conn->stash.now, conn->egress.packet_number,
5767
0
                             conn->super.stats.jumpstart.new_rtt, conn->egress.cc.cwnd, conn->super.stats.jumpstart.cwnd);
5768
0
                QUICLY_LOG_CONN(enter_jumpstart, conn, {
5769
0
                    PTLS_LOG_ELEMENT_UNSIGNED(pn, conn->egress.packet_number);
5770
0
                    PTLS_LOG_ELEMENT_UNSIGNED(rtt, conn->super.stats.jumpstart.new_rtt);
5771
0
                    PTLS_LOG_ELEMENT_UNSIGNED(cwnd, conn->egress.cc.cwnd);
5772
0
                    PTLS_LOG_ELEMENT_UNSIGNED(jumpstart_cwnd, conn->super.stats.jumpstart.cwnd);
5773
0
                });
5774
0
            }
5775
0
            if (conn->super.stats.jumpstart.cwnd > 0)
5776
0
                conn->egress.cc.type->cc_jumpstart(&conn->egress.cc, conn->super.stats.jumpstart.cwnd, conn->egress.packet_number);
5777
0
        }
5778
0
    }
5779
0
    if (ret == 0 && s->target.first_byte_at != NULL) {
5780
        /* last packet can be small-sized, unless it is the first flight sent from the client */
5781
0
        if ((s->payload_buf.datagram[0] & QUICLY_PACKET_TYPE_BITMASK) == QUICLY_PACKET_TYPE_INITIAL &&
5782
0
            (quicly_is_client(conn) || !ack_only))
5783
0
            s->target.full_size = 1;
5784
0
        commit_send_packet(conn, s, 0);
5785
0
    }
5786
0
    if (ret == 0) {
5787
        /* update timers, cc and delivery rate estimator states */
5788
0
        if (conn->application == NULL || conn->application->super.unacked_count == 0)
5789
0
            conn->egress.send_ack_at = INT64_MAX; /* we have sent ACKs for every epoch (or before address validation) */
5790
0
        int can_send_stream_data = scheduler_can_send(conn);
5791
0
        update_send_alarm(conn, can_send_stream_data, s->path_index == 0);
5792
0
        update_ratemeter(conn, can_send_stream_data && conn->super.remote.address_validation.validated &&
5793
0
                                   (s->num_datagrams == s->max_datagrams ||
5794
0
                                    conn->egress.loss.sentmap.bytes_in_flight >= conn->egress.cc.cwnd ||
5795
0
                                    pacer_can_send_at(conn) > conn->stash.now));
5796
0
        if (s->num_datagrams != 0)
5797
0
            update_idle_timeout(conn, 0);
5798
0
    }
5799
0
    return ret;
5800
5801
0
CloseNow:
5802
0
    conn->super.state = QUICLY_STATE_DRAINING;
5803
0
    destroy_all_streams(conn, 0, 0);
5804
0
    return QUICLY_ERROR_FREE_CONNECTION;
5805
0
}
5806
5807
void quicly_send_datagram_frames(quicly_conn_t *conn, ptls_iovec_t *datagrams, size_t num_datagrams)
5808
0
{
5809
0
    for (size_t i = 0; i != num_datagrams; ++i) {
5810
0
        if (conn->egress.datagram_frame_payloads.count == PTLS_ELEMENTSOF(conn->egress.datagram_frame_payloads.payloads))
5811
0
            break;
5812
0
        void *copied;
5813
0
        if ((copied = malloc(datagrams[i].len)) == NULL)
5814
0
            break;
5815
0
        memcpy(copied, datagrams[i].base, datagrams[i].len);
5816
0
        conn->egress.datagram_frame_payloads.payloads[conn->egress.datagram_frame_payloads.count++] =
5817
0
            ptls_iovec_init(copied, datagrams[i].len);
5818
0
    }
5819
0
}
5820
5821
int quicly_set_cc(quicly_conn_t *conn, quicly_cc_type_t *cc)
5822
0
{
5823
0
    return cc->cc_switch(&conn->egress.cc);
5824
0
}
5825
5826
static quicly_error_t do_send_closed(quicly_conn_t *conn, quicly_send_context_t *s)
5827
0
{
5828
0
    assert(s->path_index == 0);
5829
5830
0
    quicly_sentmap_iter_t iter;
5831
0
    quicly_error_t ret;
5832
5833
0
    if ((ret = init_acks_iter(conn, &iter)) != 0)
5834
0
        goto Exit;
5835
5836
    /* check if the connection can be closed now (after 3 pto) */
5837
0
    if (conn->super.state == QUICLY_STATE_DRAINING ||
5838
0
        conn->super.stats.num_frames_sent.transport_close + conn->super.stats.num_frames_sent.application_close != 0) {
5839
0
        if (quicly_sentmap_get(&iter)->packet_number == UINT64_MAX) {
5840
0
            assert(quicly_num_streams(conn) == 0);
5841
0
            ret = QUICLY_ERROR_FREE_CONNECTION;
5842
0
            goto Exit;
5843
0
        }
5844
0
    }
5845
5846
0
    if (conn->super.state == QUICLY_STATE_CLOSING && conn->egress.send_ack_at <= conn->stash.now) {
5847
        /* destroy all streams; doing so is delayed until the emission of CONNECTION_CLOSE frame to allow quicly_close to be called
5848
         * from a stream handler */
5849
0
        destroy_all_streams(conn, 0, 0);
5850
        /* send CONNECTION_CLOSE in all possible epochs */
5851
0
        s->dcid = get_dcid(conn, 0);
5852
0
        for (size_t epoch = 0; epoch < QUICLY_NUM_EPOCHS; ++epoch) {
5853
0
            if ((ret = send_connection_close(conn, epoch, s)) != 0)
5854
0
                goto Exit;
5855
0
        }
5856
0
        if ((ret = commit_send_packet(conn, s, 0)) != 0)
5857
0
            goto Exit;
5858
0
    }
5859
5860
    /* wait at least 1ms */
5861
0
    if ((conn->egress.send_ack_at = quicly_sentmap_get(&iter)->sent_at + get_sentmap_expiration_time(conn)) <= conn->stash.now)
5862
0
        conn->egress.send_ack_at = conn->stash.now + 1;
5863
5864
0
    ret = 0;
5865
5866
0
Exit:
5867
0
    return ret;
5868
0
}
5869
5870
quicly_error_t quicly_send(quicly_conn_t *conn, quicly_address_t *dest, quicly_address_t *src, struct iovec *datagrams,
5871
                           size_t *num_datagrams, void *buf, size_t bufsize)
5872
0
{
5873
0
    quicly_send_context_t s = {.current = {.first_byte = -1},
5874
0
                               .datagrams = datagrams,
5875
0
                               .max_datagrams = *num_datagrams,
5876
0
                               .payload_buf = {.datagram = buf, .end = (uint8_t *)buf + bufsize}};
5877
0
    quicly_error_t ret;
5878
5879
0
    lock_now(conn, 0);
5880
5881
    /* bail out if there's nothing scheduled to be sent */
5882
0
    if (conn->stash.now < quicly_get_first_timeout(conn)) {
5883
0
        ret = 0;
5884
0
        goto Exit;
5885
0
    }
5886
5887
    /* determine DCID of active path; doing so is guaranteed to succeed as the protocol guarantees that there will always be at
5888
     * least one non-retired CID available */
5889
0
    if (conn->paths[0]->dcid == UINT64_MAX) {
5890
0
        int success = setup_path_dcid(conn, 0);
5891
0
        assert(success);
5892
0
    }
5893
5894
0
    PTLS_LOG_DEFINE_POINT(quicly, send, send_logpoint);
5895
0
    if (QUICLY_PROBE_ENABLED(SEND) ||
5896
0
        (ptls_log_point_maybe_active(&send_logpoint) &
5897
0
         ptls_log_conn_maybe_active(ptls_get_log_state(conn->crypto.tls), ptls_log_getsni_ptls(conn->crypto.tls))) != 0) {
5898
0
        const quicly_cid_t *dcid = get_dcid(conn, 0);
5899
0
        QUICLY_PROBE(SEND, conn, conn->stash.now, conn->super.state, QUICLY_PROBE_HEXDUMP(dcid->cid, dcid->len));
5900
0
        QUICLY_LOG_CONN(send, conn, {
5901
0
            PTLS_LOG_ELEMENT_SIGNED(state, conn->super.state);
5902
0
            PTLS_LOG_ELEMENT_HEXDUMP(dcid, dcid->cid, dcid->len);
5903
0
        });
5904
0
    }
5905
5906
0
    if (conn->super.state >= QUICLY_STATE_CLOSING) {
5907
0
        ret = do_send_closed(conn, &s);
5908
0
        goto Exit;
5909
0
    }
5910
5911
    /* try emitting one probe packet on one of the backup paths, or ... (note: API of `quicly_send` allows us to send packets on no
5912
     * more than one path at a time) */
5913
0
    if (conn->egress.send_probe_at <= conn->stash.now) {
5914
0
        for (s.path_index = 1; s.path_index < PTLS_ELEMENTSOF(conn->paths); ++s.path_index) {
5915
0
            if (conn->paths[s.path_index] == NULL || !(conn->stash.now >= conn->paths[s.path_index]->path_challenge.send_at ||
5916
0
                                                       conn->paths[s.path_index]->path_response.send_))
5917
0
                continue;
5918
0
            if (conn->paths[s.path_index]->path_challenge.num_sent > conn->super.ctx->max_probe_packets) {
5919
0
                if ((ret = delete_path(conn, s.path_index)) != 0) {
5920
0
                    initiate_close(conn, ret, QUICLY_FRAME_TYPE_PADDING, NULL);
5921
0
                    assert(conn->super.state >= QUICLY_STATE_CLOSING);
5922
0
                    s.path_index = 0;
5923
0
                    ret = do_send_closed(conn, &s);
5924
0
                    goto Exit;
5925
0
                }
5926
0
                s.recalc_send_probe_at = 1;
5927
0
                continue;
5928
0
            }
5929
            /* determine DCID to be used, if not yet been done; upon failure, this path (being secondary) is discarded */
5930
0
            if (conn->paths[s.path_index]->dcid == UINT64_MAX && !setup_path_dcid(conn, s.path_index)) {
5931
0
                ret = delete_path(conn, s.path_index);
5932
0
                assert(ret == 0 && "path->dcid is UINT64_MAX and therefore does not trigger an error");
5933
0
                s.recalc_send_probe_at = 1;
5934
0
                conn->super.stats.num_paths.closed_no_dcid += 1;
5935
0
                continue;
5936
0
            }
5937
0
            if ((ret = do_send(conn, &s)) != 0)
5938
0
                goto Exit;
5939
0
            assert(conn->stash.now < conn->paths[s.path_index]->path_challenge.send_at);
5940
0
            if (s.num_datagrams != 0)
5941
0
                break;
5942
0
        }
5943
0
    }
5944
    /* otherwise, emit non-probing packets */
5945
0
    if (s.num_datagrams == 0) {
5946
0
        s.path_index = 0;
5947
0
        if ((ret = do_send(conn, &s)) != 0)
5948
0
            goto Exit;
5949
0
    } else {
5950
0
        ret = 0;
5951
0
    }
5952
5953
0
    assert_consistency(conn, s.path_index == 0);
5954
5955
0
Exit:
5956
0
    if (s.path_index == 0)
5957
0
        clear_datagram_frame_payloads(conn);
5958
0
    if (s.recalc_send_probe_at)
5959
0
        recalc_send_probe_at(conn);
5960
0
    if (s.num_datagrams != 0) {
5961
0
        *dest = conn->paths[s.path_index]->address.remote;
5962
0
        *src = conn->paths[s.path_index]->address.local;
5963
0
    }
5964
0
    *num_datagrams = s.num_datagrams;
5965
0
    unlock_now(conn);
5966
0
    return ret;
5967
0
}
5968
5969
uint8_t quicly_send_get_ecn_bits(quicly_conn_t *conn)
5970
0
{
5971
0
    return conn->egress.ecn.state == QUICLY_ECN_OFF ? 0 : 2; /* NON-ECT or ECT(0) */
5972
0
}
5973
5974
size_t quicly_send_close_invalid_token(quicly_context_t *ctx, uint32_t protocol_version, ptls_iovec_t dest_cid,
5975
                                       ptls_iovec_t src_cid, const char *err_desc, void *datagram)
5976
0
{
5977
0
    struct st_quicly_cipher_context_t egress = {};
5978
0
    const quicly_salt_t *salt;
5979
5980
    /* setup keys */
5981
0
    if ((salt = quicly_get_salt(protocol_version)) == NULL)
5982
0
        return SIZE_MAX;
5983
0
    if (setup_initial_encryption(get_aes128gcmsha256(ctx), NULL, &egress, src_cid, 0,
5984
0
                                 ptls_iovec_init(salt->initial, sizeof(salt->initial)), NULL) != 0)
5985
0
        return SIZE_MAX;
5986
5987
0
    uint8_t *dst = datagram, *length_at;
5988
5989
    /* build packet */
5990
0
    PTLS_BUILD_ASSERT(QUICLY_SEND_PN_SIZE == 2);
5991
0
    *dst++ = QUICLY_PACKET_TYPE_INITIAL | 0x1 /* 2-byte PN */;
5992
0
    dst = quicly_encode32(dst, protocol_version);
5993
0
    *dst++ = dest_cid.len;
5994
0
    memcpy(dst, dest_cid.base, dest_cid.len);
5995
0
    dst += dest_cid.len;
5996
0
    *dst++ = src_cid.len;
5997
0
    memcpy(dst, src_cid.base, src_cid.len);
5998
0
    dst += src_cid.len;
5999
0
    *dst++ = 0;        /* token_length = 0 */
6000
0
    length_at = dst++; /* length_at to be filled in later as 1-byte varint */
6001
0
    *dst++ = 0;        /* PN = 0 */
6002
0
    *dst++ = 0;        /* ditto */
6003
0
    uint8_t *payload_from = dst;
6004
0
    dst = quicly_encode_close_frame(dst, QUICLY_ERROR_GET_ERROR_CODE(QUICLY_TRANSPORT_ERROR_INVALID_TOKEN),
6005
0
                                    QUICLY_FRAME_TYPE_PADDING, err_desc);
6006
6007
    /* determine the size of the packet, make adjustments */
6008
0
    dst += egress.aead->algo->tag_size;
6009
0
    assert(dst - (uint8_t *)datagram <= QUICLY_MIN_CLIENT_INITIAL_SIZE);
6010
0
    assert(dst - length_at - 1 < 64);
6011
0
    *length_at = dst - length_at - 1;
6012
0
    size_t datagram_len = dst - (uint8_t *)datagram;
6013
6014
    /* encrypt packet */
6015
0
    quicly_default_crypto_engine.encrypt_packet(&quicly_default_crypto_engine, NULL, egress.header_protection, egress.aead,
6016
0
                                                ptls_iovec_init(datagram, datagram_len), 0, payload_from - (uint8_t *)datagram, 0,
6017
0
                                                0);
6018
6019
0
    dispose_cipher(&egress);
6020
0
    return datagram_len;
6021
0
}
6022
6023
size_t quicly_send_stateless_reset(quicly_context_t *ctx, const void *src_cid, void *payload)
6024
0
{
6025
0
    uint8_t *base = payload;
6026
6027
    /* build stateless reset packet */
6028
0
    ctx->tls->random_bytes(base, QUICLY_STATELESS_RESET_PACKET_MIN_LEN - QUICLY_STATELESS_RESET_TOKEN_LEN);
6029
0
    base[0] = (base[0] & ~QUICLY_LONG_HEADER_BIT) | QUICLY_QUIC_BIT;
6030
0
    if (!ctx->cid_encryptor->generate_stateless_reset_token(
6031
0
            ctx->cid_encryptor, base + QUICLY_STATELESS_RESET_PACKET_MIN_LEN - QUICLY_STATELESS_RESET_TOKEN_LEN, src_cid))
6032
0
        return SIZE_MAX;
6033
6034
0
    return QUICLY_STATELESS_RESET_PACKET_MIN_LEN;
6035
0
}
6036
6037
quicly_error_t quicly_send_resumption_token(quicly_conn_t *conn)
6038
{
6039
    assert(!quicly_is_client(conn));
6040
6041
    if (conn->super.state <= QUICLY_STATE_CONNECTED) {
6042
        ++conn->egress.new_token.generation;
6043
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_NEW_TOKEN_BIT;
6044
    }
6045
    return 0;
6046
}
6047
6048
static quicly_error_t on_end_closing(quicly_sentmap_t *map, const quicly_sent_packet_t *packet, int acked, quicly_sent_t *sent)
6049
0
{
6050
    /* we stop accepting frames by the time this ack callback is being registered */
6051
0
    assert(!acked);
6052
0
    return 0;
6053
0
}
6054
6055
static quicly_error_t enter_close(quicly_conn_t *conn, int local_is_initiating, int wait_draining)
6056
0
{
6057
0
    quicly_error_t ret;
6058
6059
0
    assert(conn->super.state < QUICLY_STATE_CLOSING);
6060
6061
    /* release all inflight info, register a close timeout */
6062
0
    if ((ret = discard_sentmap_by_epoch(conn, ~0u)) != 0)
6063
0
        return ret;
6064
0
    if ((ret = quicly_sentmap_prepare(&conn->egress.loss.sentmap, conn->egress.packet_number, conn->stash.now,
6065
0
                                      QUICLY_EPOCH_INITIAL)) != 0)
6066
0
        return ret;
6067
0
    if (quicly_sentmap_allocate(&conn->egress.loss.sentmap, on_end_closing) == NULL)
6068
0
        return PTLS_ERROR_NO_MEMORY;
6069
0
    quicly_sentmap_commit(&conn->egress.loss.sentmap, 0, 0, 0);
6070
0
    ++conn->egress.packet_number;
6071
6072
0
    if (local_is_initiating) {
6073
0
        conn->super.state = QUICLY_STATE_CLOSING;
6074
0
        conn->egress.send_ack_at = 0;
6075
0
    } else {
6076
0
        conn->super.state = QUICLY_STATE_DRAINING;
6077
0
        conn->egress.send_ack_at = wait_draining ? conn->stash.now + get_sentmap_expiration_time(conn) : 0;
6078
0
    }
6079
6080
0
    setup_next_send(conn);
6081
6082
0
    return 0;
6083
0
}
6084
6085
static int set_connection_close(quicly_conn_t *conn, quicly_error_t err, uint64_t frame_type, const char *reason_phrase,
6086
                                size_t reason_phrase_len, int is_remote)
6087
0
{
6088
0
    assert(conn->connection_close.reason_phrase == NULL && "never called twice");
6089
6090
0
    if (QUICLY_ERROR_IS_QUIC_APPLICATION(err)) {
6091
0
        conn->connection_close.err = err;
6092
0
        conn->connection_close.frame_type = UINT64_MAX;
6093
0
    } else {
6094
0
        assert(frame_type != UINT64_MAX);
6095
0
        if (QUICLY_TRANSPORT_ERROR_CRYPTO(0) <= err && err <= QUICLY_TRANSPORT_ERROR_CRYPTO(0xff)) {
6096
            /* TLS alerts use the that of picotls */
6097
0
            uint8_t tls_alert = err - QUICLY_TRANSPORT_ERROR_CRYPTO(0);
6098
0
            conn->connection_close.err = is_remote ? PTLS_ALERT_TO_PEER_ERROR(tls_alert) : PTLS_ALERT_TO_SELF_ERROR(tls_alert);
6099
0
        } else {
6100
0
            conn->connection_close.err = err;
6101
0
        }
6102
0
        conn->connection_close.frame_type = frame_type;
6103
0
    }
6104
0
    if ((conn->connection_close.reason_phrase = malloc(reason_phrase_len + 1)) == NULL)
6105
0
        return PTLS_ERROR_NO_MEMORY;
6106
0
    memcpy(conn->connection_close.reason_phrase, reason_phrase, reason_phrase_len);
6107
0
    conn->connection_close.reason_phrase[reason_phrase_len] = '\0';
6108
0
    conn->connection_close.is_remote = is_remote;
6109
6110
0
    return 0;
6111
0
}
6112
6113
quicly_error_t initiate_close(quicly_conn_t *conn, quicly_error_t err, uint64_t frame_type, const char *reason_phrase)
6114
0
{
6115
0
    quicly_error_t ret;
6116
6117
0
    if (conn->super.state >= QUICLY_STATE_CLOSING)
6118
0
        return 0;
6119
6120
0
    if (reason_phrase == NULL)
6121
0
        reason_phrase = "";
6122
6123
0
    if ((ret = enter_close(conn, 1, 0)) != 0 ||
6124
0
        (ret = set_connection_close(conn, err, frame_type, reason_phrase, strlen(reason_phrase), 0)) != 0)
6125
0
        return ret;
6126
0
    if (conn->super.ctx->closed != NULL)
6127
0
        conn->super.ctx->closed->cb(conn->super.ctx->closed, conn);
6128
0
    return 0;
6129
0
}
6130
6131
quicly_error_t quicly_close(quicly_conn_t *conn, quicly_error_t err, const char *reason_phrase)
6132
{
6133
    quicly_error_t ret;
6134
6135
    assert(err == 0 || QUICLY_ERROR_IS_QUIC_APPLICATION(err) || QUICLY_ERROR_IS_CONCEALED(err));
6136
6137
    lock_now(conn, 1);
6138
    ret = initiate_close(conn, err, QUICLY_FRAME_TYPE_PADDING /* used when err == 0 */, reason_phrase);
6139
    unlock_now(conn);
6140
6141
    return ret;
6142
}
6143
6144
quicly_error_t quicly_get_or_open_stream(quicly_conn_t *conn, uint64_t stream_id, quicly_stream_t **stream)
6145
0
{
6146
0
    quicly_error_t ret = 0;
6147
6148
0
    if ((*stream = quicly_get_stream(conn, stream_id)) != NULL)
6149
0
        goto Exit;
6150
6151
0
    if (quicly_stream_is_client_initiated(stream_id) != quicly_is_client(conn)) {
6152
        /* check if stream id is within the bounds */
6153
0
        if (stream_id / 4 >= quicly_get_ingress_max_streams(conn, quicly_stream_is_unidirectional(stream_id))) {
6154
0
            ret = QUICLY_TRANSPORT_ERROR_STREAM_LIMIT;
6155
0
            goto Exit;
6156
0
        }
6157
        /* open new streams upto given id */
6158
0
        struct st_quicly_conn_streamgroup_state_t *group = get_streamgroup_state(conn, stream_id);
6159
0
        if (group->next_stream_id <= stream_id) {
6160
0
            uint64_t max_stream_data_local, max_stream_data_remote;
6161
0
            if (quicly_stream_is_unidirectional(stream_id)) {
6162
0
                max_stream_data_local = conn->super.ctx->transport_params.max_stream_data.uni;
6163
0
                max_stream_data_remote = 0;
6164
0
            } else {
6165
0
                max_stream_data_local = conn->super.ctx->transport_params.max_stream_data.bidi_remote;
6166
0
                max_stream_data_remote = conn->super.remote.transport_params.max_stream_data.bidi_local;
6167
0
            }
6168
0
            do {
6169
0
                if ((*stream = open_stream(conn, group->next_stream_id, (uint32_t)max_stream_data_local, max_stream_data_remote)) ==
6170
0
                    NULL) {
6171
0
                    ret = PTLS_ERROR_NO_MEMORY;
6172
0
                    goto Exit;
6173
0
                }
6174
0
                QUICLY_PROBE(STREAM_ON_OPEN, conn, conn->stash.now, *stream);
6175
0
                QUICLY_LOG_CONN(stream_on_open, conn, { PTLS_LOG_ELEMENT_SIGNED(stream_id, (*stream)->stream_id); });
6176
0
                if ((ret = conn->super.ctx->stream_open->cb(conn->super.ctx->stream_open, *stream)) != 0) {
6177
0
                    *stream = NULL;
6178
0
                    goto Exit;
6179
0
                }
6180
0
                ++group->num_streams;
6181
0
                group->next_stream_id += 4;
6182
0
            } while (stream_id != (*stream)->stream_id);
6183
0
        }
6184
0
    }
6185
6186
0
Exit:
6187
0
    return ret;
6188
0
}
6189
6190
static quicly_error_t handle_crypto_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6191
0
{
6192
0
    quicly_stream_frame_t frame;
6193
0
    quicly_stream_t *stream;
6194
0
    quicly_error_t ret;
6195
6196
0
    if ((ret = quicly_decode_crypto_frame(&state->src, state->end, &frame)) != 0)
6197
0
        return ret;
6198
0
    stream = quicly_get_stream(conn, -(quicly_stream_id_t)(1 + state->epoch));
6199
0
    assert(stream != NULL);
6200
0
    return apply_stream_frame(stream, &frame);
6201
0
}
6202
6203
static quicly_error_t handle_stream_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6204
0
{
6205
0
    quicly_stream_frame_t frame;
6206
0
    quicly_stream_t *stream;
6207
0
    quicly_error_t ret;
6208
6209
0
    if ((ret = quicly_decode_stream_frame(state->frame_type, &state->src, state->end, &frame)) != 0)
6210
0
        return ret;
6211
0
    QUICLY_PROBE(QUICTRACE_RECV_STREAM, conn, conn->stash.now, frame.stream_id, frame.offset, frame.data.len, (int)frame.is_fin);
6212
0
    if ((ret = quicly_get_or_open_stream(conn, frame.stream_id, &stream)) != 0 || stream == NULL)
6213
0
        return ret;
6214
0
    return apply_stream_frame(stream, &frame);
6215
0
}
6216
6217
static quicly_error_t handle_reset_stream_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6218
0
{
6219
0
    quicly_reset_stream_frame_t frame;
6220
0
    quicly_stream_t *stream;
6221
0
    quicly_error_t ret;
6222
6223
0
    if ((ret = quicly_decode_reset_stream_frame(&state->src, state->end, &frame)) != 0)
6224
0
        return ret;
6225
0
    QUICLY_PROBE(RESET_STREAM_RECEIVE, conn, conn->stash.now, frame.stream_id, frame.app_error_code, frame.final_size);
6226
0
    QUICLY_LOG_CONN(reset_stream_receive, conn, {
6227
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, (quicly_stream_id_t)frame.stream_id);
6228
0
        PTLS_LOG_ELEMENT_UNSIGNED(app_error_code, frame.app_error_code);
6229
0
        PTLS_LOG_ELEMENT_UNSIGNED(final_size, frame.final_size);
6230
0
    });
6231
6232
0
    if ((ret = quicly_get_or_open_stream(conn, frame.stream_id, &stream)) != 0 || stream == NULL)
6233
0
        return ret;
6234
6235
0
    if (frame.final_size > stream->recvstate.data_off + stream->_recv_aux.window)
6236
0
        return QUICLY_TRANSPORT_ERROR_FLOW_CONTROL;
6237
6238
0
    if (!quicly_recvstate_transfer_complete(&stream->recvstate)) {
6239
0
        uint64_t bytes_missing;
6240
0
        if ((ret = quicly_recvstate_reset(&stream->recvstate, frame.final_size, &bytes_missing)) != 0)
6241
0
            return ret;
6242
0
        stream->conn->ingress.max_data.bytes_consumed += bytes_missing;
6243
0
        quicly_error_t err = QUICLY_ERROR_FROM_APPLICATION_ERROR_CODE(frame.app_error_code);
6244
0
        QUICLY_PROBE(STREAM_ON_RECEIVE_RESET, stream->conn, stream->conn->stash.now, stream, err);
6245
0
        QUICLY_LOG_CONN(stream_on_receive_reset, stream->conn, {
6246
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
6247
0
            PTLS_LOG_ELEMENT_SIGNED(err, err);
6248
0
        });
6249
0
        stream->callbacks->on_receive_reset(stream, err);
6250
0
        if (stream->conn->super.state >= QUICLY_STATE_CLOSING)
6251
0
            return QUICLY_ERROR_IS_CLOSING;
6252
0
        if (stream_is_destroyable(stream))
6253
0
            destroy_stream(stream, 0);
6254
0
    }
6255
6256
0
    return 0;
6257
0
}
6258
6259
static quicly_error_t handle_ack_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6260
0
{
6261
0
    quicly_ack_frame_t frame;
6262
0
    quicly_sentmap_iter_t iter;
6263
0
    struct {
6264
0
        uint64_t pn;
6265
0
        int64_t sent_at;
6266
0
    } largest_newly_acked = {UINT64_MAX, INT64_MAX};
6267
0
    size_t bytes_acked = 0;
6268
0
    int includes_ack_eliciting = 0, includes_late_ack = 0;
6269
0
    uint64_t largest_late_acked = UINT64_MAX;
6270
0
    quicly_error_t ret;
6271
6272
    /* The flow is considered CC-limited if the packet was sent while `inflight >= 1/2 * CNWD` or acked under the same condition.
6273
     * 1/2 of CWND is adopted for fairness with RFC 7661, and also provides correct increase; i.e., if an idle flow goes into
6274
     * CC-limited state for X round-trips then becomes idle again, all packets sent during that X round-trips will be considered as
6275
     * CC-limited. */
6276
0
    int cc_limited =
6277
0
        conn->super.stats.num_respected_app_limited == 0 || conn->egress.loss.sentmap.bytes_in_flight >= conn->egress.cc.cwnd / 2;
6278
6279
0
    if ((ret = quicly_decode_ack_frame(&state->src, state->end, &frame, state->frame_type == QUICLY_FRAME_TYPE_ACK_ECN)) != 0)
6280
0
        return ret;
6281
6282
    /* early bail out if the peer is acking a PN that would have never been sent */
6283
0
    if (frame.largest_acknowledged > conn->egress.packet_number)
6284
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6285
6286
0
    uint64_t pn_acked = frame.smallest_acknowledged;
6287
6288
0
    switch (state->epoch) {
6289
0
    case QUICLY_EPOCH_0RTT:
6290
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6291
0
    case QUICLY_EPOCH_HANDSHAKE:
6292
0
        conn->super.remote.address_validation.send_probe = 0;
6293
0
        break;
6294
0
    default:
6295
0
        break;
6296
0
    }
6297
6298
0
    if ((ret = init_acks_iter(conn, &iter)) != 0)
6299
0
        return ret;
6300
6301
    /* TODO log PNs being ACKed too late */
6302
6303
0
    size_t gap_index = frame.num_gaps;
6304
0
    while (1) {
6305
0
        assert(frame.ack_block_lengths[gap_index] != 0);
6306
        /* Ack blocks are organized in the ACK frame and consequently in the ack_block_lengths array from the largest acked down.
6307
         * Processing acks in packet number order requires processing the ack blocks in reverse order. */
6308
0
        uint64_t pn_block_max = pn_acked + frame.ack_block_lengths[gap_index] - 1;
6309
0
        QUICLY_PROBE(ACK_BLOCK_RECEIVED, conn, conn->stash.now, pn_acked, pn_block_max);
6310
0
        QUICLY_LOG_CONN(ack_block_received, conn, {
6311
0
            PTLS_LOG_ELEMENT_UNSIGNED(ack_block_begin, pn_acked);
6312
0
            PTLS_LOG_ELEMENT_UNSIGNED(ack_block_end, pn_block_max);
6313
0
        });
6314
0
        while (quicly_sentmap_get(&iter)->packet_number < pn_acked)
6315
0
            quicly_sentmap_skip(&iter);
6316
0
        do {
6317
0
            const quicly_sent_packet_t *sent = quicly_sentmap_get(&iter);
6318
0
            uint64_t pn_sent = sent->packet_number;
6319
0
            assert(pn_acked <= pn_sent);
6320
0
            if (pn_acked < pn_sent) {
6321
                /* set pn_acked to pn_sent; or past the end of the ack block, for use with the next ack block */
6322
0
                if (pn_sent <= pn_block_max) {
6323
0
                    pn_acked = pn_sent;
6324
0
                } else {
6325
0
                    pn_acked = pn_block_max + 1;
6326
0
                    break;
6327
0
                }
6328
0
            }
6329
            /* process newly acked packet */
6330
0
            if (state->epoch != sent->ack_epoch)
6331
0
                return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6332
0
            int is_late_ack = 0;
6333
0
            if (sent->ack_eliciting) {
6334
0
                includes_ack_eliciting = 1;
6335
0
                if (sent->cc_bytes_in_flight == 0) {
6336
0
                    is_late_ack = 1;
6337
0
                    includes_late_ack = 1;
6338
0
                    largest_late_acked = pn_acked;
6339
0
                    ++conn->super.stats.num_packets.late_acked;
6340
0
                    if (conn->egress.pn_path_start <= pn_acked && conn->egress.cc.type->cc_on_late_ack != NULL)
6341
0
                        conn->egress.cc.type->cc_on_late_ack(&conn->egress.cc, pn_acked, conn->stash.now);
6342
0
                }
6343
0
            }
6344
0
            ++conn->super.stats.num_packets.ack_received;
6345
0
            if (sent->promoted_path)
6346
0
                ++conn->super.stats.num_packets.ack_received_promoted_paths;
6347
0
            if (conn->egress.pn_path_start <= pn_acked) {
6348
0
                largest_newly_acked.pn = pn_acked;
6349
0
                largest_newly_acked.sent_at = sent->sent_at;
6350
0
            }
6351
0
            QUICLY_PROBE(PACKET_ACKED, conn, conn->stash.now, pn_acked, is_late_ack);
6352
0
            QUICLY_LOG_CONN(packet_acked, conn, {
6353
0
                PTLS_LOG_ELEMENT_UNSIGNED(pn, pn_acked);
6354
0
                PTLS_LOG_ELEMENT_BOOL(is_late_ack, is_late_ack);
6355
0
            });
6356
0
            if (sent->cc_bytes_in_flight != 0) {
6357
0
                if (conn->egress.pn_path_start <= pn_acked) {
6358
0
                    bytes_acked += sent->cc_bytes_in_flight;
6359
0
                    if (sent->cc_limited)
6360
0
                        cc_limited = 1;
6361
0
                }
6362
0
                conn->super.stats.num_bytes.ack_received += sent->cc_bytes_in_flight;
6363
0
            }
6364
0
            if ((ret = quicly_sentmap_update(&conn->egress.loss.sentmap, &iter, QUICLY_SENTMAP_EVENT_ACKED)) != 0)
6365
0
                return ret;
6366
0
            if (state->epoch == QUICLY_EPOCH_1RTT) {
6367
0
                struct st_quicly_application_space_t *space = conn->application;
6368
0
                if (space->cipher.egress.key_update_pn.last <= pn_acked) {
6369
0
                    space->cipher.egress.key_update_pn.last = UINT64_MAX;
6370
0
                    space->cipher.egress.key_update_pn.next = conn->egress.packet_number + conn->super.ctx->max_packets_per_key;
6371
0
                    QUICLY_PROBE(CRYPTO_SEND_KEY_UPDATE_CONFIRMED, conn, conn->stash.now, space->cipher.egress.key_update_pn.next);
6372
0
                    QUICLY_LOG_CONN(crypto_send_key_update_confirmed, conn,
6373
0
                                    { PTLS_LOG_ELEMENT_UNSIGNED(next_pn, space->cipher.egress.key_update_pn.next); });
6374
0
                }
6375
0
            }
6376
0
            ++pn_acked;
6377
0
        } while (pn_acked <= pn_block_max);
6378
0
        assert(pn_acked == pn_block_max + 1);
6379
0
        if (gap_index-- == 0)
6380
0
            break;
6381
0
        pn_acked += frame.gaps[gap_index];
6382
0
    }
6383
6384
0
    if ((ret = on_ack_stream_ack_cached(conn)) != 0)
6385
0
        return ret;
6386
6387
0
    QUICLY_PROBE(ACK_DELAY_RECEIVED, conn, conn->stash.now, frame.ack_delay);
6388
0
    QUICLY_LOG_CONN(ack_delay_received, conn, { PTLS_LOG_ELEMENT_UNSIGNED(ack_delay, frame.ack_delay); });
6389
6390
0
    if (largest_newly_acked.pn != UINT64_MAX)
6391
0
        quicly_ratemeter_on_ack(&conn->egress.ratemeter, conn->stash.now, conn->super.stats.num_bytes.ack_received,
6392
0
                                largest_newly_acked.pn);
6393
6394
    /* Update loss detection engine on ack. The function uses ack_delay only when the largest_newly_acked is also the largest acked
6395
     * so far. So, it does not matter if the ack_delay being passed in does not apply to the largest_newly_acked. */
6396
0
    quicly_loss_on_ack_received(&conn->egress.loss, largest_newly_acked.pn, largest_late_acked, conn->egress.packet_number,
6397
0
                                state->epoch, conn->stash.now, largest_newly_acked.sent_at, frame.ack_delay,
6398
0
                                includes_ack_eliciting ? includes_late_ack ? QUICLY_LOSS_ACK_RECEIVED_KIND_ACK_ELICITING_LATE_ACK
6399
0
                                                                           : QUICLY_LOSS_ACK_RECEIVED_KIND_ACK_ELICITING
6400
0
                                                       : QUICLY_LOSS_ACK_RECEIVED_KIND_NON_ACK_ELICITING);
6401
6402
    /* OnPacketAcked and OnPacketAckedCC */
6403
0
    if (bytes_acked > 0) {
6404
0
        conn->egress.cc.type->cc_on_acked(&conn->egress.cc, &conn->egress.loss, (uint32_t)bytes_acked, frame.largest_acknowledged,
6405
0
                                          (uint32_t)(conn->egress.loss.sentmap.bytes_in_flight + bytes_acked), cc_limited,
6406
0
                                          conn->egress.packet_number, conn->stash.now, conn->egress.max_udp_payload_size);
6407
0
        QUICLY_PROBE(QUICTRACE_CC_ACK, conn, conn->stash.now, &conn->egress.loss.rtt, conn->egress.cc.cwnd,
6408
0
                     conn->egress.loss.sentmap.bytes_in_flight);
6409
0
    }
6410
6411
0
    QUICLY_PROBE(CC_ACK_RECEIVED, conn, conn->stash.now, frame.largest_acknowledged, bytes_acked, conn->egress.cc.cwnd,
6412
0
                 conn->egress.loss.sentmap.bytes_in_flight);
6413
0
    QUICLY_LOG_CONN(cc_ack_received, conn, {
6414
0
        PTLS_LOG_ELEMENT_UNSIGNED(largest_acked, frame.largest_acknowledged);
6415
0
        PTLS_LOG_ELEMENT_UNSIGNED(bytes_acked, bytes_acked);
6416
0
        PTLS_LOG_ELEMENT_UNSIGNED(cwnd, conn->egress.cc.cwnd);
6417
0
        PTLS_LOG_ELEMENT_UNSIGNED(inflight, conn->egress.loss.sentmap.bytes_in_flight);
6418
0
    });
6419
6420
    /* loss-detection  */
6421
0
    if ((ret = quicly_loss_detect_loss(&conn->egress.loss, conn->stash.now, conn->super.remote.transport_params.max_ack_delay,
6422
0
                                       conn->initial == NULL && conn->handshake == NULL, on_loss_detected)) != 0)
6423
0
        return ret;
6424
6425
    /* ECN */
6426
0
    if (conn->egress.ecn.state != QUICLY_ECN_OFF && largest_newly_acked.pn != UINT64_MAX) {
6427
        /* if things look suspicious (ECT(1) count becoming non-zero), turn ECN off */
6428
0
        if (frame.ecn_counts[1] != 0)
6429
0
            update_ecn_state(conn, QUICLY_ECN_OFF);
6430
        /* TODO: maybe compare num_packets.acked vs. sum(ecn_counts) to see if any packet has been received as NON-ECT? */
6431
6432
        /* ECN validation succeeds if at least one packet is acked using one of the expected marks during the probing period */
6433
0
        if (conn->egress.ecn.state == QUICLY_ECN_PROBING && frame.ecn_counts[0] + frame.ecn_counts[2] > 0)
6434
0
            update_ecn_state(conn, QUICLY_ECN_ON);
6435
6436
        /* check if congestion should be reported */
6437
0
        int report_congestion =
6438
0
            conn->egress.ecn.state != QUICLY_ECN_OFF && frame.ecn_counts[2] > conn->egress.ecn.counts[state->epoch][2];
6439
6440
        /* update counters */
6441
0
        for (size_t i = 0; i < PTLS_ELEMENTSOF(frame.ecn_counts); ++i) {
6442
0
            if (frame.ecn_counts[i] > conn->egress.ecn.counts[state->epoch][i]) {
6443
0
                conn->super.stats.num_packets.acked_ecn_counts[i] += frame.ecn_counts[i] - conn->egress.ecn.counts[state->epoch][i];
6444
0
                conn->egress.ecn.counts[state->epoch][i] = frame.ecn_counts[i];
6445
0
            }
6446
0
        }
6447
6448
        /* report congestion */
6449
0
        if (report_congestion) {
6450
0
            QUICLY_PROBE(ECN_CONGESTION, conn, conn->stash.now, conn->super.stats.num_packets.acked_ecn_counts[2]);
6451
0
            QUICLY_LOG_CONN(ecn_congestion, conn,
6452
0
                            { PTLS_LOG_ELEMENT_UNSIGNED(ce_count, conn->super.stats.num_packets.acked_ecn_counts[2]); });
6453
0
            notify_congestion_to_cc(conn, 0, largest_newly_acked.pn);
6454
0
        }
6455
0
    }
6456
6457
0
    setup_next_send(conn);
6458
6459
0
    return 0;
6460
0
}
6461
6462
static quicly_error_t handle_max_stream_data_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6463
0
{
6464
0
    quicly_max_stream_data_frame_t frame;
6465
0
    quicly_stream_t *stream;
6466
0
    quicly_error_t ret;
6467
6468
0
    if ((ret = quicly_decode_max_stream_data_frame(&state->src, state->end, &frame)) != 0)
6469
0
        return ret;
6470
6471
0
    QUICLY_PROBE(MAX_STREAM_DATA_RECEIVE, conn, conn->stash.now, frame.stream_id, frame.max_stream_data);
6472
0
    QUICLY_LOG_CONN(max_stream_data_receive, conn, {
6473
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, (quicly_stream_id_t)frame.stream_id);
6474
0
        PTLS_LOG_ELEMENT_UNSIGNED(max_stream_data, frame.max_stream_data);
6475
0
    });
6476
6477
0
    if (!quicly_stream_has_send_side(quicly_is_client(conn), frame.stream_id))
6478
0
        return QUICLY_TRANSPORT_ERROR_FRAME_ENCODING;
6479
6480
0
    if ((stream = quicly_get_stream(conn, frame.stream_id)) == NULL)
6481
0
        return 0;
6482
6483
0
    if (frame.max_stream_data <= stream->_send_aux.max_stream_data)
6484
0
        return 0;
6485
0
    stream->_send_aux.max_stream_data = frame.max_stream_data;
6486
0
    stream->_send_aux.blocked = QUICLY_SENDER_STATE_NONE;
6487
6488
0
    if (stream->_send_aux.reset_stream.sender_state == QUICLY_SENDER_STATE_NONE)
6489
0
        resched_stream_data(stream);
6490
6491
0
    return 0;
6492
0
}
6493
6494
static quicly_error_t handle_data_blocked_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6495
0
{
6496
0
    quicly_data_blocked_frame_t frame;
6497
0
    quicly_error_t ret;
6498
6499
0
    if ((ret = quicly_decode_data_blocked_frame(&state->src, state->end, &frame)) != 0)
6500
0
        return ret;
6501
6502
0
    QUICLY_PROBE(DATA_BLOCKED_RECEIVE, conn, conn->stash.now, frame.offset);
6503
0
    QUICLY_LOG_CONN(data_blocked_receive, conn, { PTLS_LOG_ELEMENT_UNSIGNED(off, frame.offset); });
6504
6505
0
    quicly_maxsender_request_transmit(&conn->ingress.max_data.sender);
6506
0
    if (should_send_max_data(conn))
6507
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
6508
6509
0
    return 0;
6510
0
}
6511
6512
static quicly_error_t handle_stream_data_blocked_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6513
0
{
6514
0
    quicly_stream_data_blocked_frame_t frame;
6515
0
    quicly_stream_t *stream;
6516
0
    quicly_error_t ret;
6517
6518
0
    if ((ret = quicly_decode_stream_data_blocked_frame(&state->src, state->end, &frame)) != 0)
6519
0
        return ret;
6520
6521
0
    QUICLY_PROBE(STREAM_DATA_BLOCKED_RECEIVE, conn, conn->stash.now, frame.stream_id, frame.offset);
6522
0
    QUICLY_LOG_CONN(stream_data_blocked_receive, conn, {
6523
0
        PTLS_LOG_ELEMENT_SIGNED(stream_id, frame.stream_id);
6524
0
        PTLS_LOG_ELEMENT_UNSIGNED(maximum, frame.offset);
6525
0
    });
6526
6527
0
    if (!quicly_stream_has_receive_side(quicly_is_client(conn), frame.stream_id))
6528
0
        return QUICLY_TRANSPORT_ERROR_FRAME_ENCODING;
6529
6530
0
    if ((stream = quicly_get_stream(conn, frame.stream_id)) != NULL) {
6531
0
        quicly_maxsender_request_transmit(&stream->_send_aux.max_stream_data_sender);
6532
0
        if (should_send_max_stream_data(stream))
6533
0
            sched_stream_control(stream);
6534
0
    }
6535
6536
0
    return 0;
6537
0
}
6538
6539
static quicly_error_t handle_streams_blocked_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6540
0
{
6541
0
    quicly_streams_blocked_frame_t frame;
6542
0
    int uni = state->frame_type == QUICLY_FRAME_TYPE_STREAMS_BLOCKED_UNI;
6543
0
    quicly_error_t ret;
6544
6545
0
    if ((ret = quicly_decode_streams_blocked_frame(&state->src, state->end, &frame)) != 0)
6546
0
        return ret;
6547
6548
0
    QUICLY_PROBE(STREAMS_BLOCKED_RECEIVE, conn, conn->stash.now, frame.count, uni);
6549
0
    QUICLY_LOG_CONN(streams_blocked_receive, conn, {
6550
0
        PTLS_LOG_ELEMENT_UNSIGNED(maximum, frame.count);
6551
0
        PTLS_LOG_ELEMENT_BOOL(is_unidirectional, uni);
6552
0
    });
6553
6554
0
    if (should_send_max_streams(conn, uni)) {
6555
0
        quicly_maxsender_t *maxsender = uni ? &conn->ingress.max_streams.uni : &conn->ingress.max_streams.bidi;
6556
0
        quicly_maxsender_request_transmit(maxsender);
6557
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
6558
0
    }
6559
6560
0
    return 0;
6561
0
}
6562
6563
static quicly_error_t handle_max_streams_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state, int uni)
6564
0
{
6565
0
    quicly_max_streams_frame_t frame;
6566
0
    quicly_error_t ret;
6567
6568
0
    if ((ret = quicly_decode_max_streams_frame(&state->src, state->end, &frame)) != 0)
6569
0
        return ret;
6570
6571
0
    QUICLY_PROBE(MAX_STREAMS_RECEIVE, conn, conn->stash.now, frame.count, uni);
6572
0
    QUICLY_LOG_CONN(max_streams_receive, conn, {
6573
0
        PTLS_LOG_ELEMENT_UNSIGNED(maximum, frame.count);
6574
0
        PTLS_LOG_ELEMENT_BOOL(is_unidirectional, uni);
6575
0
    });
6576
6577
0
    if ((ret = update_max_streams(uni ? &conn->egress.max_streams.uni : &conn->egress.max_streams.bidi, frame.count)) != 0)
6578
0
        return ret;
6579
6580
0
    open_blocked_streams(conn, uni);
6581
6582
0
    return 0;
6583
0
}
6584
6585
static quicly_error_t handle_max_streams_bidi_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6586
0
{
6587
0
    return handle_max_streams_frame(conn, state, 0);
6588
0
}
6589
6590
static quicly_error_t handle_max_streams_uni_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6591
0
{
6592
0
    return handle_max_streams_frame(conn, state, 1);
6593
0
}
6594
6595
static quicly_error_t handle_path_challenge_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6596
0
{
6597
0
    quicly_path_challenge_frame_t frame;
6598
0
    quicly_error_t ret;
6599
6600
0
    if ((ret = quicly_decode_path_challenge_frame(&state->src, state->end, &frame)) != 0)
6601
0
        return ret;
6602
6603
0
    QUICLY_PROBE(PATH_CHALLENGE_RECEIVE, conn, conn->stash.now, frame.data, QUICLY_PATH_CHALLENGE_DATA_LEN);
6604
0
    QUICLY_LOG_CONN(path_challenge_receive, conn, { PTLS_LOG_ELEMENT_HEXDUMP(data, frame.data, QUICLY_PATH_CHALLENGE_DATA_LEN); });
6605
6606
    /* schedule the emission of PATH_RESPONSE frame */
6607
0
    struct st_quicly_conn_path_t *path = conn->paths[state->path_index];
6608
0
    memcpy(path->path_response.data, frame.data, QUICLY_PATH_CHALLENGE_DATA_LEN);
6609
0
    path->path_response.send_ = 1;
6610
0
    conn->egress.send_probe_at = 0;
6611
6612
0
    return 0;
6613
0
}
6614
6615
static quicly_error_t handle_path_response_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6616
0
{
6617
0
    quicly_path_challenge_frame_t frame;
6618
0
    quicly_error_t ret;
6619
6620
0
    if ((ret = quicly_decode_path_challenge_frame(&state->src, state->end, &frame)) != 0)
6621
0
        return ret;
6622
6623
0
    QUICLY_PROBE(PATH_RESPONSE_RECEIVE, conn, conn->stash.now, frame.data, QUICLY_PATH_CHALLENGE_DATA_LEN);
6624
0
    QUICLY_LOG_CONN(path_response_receive, conn, { PTLS_LOG_ELEMENT_HEXDUMP(data, frame.data, QUICLY_PATH_CHALLENGE_DATA_LEN); });
6625
6626
0
    struct st_quicly_conn_path_t *path = conn->paths[state->path_index];
6627
6628
0
    if (ptls_mem_equal(path->path_challenge.data, frame.data, QUICLY_PATH_CHALLENGE_DATA_LEN)) {
6629
        /* Path validation succeeded, stop sending PATH_CHALLENGEs. Active path might become changed in `quicly_receive`. */
6630
0
        path->path_challenge.send_at = INT64_MAX;
6631
0
        recalc_send_probe_at(conn);
6632
0
        conn->super.stats.num_paths.validated += 1;
6633
0
    }
6634
6635
0
    return 0;
6636
0
}
6637
6638
static quicly_error_t handle_new_token_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6639
0
{
6640
0
    quicly_new_token_frame_t frame;
6641
0
    quicly_error_t ret;
6642
6643
0
    if (!quicly_is_client(conn))
6644
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6645
0
    if ((ret = quicly_decode_new_token_frame(&state->src, state->end, &frame)) != 0)
6646
0
        return ret;
6647
0
    QUICLY_PROBE(NEW_TOKEN_RECEIVE, conn, conn->stash.now, frame.token.base, frame.token.len);
6648
0
    QUICLY_LOG_CONN(new_token_receive, conn, { PTLS_LOG_ELEMENT_HEXDUMP(token, frame.token.base, frame.token.len); });
6649
0
    if (conn->super.ctx->save_resumption_token == NULL)
6650
0
        return 0;
6651
0
    return conn->super.ctx->save_resumption_token->cb(conn->super.ctx->save_resumption_token, conn, frame.token);
6652
0
}
6653
6654
static quicly_error_t handle_stop_sending_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6655
0
{
6656
0
    quicly_stop_sending_frame_t frame;
6657
0
    quicly_stream_t *stream;
6658
0
    quicly_error_t ret;
6659
6660
0
    if ((ret = quicly_decode_stop_sending_frame(&state->src, state->end, &frame)) != 0)
6661
0
        return ret;
6662
0
    QUICLY_PROBE(STOP_SENDING_RECEIVE, conn, conn->stash.now, frame.stream_id, frame.app_error_code);
6663
0
    QUICLY_LOG_CONN(stop_sending_receive, conn, {
6664
0
        PTLS_LOG_ELEMENT_UNSIGNED(stream_id, (quicly_stream_id_t)frame.stream_id);
6665
0
        PTLS_LOG_ELEMENT_UNSIGNED(error_code, frame.app_error_code);
6666
0
    });
6667
6668
0
    if ((ret = quicly_get_or_open_stream(conn, frame.stream_id, &stream)) != 0 || stream == NULL)
6669
0
        return ret;
6670
6671
0
    if (quicly_sendstate_is_open(&stream->sendstate)) {
6672
        /* reset the stream, then notify the application */
6673
0
        quicly_error_t err = QUICLY_ERROR_FROM_APPLICATION_ERROR_CODE(frame.app_error_code);
6674
0
        quicly_reset_stream(stream, err);
6675
0
        QUICLY_PROBE(STREAM_ON_SEND_STOP, stream->conn, stream->conn->stash.now, stream, err);
6676
0
        QUICLY_LOG_CONN(stream_on_send_stop, stream->conn, {
6677
0
            PTLS_LOG_ELEMENT_SIGNED(stream_id, stream->stream_id);
6678
0
            PTLS_LOG_ELEMENT_SIGNED(err, err);
6679
0
        });
6680
0
        stream->callbacks->on_send_stop(stream, err);
6681
0
        if (stream->conn->super.state >= QUICLY_STATE_CLOSING)
6682
0
            return QUICLY_ERROR_IS_CLOSING;
6683
0
    }
6684
6685
0
    return 0;
6686
0
}
6687
6688
static quicly_error_t handle_max_data_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6689
0
{
6690
0
    quicly_max_data_frame_t frame;
6691
0
    quicly_error_t ret;
6692
6693
0
    if ((ret = quicly_decode_max_data_frame(&state->src, state->end, &frame)) != 0)
6694
0
        return ret;
6695
6696
0
    QUICLY_PROBE(MAX_DATA_RECEIVE, conn, conn->stash.now, frame.max_data);
6697
0
    QUICLY_LOG_CONN(max_data_receive, conn, { PTLS_LOG_ELEMENT_UNSIGNED(maximum, frame.max_data); });
6698
6699
0
    if (frame.max_data <= conn->egress.max_data.permitted)
6700
0
        return 0;
6701
0
    conn->egress.max_data.permitted = frame.max_data;
6702
0
    conn->egress.data_blocked = QUICLY_SENDER_STATE_NONE; /* DATA_BLOCKED has not been sent for the new limit */
6703
6704
0
    return 0;
6705
0
}
6706
6707
static quicly_error_t negotiate_using_version(quicly_conn_t *conn, uint32_t version)
6708
0
{
6709
0
    quicly_error_t ret;
6710
6711
    /* set selected version, update transport parameters extension ID */
6712
0
    conn->super.version = version;
6713
0
    QUICLY_PROBE(VERSION_SWITCH, conn, conn->stash.now, version);
6714
0
    QUICLY_LOG_CONN(version_switch, conn, { PTLS_LOG_ELEMENT_UNSIGNED(new_version, version); });
6715
6716
    /* replace initial keys */
6717
0
    if ((ret = reinstall_initial_encryption(conn, PTLS_ERROR_LIBRARY)) != 0)
6718
0
        return ret;
6719
6720
    /* reschedule all the packets that have been sent for immediate resend */
6721
0
    if ((ret = discard_sentmap_by_epoch(conn, ~0u)) != 0)
6722
0
        return ret;
6723
6724
0
    return 0;
6725
0
}
6726
6727
static quicly_error_t handle_version_negotiation_packet(quicly_conn_t *conn, quicly_decoded_packet_t *packet)
6728
0
{
6729
0
    const uint8_t *src = packet->octets.base + packet->encrypted_off, *end = packet->octets.base + packet->octets.len;
6730
0
    uint32_t selected_version = 0;
6731
6732
0
    if (src == end || (end - src) % 4 != 0)
6733
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6734
6735
    /* select in the precedence of V1 -> draft29 -> draft27 -> fail */
6736
0
    while (src != end) {
6737
0
        uint32_t supported_version = quicly_decode32(&src);
6738
0
        switch (supported_version) {
6739
0
        case QUICLY_PROTOCOL_VERSION_1:
6740
0
            selected_version = QUICLY_PROTOCOL_VERSION_1;
6741
0
            break;
6742
0
        case QUICLY_PROTOCOL_VERSION_DRAFT29:
6743
0
            if (selected_version == 0 || selected_version == QUICLY_PROTOCOL_VERSION_DRAFT27)
6744
0
                selected_version = QUICLY_PROTOCOL_VERSION_DRAFT29;
6745
0
            break;
6746
0
        case QUICLY_PROTOCOL_VERSION_DRAFT27:
6747
0
            if (selected_version == 0)
6748
0
                selected_version = QUICLY_PROTOCOL_VERSION_DRAFT27;
6749
0
            break;
6750
0
        }
6751
0
    }
6752
0
    if (selected_version == 0)
6753
0
        return handle_close(conn, QUICLY_ERROR_NO_COMPATIBLE_VERSION, QUICLY_FRAME_TYPE_PADDING, ptls_iovec_init("", 0));
6754
6755
0
    return negotiate_using_version(conn, selected_version);
6756
0
}
6757
6758
static int compare_socket_address(struct sockaddr *x, struct sockaddr *y)
6759
0
{
6760
0
#define CMP(a, b)                                                                                                                  \
6761
0
    if (a != b)                                                                                                                    \
6762
0
    return a < b ? -1 : 1
6763
6764
0
    CMP(x->sa_family, y->sa_family);
6765
6766
0
    if (x->sa_family == AF_INET) {
6767
0
        struct sockaddr_in *xin = (void *)x, *yin = (void *)y;
6768
0
        CMP(ntohl(xin->sin_addr.s_addr), ntohl(yin->sin_addr.s_addr));
6769
0
        CMP(ntohs(xin->sin_port), ntohs(yin->sin_port));
6770
0
    } else if (x->sa_family == AF_INET6) {
6771
0
        struct sockaddr_in6 *xin6 = (void *)x, *yin6 = (void *)y;
6772
0
        int r = memcmp(xin6->sin6_addr.s6_addr, yin6->sin6_addr.s6_addr, sizeof(xin6->sin6_addr.s6_addr));
6773
0
        if (r != 0)
6774
0
            return r;
6775
0
        CMP(ntohs(xin6->sin6_port), ntohs(yin6->sin6_port));
6776
0
        CMP(xin6->sin6_scope_id, yin6->sin6_scope_id);
6777
0
    } else if (x->sa_family == AF_UNSPEC) {
6778
0
        return 1;
6779
0
    } else {
6780
0
        assert(!"unknown sa_family");
6781
0
    }
6782
6783
0
#undef CMP
6784
0
    return 0;
6785
0
}
6786
6787
static int is_stateless_reset(quicly_conn_t *conn, quicly_decoded_packet_t *decoded)
6788
0
{
6789
0
    switch (decoded->_is_stateless_reset_cached) {
6790
0
    case QUICLY__DECODED_PACKET_CACHED_IS_STATELESS_RESET:
6791
0
        return 1;
6792
0
    case QUICLY__DECODED_PACKET_CACHED_NOT_STATELESS_RESET:
6793
0
        return 0;
6794
0
    default:
6795
0
        break;
6796
0
    }
6797
6798
0
    if (decoded->octets.len < QUICLY_STATELESS_RESET_PACKET_MIN_LEN)
6799
0
        return 0;
6800
6801
0
    for (size_t i = 0; i < PTLS_ELEMENTSOF(conn->super.remote.cid_set.cids); ++i) {
6802
0
        if (conn->super.remote.cid_set.cids[i].state == QUICLY_REMOTE_CID_UNAVAILABLE)
6803
0
            continue;
6804
0
        if (memcmp(decoded->octets.base + decoded->octets.len - QUICLY_STATELESS_RESET_TOKEN_LEN,
6805
0
                   conn->super.remote.cid_set.cids[i].stateless_reset_token, QUICLY_STATELESS_RESET_TOKEN_LEN) == 0)
6806
0
            return 1;
6807
0
    }
6808
6809
0
    return 0;
6810
0
}
6811
6812
int quicly_is_destination(quicly_conn_t *conn, struct sockaddr *dest_addr, struct sockaddr *src_addr,
6813
                          quicly_decoded_packet_t *decoded)
6814
0
{
6815
0
    if (QUICLY_PACKET_IS_LONG_HEADER(decoded->octets.base[0])) {
6816
        /* long header: validate address, then consult the CID */
6817
0
        if (compare_socket_address(&conn->paths[0]->address.remote.sa, src_addr) != 0)
6818
0
            return 0;
6819
0
        if (conn->paths[0]->address.local.sa.sa_family != AF_UNSPEC &&
6820
0
            compare_socket_address(&conn->paths[0]->address.local.sa, dest_addr) != 0)
6821
0
            return 0;
6822
        /* server may see the CID generated by the client for Initial and 0-RTT packets */
6823
0
        if (!quicly_is_client(conn) && decoded->cid.dest.might_be_client_generated) {
6824
0
            const quicly_cid_t *odcid = is_retry(conn) ? &conn->retry_scid : &conn->super.original_dcid;
6825
0
            if (quicly_cid_is_equal(odcid, decoded->cid.dest.encrypted))
6826
0
                goto Found;
6827
0
        }
6828
0
    }
6829
6830
0
    if (conn->super.ctx->cid_encryptor != NULL) {
6831
        /* Note on multiple CIDs
6832
         * Multiple CIDs issued by this host are always based on the same 3-tuple (master_id, thread_id, node_id)
6833
         * and the only difference is path_id. Therefore comparing the 3-tuple is enough to cover all CIDs issued by
6834
         * this host.
6835
         */
6836
0
        if (conn->super.local.cid_set.plaintext.master_id == decoded->cid.dest.plaintext.master_id &&
6837
0
            conn->super.local.cid_set.plaintext.thread_id == decoded->cid.dest.plaintext.thread_id &&
6838
0
            conn->super.local.cid_set.plaintext.node_id == decoded->cid.dest.plaintext.node_id)
6839
0
            goto Found;
6840
0
        if (is_stateless_reset(conn, decoded))
6841
0
            goto Found_StatelessReset;
6842
0
    } else {
6843
0
        if (compare_socket_address(&conn->paths[0]->address.remote.sa, src_addr) == 0)
6844
0
            goto Found;
6845
0
        if (conn->paths[0]->address.local.sa.sa_family != AF_UNSPEC &&
6846
0
            compare_socket_address(&conn->paths[0]->address.local.sa, dest_addr) != 0)
6847
0
            return 0;
6848
0
    }
6849
6850
    /* not found */
6851
0
    return 0;
6852
6853
0
Found:
6854
0
    decoded->_is_stateless_reset_cached = QUICLY__DECODED_PACKET_CACHED_NOT_STATELESS_RESET;
6855
0
    return 1;
6856
6857
0
Found_StatelessReset:
6858
0
    decoded->_is_stateless_reset_cached = QUICLY__DECODED_PACKET_CACHED_IS_STATELESS_RESET;
6859
0
    return 1;
6860
0
}
6861
6862
quicly_error_t handle_close(quicly_conn_t *conn, quicly_error_t err, uint64_t frame_type, ptls_iovec_t reason_phrase)
6863
0
{
6864
0
    quicly_error_t ret;
6865
6866
0
    if (conn->super.state >= QUICLY_STATE_CLOSING)
6867
0
        return 0;
6868
6869
    /* switch to closing state, notify the app (at this moment the streams are accessible), then destroy the streams */
6870
0
    if ((ret = enter_close(conn, 0,
6871
0
                           !(err == QUICLY_ERROR_RECEIVED_STATELESS_RESET || err == QUICLY_ERROR_NO_COMPATIBLE_VERSION))) != 0 ||
6872
0
        (ret = set_connection_close(conn, err, frame_type, (const char *)reason_phrase.base, reason_phrase.len, 1)) != 0)
6873
0
        return ret;
6874
0
    if (conn->super.ctx->closed != NULL)
6875
0
        conn->super.ctx->closed->cb(conn->super.ctx->closed, conn);
6876
0
    destroy_all_streams(conn, err, 0);
6877
6878
0
    return QUICLY_ERROR_IS_CLOSING;
6879
0
}
6880
6881
static quicly_error_t handle_transport_close_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6882
0
{
6883
0
    quicly_transport_close_frame_t frame;
6884
0
    quicly_error_t ret;
6885
6886
0
    if ((ret = quicly_decode_transport_close_frame(&state->src, state->end, &frame)) != 0)
6887
0
        return ret;
6888
6889
0
    QUICLY_PROBE(TRANSPORT_CLOSE_RECEIVE, conn, conn->stash.now, frame.error_code, frame.frame_type,
6890
0
                 QUICLY_PROBE_ESCAPE_UNSAFE_STRING(frame.reason_phrase.base, frame.reason_phrase.len));
6891
0
    QUICLY_LOG_CONN(transport_close_receive, conn, {
6892
0
        PTLS_LOG_ELEMENT_UNSIGNED(error_code, frame.error_code);
6893
0
        PTLS_LOG_ELEMENT_UNSIGNED(frame_type, frame.frame_type);
6894
0
        PTLS_LOG_ELEMENT_UNSAFESTR(reason_phrase, (const char *)frame.reason_phrase.base, frame.reason_phrase.len);
6895
0
    });
6896
0
    return handle_close(conn, QUICLY_ERROR_FROM_TRANSPORT_ERROR_CODE(frame.error_code), frame.frame_type, frame.reason_phrase);
6897
0
}
6898
6899
static quicly_error_t handle_application_close_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6900
0
{
6901
0
    quicly_application_close_frame_t frame;
6902
0
    quicly_error_t ret;
6903
6904
0
    if ((ret = quicly_decode_application_close_frame(&state->src, state->end, &frame)) != 0)
6905
0
        return ret;
6906
6907
0
    QUICLY_PROBE(APPLICATION_CLOSE_RECEIVE, conn, conn->stash.now, frame.error_code,
6908
0
                 QUICLY_PROBE_ESCAPE_UNSAFE_STRING(frame.reason_phrase.base, frame.reason_phrase.len));
6909
0
    QUICLY_LOG_CONN(application_close_receive, conn, {
6910
0
        PTLS_LOG_ELEMENT_UNSIGNED(error_code, frame.error_code);
6911
0
        PTLS_LOG_ELEMENT_UNSAFESTR(reason_phrase, (const char *)frame.reason_phrase.base, frame.reason_phrase.len);
6912
0
    });
6913
0
    return handle_close(conn, QUICLY_ERROR_FROM_APPLICATION_ERROR_CODE(frame.error_code), UINT64_MAX, frame.reason_phrase);
6914
0
}
6915
6916
static quicly_error_t handle_padding_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6917
0
{
6918
0
    return 0;
6919
0
}
6920
6921
static quicly_error_t handle_ping_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6922
0
{
6923
0
    QUICLY_PROBE(PING_RECEIVE, conn, conn->stash.now);
6924
0
    QUICLY_LOG_CONN(ping_receive, conn, {});
6925
6926
0
    return 0;
6927
0
}
6928
6929
static quicly_error_t handle_new_connection_id_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6930
0
{
6931
0
    quicly_new_connection_id_frame_t frame;
6932
0
    quicly_error_t ret;
6933
6934
    /* TODO: return error when using zero-length CID */
6935
6936
0
    if ((ret = quicly_decode_new_connection_id_frame(&state->src, state->end, &frame)) != 0)
6937
0
        return ret;
6938
6939
0
    QUICLY_PROBE(NEW_CONNECTION_ID_RECEIVE, conn, conn->stash.now, frame.sequence, frame.retire_prior_to,
6940
0
                 QUICLY_PROBE_HEXDUMP(frame.cid.base, frame.cid.len),
6941
0
                 QUICLY_PROBE_HEXDUMP(frame.stateless_reset_token, QUICLY_STATELESS_RESET_TOKEN_LEN));
6942
0
    QUICLY_LOG_CONN(new_connection_id_receive, conn, {
6943
0
        PTLS_LOG_ELEMENT_UNSIGNED(sequence, frame.sequence);
6944
0
        PTLS_LOG_ELEMENT_UNSIGNED(retire_prior_to, frame.retire_prior_to);
6945
0
        PTLS_LOG_ELEMENT_HEXDUMP(cid, frame.cid.base, frame.cid.len);
6946
0
        PTLS_LOG_ELEMENT_HEXDUMP(stateless_reset_token, frame.stateless_reset_token, QUICLY_STATELESS_RESET_TOKEN_LEN);
6947
0
    });
6948
6949
0
    size_t orig_num_retired = conn->super.remote.cid_set.retired.count;
6950
0
    if ((ret = quicly_remote_cid_register(&conn->super.remote.cid_set, frame.sequence, frame.cid.base, frame.cid.len,
6951
0
                                          frame.stateless_reset_token, frame.retire_prior_to)) != 0)
6952
0
        return ret;
6953
0
    if (orig_num_retired != conn->super.remote.cid_set.retired.count) {
6954
0
        for (size_t i = orig_num_retired; i < conn->super.remote.cid_set.retired.count; ++i)
6955
0
            dissociate_cid(conn, conn->super.remote.cid_set.retired.cids[i]);
6956
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
6957
0
    }
6958
6959
0
    return 0;
6960
0
}
6961
6962
static quicly_error_t handle_retire_connection_id_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6963
0
{
6964
0
    int has_pending;
6965
0
    quicly_retire_connection_id_frame_t frame;
6966
0
    quicly_error_t ret;
6967
6968
0
    if ((ret = quicly_decode_retire_connection_id_frame(&state->src, state->end, &frame)) != 0)
6969
0
        return ret;
6970
6971
0
    QUICLY_PROBE(RETIRE_CONNECTION_ID_RECEIVE, conn, conn->stash.now, frame.sequence);
6972
0
    QUICLY_LOG_CONN(retire_connection_id_receive, conn, { PTLS_LOG_ELEMENT_UNSIGNED(sequence, frame.sequence); });
6973
6974
0
    if (frame.sequence >= conn->super.local.cid_set.plaintext.path_id) {
6975
        /* Receipt of a RETIRE_CONNECTION_ID frame containing a sequence number greater than any previously sent to the remote peer
6976
         * MUST be treated as a connection error of type PROTOCOL_VIOLATION. (19.16) */
6977
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6978
0
    }
6979
6980
0
    if ((ret = quicly_local_cid_retire(&conn->super.local.cid_set, frame.sequence, &has_pending)) != 0)
6981
0
        return ret;
6982
0
    if (has_pending)
6983
0
        conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
6984
6985
0
    return 0;
6986
0
}
6987
6988
static quicly_error_t handle_handshake_done_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
6989
0
{
6990
0
    quicly_error_t ret;
6991
6992
0
    QUICLY_PROBE(HANDSHAKE_DONE_RECEIVE, conn, conn->stash.now);
6993
0
    QUICLY_LOG_CONN(handshake_done_receive, conn, {});
6994
6995
0
    if (!quicly_is_client(conn))
6996
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
6997
6998
0
    assert(conn->initial == NULL);
6999
0
    if (conn->handshake == NULL)
7000
0
        return 0;
7001
7002
0
    conn->super.remote.address_validation.send_probe = 0;
7003
0
    if ((ret = discard_handshake_context(conn, QUICLY_EPOCH_HANDSHAKE)) != 0)
7004
0
        return ret;
7005
0
    setup_next_send(conn);
7006
0
    return 0;
7007
0
}
7008
7009
static quicly_error_t handle_datagram_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
7010
0
{
7011
0
    quicly_datagram_frame_t frame;
7012
0
    quicly_error_t ret;
7013
7014
    /* check if we advertised support for DATAGRAM frames on this connection */
7015
0
    if (conn->super.ctx->transport_params.max_datagram_frame_size == 0)
7016
0
        return QUICLY_TRANSPORT_ERROR_FRAME_ENCODING;
7017
7018
    /* decode the frame */
7019
0
    if ((ret = quicly_decode_datagram_frame(state->frame_type, &state->src, state->end, &frame)) != 0)
7020
0
        return ret;
7021
0
    QUICLY_PROBE(DATAGRAM_RECEIVE, conn, conn->stash.now, frame.payload.base, frame.payload.len);
7022
0
    QUICLY_LOG_CONN(datagram_receive, conn, { PTLS_LOG_ELEMENT_UNSIGNED(payload_len, frame.payload.len); });
7023
7024
    /* handle the frame. Applications might call quicly_close or other functions that modify the connection state. */
7025
0
    conn->super.ctx->receive_datagram_frame->cb(conn->super.ctx->receive_datagram_frame, conn, frame.payload);
7026
7027
0
    return 0;
7028
0
}
7029
7030
static quicly_error_t handle_ack_frequency_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
7031
0
{
7032
0
    quicly_ack_frequency_frame_t frame;
7033
0
    quicly_error_t ret;
7034
7035
    /* recognize the frame only when the support has been advertised */
7036
0
    if (conn->super.ctx->transport_params.min_ack_delay_usec == UINT64_MAX)
7037
0
        return QUICLY_TRANSPORT_ERROR_FRAME_ENCODING;
7038
7039
0
    if ((ret = quicly_decode_ack_frequency_frame(&state->src, state->end, &frame)) != 0)
7040
0
        return ret;
7041
7042
0
    QUICLY_PROBE(ACK_FREQUENCY_RECEIVE, conn, conn->stash.now, frame.sequence, frame.packet_tolerance, frame.max_ack_delay,
7043
0
                 frame.reordering_threshold);
7044
0
    QUICLY_LOG_CONN(ack_frequency_receive, conn, {
7045
0
        PTLS_LOG_ELEMENT_UNSIGNED(sequence, frame.sequence);
7046
0
        PTLS_LOG_ELEMENT_UNSIGNED(packet_tolerance, frame.packet_tolerance);
7047
0
        PTLS_LOG_ELEMENT_UNSIGNED(max_ack_delay, frame.max_ack_delay);
7048
0
        PTLS_LOG_ELEMENT_UNSIGNED(reordering_threshold, frame.reordering_threshold);
7049
0
    });
7050
7051
    /* Reject Request Max Ack Delay below our TP.min_ack_delay (which is at the moment equal to LOCAL_MAX_ACK_DELAY). */
7052
0
    if (frame.max_ack_delay < QUICLY_LOCAL_MAX_ACK_DELAY * 1000 || frame.max_ack_delay >= (1 << 14) * 1000)
7053
0
        return QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
7054
7055
    // TODO: use received frame.max_ack_delay. We currently use a constant (25 ms) and
7056
    // ignore the value set by our transport parameter (see max_ack_delay field comment).
7057
7058
0
    if (frame.sequence >= conn->ingress.ack_frequency.next_sequence) {
7059
0
        conn->ingress.ack_frequency.next_sequence = frame.sequence + 1;
7060
0
        conn->application->super.packet_tolerance =
7061
0
            (uint32_t)(frame.packet_tolerance < QUICLY_MAX_PACKET_TOLERANCE ? frame.packet_tolerance : QUICLY_MAX_PACKET_TOLERANCE);
7062
0
        conn->application->super.reordering_threshold = frame.reordering_threshold;
7063
0
    }
7064
7065
0
    return 0;
7066
0
}
7067
7068
static quicly_error_t handle_immediate_ack_frame(quicly_conn_t *conn, struct st_quicly_handle_payload_state_t *state)
7069
0
{
7070
    /* recognize the frame only when the support has been advertised */
7071
0
    if (conn->super.ctx->transport_params.min_ack_delay_usec == UINT64_MAX)
7072
0
        return QUICLY_TRANSPORT_ERROR_FRAME_ENCODING;
7073
0
    conn->egress.send_ack_at = conn->stash.now;
7074
0
    return 0;
7075
0
}
7076
7077
static quicly_error_t handle_payload(quicly_conn_t *conn, size_t epoch, size_t path_index, const uint8_t *_src, size_t _len,
7078
                                     uint64_t *offending_frame_type, int *is_ack_only, int *is_probe_only)
7079
0
{
7080
    /* clang-format off */
7081
7082
    /* `frame_handlers` is an array of frame handlers and the properties of the frames, indexed by the ID of the frame. */
7083
0
    static const struct st_quicly_frame_handler_t {
7084
0
        quicly_error_t (*cb)(quicly_conn_t *, struct st_quicly_handle_payload_state_t *); /* callback function that handles the
7085
                                                                                           * frame */
7086
0
        uint8_t permitted_epochs;  /* the epochs the frame can appear, calculated as bitwise-or of `1 << epoch` */
7087
0
        uint8_t ack_eliciting;     /* boolean indicating if the frame is ack-eliciting */
7088
0
        uint8_t probing;           /* boolean indicating if the frame is a "probing frame" */
7089
0
        size_t counter_offset;     /* offset of corresponding `conn->super.stats.num_frames_received.type` within quicly_conn_t */
7090
0
    } frame_handlers[] = {
7091
0
#define FRAME(n, i, z, h, o, ae, p)                                                                                                \
7092
0
    {                                                                                                                              \
7093
0
        handle_##n##_frame,                                                                                                        \
7094
0
        (i << QUICLY_EPOCH_INITIAL) | (z << QUICLY_EPOCH_0RTT) | (h << QUICLY_EPOCH_HANDSHAKE) | (o << QUICLY_EPOCH_1RTT),         \
7095
0
        ae,                                                                                                                        \
7096
0
        p,                                                                                                                         \
7097
0
        offsetof(quicly_conn_t, super.stats.num_frames_received.n)                                                                 \
7098
0
    }
7099
        /*   +----------------------+-------------------+---------------+---------+
7100
         *   |                      |  permitted epochs |               |         |
7101
         *   |        frame         +----+----+----+----+ ack-eliciting | probing |
7102
         *   |                      | IN | 0R | HS | 1R |               |         |
7103
         *   +----------------------+----+----+----+----+---------------+---------+ */
7104
0
        FRAME( padding              ,  1 ,  1 ,  1 ,  1 ,             0 ,       1 ), /* 0 */
7105
0
        FRAME( ping                 ,  1 ,  1 ,  1 ,  1 ,             1 ,       0 ),
7106
0
        FRAME( ack                  ,  1 ,  0 ,  1 ,  1 ,             0 ,       0 ),
7107
0
        FRAME( ack                  ,  1 ,  0 ,  1 ,  1 ,             0 ,       0 ),
7108
0
        FRAME( reset_stream         ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7109
0
        FRAME( stop_sending         ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7110
0
        FRAME( crypto               ,  1 ,  0 ,  1 ,  1 ,             1 ,       0 ),
7111
0
        FRAME( new_token            ,  0 ,  0 ,  0 ,  1 ,             1 ,       0 ),
7112
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ), /* 8 */
7113
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7114
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7115
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7116
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7117
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7118
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7119
0
        FRAME( stream               ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7120
0
        FRAME( max_data             ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ), /* 16 */
7121
0
        FRAME( max_stream_data      ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7122
0
        FRAME( max_streams_bidi     ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7123
0
        FRAME( max_streams_uni      ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7124
0
        FRAME( data_blocked         ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7125
0
        FRAME( stream_data_blocked  ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7126
0
        FRAME( streams_blocked      ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7127
0
        FRAME( streams_blocked      ,  0 ,  1 ,  0 ,  1 ,             1 ,       0 ),
7128
0
        FRAME( new_connection_id    ,  0 ,  1 ,  0 ,  1 ,             1 ,       1 ), /* 24 */
7129
0
        FRAME( retire_connection_id ,  0 ,  0 ,  0 ,  1 ,             1 ,       0 ),
7130
0
        FRAME( path_challenge       ,  0 ,  1 ,  0 ,  1 ,             1 ,       1 ),
7131
0
        FRAME( path_response        ,  0 ,  0 ,  0 ,  1 ,             1 ,       1 ),
7132
0
        FRAME( transport_close      ,  1 ,  1 ,  1 ,  1 ,             0 ,       0 ),
7133
0
        FRAME( application_close    ,  0 ,  1 ,  0 ,  1 ,             0 ,       0 ),
7134
0
        FRAME( handshake_done       ,  0,   0 ,  0 ,  1 ,             1 ,       0 ),
7135
0
        FRAME( immediate_ack        ,  0,   0 ,  0 ,  1 ,             1 ,       0 ),
7136
        /*   +----------------------+----+----+----+----+---------------+---------+ */
7137
0
#undef FRAME
7138
0
    };
7139
0
    static const struct {
7140
0
        uint64_t type;
7141
0
        struct st_quicly_frame_handler_t _;
7142
0
    } ex_frame_handlers[] = {
7143
0
#define FRAME(uc, lc, i, z, h, o, ae, p)                                                                                           \
7144
0
    {                                                                                                                              \
7145
0
        QUICLY_FRAME_TYPE_##uc,                                                                                                    \
7146
0
        {                                                                                                                          \
7147
0
            handle_##lc##_frame,                                                                                                   \
7148
0
            (i << QUICLY_EPOCH_INITIAL) | (z << QUICLY_EPOCH_0RTT) | (h << QUICLY_EPOCH_HANDSHAKE) | (o << QUICLY_EPOCH_1RTT),     \
7149
0
            ae,                                                                                                                    \
7150
0
            p,                                                                                                                     \
7151
0
            offsetof(quicly_conn_t, super.stats.num_frames_received.lc)                                                            \
7152
0
        },                                                                                                                         \
7153
0
    }
7154
        /*   +----------------------------------+-------------------+---------------+---------+
7155
         *   |               frame              |  permitted epochs |               |         |
7156
         *   |------------------+---------------+----+----+----+----+ ack-eliciting | probing |
7157
         *   |    upper-case    |  lower-case   | IN | 0R | HS | 1R |               |         |
7158
         *   +------------------+---------------+----+----+----+----+---------------+---------+ */
7159
0
        FRAME( DATAGRAM_NOLEN   , datagram      ,  0 ,  1,   0,   1 ,             1 ,       0 ),
7160
0
        FRAME( DATAGRAM_WITHLEN , datagram      ,  0 ,  1,   0,   1 ,             1 ,       0 ),
7161
0
        FRAME( ACK_FREQUENCY    , ack_frequency ,  0 ,  0 ,  0 ,  1 ,             1 ,       0 ),
7162
        /*   +------------------+---------------+-------------------+---------------+---------+ */
7163
0
#undef FRAME
7164
0
        {UINT64_MAX},
7165
0
    };
7166
    /* clang-format on */
7167
7168
0
    struct st_quicly_handle_payload_state_t state = {.epoch = epoch, .path_index = path_index, .src = _src, .end = _src + _len};
7169
0
    size_t num_frames_ack_eliciting = 0, num_frames_non_probing = 0;
7170
0
    quicly_error_t ret;
7171
7172
0
    do {
7173
        /* determine the frame type; fast path is available for frame types below 64 */
7174
0
        const struct st_quicly_frame_handler_t *frame_handler;
7175
0
        state.frame_type = *state.src++;
7176
0
        if (state.frame_type < PTLS_ELEMENTSOF(frame_handlers)) {
7177
0
            frame_handler = frame_handlers + state.frame_type;
7178
0
        } else {
7179
            /* slow path */
7180
0
            --state.src;
7181
0
            if ((state.frame_type = quicly_decodev(&state.src, state.end)) == UINT64_MAX) {
7182
0
                state.frame_type =
7183
0
                    QUICLY_FRAME_TYPE_PADDING; /* we cannot signal the offending frame type when failing to decode the frame type */
7184
0
                ret = QUICLY_TRANSPORT_ERROR_FRAME_ENCODING;
7185
0
                break;
7186
0
            }
7187
0
            size_t i;
7188
0
            for (i = 0; ex_frame_handlers[i].type < state.frame_type; ++i)
7189
0
                ;
7190
0
            if (ex_frame_handlers[i].type != state.frame_type) {
7191
0
                ret = QUICLY_TRANSPORT_ERROR_FRAME_ENCODING; /* not found */
7192
0
                break;
7193
0
            }
7194
0
            frame_handler = &ex_frame_handlers[i]._;
7195
0
        }
7196
        /* check if frame is allowed, then process */
7197
0
        if ((frame_handler->permitted_epochs & (1 << epoch)) == 0) {
7198
0
            ret = QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
7199
0
            break;
7200
0
        }
7201
0
        ++*(uint64_t *)((uint8_t *)conn + frame_handler->counter_offset);
7202
0
        if (frame_handler->ack_eliciting)
7203
0
            ++num_frames_ack_eliciting;
7204
0
        if (!frame_handler->probing)
7205
0
            ++num_frames_non_probing;
7206
0
        if ((ret = frame_handler->cb(conn, &state)) != 0)
7207
0
            break;
7208
0
    } while (state.src != state.end);
7209
7210
0
    *is_ack_only = num_frames_ack_eliciting == 0;
7211
0
    *is_probe_only = num_frames_non_probing == 0;
7212
0
    if (ret != 0)
7213
0
        *offending_frame_type = state.frame_type;
7214
0
    return ret;
7215
0
}
7216
7217
static quicly_error_t handle_stateless_reset(quicly_conn_t *conn)
7218
0
{
7219
0
    QUICLY_PROBE(STATELESS_RESET_RECEIVE, conn, conn->stash.now);
7220
0
    QUICLY_LOG_CONN(stateless_reset_receive, conn, {});
7221
0
    return handle_close(conn, QUICLY_ERROR_RECEIVED_STATELESS_RESET, QUICLY_FRAME_TYPE_PADDING, ptls_iovec_init("", 0));
7222
0
}
7223
7224
static int validate_retry_tag(quicly_decoded_packet_t *packet, quicly_cid_t *odcid, ptls_aead_context_t *retry_aead)
7225
0
{
7226
0
    size_t pseudo_packet_len = 1 + odcid->len + packet->encrypted_off;
7227
0
    uint8_t pseudo_packet[pseudo_packet_len];
7228
0
    pseudo_packet[0] = odcid->len;
7229
0
    memcpy(pseudo_packet + 1, odcid->cid, odcid->len);
7230
0
    memcpy(pseudo_packet + 1 + odcid->len, packet->octets.base, packet->encrypted_off);
7231
0
    return ptls_aead_decrypt(retry_aead, packet->octets.base + packet->encrypted_off, packet->octets.base + packet->encrypted_off,
7232
0
                             PTLS_AESGCM_TAG_SIZE, 0, pseudo_packet, pseudo_packet_len) == 0;
7233
0
}
7234
7235
quicly_error_t quicly_accept(quicly_conn_t **conn, quicly_context_t *ctx, struct sockaddr *dest_addr, struct sockaddr *src_addr,
7236
                             quicly_decoded_packet_t *packet, quicly_address_token_plaintext_t *address_token,
7237
                             const quicly_cid_plaintext_t *new_cid, ptls_handshake_properties_t *handshake_properties,
7238
                             void *appdata)
7239
0
{
7240
0
    const quicly_salt_t *salt;
7241
0
    struct {
7242
0
        struct st_quicly_cipher_context_t ingress, egress;
7243
0
        int alive;
7244
0
    } cipher = {};
7245
0
    ptls_iovec_t payload;
7246
0
    uint64_t next_expected_pn, pn, offending_frame_type = QUICLY_FRAME_TYPE_PADDING;
7247
0
    int is_ack_only, is_probe_only;
7248
0
    quicly_error_t ret;
7249
7250
0
    *conn = NULL;
7251
7252
    /* process initials only */
7253
0
    if ((packet->octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) != QUICLY_PACKET_TYPE_INITIAL) {
7254
0
        ret = QUICLY_ERROR_PACKET_IGNORED;
7255
0
        goto Exit;
7256
0
    }
7257
0
    if ((salt = quicly_get_salt(packet->version)) == NULL) {
7258
0
        ret = QUICLY_ERROR_PACKET_IGNORED;
7259
0
        goto Exit;
7260
0
    }
7261
0
    if (packet->datagram_size < QUICLY_MIN_CLIENT_INITIAL_SIZE) {
7262
0
        ret = QUICLY_ERROR_PACKET_IGNORED;
7263
0
        goto Exit;
7264
0
    }
7265
0
    if (packet->cid.dest.encrypted.len < 8) {
7266
0
        ret = QUICLY_TRANSPORT_ERROR_PROTOCOL_VIOLATION;
7267
0
        goto Exit;
7268
0
    }
7269
0
    if ((ret = setup_initial_encryption(get_aes128gcmsha256(ctx), &cipher.ingress, &cipher.egress, packet->cid.dest.encrypted, 0,
7270
0
                                        ptls_iovec_init(salt->initial, sizeof(salt->initial)), NULL)) != 0)
7271
0
        goto Exit;
7272
0
    cipher.alive = 1;
7273
0
    next_expected_pn = 0; /* is this correct? do we need to take care of underflow? */
7274
0
    if ((ret = decrypt_packet(cipher.ingress.header_protection, aead_decrypt_fixed_key, cipher.ingress.aead, &next_expected_pn,
7275
0
                              packet, &pn, &payload)) != 0) {
7276
0
        ret = QUICLY_ERROR_DECRYPTION_FAILED;
7277
0
        goto Exit;
7278
0
    }
7279
7280
    /* create connection */
7281
0
    if ((*conn = create_connection(
7282
0
             ctx, packet->version, NULL, src_addr, dest_addr, &packet->cid.src, new_cid, handshake_properties, appdata,
7283
0
             quicly_cc_calc_initial_cwnd(ctx->initcwnd_packets, ctx->transport_params.max_udp_payload_size))) == NULL) {
7284
0
        ret = PTLS_ERROR_NO_MEMORY;
7285
0
        goto Exit;
7286
0
    }
7287
0
    (*conn)->super.state = QUICLY_STATE_ACCEPTING;
7288
0
    quicly_set_cid(&(*conn)->super.original_dcid, packet->cid.dest.encrypted);
7289
0
    if (address_token != NULL) {
7290
0
        (*conn)->super.remote.address_validation.validated = !address_token->address_mismatch;
7291
0
        switch (address_token->type) {
7292
0
        case QUICLY_ADDRESS_TOKEN_TYPE_RETRY:
7293
0
            if (!address_token->address_mismatch) {
7294
0
                (*conn)->retry_scid = (*conn)->super.original_dcid;
7295
0
                (*conn)->super.original_dcid = address_token->retry.original_dcid;
7296
0
            }
7297
0
            break;
7298
0
        case QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION:
7299
0
            if (decode_resumption_info(address_token->resumption.bytes, address_token->resumption.len,
7300
0
                                       &(*conn)->super.stats.jumpstart.prev_rate, &(*conn)->super.stats.jumpstart.prev_rtt) != 0) {
7301
0
                (*conn)->super.stats.jumpstart.prev_rate = 0;
7302
0
                (*conn)->super.stats.jumpstart.prev_rtt = 0;
7303
0
            }
7304
0
            break;
7305
0
        default:
7306
            /* We might not get here as tokens are integrity-protected, but as this is information supplied via network, potentially
7307
             * from broken quicly instances, we drop anything unexpected rather than calling abort(). */
7308
0
            break;
7309
0
        }
7310
0
    }
7311
0
    if ((ret = setup_handshake_space_and_flow(*conn, QUICLY_EPOCH_INITIAL)) != 0)
7312
0
        goto Exit;
7313
0
    (*conn)->initial->super.next_expected_packet_number = next_expected_pn;
7314
0
    (*conn)->initial->cipher.ingress = cipher.ingress;
7315
0
    (*conn)->initial->cipher.egress = cipher.egress;
7316
0
    cipher.alive = 0;
7317
0
    (*conn)->crypto.handshake_properties.collected_extensions = server_collected_extensions;
7318
0
    (*conn)->initial->largest_ingress_udp_payload_size = packet->datagram_size;
7319
7320
0
    QUICLY_PROBE(ACCEPT, *conn, (*conn)->stash.now,
7321
0
                 QUICLY_PROBE_HEXDUMP(packet->cid.dest.encrypted.base, packet->cid.dest.encrypted.len), address_token);
7322
0
    QUICLY_LOG_CONN(accept, *conn, {
7323
0
        PTLS_LOG_ELEMENT_HEXDUMP(dcid, packet->cid.dest.encrypted.base, packet->cid.dest.encrypted.len);
7324
0
        if (address_token != NULL) {
7325
0
            PTLS_LOG_ELEMENT_UNSIGNED(type, address_token->type);
7326
0
            PTLS_LOG_ELEMENT_UNSIGNED(issued_at, address_token->issued_at);
7327
0
            PTLS_LOG_ELEMENT_BOOL(address_mismatch, address_token->address_mismatch);
7328
0
            switch (address_token->type) {
7329
0
            case QUICLY_ADDRESS_TOKEN_TYPE_RETRY:
7330
0
                PTLS_LOG_ELEMENT_HEXDUMP(original_dcid, address_token->retry.original_dcid.cid,
7331
0
                                         address_token->retry.original_dcid.len);
7332
0
                PTLS_LOG_ELEMENT_HEXDUMP(client_cid, address_token->retry.client_cid.cid, address_token->retry.client_cid.len);
7333
0
                PTLS_LOG_ELEMENT_HEXDUMP(server_cid, address_token->retry.server_cid.cid, address_token->retry.server_cid.len);
7334
0
                break;
7335
0
            case QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION:
7336
0
                PTLS_LOG_ELEMENT_UNSIGNED(rate, (*conn)->super.stats.jumpstart.prev_rate);
7337
0
                PTLS_LOG_ELEMENT_UNSIGNED(rtt, (*conn)->super.stats.jumpstart.prev_rtt);
7338
0
                break;
7339
0
            }
7340
0
        }
7341
0
    });
7342
0
    QUICLY_PROBE(PACKET_RECEIVED, *conn, (*conn)->stash.now, pn, payload.base, payload.len, get_epoch(packet->octets.base[0]));
7343
0
    QUICLY_LOG_CONN(packet_received, *conn, {
7344
0
        PTLS_LOG_ELEMENT_UNSIGNED(pn, pn);
7345
0
        PTLS_LOG_APPDATA_ELEMENT_HEXDUMP(decrypted, payload.base, payload.len);
7346
0
        PTLS_LOG_ELEMENT_UNSIGNED(packet_type, get_epoch(packet->octets.base[0]));
7347
0
    });
7348
7349
    /* handle the input; we ignore is_ack_only, we consult if there's any output from TLS in response to CH anyways */
7350
0
    (*conn)->super.stats.num_packets.received += 1;
7351
0
    (*conn)->super.stats.num_packets.initial_received += 1;
7352
0
    if (packet->ecn != 0)
7353
0
        (*conn)->super.stats.num_packets.received_ecn_counts[get_ecn_index_from_bits(packet->ecn)] += 1;
7354
0
    (*conn)->super.stats.num_bytes.received += packet->datagram_size;
7355
0
    if ((ret = handle_payload(*conn, QUICLY_EPOCH_INITIAL, 0, payload.base, payload.len, &offending_frame_type, &is_ack_only,
7356
0
                              &is_probe_only)) != 0)
7357
0
        goto Exit;
7358
0
    if ((ret = record_receipt(&(*conn)->initial->super, pn, packet->ecn, 0, (*conn)->stash.now, &(*conn)->egress.send_ack_at,
7359
0
                              &(*conn)->super.stats.num_packets.received_out_of_order)) != 0)
7360
0
        goto Exit;
7361
7362
0
Exit:
7363
0
    if (*conn != NULL) {
7364
0
        if (ret == 0) {
7365
            /* if CONNECTION_CLOSE was found and the state advanced to DRAINING, we need to retain that state */
7366
0
            if ((*conn)->super.state < QUICLY_STATE_CONNECTED)
7367
0
                (*conn)->super.state = QUICLY_STATE_CONNECTED;
7368
0
        } else {
7369
0
            initiate_close(*conn, ret, offending_frame_type, "");
7370
0
            ret = 0;
7371
0
        }
7372
0
        unlock_now(*conn);
7373
0
    }
7374
0
    if (cipher.alive) {
7375
0
        dispose_cipher(&cipher.ingress);
7376
0
        dispose_cipher(&cipher.egress);
7377
0
    }
7378
0
    return ret;
7379
0
}
7380
7381
/**
7382
 * @param receive_delay  set to -1 when received for the first time, but if buffered for replay, contains how long the packet has
7383
 *                       been delayed
7384
 */
7385
static quicly_error_t do_receive(quicly_conn_t *conn, struct sockaddr *dest_addr, struct sockaddr *src_addr,
7386
                                 quicly_decoded_packet_t *packet, int64_t receive_delay, int *might_be_reorder)
7387
0
{
7388
0
    ptls_cipher_context_t *header_protection;
7389
0
    struct {
7390
0
        int (*cb)(void *, uint64_t, quicly_decoded_packet_t *, size_t, size_t *);
7391
0
        void *ctx;
7392
0
    } aead;
7393
0
    struct st_quicly_pn_space_t **space;
7394
0
    size_t epoch, path_index;
7395
0
    ptls_iovec_t payload;
7396
0
    uint64_t pn, offending_frame_type = QUICLY_FRAME_TYPE_PADDING;
7397
0
    int is_ack_only, is_probe_only;
7398
0
    quicly_error_t ret;
7399
7400
0
    assert(src_addr->sa_family == AF_INET || src_addr->sa_family == AF_INET6);
7401
7402
0
    *might_be_reorder = 0;
7403
7404
0
    QUICLY_PROBE(RECEIVE, conn, conn->stash.now,
7405
0
                 QUICLY_PROBE_HEXDUMP(packet->cid.dest.encrypted.base, packet->cid.dest.encrypted.len), packet->octets.base,
7406
0
                 packet->octets.len, receive_delay);
7407
0
    QUICLY_LOG_CONN(receive, conn, {
7408
0
        PTLS_LOG_ELEMENT_HEXDUMP(dcid, packet->cid.dest.encrypted.base, packet->cid.dest.encrypted.len);
7409
0
        PTLS_LOG_ELEMENT_HEXDUMP(bytes, packet->octets.base, packet->octets.len);
7410
0
        PTLS_LOG_ELEMENT_SIGNED(receive_delay, receive_delay);
7411
0
    });
7412
7413
    /* drop packets with invalid server tuple (note: when running as a server, `dest_addr` may not be available depending on the
7414
     * socket option being used */
7415
0
    if (quicly_is_client(conn)) {
7416
0
        if (compare_socket_address(src_addr, &conn->paths[0]->address.remote.sa) != 0) {
7417
0
            ret = QUICLY_ERROR_PACKET_IGNORED;
7418
0
            goto Exit;
7419
0
        }
7420
0
    } else if (dest_addr != NULL && dest_addr->sa_family != AF_UNSPEC) {
7421
0
        assert(conn->paths[0]->address.local.sa.sa_family != AF_UNSPEC);
7422
0
        if (compare_socket_address(dest_addr, &conn->paths[0]->address.local.sa) != 0) {
7423
0
            ret = QUICLY_ERROR_PACKET_IGNORED;
7424
0
            goto Exit;
7425
0
        }
7426
0
    }
7427
7428
0
    if (is_stateless_reset(conn, packet)) {
7429
0
        ret = handle_stateless_reset(conn);
7430
0
        goto Exit;
7431
0
    }
7432
7433
    /* Determine the incoming path. path_index may be set to PTLS_ELEMENTSOF(conn->paths), which indicates that a new path needs to
7434
     * be created once packet decryption succeeds. */
7435
0
    for (path_index = 0; path_index < PTLS_ELEMENTSOF(conn->paths); ++path_index)
7436
0
        if (conn->paths[path_index] != NULL && compare_socket_address(src_addr, &conn->paths[path_index]->address.remote.sa) == 0)
7437
0
            break;
7438
0
    if (path_index != 0 && !quicly_is_client(conn) &&
7439
0
        (QUICLY_PACKET_IS_LONG_HEADER(packet->octets.base[0]) || !conn->super.remote.address_validation.validated)) {
7440
0
        ret = QUICLY_ERROR_PACKET_IGNORED;
7441
0
        goto Exit;
7442
0
    }
7443
0
    if (path_index == PTLS_ELEMENTSOF(conn->paths) &&
7444
0
        conn->super.stats.num_paths.validation_failed >= conn->super.ctx->max_path_validation_failures) {
7445
0
        ret = QUICLY_ERROR_PACKET_IGNORED;
7446
0
        goto Exit;
7447
0
    }
7448
7449
    /* update num_bytes.received which is counted at the datagram-level */
7450
0
    if (packet->first_packet)
7451
0
        conn->super.stats.num_bytes.received += packet->datagram_size;
7452
7453
0
    switch (conn->super.state) {
7454
0
    case QUICLY_STATE_CLOSING:
7455
0
        ++conn->connection_close.num_packets_received;
7456
        /* respond with a CONNECTION_CLOSE frame using exponential back-off */
7457
0
        if (__builtin_popcountl(conn->connection_close.num_packets_received) == 1)
7458
0
            conn->egress.send_ack_at = 0;
7459
0
        ret = 0;
7460
0
        goto Exit;
7461
0
    case QUICLY_STATE_DRAINING:
7462
0
        ret = 0;
7463
0
        goto Exit;
7464
0
    default:
7465
0
        break;
7466
0
    }
7467
7468
0
    if (QUICLY_PACKET_IS_LONG_HEADER(packet->octets.base[0])) {
7469
0
        if (conn->super.state == QUICLY_STATE_FIRSTFLIGHT) {
7470
0
            if (packet->version == 0) {
7471
0
                ret = handle_version_negotiation_packet(conn, packet);
7472
0
                goto Exit;
7473
0
            }
7474
0
        }
7475
0
        if (packet->version != conn->super.version) {
7476
0
            ret = QUICLY_ERROR_PACKET_IGNORED;
7477
0
            goto Exit;
7478
0
        }
7479
0
        switch (packet->octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) {
7480
0
        case QUICLY_PACKET_TYPE_RETRY: {
7481
0
            assert(packet->encrypted_off + PTLS_AESGCM_TAG_SIZE == packet->octets.len);
7482
            /* handle only if the connection is the client */
7483
0
            if (!quicly_is_client(conn)) {
7484
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7485
0
                goto Exit;
7486
0
            }
7487
            /* server CID has to change */
7488
0
            if (quicly_cid_is_equal(&conn->super.remote.cid_set.cids[0].cid, packet->cid.src)) {
7489
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7490
0
                goto Exit;
7491
0
            }
7492
            /* do not accept a second Retry */
7493
0
            if (is_retry(conn)) {
7494
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7495
0
                goto Exit;
7496
0
            }
7497
0
            ptls_aead_context_t *retry_aead = create_retry_aead(conn->super.ctx, conn->super.version, 0);
7498
0
            int retry_ok = validate_retry_tag(packet, &conn->super.remote.cid_set.cids[0].cid, retry_aead);
7499
0
            ptls_aead_free(retry_aead);
7500
0
            if (!retry_ok) {
7501
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7502
0
                goto Exit;
7503
0
            }
7504
            /* check size of the Retry packet */
7505
0
            if (packet->token.len > QUICLY_MAX_TOKEN_LEN) {
7506
0
                ret = QUICLY_ERROR_PACKET_IGNORED; /* TODO this is a immediate fatal error, chose a better error code */
7507
0
                goto Exit;
7508
0
            }
7509
            /* store token and ODCID */
7510
0
            free(conn->token.base);
7511
0
            if ((conn->token.base = malloc(packet->token.len)) == NULL) {
7512
0
                ret = PTLS_ERROR_NO_MEMORY;
7513
0
                goto Exit;
7514
0
            }
7515
0
            memcpy(conn->token.base, packet->token.base, packet->token.len);
7516
0
            conn->token.len = packet->token.len;
7517
            /* update DCID */
7518
0
            quicly_set_cid(&conn->super.remote.cid_set.cids[0].cid, packet->cid.src);
7519
0
            conn->retry_scid = conn->super.remote.cid_set.cids[0].cid;
7520
            /* replace initial keys, or drop the keys if this is a response packet to a greased version */
7521
0
            if ((ret = reinstall_initial_encryption(conn, QUICLY_ERROR_PACKET_IGNORED)) != 0)
7522
0
                goto Exit;
7523
            /* schedule retransmit */
7524
0
            ret = discard_sentmap_by_epoch(conn, ~0u);
7525
0
            goto Exit;
7526
0
        } break;
7527
0
        case QUICLY_PACKET_TYPE_INITIAL:
7528
0
            if (conn->initial == NULL || (header_protection = conn->initial->cipher.ingress.header_protection) == NULL) {
7529
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7530
0
                goto Exit;
7531
0
            }
7532
0
            if (quicly_is_client(conn)) {
7533
                /* client: update cid if this is the first Initial packet that's being received */
7534
0
                if (conn->super.state == QUICLY_STATE_FIRSTFLIGHT)
7535
0
                    quicly_set_cid(&conn->super.remote.cid_set.cids[0].cid, packet->cid.src);
7536
0
            } else {
7537
                /* server: ignore packets that are too small */
7538
0
                if (packet->datagram_size < QUICLY_MIN_CLIENT_INITIAL_SIZE) {
7539
0
                    ret = QUICLY_ERROR_PACKET_IGNORED;
7540
0
                    goto Exit;
7541
0
                }
7542
0
            }
7543
0
            aead.cb = aead_decrypt_fixed_key;
7544
0
            aead.ctx = conn->initial->cipher.ingress.aead;
7545
0
            space = (void *)&conn->initial;
7546
0
            epoch = QUICLY_EPOCH_INITIAL;
7547
0
            break;
7548
0
        case QUICLY_PACKET_TYPE_HANDSHAKE:
7549
0
            if (conn->handshake == NULL || (header_protection = conn->handshake->cipher.ingress.header_protection) == NULL) {
7550
0
                if (!(conn->application != NULL && conn->application->cipher.ingress.header_protection.one_rtt != NULL))
7551
0
                    *might_be_reorder = 1;
7552
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7553
0
                goto Exit;
7554
0
            }
7555
0
            aead.cb = aead_decrypt_fixed_key;
7556
0
            aead.ctx = conn->handshake->cipher.ingress.aead;
7557
0
            space = (void *)&conn->handshake;
7558
0
            epoch = QUICLY_EPOCH_HANDSHAKE;
7559
0
            break;
7560
0
        case QUICLY_PACKET_TYPE_0RTT:
7561
0
            if (quicly_is_client(conn)) {
7562
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7563
0
                goto Exit;
7564
0
            }
7565
0
            if (conn->application == NULL ||
7566
0
                (header_protection = conn->application->cipher.ingress.header_protection.zero_rtt) == NULL) {
7567
0
                if (!(conn->application != NULL && conn->application->cipher.ingress.header_protection.one_rtt != NULL))
7568
0
                    *might_be_reorder = 1;
7569
0
                ret = QUICLY_ERROR_PACKET_IGNORED;
7570
0
                goto Exit;
7571
0
            }
7572
0
            aead.cb = aead_decrypt_fixed_key;
7573
0
            aead.ctx = conn->application->cipher.ingress.aead[1];
7574
0
            space = (void *)&conn->application;
7575
0
            epoch = QUICLY_EPOCH_0RTT;
7576
0
            break;
7577
0
        default:
7578
0
            ret = QUICLY_ERROR_PACKET_IGNORED;
7579
0
            goto Exit;
7580
0
        }
7581
0
    } else {
7582
        /* short header packet */
7583
0
        if (conn->application == NULL ||
7584
0
            (header_protection = conn->application->cipher.ingress.header_protection.one_rtt) == NULL) {
7585
0
            *might_be_reorder = 1;
7586
0
            ret = QUICLY_ERROR_PACKET_IGNORED;
7587
0
            goto Exit;
7588
0
        }
7589
0
        aead.cb = aead_decrypt_1rtt;
7590
0
        aead.ctx = conn;
7591
0
        space = (void *)&conn->application;
7592
0
        epoch = QUICLY_EPOCH_1RTT;
7593
0
    }
7594
7595
    /* decrypt */
7596
0
    if ((ret = decrypt_packet(header_protection, aead.cb, aead.ctx, &(*space)->next_expected_packet_number, packet, &pn,
7597
0
                              &payload)) != 0) {
7598
0
        ++conn->super.stats.num_packets.decryption_failed;
7599
0
        QUICLY_PROBE(PACKET_DECRYPTION_FAILED, conn, conn->stash.now, pn);
7600
0
        goto Exit;
7601
0
    }
7602
7603
0
    QUICLY_PROBE(PACKET_RECEIVED, conn, conn->stash.now, pn, payload.base, payload.len, get_epoch(packet->octets.base[0]));
7604
0
    QUICLY_LOG_CONN(packet_received, conn, {
7605
0
        PTLS_LOG_ELEMENT_UNSIGNED(pn, pn);
7606
0
        PTLS_LOG_ELEMENT_UNSIGNED(decrypted_len, payload.len);
7607
0
        PTLS_LOG_ELEMENT_UNSIGNED(packet_type, get_epoch(packet->octets.base[0]));
7608
0
    });
7609
7610
    /* open a new path if necessary, now that decryption succeeded */
7611
0
    if (path_index == PTLS_ELEMENTSOF(conn->paths) && (ret = open_path(conn, &path_index, src_addr, dest_addr)) != 0)
7612
0
        goto Exit;
7613
7614
    /* update states */
7615
0
    if (conn->super.state == QUICLY_STATE_FIRSTFLIGHT)
7616
0
        conn->super.state = QUICLY_STATE_CONNECTED;
7617
0
    conn->super.stats.num_packets.received += 1;
7618
0
    conn->paths[path_index]->packet_last_received = conn->super.stats.num_packets.received;
7619
0
    conn->paths[path_index]->num_packets.received += 1;
7620
0
    if (QUICLY_PACKET_IS_LONG_HEADER(packet->octets.base[0])) {
7621
0
        switch (packet->octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) {
7622
0
        case QUICLY_PACKET_TYPE_INITIAL:
7623
0
            conn->super.stats.num_packets.initial_received += 1;
7624
0
            break;
7625
0
        case QUICLY_PACKET_TYPE_0RTT:
7626
0
            conn->super.stats.num_packets.zero_rtt_received += 1;
7627
0
            break;
7628
0
        case QUICLY_PACKET_TYPE_HANDSHAKE:
7629
0
            conn->super.stats.num_packets.handshake_received += 1;
7630
0
            break;
7631
0
        }
7632
0
    }
7633
0
    if (packet->ecn != 0)
7634
0
        conn->super.stats.num_packets.received_ecn_counts[get_ecn_index_from_bits(packet->ecn)] += 1;
7635
7636
    /* state updates, that are triggered by the receipt of a packet */
7637
0
    switch (epoch) {
7638
0
    case QUICLY_EPOCH_INITIAL:
7639
        /* update max_ingress_udp_payload_size if necessary */
7640
0
        if (conn->initial->largest_ingress_udp_payload_size < packet->datagram_size)
7641
0
            conn->initial->largest_ingress_udp_payload_size = packet->datagram_size;
7642
0
        break;
7643
0
    case QUICLY_EPOCH_HANDSHAKE:
7644
        /* Discard Initial space before processing the payload of the Handshake packet to avoid the chance of an ACK frame included
7645
         * in the Handshake packet setting a loss timer for the Initial packet. */
7646
0
        if (conn->initial != NULL) {
7647
0
            if ((ret = discard_handshake_context(conn, QUICLY_EPOCH_INITIAL)) != 0)
7648
0
                goto Exit;
7649
0
            setup_next_send(conn);
7650
0
            conn->super.remote.address_validation.validated = 1;
7651
0
        }
7652
0
        break;
7653
0
    default:
7654
0
        break;
7655
0
    }
7656
7657
    /* handle the payload */
7658
0
    if ((ret = handle_payload(conn, epoch, path_index, payload.base, payload.len, &offending_frame_type, &is_ack_only,
7659
0
                              &is_probe_only)) != 0)
7660
0
        goto Exit;
7661
0
    if (!is_probe_only && conn->paths[path_index]->probe_only) {
7662
0
        assert(path_index != 0);
7663
0
        conn->paths[path_index]->probe_only = 0;
7664
0
        ++conn->super.stats.num_paths.migration_elicited;
7665
0
        QUICLY_PROBE(ELICIT_PATH_MIGRATION, conn, conn->stash.now, path_index);
7666
0
        QUICLY_LOG_CONN(elicit_path_migration, conn, { PTLS_LOG_ELEMENT_UNSIGNED(path_index, path_index); });
7667
0
    }
7668
0
    if (*space != NULL && conn->super.state < QUICLY_STATE_CLOSING) {
7669
0
        if ((ret = record_receipt(*space, pn, packet->ecn, is_ack_only, conn->stash.now - (receive_delay >= 0 ? receive_delay : 0),
7670
0
                                  &conn->egress.send_ack_at, &conn->super.stats.num_packets.received_out_of_order)) != 0)
7671
0
            goto Exit;
7672
0
    }
7673
7674
    /* state updates post payload processing */
7675
0
    switch (epoch) {
7676
0
    case QUICLY_EPOCH_INITIAL:
7677
0
        assert(conn->initial != NULL);
7678
0
        if (quicly_is_client(conn) && conn->handshake != NULL && conn->handshake->cipher.egress.aead != NULL) {
7679
0
            if ((ret = discard_handshake_context(conn, QUICLY_EPOCH_INITIAL)) != 0)
7680
0
                goto Exit;
7681
0
            setup_next_send(conn);
7682
0
        }
7683
0
        break;
7684
0
    case QUICLY_EPOCH_HANDSHAKE:
7685
0
        if (quicly_is_client(conn)) {
7686
            /* Running as a client.
7687
             * Respect "disable_migration" TP sent by the remote peer at the end of the TLS handshake. */
7688
0
            if (conn->paths[0]->address.local.sa.sa_family == AF_UNSPEC && dest_addr != NULL && dest_addr->sa_family != AF_UNSPEC &&
7689
0
                ptls_handshake_is_complete(conn->crypto.tls) && conn->super.remote.transport_params.disable_active_migration)
7690
0
                set_address(&conn->paths[0]->address.local, dest_addr);
7691
0
        } else {
7692
            /* Running as a server.
7693
             * If handshake was just completed, drop handshake context, schedule the first emission of HANDSHAKE_DONE frame. */
7694
0
            if (ptls_handshake_is_complete(conn->crypto.tls)) {
7695
0
                if ((ret = discard_handshake_context(conn, QUICLY_EPOCH_HANDSHAKE)) != 0)
7696
0
                    goto Exit;
7697
0
                assert(conn->handshake == NULL);
7698
0
                conn->egress.pending_flows |= QUICLY_PENDING_FLOW_HANDSHAKE_DONE_BIT;
7699
0
                setup_next_send(conn);
7700
0
            }
7701
0
        }
7702
0
        break;
7703
0
    case QUICLY_EPOCH_1RTT:
7704
0
        if (!is_ack_only && should_send_max_data(conn))
7705
0
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
7706
        /* switch active path to current path, if current path is validated and not probe-only */
7707
0
        if (path_index != 0 && conn->paths[path_index]->path_challenge.send_at == INT64_MAX &&
7708
0
            !conn->paths[path_index]->probe_only) {
7709
0
            if ((ret = promote_path(conn, path_index)) != 0)
7710
0
                goto Exit;
7711
0
            recalc_send_probe_at(conn);
7712
0
        }
7713
0
        break;
7714
0
    default:
7715
0
        break;
7716
0
    }
7717
7718
0
    update_idle_timeout(conn, 1);
7719
7720
0
Exit:
7721
0
    switch (ret) {
7722
0
    case 0:
7723
        /* Avoid time in the past being emitted by quicly_get_first_timeout. We hit the condition below when retransmission is
7724
         * suspended by the 3x limit (in which case we have loss.alarm_at set but return INT64_MAX from quicly_get_first_timeout
7725
         * until we receive something from the client).
7726
         */
7727
0
        if (conn->egress.loss.alarm_at < conn->stash.now)
7728
0
            conn->egress.loss.alarm_at = conn->stash.now;
7729
0
        assert_consistency(conn, 0);
7730
0
        break;
7731
0
    case PTLS_ERROR_NO_MEMORY:
7732
0
    case QUICLY_ERROR_STATE_EXHAUSTION:
7733
0
    case QUICLY_ERROR_PACKET_IGNORED:
7734
0
        break;
7735
0
    default: /* close connection */
7736
0
        initiate_close(conn, ret, offending_frame_type, "");
7737
0
        ret = 0;
7738
0
        break;
7739
0
    }
7740
0
    return ret;
7741
0
}
7742
7743
quicly_error_t quicly_receive(quicly_conn_t *conn, struct sockaddr *dest_addr, struct sockaddr *src_addr,
7744
                              quicly_decoded_packet_t *packet)
7745
0
{
7746
0
    lock_now(conn, 0);
7747
7748
0
    int might_be_reorder;
7749
0
    quicly_error_t ret = do_receive(conn, dest_addr, src_addr, packet, -1, &might_be_reorder);
7750
7751
0
    if (might_be_reorder) {
7752
7753
0
        if (conn->delayed_packets.num_packets < QUICLY_MAX_DELAYED_PACKETS &&
7754
0
            compare_socket_address(&conn->paths[0]->address.remote.sa, src_addr) == 0) {
7755
            /* instantiate the delayed packet */
7756
0
            struct st_quicly_delayed_packet_t *delayed;
7757
0
            if ((delayed = malloc(offsetof(struct st_quicly_delayed_packet_t, bytes) + packet->octets.len)) == NULL) {
7758
0
                ret = PTLS_ERROR_NO_MEMORY;
7759
0
                goto Exit;
7760
0
            }
7761
0
            delayed->next = NULL;
7762
0
            delayed->at = conn->stash.now;
7763
0
            delayed->packet = *packet;
7764
0
            memcpy(delayed->bytes, packet->octets.base, packet->octets.len);
7765
0
            adjust_pointers_of_decoded_packet(&delayed->packet, delayed->bytes);
7766
            /* attach */
7767
0
            size_t slot;
7768
0
            if ((delayed->packet.octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) == QUICLY_PACKET_TYPE_0RTT) {
7769
0
                slot = &conn->delayed_packets.zero_rtt - conn->delayed_packets.as_array;
7770
0
            } else if ((delayed->packet.octets.base[0] & QUICLY_PACKET_TYPE_BITMASK) == QUICLY_PACKET_TYPE_HANDSHAKE) {
7771
0
                slot = &conn->delayed_packets.handshake - conn->delayed_packets.as_array;
7772
0
            } else {
7773
0
                assert(!QUICLY_PACKET_IS_LONG_HEADER(delayed->packet.octets.base[0]));
7774
0
                slot = &conn->delayed_packets.one_rtt - conn->delayed_packets.as_array;
7775
0
            }
7776
0
            *conn->delayed_packets.as_array[slot].tail = delayed;
7777
0
            conn->delayed_packets.as_array[slot].tail = &delayed->next;
7778
0
            ++conn->delayed_packets.num_packets;
7779
0
            if (conn->super.stats.num_packets.max_delayed < conn->delayed_packets.num_packets)
7780
0
                conn->super.stats.num_packets.max_delayed = conn->delayed_packets.num_packets;
7781
0
        }
7782
7783
0
    } else if (ret == 0) { /* if state has advanced, process delayed slots that have become processible */
7784
7785
0
        for (size_t slot = 0; conn->delayed_packets.slots_newly_processible != 0; ++slot) {
7786
0
            if ((conn->delayed_packets.slots_newly_processible & (1 << slot)) == 0)
7787
0
                continue;
7788
0
            conn->delayed_packets.slots_newly_processible ^= 1 << slot;
7789
7790
            /* processes each delayed packet */
7791
0
            struct st_quicly_delayed_packet_t *delayed;
7792
0
            while ((delayed = conn->delayed_packets.as_array[slot].head) != NULL) {
7793
                /* detach */
7794
0
                if ((conn->delayed_packets.as_array[slot].head = delayed->next) == NULL)
7795
0
                    conn->delayed_packets.as_array[slot].tail = &conn->delayed_packets.as_array[slot].head;
7796
0
                --conn->delayed_packets.num_packets;
7797
                /* process the packet and free */
7798
0
                int might_be_reorder;
7799
0
                ret = do_receive(conn, NULL, &conn->paths[0]->address.remote.sa, &delayed->packet, conn->stash.now - delayed->at,
7800
0
                                 &might_be_reorder);
7801
0
                free(delayed);
7802
0
                switch (ret) {
7803
0
                case 0:
7804
0
                    conn->super.stats.num_packets.delayed_used += 1;
7805
0
                    break;
7806
0
                case QUICLY_ERROR_PACKET_IGNORED:
7807
0
                case QUICLY_ERROR_DECRYPTION_FAILED:
7808
0
                    break;
7809
0
                default: /* bail out if a fatal error has been raised */
7810
0
                    goto Exit;
7811
0
                }
7812
0
            }
7813
0
        }
7814
0
    }
7815
7816
0
Exit:
7817
0
    unlock_now(conn);
7818
0
    return ret;
7819
0
}
7820
7821
quicly_error_t quicly_open_stream(quicly_conn_t *conn, quicly_stream_t **_stream, int uni)
7822
{
7823
    quicly_stream_t *stream;
7824
    struct st_quicly_conn_streamgroup_state_t *group;
7825
    uint64_t *max_stream_count;
7826
    uint32_t max_stream_data_local;
7827
    uint64_t max_stream_data_remote;
7828
    quicly_error_t ret;
7829
7830
    /* determine the states */
7831
    if (uni) {
7832
        group = &conn->super.local.uni;
7833
        max_stream_count = &conn->egress.max_streams.uni.count;
7834
        max_stream_data_local = 0;
7835
        max_stream_data_remote = conn->super.remote.transport_params.max_stream_data.uni;
7836
    } else {
7837
        group = &conn->super.local.bidi;
7838
        max_stream_count = &conn->egress.max_streams.bidi.count;
7839
        max_stream_data_local = (uint32_t)conn->super.ctx->transport_params.max_stream_data.bidi_local;
7840
        max_stream_data_remote = conn->super.remote.transport_params.max_stream_data.bidi_remote;
7841
    }
7842
7843
    /* open */
7844
    if ((stream = open_stream(conn, group->next_stream_id, max_stream_data_local, max_stream_data_remote)) == NULL)
7845
        return PTLS_ERROR_NO_MEMORY;
7846
    ++group->num_streams;
7847
    group->next_stream_id += 4;
7848
7849
    /* adjust blocked */
7850
    if (stream->stream_id / 4 >= *max_stream_count) {
7851
        stream->streams_blocked = 1;
7852
        quicly_linklist_insert((uni ? &conn->egress.pending_streams.blocked.uni : &conn->egress.pending_streams.blocked.bidi)->prev,
7853
                               &stream->_send_aux.pending_link.control);
7854
        /* schedule the emission of STREAMS_BLOCKED if application write key is available (otherwise the scheduling is done when
7855
         * the key becomes available) */
7856
        if (stream->conn->application != NULL && stream->conn->application->cipher.egress.key.aead != NULL)
7857
            conn->egress.pending_flows |= QUICLY_PENDING_FLOW_OTHERS_BIT;
7858
    }
7859
7860
    /* application-layer initialization */
7861
    QUICLY_PROBE(STREAM_ON_OPEN, conn, conn->stash.now, stream);
7862
    QUICLY_LOG_CONN(stream_on_open, conn, {});
7863
7864
    if ((ret = conn->super.ctx->stream_open->cb(conn->super.ctx->stream_open, stream)) != 0)
7865
        return ret;
7866
7867
    *_stream = stream;
7868
    return 0;
7869
}
7870
7871
void quicly_reset_stream(quicly_stream_t *stream, quicly_error_t err)
7872
{
7873
    assert(quicly_stream_has_send_side(quicly_is_client(stream->conn), stream->stream_id));
7874
    assert(QUICLY_ERROR_IS_QUIC_APPLICATION(err));
7875
    assert(stream->_send_aux.reset_stream.sender_state == QUICLY_SENDER_STATE_NONE);
7876
    assert(!quicly_sendstate_transfer_complete(&stream->sendstate));
7877
7878
    /* dispose sendbuf state */
7879
    quicly_sendstate_reset(&stream->sendstate);
7880
7881
    /* setup RESET_STREAM */
7882
    stream->_send_aux.reset_stream.sender_state = QUICLY_SENDER_STATE_SEND;
7883
    stream->_send_aux.reset_stream.error_code = QUICLY_ERROR_GET_ERROR_CODE(err);
7884
7885
    /* schedule for delivery */
7886
    sched_stream_control(stream);
7887
    resched_stream_data(stream);
7888
}
7889
7890
void quicly_request_stop(quicly_stream_t *stream, quicly_error_t err)
7891
{
7892
    assert(quicly_stream_has_receive_side(quicly_is_client(stream->conn), stream->stream_id));
7893
    assert(QUICLY_ERROR_IS_QUIC_APPLICATION(err));
7894
7895
    /* send STOP_SENDING if the incoming side of the stream is still open */
7896
    if (stream->recvstate.eos == UINT64_MAX && stream->_send_aux.stop_sending.sender_state == QUICLY_SENDER_STATE_NONE) {
7897
        stream->_send_aux.stop_sending.sender_state = QUICLY_SENDER_STATE_SEND;
7898
        stream->_send_aux.stop_sending.error_code = QUICLY_ERROR_GET_ERROR_CODE(err);
7899
        sched_stream_control(stream);
7900
    }
7901
}
7902
7903
socklen_t quicly_get_socklen(struct sockaddr *sa)
7904
409
{
7905
409
    switch (sa->sa_family) {
7906
409
    case AF_INET:
7907
409
        return sizeof(struct sockaddr_in);
7908
0
    case AF_INET6:
7909
0
        return sizeof(struct sockaddr_in6);
7910
0
    default:
7911
0
        assert(!"unexpected socket type");
7912
0
        return 0;
7913
409
    }
7914
409
}
7915
7916
char *quicly_escape_unsafe_string(char *buf, const void *bytes, size_t len)
7917
0
{
7918
0
    char *dst = buf;
7919
0
    const char *src = bytes, *end = src + len;
7920
7921
0
    for (; src != end; ++src) {
7922
0
        if ((0x20 <= *src && *src <= 0x7e) && !(*src == '"' || *src == '\'' || *src == '\\')) {
7923
0
            *dst++ = *src;
7924
0
        } else {
7925
0
            *dst++ = '\\';
7926
0
            *dst++ = 'x';
7927
0
            quicly_byte_to_hex(dst, (uint8_t)*src);
7928
0
            dst += 2;
7929
0
        }
7930
0
    }
7931
0
    *dst = '\0';
7932
7933
0
    return buf;
7934
0
}
7935
7936
char *quicly_hexdump(const uint8_t *bytes, size_t len, size_t indent)
7937
0
{
7938
0
    size_t i, line, row, bufsize = indent == SIZE_MAX ? len * 2 + 1 : (indent + 5 + 3 * 16 + 2 + 16 + 1) * ((len + 15) / 16) + 1;
7939
0
    char *buf, *p;
7940
7941
0
    if ((buf = malloc(bufsize)) == NULL)
7942
0
        return NULL;
7943
0
    p = buf;
7944
0
    if (indent == SIZE_MAX) {
7945
0
        for (i = 0; i != len; ++i) {
7946
0
            quicly_byte_to_hex(p, bytes[i]);
7947
0
            p += 2;
7948
0
        }
7949
0
    } else {
7950
0
        for (line = 0; line * 16 < len; ++line) {
7951
0
            for (i = 0; i < indent; ++i)
7952
0
                *p++ = ' ';
7953
0
            quicly_byte_to_hex(p, (line >> 4) & 0xff);
7954
0
            p += 2;
7955
0
            quicly_byte_to_hex(p, (line << 4) & 0xff);
7956
0
            p += 2;
7957
0
            *p++ = ' ';
7958
0
            for (row = 0; row < 16; ++row) {
7959
0
                *p++ = row == 8 ? '-' : ' ';
7960
0
                if (line * 16 + row < len) {
7961
0
                    quicly_byte_to_hex(p, bytes[line * 16 + row]);
7962
0
                    p += 2;
7963
0
                } else {
7964
0
                    *p++ = ' ';
7965
0
                    *p++ = ' ';
7966
0
                }
7967
0
            }
7968
0
            *p++ = ' ';
7969
0
            *p++ = ' ';
7970
0
            for (row = 0; row < 16; ++row) {
7971
0
                if (line * 16 + row < len) {
7972
0
                    int ch = bytes[line * 16 + row];
7973
0
                    *p++ = 0x20 <= ch && ch < 0x7f ? ch : '.';
7974
0
                } else {
7975
0
                    *p++ = ' ';
7976
0
                }
7977
0
            }
7978
0
            *p++ = '\n';
7979
0
        }
7980
0
    }
7981
0
    *p++ = '\0';
7982
7983
0
    assert(p - buf <= bufsize);
7984
7985
0
    return buf;
7986
0
}
7987
7988
void quicly_amend_ptls_context(ptls_context_t *ptls)
7989
0
{
7990
0
    static ptls_update_traffic_key_t update_traffic_key = {update_traffic_key_cb};
7991
7992
0
    ptls->omit_end_of_early_data = 1;
7993
0
    ptls->update_traffic_key = &update_traffic_key;
7994
7995
    /* if TLS 1.3 config permits use of early data, convert the value to 0xffffffff in accordance with QUIC-TLS */
7996
0
    if (ptls->max_early_data_size != 0)
7997
0
        ptls->max_early_data_size = UINT32_MAX;
7998
0
}
7999
8000
quicly_error_t quicly_encrypt_address_token(void (*random_bytes)(void *, size_t), ptls_aead_context_t *aead, ptls_buffer_t *buf,
8001
                                            size_t start_off, const quicly_address_token_plaintext_t *plaintext)
8002
0
{
8003
0
    quicly_error_t ret;
8004
8005
    /* type and IV */
8006
0
    if ((ret = ptls_buffer_reserve(buf, 1 + aead->algo->iv_size)) != 0)
8007
0
        goto Exit;
8008
0
    buf->base[buf->off++] = plaintext->type;
8009
0
    random_bytes(buf->base + buf->off, aead->algo->iv_size);
8010
0
    buf->off += aead->algo->iv_size;
8011
8012
0
    size_t enc_start = buf->off;
8013
8014
    /* data */
8015
0
    ptls_buffer_push64(buf, plaintext->issued_at);
8016
0
    {
8017
0
        uint16_t port;
8018
0
        ptls_buffer_push_block(buf, 1, {
8019
0
            switch (plaintext->remote.sa.sa_family) {
8020
0
            case AF_INET:
8021
0
                ptls_buffer_pushv(buf, &plaintext->remote.sin.sin_addr.s_addr, 4);
8022
0
                port = ntohs(plaintext->remote.sin.sin_port);
8023
0
                break;
8024
0
            case AF_INET6:
8025
0
                ptls_buffer_pushv(buf, &plaintext->remote.sin6.sin6_addr, 16);
8026
0
                ptls_buffer_push32(buf, plaintext->remote.sin6.sin6_scope_id);
8027
0
                port = ntohs(plaintext->remote.sin6.sin6_port);
8028
0
                break;
8029
0
            default:
8030
0
                assert(!"unsupported address type");
8031
0
                break;
8032
0
            }
8033
0
        });
8034
0
        ptls_buffer_push16(buf, port);
8035
0
    }
8036
0
    switch (plaintext->type) {
8037
0
    case QUICLY_ADDRESS_TOKEN_TYPE_RETRY:
8038
0
        ptls_buffer_push_block(buf, 1,
8039
0
                               { ptls_buffer_pushv(buf, plaintext->retry.original_dcid.cid, plaintext->retry.original_dcid.len); });
8040
0
        ptls_buffer_push_block(buf, 1,
8041
0
                               { ptls_buffer_pushv(buf, plaintext->retry.client_cid.cid, plaintext->retry.client_cid.len); });
8042
0
        ptls_buffer_push_block(buf, 1,
8043
0
                               { ptls_buffer_pushv(buf, plaintext->retry.server_cid.cid, plaintext->retry.server_cid.len); });
8044
0
        break;
8045
0
    case QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION:
8046
0
        ptls_buffer_push_block(buf, 1, { ptls_buffer_pushv(buf, plaintext->resumption.bytes, plaintext->resumption.len); });
8047
0
        break;
8048
0
    default:
8049
0
        assert(!"unexpected token type");
8050
0
        abort();
8051
0
    }
8052
0
    ptls_buffer_push_block(buf, 1, { ptls_buffer_pushv(buf, plaintext->appdata.bytes, plaintext->appdata.len); });
8053
8054
    /* encrypt, supplying full IV */
8055
0
    if ((ret = ptls_buffer_reserve(buf, aead->algo->tag_size)) != 0)
8056
0
        goto Exit;
8057
0
    ptls_aead_set_iv(aead, buf->base + enc_start - aead->algo->iv_size);
8058
0
    ptls_aead_encrypt(aead, buf->base + enc_start, buf->base + enc_start, buf->off - enc_start, 0, buf->base + start_off,
8059
0
                      enc_start - start_off);
8060
0
    buf->off += aead->algo->tag_size;
8061
8062
0
Exit:
8063
0
    return ret;
8064
0
}
8065
8066
quicly_error_t quicly_decrypt_address_token(ptls_aead_context_t *aead, quicly_address_token_plaintext_t *plaintext,
8067
                                            const void *_token, size_t len, size_t prefix_len, const char **err_desc)
8068
0
{
8069
0
    const uint8_t *const token = _token;
8070
0
    uint8_t ptbuf[QUICLY_MIN_CLIENT_INITIAL_SIZE];
8071
0
    size_t ptlen;
8072
8073
0
    *err_desc = NULL;
8074
8075
    /* check if we can get type and decrypt */
8076
0
    if (len < prefix_len + 1 + aead->algo->iv_size + aead->algo->tag_size) {
8077
0
        *err_desc = "token too small";
8078
0
        return PTLS_ALERT_DECODE_ERROR;
8079
0
    }
8080
0
    if (prefix_len + 1 + aead->algo->iv_size + sizeof(ptbuf) + aead->algo->tag_size < len) {
8081
0
        *err_desc = "token too large";
8082
0
        return PTLS_ALERT_DECODE_ERROR;
8083
0
    }
8084
8085
    /* check type */
8086
0
    switch (token[prefix_len]) {
8087
0
    case QUICLY_ADDRESS_TOKEN_TYPE_RETRY:
8088
0
        plaintext->type = QUICLY_ADDRESS_TOKEN_TYPE_RETRY;
8089
0
        break;
8090
0
    case QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION:
8091
0
        plaintext->type = QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION;
8092
0
        break;
8093
0
    default:
8094
0
        *err_desc = "unknown token type";
8095
0
        return PTLS_ALERT_DECODE_ERROR;
8096
0
    }
8097
8098
    /* `goto Exit` can only happen below this line, and that is guaranteed by declaring `ret` here */
8099
0
    quicly_error_t ret;
8100
8101
    /* decrypt */
8102
0
    ptls_aead_set_iv(aead, token + prefix_len + 1);
8103
0
    if ((ptlen = ptls_aead_decrypt(aead, ptbuf, token + prefix_len + 1 + aead->algo->iv_size,
8104
0
                                   len - (prefix_len + 1 + aead->algo->iv_size), 0, token, prefix_len + 1 + aead->algo->iv_size)) ==
8105
0
        SIZE_MAX) {
8106
0
        ret = PTLS_ALERT_DECRYPT_ERROR;
8107
0
        *err_desc = "token decryption failure";
8108
0
        goto Exit;
8109
0
    }
8110
8111
    /* parse */
8112
0
    const uint8_t *src = ptbuf, *end = src + ptlen;
8113
0
    if ((ret = ptls_decode64(&plaintext->issued_at, &src, end)) != 0)
8114
0
        goto Exit;
8115
0
    {
8116
0
        in_port_t *portaddr;
8117
0
        ptls_decode_open_block(src, end, 1, {
8118
0
            switch (end - src) {
8119
0
            case 4: /* ipv4 */
8120
0
                plaintext->remote.sin.sin_family = AF_INET;
8121
0
                memcpy(&plaintext->remote.sin.sin_addr.s_addr, src, 4);
8122
0
                portaddr = &plaintext->remote.sin.sin_port;
8123
0
                break;
8124
0
            case 20: /* ipv6 */
8125
0
                plaintext->remote.sin6 = (struct sockaddr_in6){.sin6_family = AF_INET6};
8126
0
                memcpy(&plaintext->remote.sin6.sin6_addr, src, 16);
8127
0
                if ((ret = ptls_decode32(&plaintext->remote.sin6.sin6_scope_id, &src, end)) != 0)
8128
0
                    goto Exit;
8129
0
                portaddr = &plaintext->remote.sin6.sin6_port;
8130
0
                break;
8131
0
            default:
8132
0
                ret = PTLS_ALERT_DECODE_ERROR;
8133
0
                goto Exit;
8134
0
            }
8135
0
            src = end;
8136
0
        });
8137
0
        uint16_t port;
8138
0
        if ((ret = ptls_decode16(&port, &src, end)) != 0)
8139
0
            goto Exit;
8140
0
        *portaddr = htons(port);
8141
0
    }
8142
0
    switch (plaintext->type) {
8143
0
    case QUICLY_ADDRESS_TOKEN_TYPE_RETRY:
8144
0
#define DECODE_CID(field)                                                                                                          \
8145
0
    do {                                                                                                                           \
8146
0
        ptls_decode_open_block(src, end, 1, {                                                                                      \
8147
0
            if (end - src > sizeof(plaintext->retry.field.cid)) {                                                                  \
8148
0
                ret = PTLS_ALERT_DECODE_ERROR;                                                                                     \
8149
0
                goto Exit;                                                                                                         \
8150
0
            }                                                                                                                      \
8151
0
            quicly_set_cid(&plaintext->retry.field, ptls_iovec_init(src, end - src));                                              \
8152
0
            src = end;                                                                                                             \
8153
0
        });                                                                                                                        \
8154
0
    } while (0)
8155
0
        DECODE_CID(original_dcid);
8156
0
        DECODE_CID(client_cid);
8157
0
        DECODE_CID(server_cid);
8158
0
#undef DECODE_CID
8159
0
        break;
8160
0
    case QUICLY_ADDRESS_TOKEN_TYPE_RESUMPTION:
8161
0
        ptls_decode_open_block(src, end, 1, {
8162
0
            PTLS_BUILD_ASSERT(sizeof(plaintext->resumption.bytes) >= 256);
8163
0
            plaintext->resumption.len = end - src;
8164
0
            memcpy(plaintext->resumption.bytes, src, plaintext->resumption.len);
8165
0
            src = end;
8166
0
        });
8167
0
        break;
8168
0
    default:
8169
0
        assert(!"unexpected token type");
8170
0
        abort();
8171
0
    }
8172
0
    ptls_decode_block(src, end, 1, {
8173
0
        PTLS_BUILD_ASSERT(sizeof(plaintext->appdata.bytes) >= 256);
8174
0
        plaintext->appdata.len = end - src;
8175
0
        memcpy(plaintext->appdata.bytes, src, plaintext->appdata.len);
8176
0
        src = end;
8177
0
    });
8178
0
    ret = 0;
8179
8180
0
Exit:
8181
0
    if (ret != 0) {
8182
0
        if (*err_desc == NULL)
8183
0
            *err_desc = "token decode error";
8184
        /* promote the error to one that triggers the emission of INVALID_TOKEN_ERROR, if the token looked like a retry */
8185
0
        if (plaintext->type == QUICLY_ADDRESS_TOKEN_TYPE_RETRY)
8186
0
            ret = QUICLY_TRANSPORT_ERROR_INVALID_TOKEN;
8187
0
    }
8188
0
    return ret;
8189
0
}
8190
8191
int quicly_build_session_ticket_auth_data(ptls_buffer_t *auth_data, const quicly_context_t *ctx)
8192
0
{
8193
0
    int ret;
8194
8195
0
#define PUSH_TP(id, block)                                                                                                         \
8196
0
    do {                                                                                                                           \
8197
0
        ptls_buffer_push_quicint(auth_data, id);                                                                                   \
8198
0
        ptls_buffer_push_block(auth_data, -1, block);                                                                              \
8199
0
    } while (0)
8200
8201
0
    ptls_buffer_push_block(auth_data, -1, {
8202
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_ACTIVE_CONNECTION_ID_LIMIT,
8203
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.active_connection_id_limit); });
8204
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_DATA,
8205
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.max_data); });
8206
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_LOCAL,
8207
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.max_stream_data.bidi_local); });
8208
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_BIDI_REMOTE,
8209
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.max_stream_data.bidi_remote); });
8210
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAM_DATA_UNI,
8211
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.max_stream_data.uni); });
8212
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_BIDI,
8213
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.max_streams_bidi); });
8214
0
        PUSH_TP(QUICLY_TRANSPORT_PARAMETER_ID_INITIAL_MAX_STREAMS_UNI,
8215
0
                { ptls_buffer_push_quicint(auth_data, ctx->transport_params.max_streams_uni); });
8216
0
    });
8217
8218
0
#undef PUSH_TP
8219
8220
0
    ret = 0;
8221
0
Exit:
8222
0
    return ret;
8223
0
}
8224
8225
void quicly_stream_noop_on_destroy(quicly_stream_t *stream, quicly_error_t err)
8226
0
{
8227
0
}
8228
8229
void quicly_stream_noop_on_send_shift(quicly_stream_t *stream, size_t delta)
8230
0
{
8231
0
}
8232
8233
void quicly_stream_noop_on_send_emit(quicly_stream_t *stream, size_t off, void *dst, size_t *len, int *wrote_all)
8234
0
{
8235
0
}
8236
8237
void quicly_stream_noop_on_send_stop(quicly_stream_t *stream, quicly_error_t err)
8238
0
{
8239
0
}
8240
8241
void quicly_stream_noop_on_receive(quicly_stream_t *stream, size_t off, const void *src, size_t len)
8242
0
{
8243
0
}
8244
8245
void quicly_stream_noop_on_receive_reset(quicly_stream_t *stream, quicly_error_t err)
8246
0
{
8247
0
}
8248
8249
const quicly_stream_callbacks_t quicly_stream_noop_callbacks = {
8250
    quicly_stream_noop_on_destroy,   quicly_stream_noop_on_send_shift, quicly_stream_noop_on_send_emit,
8251
    quicly_stream_noop_on_send_stop, quicly_stream_noop_on_receive,    quicly_stream_noop_on_receive_reset};
8252
8253
void quicly__debug_printf(quicly_conn_t *conn, const char *function, int line, const char *fmt, ...)
8254
0
{
8255
0
    PTLS_LOG_DEFINE_POINT(quicly, debug_message, debug_message_logpoint);
8256
0
    if (QUICLY_PROBE_ENABLED(DEBUG_MESSAGE) ||
8257
0
        (ptls_log_point_maybe_active(&debug_message_logpoint) &
8258
0
         ptls_log_conn_maybe_active(ptls_get_log_state(conn->crypto.tls), ptls_log_getsni_ptls(conn->crypto.tls))) != 0) {
8259
0
        char buf[1024];
8260
0
        va_list args;
8261
8262
0
        va_start(args, fmt);
8263
0
        vsnprintf(buf, sizeof(buf), fmt, args);
8264
0
        va_end(args);
8265
8266
0
        QUICLY_PROBE(DEBUG_MESSAGE, conn, function, line, buf);
8267
0
        QUICLY_LOG_CONN(debug_message, conn, {
8268
0
            PTLS_LOG_ELEMENT_UNSAFESTR(function, function, strlen(function));
8269
0
            PTLS_LOG_ELEMENT_SIGNED(line, line);
8270
0
            PTLS_LOG_ELEMENT_UNSAFESTR(message, buf, strlen(buf));
8271
0
        });
8272
0
    }
8273
0
}
8274
8275
const uint32_t quicly_supported_versions[] = {QUICLY_PROTOCOL_VERSION_1, QUICLY_PROTOCOL_VERSION_DRAFT29,
8276
                                              QUICLY_PROTOCOL_VERSION_DRAFT27, 0};