Line data Source code
1 : #ifndef HEADER_fd_src_waltz_quic_fd_quic_private_h
2 : #define HEADER_fd_src_waltz_quic_fd_quic_private_h
3 :
4 : #include "fd_quic.h"
5 : #include "templ/fd_quic_transport_params.h"
6 : #include "fd_quic_conn_map.h"
7 : #include "fd_quic_stream.h"
8 : #include "log/fd_quic_log_tx.h"
9 : #include "fd_quic_pkt_meta.h"
10 : #include "tls/fd_quic_tls.h"
11 : #include "fd_quic_stream_pool.h"
12 : #include "fd_quic_pretty_print.h"
13 : #include "fd_quic_svc_q.h"
14 : #include <math.h>
15 :
16 : #include "../../ballet/chacha/fd_chacha_rng.h"
17 : #include "../../util/log/fd_dtrace.h"
18 : #include "../../util/net/fd_ip4.h"
19 : #include "../../util/net/fd_udp.h"
20 :
21 : /* Handshake allocator pool */
22 : #define POOL_NAME fd_quic_tls_hs_pool
23 0 : #define POOL_T fd_quic_tls_hs_t
24 : #include "../../util/tmpl/fd_pool.c"
25 :
26 : /* Handshake FIFO cache dlist */
27 : #define DLIST_NAME fd_quic_tls_hs_cache
28 : #define DLIST_ELE_T fd_quic_tls_hs_t
29 : #include "../../util/tmpl/fd_dlist.c"
30 :
31 :
32 : /* FD_QUIC_DISABLE_CRYPTO: set to 1 to disable packet protection and
33 : encryption. Only intended for testing. */
34 : #ifndef FD_QUIC_DISABLE_CRYPTO
35 : #define FD_QUIC_DISABLE_CRYPTO 0
36 : #endif
37 :
38 0 : #define FD_QUIC_PKT_NUM_UNUSED (~0ul)
39 0 : #define FD_QUIC_PKT_NUM_PENDING (~1ul)
40 :
41 : /* FD_QUIC_MAGIC is used to signal the layout of shared memory region
42 : of an fd_quic_t. */
43 :
44 0 : #define FD_QUIC_MAGIC (0xdadf8cfa01cc5460UL)
45 :
46 : /* RFC 9002 retransmit constants */
47 :
48 0 : #define FD_QUIC_K_TIME_THRESHOLD 1.125f
49 0 : #define FD_QUIC_K_GRANULARITY_NS 1000000L
50 :
51 : /* fd_quic_state_t is the internal state of an fd_quic_t. Valid for
52 : lifetime of join. */
53 :
54 : struct __attribute__((aligned(16UL))) fd_quic_state_private {
55 : /* Flags */
56 : ulong flags;
57 :
58 : long now; /* recent timestamp, assumed in ns */
59 :
60 : /* transport_params: Template for QUIC-TLS transport params extension.
61 : Contains a mix of mutable and immutable fields. Immutable fields
62 : are set on join. Mutable fields may be modified during packet
63 : processing. Any code using this struct must ensure that the
64 : mutable fields are cleared before using (otherwise would leak a
65 : side channel).
66 :
67 : Mutable fields include:
68 : - original_destination_connection_id
69 : - initial_source_conn_id */
70 :
71 : fd_quic_transport_params_t transport_params;
72 :
73 : ulong max_inflight_frame_cnt_conn; /* per-conn max, computed from limits */
74 :
75 : /* Various internal state */
76 :
77 : fd_quic_log_tx_t log_tx[1];
78 : uint free_conn_list; /* free list of unused connections */
79 : fd_quic_conn_map_t * conn_map; /* map connection ids -> connection */
80 :
81 : fd_quic_tls_t tls[1];
82 : fd_quic_tls_hs_t * hs_pool;
83 : fd_quic_tls_hs_cache_t hs_cache; /* dlist <> dlist_private */
84 :
85 : fd_quic_stream_pool_t * stream_pool; /* stream pool, nullable */
86 : fd_quic_pkt_meta_t * pkt_meta_pool;
87 : fd_chacha_rng_t _rng[1]; /* CSPRNG, see fd_quic_rng_ulong */
88 : long rng_reseed_at; /* rekey _rng at the first refill due at/after this timestamp */
89 :
90 : /* need to be able to access connections by index */
91 : ulong conn_base; /* address of array of all connections */
92 : /* not using fd_quic_conn_t* to avoid confusion */
93 : /* use fd_quic_conn_at_idx instead */
94 : ulong conn_sz; /* size of one connection element */
95 :
96 : /* flow control - configured initial limits */
97 : ulong initial_max_data; /* directly from transport params */
98 : ulong initial_max_stream_data[4]; /* from 4 transport params indexed by stream type */
99 :
100 : /* last arp/routing tables update */
101 : ulong ip_table_upd;
102 :
103 : /* secret for generating RETRY tokens */
104 : uchar retry_secret[FD_QUIC_RETRY_SECRET_SZ];
105 : uchar retry_iv [FD_QUIC_RETRY_IV_SZ];
106 :
107 : /* Scratch space for packet protection */
108 : uchar crypt_scratch[FD_QUIC_MTU];
109 :
110 : /* the timer structs, large private fields / data follow */
111 : fd_quic_svc_timers_t * svc_timers;
112 : };
113 :
114 : /* FD_QUIC_STATE_OFF is the offset of fd_quic_state_t within fd_quic_t. */
115 0 : #define FD_QUIC_STATE_OFF (fd_ulong_align_up( sizeof(fd_quic_t), alignof(fd_quic_state_t) ))
116 :
117 : struct fd_quic_pkt {
118 : fd_ip4_hdr_t ip4[1];
119 : fd_udp_hdr_t udp[1];
120 :
121 : /* the following are the "current" values only. There may be more QUIC packets
122 : in a UDP datagram */
123 : ulong pkt_number; /* quic packet number currently being decoded/parsed */
124 : long rcv_time; /* time packet was received */
125 : uint enc_level; /* encryption level */
126 : uint datagram_sz; /* length of the original datagram */
127 : uint ack_flag; /* ORed together: 0-don't ack 1-ack 2-cancel ack */
128 0 : # define ACK_FLAG_RQD 1
129 0 : # define ACK_FLAG_CANCEL 2
130 :
131 : ulong rtt_pkt_number; /* packet number used for rtt */
132 : long rtt_ack_time;
133 : ulong rtt_ack_delay;
134 : };
135 :
136 : struct fd_quic_frame_ctx {
137 : fd_quic_t * quic;
138 : fd_quic_conn_t * conn;
139 : fd_quic_pkt_t * pkt;
140 : ulong frame_sz;
141 : };
142 :
143 : typedef struct fd_quic_frame_ctx fd_quic_frame_ctx_t;
144 :
145 : FD_PROTOTYPES_BEGIN
146 :
147 : /* fd_quic_get_state returns a pointer to private state area given a
148 : pointer to fd_quic_t. Const func, guaranteed to not access memory. */
149 :
150 : FD_FN_CONST static inline fd_quic_state_t *
151 0 : fd_quic_get_state( fd_quic_t * quic ) {
152 0 : return (fd_quic_state_t *)( (ulong)quic + FD_QUIC_STATE_OFF );
153 0 : }
154 :
155 : FD_FN_CONST static inline fd_quic_state_t const *
156 0 : fd_quic_get_state_const( fd_quic_t const * quic ) {
157 0 : return (fd_quic_state_t const *)( (ulong)quic + FD_QUIC_STATE_OFF );
158 0 : }
159 :
160 0 : #define FD_QUIC_RNG_RESEED_INTERVAL ((long)300e9) /* 5 minutes */
161 :
162 : FD_FN_SENSITIVE void
163 : fd_quic_rng_reseed( fd_quic_state_t * state );
164 :
165 : static inline ulong
166 0 : fd_quic_rng_ulong( fd_quic_state_t * state ) {
167 0 : if( FD_UNLIKELY( state->now >= state->rng_reseed_at ) ) {
168 0 : fd_quic_rng_reseed( state );
169 0 : }
170 0 : return fd_chacha_rng_ulong( state->_rng );
171 0 : }
172 :
173 : /* fd_quic_conn_service is called periodically to perform pending
174 : operations and time based operations.
175 :
176 : args
177 : quic managing quic
178 : conn connection to service
179 : now the current timestamp */
180 : void
181 : fd_quic_conn_service( fd_quic_t * quic,
182 : fd_quic_conn_t * conn,
183 : long now );
184 :
185 :
186 : /* Memory management **************************************************/
187 :
188 : fd_quic_conn_t *
189 : fd_quic_conn_create( fd_quic_t * quic,
190 : ulong our_conn_id,
191 : fd_quic_conn_id_t const * peer_conn_id,
192 : uint peer_ip_addr,
193 : ushort peer_udp_port,
194 : uint self_ip_addr,
195 : ushort self_udp_port,
196 : int server );
197 :
198 : void
199 : fd_quic_tx_stream_free( fd_quic_t * quic,
200 : fd_quic_conn_t * conn,
201 : fd_quic_stream_t * stream,
202 : int code );
203 :
204 : /* Callbacks provided by fd_quic **************************************/
205 :
206 : /* used by quic to receive data from network */
207 : int
208 : fd_quic_aio_cb_receive( void * context,
209 : fd_aio_pkt_info_t const * batch,
210 : ulong batch_sz,
211 : ulong * opt_batch_idx,
212 : int flush );
213 :
214 : /* declare callbacks from quic-tls into quic */
215 : int
216 : fd_quic_tls_cb_client_hello( fd_quic_tls_hs_t * hs,
217 : void * context );
218 :
219 : int
220 : fd_quic_tls_cb_handshake_data( fd_quic_tls_hs_t * hs,
221 : void * context,
222 : uint enc_level,
223 : uchar const * data,
224 : ulong data_sz );
225 :
226 : void
227 : fd_quic_tls_cb_alert( fd_quic_tls_hs_t * hs,
228 : void * context,
229 : int alert );
230 :
231 : void
232 : fd_quic_tls_cb_secret( fd_quic_tls_hs_t * hs,
233 : void * context,
234 : fd_quic_tls_secret_t const * secret );
235 :
236 : void
237 : fd_quic_tls_cb_handshake_complete( fd_quic_tls_hs_t * hs,
238 : void * context );
239 :
240 : void
241 : fd_quic_tls_cb_peer_params( void * context,
242 : uchar const * peer_tp_enc,
243 : ulong peer_tp_enc_sz );
244 :
245 : void
246 : fd_quic_apply_peer_params( fd_quic_conn_t * conn,
247 : fd_quic_transport_params_t const * peer_tp );
248 :
249 : /* Helpers for calling callbacks **************************************/
250 :
251 : static inline void
252 : fd_quic_cb_conn_new( fd_quic_t * quic,
253 0 : fd_quic_conn_t * conn ) {
254 0 : if( conn->called_conn_new ) return;
255 0 : conn->called_conn_new = 1;
256 0 : if( !quic->cb.conn_new ) return;
257 :
258 0 : quic->cb.conn_new( conn, quic->cb.quic_ctx );
259 0 : }
260 :
261 : static inline void
262 : fd_quic_cb_conn_hs_complete( fd_quic_t * quic,
263 0 : fd_quic_conn_t * conn ) {
264 0 : if( !quic->cb.conn_hs_complete ) return;
265 0 : quic->cb.conn_hs_complete( conn, quic->cb.quic_ctx );
266 0 : }
267 :
268 : static inline void
269 : fd_quic_cb_conn_final( fd_quic_t * quic,
270 0 : fd_quic_conn_t * conn ) {
271 0 : if( !quic->cb.conn_final || !conn->called_conn_new ) return;
272 0 : quic->cb.conn_final( conn, quic->cb.quic_ctx );
273 0 : }
274 :
275 : static inline int
276 : fd_quic_cb_stream_rx( fd_quic_t * quic,
277 : fd_quic_conn_t * conn,
278 : ulong stream_id,
279 : ulong offset,
280 : uchar const * data,
281 : ulong data_sz,
282 0 : int fin ) {
283 0 : quic->metrics.stream_rx_event_cnt++;
284 0 : quic->metrics.stream_rx_byte_cnt += data_sz;
285 :
286 0 : if( !quic->cb.stream_rx ) return FD_QUIC_SUCCESS;
287 0 : return quic->cb.stream_rx( conn, stream_id, offset, data, data_sz, fin );
288 0 : }
289 :
290 : static inline void
291 : fd_quic_cb_stream_notify( fd_quic_t * quic,
292 : fd_quic_stream_t * stream,
293 : void * stream_ctx,
294 0 : int event ) {
295 0 : quic->metrics.stream_closed_cnt[ event ]++;
296 0 : quic->metrics.stream_active_cnt--;
297 :
298 0 : if( !quic->cb.stream_notify ) return;
299 0 : quic->cb.stream_notify( stream, stream_ctx, event );
300 0 : }
301 :
302 : static inline void
303 : fd_quic_cb_ack_range( fd_quic_t * quic,
304 : fd_quic_conn_t * conn,
305 : ulong pkt_num_lo,
306 0 : ulong pkt_num_hi ) {
307 0 : if( !quic->cb.ack_range ) return;
308 0 : quic->cb.ack_range( conn, pkt_num_lo, pkt_num_hi, quic->cb.quic_ctx );
309 0 : }
310 :
311 :
312 : FD_FN_CONST ulong
313 : fd_quic_reconstruct_pkt_num( ulong pktnum_comp,
314 : ulong pktnum_sz,
315 : ulong exp_pkt_number );
316 :
317 : void
318 : fd_quic_pkt_meta_retry( fd_quic_t * quic,
319 : fd_quic_conn_t * conn,
320 : ulong force_below_pkt_num,
321 : uint arg_enc_level );
322 :
323 : /* reclaim resources associated with packet metadata
324 : this is called in response to received acks */
325 : void
326 : fd_quic_reclaim_pkt_meta( fd_quic_conn_t * conn,
327 : fd_quic_pkt_meta_t * pkt_meta,
328 : uint enc_level );
329 :
330 : ulong
331 : fd_quic_process_quic_packet_v1( fd_quic_t * quic,
332 : fd_quic_pkt_t * pkt,
333 : uchar * cur_ptr,
334 : ulong cur_sz );
335 :
336 : ulong
337 : fd_quic_handle_v1_initial( fd_quic_t * quic,
338 : fd_quic_conn_t ** p_conn,
339 : fd_quic_pkt_t * pkt,
340 : fd_quic_conn_id_t const * dcid,
341 : fd_quic_conn_id_t const * scid,
342 : uchar * cur_ptr,
343 : ulong cur_sz );
344 :
345 : ulong
346 : fd_quic_handle_v1_handshake( fd_quic_t * quic,
347 : fd_quic_conn_t * conn,
348 : fd_quic_pkt_t * pkt,
349 : uchar * cur_ptr,
350 : ulong cur_sz );
351 :
352 : ulong
353 : fd_quic_handle_v1_one_rtt( fd_quic_t * quic,
354 : fd_quic_conn_t * conn,
355 : fd_quic_pkt_t * pkt,
356 : uchar * cur_ptr,
357 : ulong cur_sz );
358 :
359 : /* fd_quic_handle_v1_frame is the primary entrypoint for handling of
360 : incoming QUIC frames. {quic,conn,pkt} identify the frame context.
361 : Memory region [frame_ptr,frame_ptr+frame_sz) contains the serialized
362 : QUIC frame (may contain arbitrary zero padding at the beginning).
363 :
364 : Returns value in (0,buf_sz) if the frame was successfully processed.
365 : Returns FD_QUIC_PARSE_FAIL if the frame was inherently malformed.
366 : Returns 0 or value in [buf_sz,ULONG_MAX) in case of a protocol
367 : violation. */
368 :
369 : ulong
370 : fd_quic_handle_v1_frame( fd_quic_t * quic,
371 : fd_quic_conn_t * conn,
372 : fd_quic_pkt_t * pkt,
373 : uint pkt_type,
374 : uchar const * frame_ptr,
375 : ulong frame_sz );
376 :
377 : /* fd_quic_lazy_ack_pkt enqueues future acknowledgement for the given
378 : packet. The ACK will be sent out at a fd_quic_service call. The
379 : delay is determined by the fd_quic_config_t ack_threshold and
380 : ack_delay settings. Respects pkt->ack_flag (ACK_FLAG_RQD schedules
381 : an ACK instantly, ACK_FLAG_CANCEL suppresses the ACK by making this
382 : function behave like a no-op) */
383 :
384 : int
385 : fd_quic_lazy_ack_pkt( fd_quic_t * quic,
386 : fd_quic_conn_t * conn,
387 : fd_quic_pkt_t const * pkt );
388 :
389 : static inline fd_quic_conn_t *
390 0 : fd_quic_conn_at_idx( fd_quic_state_t * quic_state, ulong idx ) {
391 0 : ulong addr = quic_state->conn_base;
392 0 : ulong sz = quic_state->conn_sz;
393 0 : return (fd_quic_conn_t*)( addr + idx * sz );
394 0 : }
395 :
396 : /* called with round-trip-time (rtt) and the ack delay (from the spec)
397 : to sample the round trip times. */
398 : static inline void
399 0 : fd_quic_sample_rtt( fd_quic_conn_t * conn, long rtt_ns, long ack_delay ) {
400 : /* for convenience */
401 0 : fd_rtt_estimate_t * rtt = conn->rtt;
402 :
403 : /* scale ack delay using peer exponent - rfc9000 19.3 */
404 0 : float ack_delay_ns = (float)ack_delay * conn->peer_ack_delay_scale;
405 :
406 : /* bound ack_delay by peer_max_ack_delay */
407 0 : ack_delay_ns = fminf( ack_delay_ns, conn->peer_max_ack_delay_ns );
408 :
409 0 : fd_rtt_sample( rtt, (float)rtt_ns, ack_delay_ns );
410 :
411 0 : FD_DEBUG({
412 0 : FD_LOG_NOTICE(( "conn_idx: %u min_rtt: %f smoothed_rtt: %f var_rtt: %f rtt_ns: %f ack_delay_ns: %f diff: %f",
413 0 : (uint)conn->conn_idx,
414 0 : (double)rtt->min_rtt,
415 0 : (double)rtt->smoothed_rtt,
416 0 : (double)rtt->var_rtt,
417 0 : (double)rtt_ns,
418 0 : (double)ack_delay_ns,
419 0 : ( (double)rtt_ns - (double)ack_delay_ns ) ));
420 0 : })
421 0 : }
422 :
423 : /* fd_quic_calc_expiry_duration returns the duration to the next expiry event.
424 : User should add the result to the base time to obtain the expiry timestamp.
425 : Uses the loss detection timeout if 'ack_driven', otherwise uses the PTO. */
426 :
427 : static inline long
428 0 : fd_quic_calc_expiry_duration( fd_quic_conn_t * conn, int ack_driven, int is_server ) {
429 : /* For server, we want to be conservative and minimize spam risk, so we stick
430 : with the hardcoded 500ms expiry. The following only applies to client. */
431 0 : if( is_server ) return 500e6L;
432 :
433 : /* If this calculation is ack-driven, use the time threshold:
434 :
435 : > 6.1.2 Time Threshold
436 : > max(kTimeThreshold * max(smoothed_rtt, latest_rtt), kGranularity)
437 : > The RECOMMENDED time threshold (kTimeThreshold), expressed as an RTT multiplier, is 9/8
438 :
439 : Otherwise, calculate the expiry time according to the PTO spec
440 :
441 : > 6.2.1. Computing PTO
442 : > When an ack-eliciting packet is transmitted, the sender schedules
443 : > a timer for the PTO period as follows:
444 : > PTO = smoothed_rtt + max(4*rttvar, kGranularity) + max_ack_delay */
445 :
446 0 : fd_rtt_estimate_t * rtt = conn->rtt;
447 :
448 0 : float pto_rttvar = fmaxf( 4.0f * rtt->var_rtt, (float)FD_QUIC_K_GRANULARITY_NS );
449 0 : long pto_duration = (long)( rtt->smoothed_rtt +
450 0 : pto_rttvar +
451 0 : conn->peer_max_ack_delay_ns );
452 :
453 0 : long loss_duration = fd_long_max(
454 0 : (long)( FD_QUIC_K_TIME_THRESHOLD * fmaxf( rtt->smoothed_rtt, rtt->latest_rtt ) ),
455 0 : FD_QUIC_K_GRANULARITY_NS );
456 :
457 0 : long duration = fd_long_if( ack_driven, loss_duration, pto_duration );
458 :
459 0 : FD_DTRACE_PROBE_3( quic_calc_expiry, conn->our_conn_id, duration, ack_driven );
460 :
461 0 : return duration;
462 0 : }
463 :
464 : uchar *
465 : fd_quic_gen_stream_frames( fd_quic_conn_t * conn,
466 : uchar * payload_ptr,
467 : uchar * payload_end,
468 : fd_quic_pkt_meta_t * pkt_meta_tmpl,
469 : fd_quic_pkt_meta_tracker_t * tracker );
470 :
471 : void
472 : fd_quic_process_ack_range( fd_quic_conn_t * conn,
473 : fd_quic_frame_ctx_t * context,
474 : uint enc_level,
475 : ulong largest_ack,
476 : ulong ack_range,
477 : int is_largest,
478 : long now,
479 : ulong ack_delay );
480 :
481 : FD_PROTOTYPES_END
482 :
483 : #endif /* HEADER_fd_src_waltz_quic_fd_quic_private_h */
|
