Line data Source code
1 : #include "fd_quic_tile.h"
2 : #include "../metrics/fd_metrics.h"
3 : #include "../stem/fd_stem.h"
4 : #include "../topo/fd_topo.h"
5 : #include "fd_tpu.h"
6 : #include "../../waltz/quic/fd_quic_private.h"
7 : #include <linux/futex.h>
8 : #include "generated/quic_seccomp.h"
9 : #include "../../util/io/fd_io.h"
10 : #include "../../util/net/fd_eth.h"
11 :
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <sys/random.h>
15 :
16 : #define OUT_IDX_VERIFY 0
17 0 : #define OUT_IDX_NET 1
18 :
19 0 : #define FD_QUIC_KEYLOG_FLUSH_INTERVAL_NS ((long)100e6)
20 :
21 : /* fd_quic_tile provides a TPU server tile.
22 :
23 : This tile handles incoming transactions that clients request to be
24 : included in blocks. Supported protocols currently include TPU/UDP
25 : and TPU/QUIC.
26 :
27 : The fd_quic tile acts as a plain old Tango producer. The tile will
28 : defragment multi-packet TPU streams coming in from QUIC, such that
29 : each frag_meta refers to a complete txn. This requires the dcache
30 : mtu to be at least that of the largest allowed serialized txn size.
31 :
32 : QUIC tiles don't service network devices directly, but rely on net
33 : tiles to send and receive packets. An arbitrary number of QUIC tiles
34 : can be run. Each UDP flow must stick to one QUIC tile. */
35 :
36 : static inline fd_quic_limits_t
37 0 : quic_limits( fd_topo_tile_t const * tile ) {
38 0 : fd_quic_limits_t limits = {
39 0 : .conn_cnt = tile->quic.max_concurrent_connections,
40 0 : .handshake_cnt = tile->quic.max_concurrent_handshakes,
41 :
42 : /* fd_quic will not issue nor use any new connection IDs after
43 : completing a handshake. Connection migration is not supported
44 : either. */
45 0 : .conn_id_cnt = FD_QUIC_MIN_CONN_ID_CNT,
46 0 : .inflight_frame_cnt = 64UL * tile->quic.max_concurrent_connections,
47 0 : .min_inflight_frame_cnt_conn = 32UL
48 0 : };
49 0 : if( FD_UNLIKELY( !fd_quic_footprint( &limits ) ) ) {
50 0 : FD_LOG_ERR(( "Invalid QUIC limits in config" ));
51 0 : }
52 0 : return limits;
53 0 : }
54 :
55 : FD_FN_CONST static inline ulong
56 0 : scratch_align( void ) {
57 0 : return fd_ulong_max( alignof(fd_quic_ctx_t), fd_quic_align() );
58 0 : }
59 :
60 : static inline ulong
61 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
62 0 : ulong out_depth = tile->quic.out_depth;
63 0 : ulong reasm_max = tile->quic.reasm_cnt;
64 :
65 0 : fd_quic_limits_t limits = quic_limits( tile ); /* May FD_LOG_ERR */
66 0 : ulong l = FD_LAYOUT_INIT;
67 0 : l = FD_LAYOUT_APPEND( l, alignof( fd_quic_ctx_t ), sizeof( fd_quic_ctx_t ) );
68 0 : l = FD_LAYOUT_APPEND( l, fd_quic_align(), fd_quic_footprint( &limits ) );
69 0 : l = FD_LAYOUT_APPEND( l, fd_tpu_reasm_align(), fd_tpu_reasm_footprint( out_depth, reasm_max ) );
70 0 : return FD_LAYOUT_FINI( l, scratch_align() );
71 0 : }
72 :
73 : /* reasm_evict_tail updates metrics for the reassembly about to be
74 : evicted by the next fd_tpu_reasm_{prepare,publish_fast}. Must be
75 : called before every such call. */
76 :
77 : static void
78 0 : reasm_evict_tail( fd_quic_ctx_t * ctx ) {
79 : /* Was the reasm buffer we evicted busy? */
80 0 : fd_tpu_reasm_slot_t * victim = fd_tpu_reasm_peek_tail( ctx->reasm );
81 0 : int victim_busy = victim->k.state == FD_TPU_REASM_STATE_BUSY;
82 0 : if( FD_LIKELY( !victim_busy ) ) return;
83 :
84 : /* If so, does the connection it refers to still exist?
85 : (Or was the buffer previously abandoned by means of conn close) */
86 0 : fd_quic_state_t * state = fd_quic_get_state( ctx->quic );
87 0 : uint victim_cidx = fd_quic_conn_uid_idx( victim->k.conn_uid );
88 0 : uint victim_gen = fd_quic_conn_uid_gen( victim->k.conn_uid );
89 0 : fd_quic_conn_t * victim_conn = fd_quic_conn_at_idx( state, victim_cidx );
90 0 : uint victim_exists = (victim_conn->conn_gen == victim_gen) &
91 0 : (victim_conn->state == FD_QUIC_CONN_STATE_ACTIVE); /* in [0,1] */
92 0 : victim_conn->srx->rx_streams_active -= victim_exists;
93 0 : ctx->metrics.reasm_overrun += victim_exists;
94 0 : ctx->metrics.reasm_active -= victim_exists;
95 0 : }
96 :
97 : /* legacy_stream_notify is called for transactions sent via TPU/UDP. For
98 : now both QUIC and non-QUIC transactions are accepted, with traffic
99 : type determined by port.
100 :
101 : UDP transactions must fit in one packet and cannot be fragmented, and
102 : notify here means the entire packet was received. */
103 :
104 : static void
105 : legacy_stream_notify( fd_quic_ctx_t * ctx,
106 : uchar * packet,
107 : ulong packet_sz,
108 0 : uint ipv4 ) {
109 :
110 0 : long tspub = ctx->now;
111 0 : fd_tpu_reasm_t * reasm = ctx->reasm;
112 0 : fd_stem_context_t * stem = ctx->stem;
113 0 : fd_frag_meta_t * mcache = stem->mcaches[0];
114 0 : void * base = ctx->verify_out_mem;
115 0 : ulong seq = stem->seqs[0];
116 :
117 0 : reasm_evict_tail( ctx );
118 0 : int err = fd_tpu_reasm_publish_fast( reasm, packet, packet_sz, mcache, base, seq, tspub, ipv4, FD_TXN_M_TPU_SOURCE_UDP );
119 0 : if( FD_LIKELY( err==FD_TPU_REASM_SUCCESS ) ) {
120 0 : fd_stem_advance( stem, 0UL );
121 0 : ctx->metrics.txns_received_udp++;
122 0 : }
123 0 : }
124 :
125 : /* This tile always publishes messages downstream, even if there are
126 : no credits available. It ignores the flow control of the downstream
127 : verify tile. This is OK as the verify tile is written to expect
128 : this behavior, and enables the QUIC tile to publish as fast as it
129 : can. It would currently be difficult trying to backpressure further
130 : up the stack to the network itself. */
131 : static inline void
132 : before_credit( fd_quic_ctx_t * ctx,
133 : fd_stem_context_t * stem,
134 0 : int * charge_busy ) {
135 0 : ctx->stem = stem;
136 :
137 : /* Publishes to mcache via callbacks */
138 0 : long now = fd_clock_tile_now( ctx->clock );
139 0 : ctx->now = now;
140 0 : *charge_busy = fd_quic_service( ctx->quic, now );
141 0 : }
142 :
143 : static inline long
144 0 : next_deadline( fd_quic_ctx_t * ctx ) {
145 0 : long next = fd_quic_get_next_wakeup( ctx->quic );
146 0 : return next==LONG_MAX ? LONG_MAX : fd_clock_tile_wallclock_to_tickcount( ctx->clock, next );
147 0 : }
148 :
149 : static inline void
150 0 : metrics_write( fd_quic_ctx_t * ctx ) {
151 0 : FD_MCNT_SET ( QUIC, TXN_RX_UDP, ctx->metrics.txns_received_udp );
152 0 : FD_MCNT_SET ( QUIC, TXN_RX_QUIC_FAST, ctx->metrics.txns_received_quic_fast );
153 0 : FD_MCNT_SET ( QUIC, TXN_RX_QUIC_FRAG, ctx->metrics.txns_received_quic_frag );
154 0 : FD_MCNT_SET ( QUIC, TXN_REASSEMBLY_CHUNK_RX, ctx->metrics.frag_ok_cnt );
155 0 : FD_MCNT_SET ( QUIC, TXN_REASSEMBLY_GAP_DROPPED, ctx->metrics.frag_gap_cnt );
156 0 : FD_MCNT_SET ( QUIC, TXN_REASSEMBLY_DUPLICATE_DROPPED, ctx->metrics.frag_dup_cnt );
157 0 : FD_MCNT_SET ( QUIC, TXN_REASSEMBLY_EVICTED, ctx->metrics.reasm_overrun );
158 0 : FD_MCNT_SET ( QUIC, TXN_REASSEMBLY_ABANDONED, ctx->metrics.reasm_abandoned );
159 0 : FD_MCNT_SET ( QUIC, TXN_REASSEMBLY_STARTED, ctx->metrics.reasm_started );
160 0 : FD_MGAUGE_SET( QUIC, TXN_REASSEMBLY_PENDING, (ulong)fd_long_max( ctx->metrics.reasm_active, 0L ) );
161 :
162 0 : FD_MCNT_SET( QUIC, UDP_TXN_UNDERSIZE_DROPPED, ctx->metrics.udp_pkt_too_small );
163 0 : FD_MCNT_SET( QUIC, UDP_TXN_OVERSIZE_DROPPED, ctx->metrics.udp_pkt_too_large );
164 0 : FD_MCNT_SET( QUIC, TXN_UNDERSIZE_DROPPED, ctx->metrics.quic_txn_too_small );
165 0 : FD_MCNT_SET( QUIC, TXN_OVERSIZE_DROPPED, ctx->metrics.quic_txn_too_large );
166 :
167 0 : FD_MCNT_SET( QUIC, PKT_RX, ctx->quic->metrics.net_rx_pkt_cnt );
168 0 : FD_MCNT_SET( QUIC, PKT_RX_BYTES, ctx->quic->metrics.net_rx_byte_cnt+sizeof(fd_eth_hdr_t)*ctx->quic->metrics.net_rx_pkt_cnt );
169 0 : FD_MCNT_SET( QUIC, PKT_TX, ctx->quic->metrics.net_tx_pkt_cnt );
170 0 : FD_MCNT_SET( QUIC, PKT_TX_BYTES, ctx->quic->metrics.net_tx_byte_cnt+sizeof(fd_eth_hdr_t)*ctx->quic->metrics.net_tx_pkt_cnt );
171 0 : FD_MCNT_SET( QUIC, RETRY_PKT_TX, ctx->quic->metrics.retry_tx_cnt );
172 0 : FD_MGAUGE_ENUM_COPY( QUIC, CONN_USED, (ctx->quic->metrics.conn_state_cnt+1) ); /* skip FD_QUIC_CONN_STATE_INVALID (free slots) */
173 0 : FD_MGAUGE_SET( QUIC, CONN_FREE, ctx->quic->limits.conn_cnt-ctx->quic->metrics.conn_alloc_cnt );
174 0 : FD_MGAUGE_SET( QUIC, CONN_CAPACITY, ctx->quic->limits.conn_cnt );
175 0 : FD_MCNT_SET( QUIC, CONN_CREATED, ctx->quic->metrics.conn_created_cnt );
176 0 : FD_MCNT_SET( QUIC, CONN_CLOSED, ctx->quic->metrics.conn_closed_cnt );
177 0 : FD_MCNT_SET( QUIC, CONN_ABORTED, ctx->quic->metrics.conn_aborted_cnt );
178 0 : FD_MCNT_SET( QUIC, CONN_TIMED_OUT, ctx->quic->metrics.conn_timeout_cnt );
179 0 : FD_MCNT_SET( QUIC, CONN_RETRIED, ctx->quic->metrics.conn_retry_cnt );
180 :
181 0 : FD_MCNT_SET( QUIC, CONN_NO_SLOT_REJECTED, ctx->quic->metrics.conn_err_no_slots_cnt );
182 0 : FD_MCNT_SET( QUIC, CONN_RETRY_REJECTED, ctx->quic->metrics.conn_err_retry_fail_cnt );
183 0 : FD_MCNT_SET( QUIC, CONN_RETRY_EXPIRED_REJECTED, ctx->quic->metrics.conn_err_retry_expired_cnt );
184 :
185 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_DECRYPT_FAILED, ctx->quic->metrics.pkt_decrypt_fail_cnt );
186 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_NO_KEY_DROPPED, ctx->quic->metrics.pkt_no_key_cnt );
187 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_NO_CONN_DROPPED, ctx->quic->metrics.pkt_no_conn_cnt );
188 0 : ulong pkt_drop[ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_CNT ] = {
189 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_UNDERSIZE_IDX ] = ctx->quic->metrics.pkt_undersz_cnt,
190 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_OVERSIZE_IDX ] = ctx->quic->metrics.pkt_oversz_cnt,
191 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_SRC_IP_MISMATCH_IDX ] = ctx->quic->metrics.pkt_wrong_src_cnt,
192 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_HEADER_INVALID_IDX ] = ctx->quic->metrics.pkt_quic_hdr_err_cnt,
193 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_NET_HEADER_INVALID_IDX ] = ctx->quic->metrics.pkt_net_hdr_err_cnt,
194 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_VERSION_UNSUPPORTED_IDX ] = ctx->quic->metrics.pkt_verneg_cnt,
195 0 : [ FD_METRICS_ENUM_QUIC_PKT_DROP_REASON_V_DUPLICATE_IDX ] = ctx->quic->metrics.pkt_dup_cnt,
196 0 : };
197 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_DROPPED, pkt_drop );
198 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_META_ACQUIRE_ATTEMPTED, ctx->quic->metrics.frame_tx_alloc_cnt );
199 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_RETRANSMITTED, ctx->quic->metrics.pkt_retransmissions_cnt );
200 0 : FD_MCNT_ENUM_COPY( QUIC, INITIAL_PKT_RX, ctx->quic->metrics.initial_token_len_cnt );
201 :
202 0 : FD_MCNT_SET( QUIC, HANDSHAKE_CREATED, ctx->quic->metrics.hs_created_cnt );
203 0 : FD_MCNT_SET( QUIC, HANDSHAKE_POOL_FULL_REJECTED, ctx->quic->metrics.hs_err_alloc_fail_cnt );
204 0 : FD_MCNT_SET( QUIC, HANDSHAKE_EVICTED, ctx->quic->metrics.hs_evicted_cnt );
205 :
206 0 : FD_MCNT_SET( QUIC, STREAM_RX, ctx->quic->metrics.stream_rx_event_cnt );
207 0 : FD_MCNT_SET( QUIC, STREAM_RX_BYTES, ctx->quic->metrics.stream_rx_byte_cnt );
208 :
209 0 : FD_MCNT_ENUM_COPY( QUIC, FRAME_RX, ctx->quic->metrics.frame_rx_cnt );
210 0 : FD_MCNT_SET ( QUIC, FRAME_PARSE_FAILED, ctx->quic->metrics.frame_rx_err_cnt );
211 :
212 0 : FD_MCNT_ENUM_COPY( QUIC, PKT_ACK_PROCESSED, ctx->quic->metrics.ack_tx );
213 :
214 0 : FD_MHIST_COPY( QUIC, SERVICE_DURATION_NANOS, ctx->quic->metrics.service_duration );
215 0 : FD_MHIST_COPY( QUIC, PKT_RX_DURATION_NANOS, ctx->quic->metrics.receive_duration );
216 0 : }
217 :
218 : static int
219 : before_frag( fd_quic_ctx_t * ctx,
220 : ulong in_idx,
221 : ulong seq,
222 0 : ulong sig ) {
223 0 : (void)in_idx;
224 0 : (void)seq;
225 :
226 0 : ulong proto = fd_disco_netmux_sig_proto( sig );
227 0 : if( FD_UNLIKELY( proto!=DST_PROTO_TPU_UDP && proto!=DST_PROTO_TPU_QUIC ) ) return 1;
228 :
229 0 : ulong hash = fd_disco_netmux_sig_hash( sig );
230 0 : if( FD_UNLIKELY( (hash % ctx->round_robin_cnt) != ctx->round_robin_id ) ) return 1;
231 :
232 0 : return 0;
233 0 : }
234 :
235 : static void
236 : during_frag( fd_quic_ctx_t * ctx,
237 : ulong in_idx,
238 : ulong seq FD_PARAM_UNUSED,
239 : ulong sig FD_PARAM_UNUSED,
240 : ulong chunk,
241 : ulong sz,
242 0 : ulong ctl ) {
243 0 : void const * src = fd_net_rx_translate_frag( &ctx->net_in_bounds[ in_idx ], chunk, ctl, sz );
244 :
245 : /* FIXME this copy could be eliminated by combining it with the decrypt operation */
246 0 : fd_memcpy( ctx->buffer, src, sz );
247 0 : }
248 :
249 : static void
250 : after_frag( fd_quic_ctx_t * ctx,
251 : ulong in_idx,
252 : ulong seq,
253 : ulong sig,
254 : ulong sz,
255 : ulong tsorig,
256 : ulong tspub,
257 0 : fd_stem_context_t * stem ) {
258 0 : (void)in_idx;
259 0 : (void)seq;
260 0 : (void)tsorig;
261 0 : (void)tspub;
262 0 : (void)stem;
263 :
264 0 : ulong proto = fd_disco_netmux_sig_proto( sig );
265 :
266 0 : if( FD_LIKELY( proto==DST_PROTO_TPU_QUIC ) ) {
267 0 : if( FD_UNLIKELY( sz<sizeof(fd_eth_hdr_t) ) ) FD_LOG_ERR(( "QUIC packet too small" ));
268 0 : uchar * ip_pkt = ctx->buffer + sizeof(fd_eth_hdr_t);
269 0 : ulong ip_sz = sz - sizeof(fd_eth_hdr_t);
270 :
271 0 : fd_quic_t * quic = ctx->quic;
272 0 : fd_quic_process_packet( quic, ip_pkt, ip_sz, ctx->now );
273 0 : } else if( FD_LIKELY( proto==DST_PROTO_TPU_UDP ) ) {
274 0 : ulong network_hdr_sz = fd_disco_netmux_sig_hdr_sz( sig );
275 0 : if( FD_UNLIKELY( sz<=network_hdr_sz ) ) {
276 : /* Transaction not valid if the packet isn't large enough for the network
277 : headers. */
278 0 : ctx->metrics.udp_pkt_too_small++;
279 0 : return;
280 0 : }
281 :
282 0 : ulong data_sz = sz - network_hdr_sz;
283 0 : if( FD_UNLIKELY( data_sz<FD_TXN_MIN_SERIALIZED_SZ ) ) {
284 : /* Smaller than the smallest possible transaction */
285 0 : ctx->metrics.udp_pkt_too_small++;
286 0 : return;
287 0 : }
288 :
289 0 : if( FD_UNLIKELY( data_sz>FD_TPU_MTU ) ) {
290 : /* Transaction couldn't possibly be valid if it's longer than transaction
291 : MTU so drop it. This is not required, as the txn will fail to parse,
292 : but it's a nice short circuit. */
293 0 : ctx->metrics.udp_pkt_too_large++;
294 0 : return;
295 0 : }
296 :
297 0 : legacy_stream_notify( ctx, ctx->buffer+network_hdr_sz, data_sz, fd_disco_netmux_sig_ip( sig ) );
298 0 : }
299 0 : }
300 :
301 : static void
302 : quic_conn_final( fd_quic_conn_t * conn,
303 0 : void * quic_ctx ) {
304 0 : fd_quic_ctx_t * ctx = quic_ctx;
305 0 : long abandon_cnt = fd_long_max( conn->srx->rx_streams_active, 0L );
306 0 : ctx->metrics.reasm_active -= abandon_cnt;
307 0 : ctx->metrics.reasm_abandoned += (ulong)abandon_cnt;
308 0 : }
309 :
310 : static int
311 : quic_stream_rx( fd_quic_conn_t * conn,
312 : ulong stream_id,
313 : ulong offset,
314 : uchar const * data,
315 : ulong data_sz,
316 0 : int fin ) {
317 :
318 0 : fd_quic_t * quic = conn->quic;
319 0 : fd_quic_ctx_t * ctx = quic->cb.quic_ctx;
320 0 : long tspub = ctx->now;
321 0 : fd_tpu_reasm_t * reasm = ctx->reasm;
322 0 : ulong conn_uid = fd_quic_conn_uid( conn );
323 0 : fd_stem_context_t * stem = ctx->stem;
324 0 : fd_frag_meta_t * mcache = stem->mcaches[0];
325 0 : void * base = ctx->verify_out_mem;
326 0 : ulong seq = stem->seqs[0];
327 :
328 0 : int oversz = offset+data_sz > FD_TPU_MTU;
329 :
330 0 : if( offset==0UL && fin ) {
331 : /* Fast path */
332 0 : if( FD_UNLIKELY( data_sz<FD_TXN_MIN_SERIALIZED_SZ ) ) {
333 0 : ctx->metrics.quic_txn_too_small++;
334 0 : return FD_QUIC_SUCCESS; /* drop */
335 0 : }
336 0 : if( FD_UNLIKELY( oversz ) ) {
337 0 : ctx->metrics.quic_txn_too_large++;
338 0 : return FD_QUIC_SUCCESS; /* drop */
339 0 : }
340 0 : reasm_evict_tail( ctx );
341 0 : int err = fd_tpu_reasm_publish_fast( reasm, data, data_sz, mcache, base, seq, tspub, conn->peer->ip_addr, FD_TXN_M_TPU_SOURCE_QUIC );
342 0 : if( FD_LIKELY( err==FD_TPU_REASM_SUCCESS ) ) {
343 0 : fd_stem_advance( stem, 0UL );
344 0 : ctx->metrics.txns_received_quic_fast++;
345 0 : }
346 0 : return FD_QUIC_SUCCESS;
347 0 : }
348 :
349 0 : if( data_sz==0UL && !fin ) return FD_QUIC_SUCCESS; /* noop */
350 :
351 0 : fd_tpu_reasm_slot_t * slot = fd_tpu_reasm_query( reasm, conn_uid, stream_id );
352 :
353 0 : if( !slot ) { /* start a new reassembly */
354 0 : if( offset>0 ) {
355 0 : ctx->metrics.frag_gap_cnt++;
356 0 : return FD_QUIC_SUCCESS;
357 0 : }
358 0 : if( data_sz==0 ) return FD_QUIC_SUCCESS; /* ignore empty */
359 0 : if( FD_UNLIKELY( !conn->called_conn_new ) ) return FD_QUIC_FAILED;
360 0 : if( FD_UNLIKELY( oversz ) ) {
361 0 : ctx->metrics.quic_txn_too_large++;
362 0 : return FD_QUIC_SUCCESS; /* drop */
363 0 : }
364 :
365 0 : reasm_evict_tail( ctx );
366 0 : slot = fd_tpu_reasm_prepare( reasm, conn_uid, stream_id, tspub ); /* infallible */
367 0 : ctx->metrics.reasm_started++;
368 0 : ctx->metrics.reasm_active++;
369 0 : conn->srx->rx_streams_active++;
370 0 : } else if( slot->k.state != FD_TPU_REASM_STATE_BUSY ) {
371 0 : ctx->metrics.frag_dup_cnt++;
372 0 : return FD_QUIC_SUCCESS;
373 0 : }
374 :
375 0 : int reasm_res = fd_tpu_reasm_frag( reasm, slot, data, data_sz, offset );
376 0 : if( FD_UNLIKELY( reasm_res != FD_TPU_REASM_SUCCESS ) ) {
377 0 : int is_gap = reasm_res==FD_TPU_REASM_ERR_SKIP;
378 0 : int is_oversz = reasm_res==FD_TPU_REASM_ERR_SZ;
379 0 : ctx->metrics.frag_gap_cnt += (ulong)is_gap;
380 0 : ctx->metrics.quic_txn_too_large += (ulong)is_oversz;
381 0 : ctx->metrics.reasm_active -= is_oversz;
382 0 : conn->srx->rx_streams_active -= is_oversz;
383 0 : return is_gap ? FD_QUIC_FAILED : FD_QUIC_SUCCESS;
384 0 : }
385 0 : ctx->metrics.frag_ok_cnt++;
386 :
387 0 : if( fin ) {
388 0 : if( FD_UNLIKELY( slot->k.sz < FD_TXN_MIN_SERIALIZED_SZ ) ) {
389 0 : ctx->metrics.quic_txn_too_small++;
390 0 : return FD_QUIC_SUCCESS; /* ignore */
391 0 : }
392 0 : int pub_err = fd_tpu_reasm_publish( reasm, slot, mcache, base, seq, tspub, conn->peer->ip_addr, FD_TXN_M_TPU_SOURCE_QUIC );
393 0 : if( FD_UNLIKELY( pub_err!=FD_TPU_REASM_SUCCESS ) ) return FD_QUIC_SUCCESS; /* unreachable */
394 0 : ulong * rcv_cnt = (offset==0UL && fin) ? &ctx->metrics.txns_received_quic_fast : &ctx->metrics.txns_received_quic_frag;
395 0 : (*rcv_cnt)++;
396 0 : ctx->metrics.reasm_active--;
397 0 : conn->srx->rx_streams_active--;
398 :
399 0 : fd_stem_advance( stem, 0UL );
400 0 : }
401 :
402 0 : return FD_QUIC_SUCCESS;
403 0 : }
404 :
405 : static int
406 : quic_tx_aio_send( void * _ctx,
407 : fd_aio_pkt_info_t const * batch,
408 : ulong batch_cnt,
409 : ulong * opt_batch_idx,
410 0 : int flush ) {
411 0 : (void)flush;
412 :
413 0 : fd_quic_ctx_t * ctx = _ctx;
414 :
415 0 : for( ulong i=0; i<batch_cnt; i++ ) {
416 0 : if( FD_UNLIKELY( batch[ i ].buf_sz<FD_NETMUX_SIG_MIN_HDR_SZ ) ) continue;
417 :
418 0 : uint const ip_dst = FD_LOAD( uint, batch[ i ].buf+offsetof( fd_ip4_hdr_t, daddr_c ) );
419 0 : uchar * packet_l2 = fd_chunk_to_laddr( ctx->net_out_mem, ctx->net_out_chunk );
420 0 : uchar * packet_l3 = packet_l2 + sizeof(fd_eth_hdr_t);
421 0 : memset( packet_l2, 0, 12 );
422 0 : FD_STORE( ushort, packet_l2+offsetof( fd_eth_hdr_t, net_type ), fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) );
423 0 : fd_memcpy( packet_l3, batch[ i ].buf, batch[ i ].buf_sz );
424 0 : ulong sz_l2 = sizeof(fd_eth_hdr_t) + batch[ i ].buf_sz;
425 :
426 : /* send packets are just round-robined by sequence number, so for now
427 : just indicate where they came from so they don't bounce back */
428 0 : ulong sig = fd_disco_netmux_sig( ip_dst, 0U, ip_dst, DST_PROTO_OUTGOING, FD_NETMUX_SIG_MIN_HDR_SZ );
429 :
430 0 : ulong chunk = ctx->net_out_chunk;
431 0 : ulong ctl = fd_frag_meta_ctl( 0UL, 1, 1, 0 );
432 0 : fd_stem_publish( ctx->stem, OUT_IDX_NET, sig, chunk, sz_l2, ctl, 0L, 0L );
433 :
434 0 : ctx->net_out_chunk = fd_dcache_compact_next( chunk, FD_NET_MTU, ctx->net_out_chunk0, ctx->net_out_wmark );
435 0 : }
436 :
437 0 : if( FD_LIKELY( opt_batch_idx ) ) {
438 0 : *opt_batch_idx = batch_cnt;
439 0 : }
440 :
441 0 : return FD_AIO_SUCCESS;
442 0 : }
443 :
444 : static void
445 : quic_tls_keylog( void * _ctx,
446 0 : char const * line ) {
447 0 : fd_quic_ctx_t * ctx = _ctx;
448 0 : fd_io_buffered_ostream_t * os = &ctx->keylog_stream;
449 :
450 : /* Lazily flush ostream */
451 0 : ulong line_sz = strlen( line )+1;
452 0 : ulong peek_sz = fd_io_buffered_ostream_peek_sz( os );
453 0 : if( FD_UNLIKELY( peek_sz<line_sz ) ) {
454 0 : int err = fd_io_buffered_ostream_flush( os );
455 0 : if( FD_UNLIKELY( err ) ) {
456 0 : FD_LOG_ERR(( "fd_io_buffered_ostream_flush(keylog) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
457 0 : }
458 0 : peek_sz = fd_io_buffered_ostream_peek_sz( os );
459 0 : }
460 0 : if( FD_UNLIKELY( peek_sz<line_sz ) ) {
461 0 : FD_LOG_ERR(( "keylog buffer too small (buf_sz=%lu, line_sz=%lu)", peek_sz, line_sz ));
462 0 : }
463 :
464 : /* Append line */
465 0 : char * cur = fd_io_buffered_ostream_peek( os );
466 0 : cur = fd_cstr_append_text( cur, line, strlen( line ) );
467 0 : cur = fd_cstr_append_char( cur, '\n' );
468 0 : fd_io_buffered_ostream_seek( os, line_sz );
469 0 : }
470 :
471 : static void
472 0 : during_housekeeping( fd_quic_ctx_t * ctx ) {
473 0 : long now = ctx->now = fd_clock_tile_now( ctx->clock );
474 0 : if( FD_UNLIKELY( now >= fd_clock_tile_recal_next( ctx->clock ) ) ) {
475 0 : fd_clock_tile_recal( ctx->clock );
476 0 : }
477 :
478 0 : if( FD_UNLIKELY( ctx->keylog_stream.wbuf ) ) {
479 0 : if( FD_UNLIKELY( now > ctx->keylog_next_flush ) ) {
480 0 : int err = fd_io_buffered_ostream_flush( &ctx->keylog_stream );
481 0 : if( FD_UNLIKELY( err ) ) {
482 0 : FD_LOG_ERR(( "fd_io_buffered_ostream_flush(keylog) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
483 0 : }
484 0 : ctx->keylog_next_flush = now + FD_QUIC_KEYLOG_FLUSH_INTERVAL_NS;
485 0 : }
486 0 : }
487 0 : }
488 :
489 : static void
490 : privileged_init( fd_topo_t const * topo,
491 0 : fd_topo_tile_t const * tile ) {
492 0 : fd_quic_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
493 0 : if( FD_UNLIKELY( topo->objs[ tile->tile_obj_id ].footprint < scratch_footprint( tile ) ) ) {
494 0 : FD_LOG_ERR(( "insufficient tile scratch space" ));
495 0 : }
496 0 : fd_memset( ctx, 0, sizeof(fd_quic_ctx_t) );
497 0 : FD_TEST( fd_rng_secure( &ctx->reasm_seed, sizeof(ctx->reasm_seed) ) );
498 0 : ctx->keylog_fd = -1;
499 :
500 0 : if( 0!=strcmp( tile->quic.key_log_path, "" ) ) {
501 0 : ctx->keylog_fd = open( tile->quic.key_log_path, O_WRONLY|O_CREAT|O_APPEND, 0644 );
502 0 : if( FD_UNLIKELY( ctx->keylog_fd<0 ) ) {
503 0 : FD_LOG_ERR(( "open(%s, O_WRONLY|O_CREAT|O_APPEND, 0644) failed (%i-%s)",
504 0 : tile->quic.key_log_path, errno, fd_io_strerror( errno ) ));
505 0 : }
506 0 : fd_io_buffered_ostream_init( &ctx->keylog_stream, ctx->keylog_fd, ctx->keylog_buf, sizeof(ctx->keylog_buf) );
507 0 : FD_LOG_WARNING(( "Logging QUIC encryption keys to %s", tile->quic.key_log_path ));
508 0 : }
509 :
510 : /* The fd_quic implementation calls fd_log_wallclock() internally
511 : which itself calls clock_gettime() which on most kernels is not a
512 : real syscall but a virtual one in the process via. the vDSO.
513 :
514 : The first time this virtual call is made to the vDSO it does an
515 : mmap(2) of some shared memory into userspace, which cannot
516 : happen while sandboxed so we need to ensure that initialization
517 : happens here. */
518 :
519 0 : fd_log_wallclock();
520 0 : }
521 :
522 : static void
523 : quic_tls_cv_sign( void * signer_ctx,
524 : uchar signature[ static FD_ED25519_SIG_SZ ],
525 0 : uchar const payload[ static FD_TLS_CV_SIGN_SZ ] ) {
526 0 : fd_quic_ctx_t * ctx = signer_ctx;
527 0 : fd_sha512_t * sha512 = fd_sha512_join( ctx->sha512 );
528 0 : fd_ed25519_sign( signature, payload, FD_TLS_CV_SIGN_SZ, ctx->tls_pub_key, ctx->tls_priv_key, sha512 );
529 0 : fd_sha512_leave( sha512 );
530 0 : }
531 :
532 : static void
533 : unprivileged_init( fd_topo_t const * topo,
534 0 : fd_topo_tile_t const * tile ) {
535 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
536 :
537 0 : if( FD_UNLIKELY( tile->in_cnt==0 ) ) {
538 0 : FD_LOG_ERR(( "quic tile has no input links" ));
539 0 : }
540 :
541 0 : if( FD_UNLIKELY( tile->out_cnt!=2UL ||
542 0 : strcmp( topo->links[ tile->out_link_id[ OUT_IDX_VERIFY ] ].name, "quic_verify" ) ||
543 0 : strcmp( topo->links[ tile->out_link_id[ OUT_IDX_NET ] ].name, "quic_net" ) ) )
544 0 : FD_LOG_ERR(( "quic tile has none or unexpected output links %lu %s %s",
545 0 : tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].name, topo->links[ tile->out_link_id[ 1 ] ].name ));
546 :
547 0 : ulong out_depth = topo->links[ tile->out_link_id[ 0 ] ].depth;
548 0 : if( FD_UNLIKELY( tile->quic.out_depth != out_depth ) )
549 0 : FD_LOG_ERR(( "tile->quic.out_depth (%u) does not match quic_verify link depth (%lu)",
550 0 : tile->quic.out_depth, out_depth ));
551 :
552 0 : void * txn_dcache = topo->links[ tile->out_link_id[ 0UL ] ].dcache;
553 0 : if( FD_UNLIKELY( !txn_dcache ) ) FD_LOG_ERR(( "Missing output dcache" ));
554 :
555 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
556 0 : fd_quic_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_quic_ctx_t ), sizeof( fd_quic_ctx_t ) );
557 0 : FD_TEST( (ulong)ctx==(ulong)scratch );
558 :
559 0 : for( ulong i=0; i<tile->in_cnt; i++ ) {
560 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
561 0 : if( FD_UNLIKELY( 0!=strcmp( link->name, "net_quic" ) ) ) {
562 0 : FD_LOG_ERR(( "unexpected input link %s", link->name ));
563 0 : }
564 0 : fd_net_rx_bounds_init( &ctx->net_in_bounds[ i ], link->dcache );
565 0 : }
566 :
567 0 : fd_clock_tile_t * clock = ctx->clock;
568 0 : fd_clock_tile_init( clock );
569 0 : ctx->now = fd_clock_tile_now( clock );
570 :
571 0 : if( FD_UNLIKELY( getrandom( ctx->tls_priv_key, ED25519_PRIV_KEY_SZ, 0 )!=ED25519_PRIV_KEY_SZ ) ) {
572 0 : FD_LOG_ERR(( "getrandom failed (%i-%s)", errno, fd_io_strerror( errno ) ));
573 0 : }
574 0 : fd_sha512_t * sha512 = fd_sha512_join( fd_sha512_new( ctx->sha512 ) );
575 0 : fd_ed25519_public_from_private( ctx->tls_pub_key, ctx->tls_priv_key, sha512 );
576 0 : fd_sha512_leave( sha512 );
577 :
578 0 : fd_aio_t * quic_tx_aio = fd_aio_join( fd_aio_new( ctx->quic_tx_aio, ctx, quic_tx_aio_send ) );
579 0 : if( FD_UNLIKELY( !quic_tx_aio ) ) FD_LOG_ERR(( "fd_aio_join failed" ));
580 :
581 0 : fd_quic_limits_t limits = quic_limits( tile );
582 0 : fd_quic_t * quic = fd_quic_join( fd_quic_new( FD_SCRATCH_ALLOC_APPEND( l, fd_quic_align(), fd_quic_footprint( &limits ) ), &limits ) );
583 0 : if( FD_UNLIKELY( !quic ) ) FD_LOG_ERR(( "fd_quic_new failed" ));
584 :
585 0 : ulong orig = 0UL; /* fd_tango origin ID */
586 0 : ulong reasm_max = tile->quic.reasm_cnt;
587 0 : void * reasm_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_tpu_reasm_align(), fd_tpu_reasm_footprint( out_depth, reasm_max ) );
588 0 : ctx->reasm = fd_tpu_reasm_join( fd_tpu_reasm_new( reasm_mem, out_depth, reasm_max, ctx->reasm_seed, orig, txn_dcache ) );
589 0 : if( FD_UNLIKELY( !ctx->reasm ) ) FD_LOG_ERR(( "fd_tpu_reasm_new failed" ));
590 :
591 0 : if( FD_UNLIKELY( tile->quic.ack_delay_millis == 0 ) ) {
592 0 : FD_LOG_ERR(( "Invalid `ack_delay_millis`: must be greater than zero" ));
593 0 : }
594 0 : if( FD_UNLIKELY( tile->quic.ack_delay_millis >= tile->quic.idle_timeout_millis ) ) {
595 0 : FD_LOG_ERR(( "Invalid `ack_delay_millis`: must be lower than `idle_timeout_millis`" ));
596 0 : }
597 :
598 0 : quic->config.role = FD_QUIC_ROLE_SERVER;
599 0 : quic->config.idle_timeout = tile->quic.idle_timeout_millis * (long)1e6;
600 0 : quic->config.ack_delay = tile->quic.ack_delay_millis * (long)1e6;
601 0 : quic->config.initial_rx_max_stream_data = FD_TXN_MTU;
602 0 : quic->config.retry = tile->quic.retry;
603 0 : fd_memcpy( quic->config.identity_public_key, ctx->tls_pub_key, ED25519_PUB_KEY_SZ );
604 :
605 0 : quic->config.sign = quic_tls_cv_sign;
606 0 : quic->config.sign_ctx = ctx;
607 :
608 0 : quic->cb.conn_final = quic_conn_final;
609 0 : quic->cb.stream_rx = quic_stream_rx;
610 0 : quic->cb.quic_ctx = ctx;
611 0 : if( ctx->keylog_fd>=0 ) {
612 0 : quic->cb.tls_keylog = quic_tls_keylog;
613 0 : ctx->keylog_next_flush = fd_log_wallclock() + FD_QUIC_KEYLOG_FLUSH_INTERVAL_NS;
614 0 : }
615 :
616 0 : fd_quic_set_aio_net_tx( quic, quic_tx_aio );
617 0 : if( FD_UNLIKELY( !fd_quic_init( quic ) ) ) FD_LOG_ERR(( "fd_quic_init failed" ));
618 :
619 0 : fd_topo_link_t const * net_out = &topo->links[ tile->out_link_id[ 1 ] ];
620 :
621 0 : ctx->net_out_mem = topo->workspaces[ topo->objs[ net_out->dcache_obj_id ].wksp_id ].wksp;
622 0 : ctx->net_out_chunk0 = fd_dcache_compact_chunk0( ctx->net_out_mem, net_out->dcache );
623 0 : ctx->net_out_wmark = fd_dcache_compact_wmark ( ctx->net_out_mem, net_out->dcache, net_out->mtu );
624 0 : ctx->net_out_chunk = ctx->net_out_chunk0;
625 :
626 0 : fd_topo_link_t const * verify_out = &topo->links[ tile->out_link_id[ 0 ] ];
627 :
628 0 : ctx->verify_out_mem = topo->workspaces[ topo->objs[ verify_out->dcache_obj_id ].wksp_id ].wksp;
629 :
630 0 : ctx->quic = quic;
631 :
632 0 : ctx->round_robin_cnt = fd_topo_tile_name_cnt( topo, tile->name );
633 0 : ctx->round_robin_id = tile->kind_id;
634 0 : if( FD_UNLIKELY( ctx->round_robin_id >= ctx->round_robin_cnt ) ) {
635 0 : FD_LOG_ERR(( "invalid round robin configuration" ));
636 0 : }
637 :
638 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
639 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
640 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
641 :
642 : /* Call new/join here rather than in fd_quic so min/max can differ across uses */
643 0 : fd_histf_join( fd_metrics_histf_new_ns( ctx->quic->metrics.service_duration, FD_MHIST_EDGES_NS( QUIC, SERVICE_DURATION_NANOS ) ) );
644 0 : fd_histf_join( fd_metrics_histf_new_ns( ctx->quic->metrics.receive_duration, FD_MHIST_EDGES_NS( QUIC, PKT_RX_DURATION_NANOS ) ) );
645 0 : }
646 :
647 : static ulong
648 : populate_allowed_seccomp( fd_topo_t const * topo,
649 : fd_topo_tile_t const * tile,
650 : ulong out_cnt,
651 0 : struct sock_filter * out ) {
652 0 : fd_quic_ctx_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
653 0 : populate_sock_filter_policy_quic( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->keylog_fd );
654 0 : return sock_filter_policy_quic_instr_cnt;
655 0 : }
656 :
657 : static ulong
658 : populate_allowed_fds( fd_topo_t const * topo,
659 : fd_topo_tile_t const * tile,
660 : ulong out_fds_cnt,
661 0 : int * out_fds ) {
662 0 : fd_quic_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
663 :
664 0 : if( FD_UNLIKELY( out_fds_cnt<3UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
665 :
666 0 : ulong out_cnt = 0UL;
667 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
668 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
669 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
670 0 : if( ctx->keylog_fd!=-1 )
671 0 : out_fds[ out_cnt++ ] = ctx->keylog_fd;
672 0 : return out_cnt;
673 0 : }
674 :
675 0 : #define STEM_BURST (1UL)
676 0 : #define STEM_LAZY ((long)10e6) /* 10ms */
677 :
678 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_quic_ctx_t
679 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_quic_ctx_t)
680 :
681 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
682 0 : #define STEM_CALLBACK_NEXT_DEADLINE next_deadline
683 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
684 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
685 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
686 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
687 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
688 :
689 : #include "../stem/fd_stem.c"
690 :
691 : fd_topo_run_tile_t fd_tile_quic = {
692 : .name = "quic",
693 : .populate_allowed_seccomp = populate_allowed_seccomp,
694 : .populate_allowed_fds = populate_allowed_fds,
695 : .scratch_align = scratch_align,
696 : .scratch_footprint = scratch_footprint,
697 : .privileged_init = privileged_init,
698 : .unprivileged_init = unprivileged_init,
699 : .run = stem_run,
700 : };
|
