Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_snapct_tile.h"
3 :
4 : #include "utils/fd_sspeer.h"
5 : #include "utils/fd_ssping.h"
6 : #include "utils/fd_ssctrl.h"
7 : #include "utils/fd_ssarchive.h"
8 : #include "utils/fd_http_resolver.h"
9 : #include "utils/fd_ssmsg.h"
10 : #include "utils/fd_wfs.h"
11 : #include "../backup/fd_snap_pool.h"
12 :
13 : #include "../../disco/topo/fd_topo.h"
14 : #include "../../disco/topo/fd_dns_resolve.h"
15 : #include "../../disco/metrics/fd_metrics.h"
16 : #include "../../disco/metrics/fd_metrics_sample.h"
17 : #include "../../disco/events/fd_event_report.h"
18 : #include "../../disco/waker/fd_waker.h"
19 : #include "../../disco/fd_clock_tile.h"
20 : #include "../../flamenco/gossip/fd_gossip_message.h"
21 : #include "../../waltz/resolv/fd_netdb.h"
22 : #include "../../waltz/resolv/fd_adns.h"
23 : #include "../../util/fd_hash32.h"
24 :
25 : #include <errno.h>
26 : #include <stdio.h>
27 : #include <fcntl.h>
28 : #include <unistd.h>
29 : #include <sys/stat.h>
30 : #include <sys/syscall.h>
31 : #include <linux/fs.h>
32 : #include <netinet/tcp.h>
33 : #include <netinet/in.h>
34 :
35 : #include <linux/futex.h>
36 : #include "generated/fd_snapct_tile_seccomp.h"
37 :
38 : /* Raw state values exported by the metrics (states= enums) */
39 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_INIT==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_INIT_IDX, metrics_state );
40 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_WAITING_FOR_PEERS==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_WAITING_FOR_PEERS_IDX, metrics_state );
41 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_WAITING_FOR_PEERS_INCREMENTAL_IDX, metrics_state );
42 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_COLLECTING_PEERS==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_COLLECTING_PEERS_IDX, metrics_state );
43 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_COLLECTING_PEERS_INCREMENTAL_IDX, metrics_state );
44 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_READING_FULL_FILE==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_READING_FULL_FILE_IDX, metrics_state );
45 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_FULL_FILE_FINI_IDX, metrics_state );
46 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_FULL_FILE_DONE_IDX, metrics_state );
47 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_FULL_FILE_RESET_IDX, metrics_state );
48 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_READING_INCREMENTAL_FILE==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_READING_INCREMENTAL_FILE_IDX, metrics_state );
49 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_INCREMENTAL_FILE_FINI_IDX, metrics_state );
50 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_INCREMENTAL_FILE_DONE_IDX, metrics_state );
51 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_INCREMENTAL_FILE_RESET_IDX, metrics_state );
52 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_READING_FULL_HTTP==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_READING_FULL_HTTP_IDX, metrics_state );
53 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_FULL_HTTP_FINI_IDX, metrics_state );
54 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_FULL_HTTP_DONE_IDX, metrics_state );
55 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_FULL_HTTP_RESET_IDX, metrics_state );
56 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_READING_INCREMENTAL_HTTP_IDX, metrics_state );
57 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_INCREMENTAL_HTTP_FINI_IDX, metrics_state );
58 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_INCREMENTAL_HTTP_DONE_IDX, metrics_state );
59 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_FLUSHING_INCREMENTAL_HTTP_RESET_IDX, metrics_state );
60 : FD_STATIC_ASSERT( FD_SNAPCT_STATE_SHUTDOWN==FD_METRICS_ENUM_SNAPCT_LOAD_STATE_V_SHUTDOWN_IDX, metrics_state );
61 :
62 : #define NAME "snapct"
63 :
64 : /* FIXME: Do a finishing pass over the default.toml config options / comments */
65 :
66 0 : #define GOSSIP_PEERS_MAX (FD_CONTACT_INFO_TABLE_SIZE)
67 0 : #define SERVER_PEERS_MAX (FD_TOPO_SNAPSHOTS_SERVERS_MAX_RESOLVED)
68 0 : #define TOTAL_PEERS_MAX (GOSSIP_PEERS_MAX + SERVER_PEERS_MAX)
69 :
70 0 : #define IN_KIND_ACK (0)
71 0 : #define IN_KIND_SNAPLD (1)
72 0 : #define IN_KIND_GOSSIP (2)
73 :
74 : struct fd_snapct_out_link {
75 : ulong idx;
76 : fd_wksp_t * mem;
77 : ulong chunk0;
78 : ulong wmark;
79 : ulong chunk;
80 : ulong mtu;
81 : };
82 : typedef struct fd_snapct_out_link fd_snapct_out_link_t;
83 :
84 0 : #define FD_SNAPCT_COLLECTING_PEERS_TIMEOUT (90L*1000L*1000L*1000L) /* 1.5 minutes */
85 :
86 : struct gossip_ci_entry {
87 : fd_pubkey_t pubkey;
88 : int allowed;
89 : fd_ip4_port_t rpc_addr;
90 : ulong map_next;
91 : };
92 : typedef struct gossip_ci_entry gossip_ci_entry_t;
93 :
94 : #define MAP_NAME gossip_ci_map
95 0 : #define MAP_KEY pubkey
96 : #define MAP_ELE_T gossip_ci_entry_t
97 : #define MAP_KEY_T fd_pubkey_t
98 0 : #define MAP_NEXT map_next
99 0 : #define MAP_KEY_EQ(k0,k1) fd_pubkey_eq( k0, k1 )
100 0 : #define MAP_KEY_HASH(key,seed) fd_hash32( key->uc, seed )
101 : #include "../../util/tmpl/fd_map_chain.c"
102 :
103 : /* Standalone blacklist keyed on fd_sspeer_key_t (peer identity).
104 : Keying on identity rather than network address is intentional:
105 : a peer's address can change freely, but its identity cannot.
106 : The identity is the gossip pubkey. Blacklisted peers are
107 : permanently banned for the bootstrap lifetime. Servers the
108 : operator listed in the config are never blacklisted. The
109 : blacklist uses its own dedicated pool so entries are never evicted
110 : by pool pressure, unlike the ssping peer pool. */
111 :
112 : struct fd_sspeer_blacklist_entry {
113 : fd_sspeer_key_t key;
114 : ulong pool_next;
115 : ulong map_next;
116 : };
117 : typedef struct fd_sspeer_blacklist_entry fd_sspeer_blacklist_entry_t;
118 :
119 : #define POOL_NAME blacklist_pool
120 0 : #define POOL_T fd_sspeer_blacklist_entry_t
121 : #define POOL_IDX_T ulong
122 0 : #define POOL_NEXT pool_next
123 : #include "../../util/tmpl/fd_pool.c"
124 :
125 : #define MAP_NAME blacklist_map
126 0 : #define MAP_KEY key
127 : #define MAP_ELE_T fd_sspeer_blacklist_entry_t
128 : #define MAP_KEY_T fd_sspeer_key_t
129 0 : #define MAP_NEXT map_next
130 0 : #define MAP_KEY_EQ(k0,k1) (fd_sspeer_key_eq(k0,k1))
131 0 : #define MAP_KEY_HASH(key,seed) (fd_sspeer_key_hash(key,seed))
132 : #include "../../util/tmpl/fd_map_chain.c"
133 :
134 : struct fd_snapct_tile {
135 : struct fd_topo_tile_snapct config; /* read only */
136 : int gossip_enabled;
137 : int download_enabled;
138 : int wfs_is_configured;
139 : int load_incremental;
140 :
141 : fd_netdb_fds_t netdb_fds[1];
142 :
143 : ulong waker_client_idx;
144 : ulong * waker_fseq;
145 :
146 : fd_clock_tile_t clock[1];
147 :
148 : fd_adns_t * adns;
149 : struct {
150 : char hostname[ FD_FQDN_BUF_MAX ];
151 : ushort port; /* net order */
152 : int is_https;
153 : int resolved;
154 : long retry_nanos; /* re-queue due time, 0 while in flight */
155 : } dns_servers[ FD_TOPO_SNAPSHOTS_SERVERS_MAX ];
156 : struct {
157 : char hostname[ FD_FQDN_BUF_MAX ];
158 : ushort port;
159 : int resolved;
160 : long retry_nanos;
161 : } dns_entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ];
162 :
163 : ulong resolved_servers_cnt;
164 : struct {
165 : fd_ip4_port_t addr;
166 : char hostname[ FD_FQDN_BUF_MAX ];
167 : int is_https;
168 : } resolved_servers[ FD_TOPO_SNAPSHOTS_SERVERS_MAX_RESOLVED ];
169 :
170 : ulong resolved_entrypoints_cnt;
171 : fd_ip4_port_t resolved_entrypoints[ FD_TOPO_GOSSIP_ENTRYPOINTS_MAX ];
172 :
173 : fd_ssping_t * ssping;
174 : fd_http_resolver_t * ssresolver;
175 : fd_sspeer_selector_t * selector;
176 : ulong ssping_seed;
177 : ulong gossip_ci_seed;
178 : ulong selector_seed;
179 : ulong blacklist_seed;
180 :
181 : fd_sspeer_blacklist_entry_t * blacklist_pool;
182 : blacklist_map_t * blacklist_map;
183 :
184 : int state;
185 : int malformed;
186 : int load_complete;
187 : long deadline_nanos;
188 : long io_due_nanos;
189 : int flush_ack;
190 : int flush_ack_cnt;
191 : int start_sent;
192 : fd_sspeer_t peer;
193 :
194 : struct {
195 : int dir_fd;
196 : int full_snapshot_fd;
197 : int incremental_snapshot_fd;
198 : char full_snapshot_name[ FD_SNAP_NAME_MAX ];
199 : char incremental_snapshot_name[ FD_SNAP_NAME_MAX ];
200 : } local_out;
201 :
202 : char http_full_snapshot_name[ PATH_MAX ];
203 : char http_incr_snapshot_name[ PATH_MAX ];
204 :
205 : void const * gossip_in_mem;
206 : void const * snapld_in_mem;
207 : uchar in_kind[ FD_TOPO_MAX_TILE_IN_LINKS ];
208 :
209 : struct {
210 : ulong full_slot;
211 : ulong slot;
212 : int pending;
213 : int committed;
214 : fd_sspeer_t peer; /* peer last picked for an incremental on full_slot */
215 : } predicted_incremental;
216 :
217 : struct {
218 : ulong full_snapshot_slot;
219 : char full_snapshot_path[ PATH_MAX ];
220 : ulong full_snapshot_size;
221 : int full_snapshot_zstd;
222 :
223 : uchar full_snapshot_hash[ FD_HASH_FOOTPRINT ];
224 : uchar incremental_snapshot_hash[ FD_HASH_FOOTPRINT ];
225 :
226 : ulong incremental_snapshot_slot;
227 : char incremental_snapshot_path[ PATH_MAX ];
228 : ulong incremental_snapshot_size;
229 : int incremental_snapshot_zstd;
230 : } local_in;
231 :
232 : struct {
233 : struct {
234 : ulong bytes_read;
235 : ulong bytes_written;
236 : ulong bytes_total;
237 : uint num_retries;
238 : } full;
239 :
240 : struct {
241 : ulong bytes_read;
242 : ulong bytes_written;
243 : ulong bytes_total;
244 : uint num_retries;
245 : } incremental;
246 : } metrics;
247 :
248 : struct {
249 : gossip_ci_entry_t * ci_table; /* flat array of all gossip entries, allowed or not */
250 : gossip_ci_map_t * ci_map; /* map from pubkey to only allowed gossip entries */
251 : ulong allowed_cnt; /* number of allowed entries in ci_map */
252 : int saturated;
253 : } gossip;
254 :
255 : long snapshot_start_timestamp_ns;
256 :
257 : fd_snapct_out_link_t out_ld;
258 : fd_snapct_out_link_t out_gui;
259 : fd_snapct_out_link_t out_rp;
260 : };
261 : typedef struct fd_snapct_tile fd_snapct_tile_t;
262 :
263 : static int
264 0 : gossip_enabled( fd_topo_tile_t const * tile ) {
265 0 : return tile->snapct.sources.gossip.allow_any || tile->snapct.sources.gossip.allow_list_cnt>0UL;
266 0 : }
267 :
268 : static int
269 0 : download_enabled( fd_topo_tile_t const * tile ) {
270 0 : return gossip_enabled( tile ) || tile->snapct.sources.servers_cnt>0UL;
271 0 : }
272 :
273 : static int
274 0 : snapct_needs_incr( fd_topo_tile_t const * tile ) {
275 0 : return fd_wfs_needs_incr( tile->snapct.incremental_snapshots,
276 0 : tile->snapct.wfs_slot,
277 0 : tile->snapct.wfs_hash_is_zero,
278 0 : (ulong)tile->snapct.wfs_shred_version );
279 0 : }
280 :
281 : /* wfs_floor is the lowest slot a boot may land on: the WFS target
282 : when configured, else 0 (fd_wfs.h). A zero floor leaves slot 0 and
283 : ULONG_MAX at or above it, so "WFS off" and "no such candidate" need
284 : no special-casing. */
285 :
286 : static ulong
287 0 : wfs_floor( fd_snapct_tile_t const * ctx ) {
288 0 : return fd_ulong_if( ctx->wfs_is_configured, ctx->config.wfs_slot, 0UL );
289 0 : }
290 :
291 : /* wfs_short_of_target returns whether booting at slot lands below the
292 : floor. ULONG_MAX, meaning no such candidate, never does. */
293 :
294 : static int
295 : wfs_short_of_target( fd_snapct_tile_t const * ctx,
296 0 : ulong slot ) {
297 0 : return slot<wfs_floor( ctx );
298 0 : }
299 :
300 : /* incr_use_local returns whether to load the incremental snapshot from
301 : disk rather than download it. local_slot and fetch_slot (= the
302 : available download slot) are the slots each option reaches,
303 : ULONG_MAX if it does not exist, and must pair with the same full.
304 : Keep the local one while it is within max_local_incremental_age of
305 : the download. */
306 :
307 : static int
308 : incr_use_local( fd_snapct_tile_t const * ctx,
309 : ulong local_slot,
310 0 : ulong fetch_slot ) {
311 0 : if( FD_UNLIKELY( local_slot==ULONG_MAX ) ) return 0;
312 :
313 : /* Before the age rule and the no-download shortcut: a local
314 : incremental below the floor can never be the boot. */
315 :
316 0 : if( FD_UNLIKELY( wfs_short_of_target( ctx, local_slot ) ) ) {
317 0 : FD_LOG_NOTICE(( "local incremental snapshot at slot %lu is short of "
318 0 : "[consensus.wait_for_supermajority_at_slot] %lu, discarding it",
319 0 : local_slot, wfs_floor( ctx ) ));
320 0 : return 0;
321 0 : }
322 :
323 0 : if( FD_UNLIKELY( fetch_slot==ULONG_MAX ) ) return 1;
324 :
325 0 : if( FD_UNLIKELY( local_slot<fd_ulong_sat_sub( fetch_slot, ctx->config.sources.max_local_incremental_age ) ) ) return 0;
326 :
327 0 : return 1;
328 0 : }
329 :
330 0 : #define ADNS_REQS_MAX (FD_TOPO_SNAPSHOTS_SERVERS_MAX+FD_TOPO_GOSSIP_ENTRYPOINTS_MAX)
331 :
332 : static ulong
333 0 : scratch_align( void ) {
334 0 : return fd_ulong_max( alignof(fd_snapct_tile_t),
335 0 : fd_ulong_max( fd_ssping_align(),
336 0 : fd_ulong_max( alignof(gossip_ci_entry_t),
337 0 : fd_ulong_max( gossip_ci_map_align(),
338 0 : fd_ulong_max( fd_http_resolver_align(),
339 0 : fd_ulong_max( fd_sspeer_selector_align(),
340 0 : fd_ulong_max( blacklist_pool_align(),
341 0 : fd_ulong_max( blacklist_map_align(),
342 0 : fd_adns_align() ) ) ) ) ) ) ) );
343 0 : }
344 :
345 : static ulong
346 0 : scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
347 0 : ulong l = FD_LAYOUT_INIT;
348 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_snapct_tile_t), sizeof(fd_snapct_tile_t) );
349 0 : l = FD_LAYOUT_APPEND( l, fd_ssping_align(), fd_ssping_footprint( TOTAL_PEERS_MAX ) );
350 0 : l = FD_LAYOUT_APPEND( l, alignof(gossip_ci_entry_t), sizeof(gossip_ci_entry_t) * GOSSIP_PEERS_MAX );
351 0 : l = FD_LAYOUT_APPEND( l, gossip_ci_map_align(), gossip_ci_map_footprint( gossip_ci_map_chain_cnt_est( GOSSIP_PEERS_MAX ) ) );
352 0 : l = FD_LAYOUT_APPEND( l, fd_http_resolver_align(), fd_http_resolver_footprint( SERVER_PEERS_MAX ) );
353 0 : l = FD_LAYOUT_APPEND( l, fd_sspeer_selector_align(), fd_sspeer_selector_footprint( TOTAL_PEERS_MAX ) );
354 0 : l = FD_LAYOUT_APPEND( l, blacklist_pool_align(), blacklist_pool_footprint( TOTAL_PEERS_MAX ) );
355 0 : l = FD_LAYOUT_APPEND( l, blacklist_map_align(), blacklist_map_footprint( blacklist_map_chain_cnt_est( TOTAL_PEERS_MAX ) ) );
356 0 : l = FD_LAYOUT_APPEND( l, fd_adns_align(), fd_adns_footprint( ADNS_REQS_MAX ) );
357 0 : l = FD_LAYOUT_APPEND( l, fd_alloc_align(), fd_alloc_footprint() );
358 0 : return FD_LAYOUT_FINI( l, scratch_align() );
359 0 : }
360 :
361 : static inline int
362 0 : should_shutdown( fd_snapct_tile_t * ctx ) {
363 0 : return ctx->state==FD_SNAPCT_STATE_SHUTDOWN;
364 0 : }
365 :
366 : static void
367 0 : metrics_write( fd_snapct_tile_t * ctx ) {
368 0 : FD_MGAUGE_SET( SNAPCT, READ_PROGRESS_BYTES_FULL, ctx->metrics.full.bytes_read );
369 0 : FD_MGAUGE_SET( SNAPCT, READ_PROGRESS_BYTES_INCREMENTAL, ctx->metrics.incremental.bytes_read );
370 0 : FD_MGAUGE_SET( SNAPCT, READ_EXPECTED_BYTES_FULL, ctx->metrics.full.bytes_total );
371 0 : FD_MGAUGE_SET( SNAPCT, READ_EXPECTED_BYTES_INCREMENTAL, ctx->metrics.incremental.bytes_total );
372 0 : FD_MGAUGE_SET( SNAPCT, SAVE_PROGRESS_BYTES_FULL, ctx->metrics.full.bytes_written );
373 0 : FD_MGAUGE_SET( SNAPCT, SAVE_PROGRESS_BYTES_INCREMENTAL, ctx->metrics.incremental.bytes_written );
374 0 : FD_MCNT_SET ( SNAPCT, LOAD_RETRIED_FULL, ctx->metrics.full.num_retries );
375 0 : FD_MCNT_SET ( SNAPCT, LOAD_RETRIED_INCREMENTAL, ctx->metrics.incremental.num_retries );
376 0 : FD_MGAUGE_SET( SNAPCT, PREDICTED_INCREMENTAL_SLOT, fd_ulong_if( ctx->predicted_incremental.slot==ULONG_MAX, 0UL, ctx->predicted_incremental.slot ) );
377 0 : FD_MGAUGE_SET( SNAPCT, LOAD_STATE, (ulong)ctx->state );
378 0 : }
379 :
380 : static void
381 : set_state( fd_snapct_tile_t * ctx,
382 0 : int state ) {
383 0 : ctx->state = state;
384 0 : FD_COMPILER_MFENCE();
385 0 : metrics_write( ctx );
386 0 : FD_COMPILER_MFENCE();
387 0 : fd_event_metrics_trigger( FD_METRICS_SAMPLE_REASON_SNAPCT_STATE, 0UL, (ulong)state );
388 0 : }
389 :
390 : static void
391 : snapshot_path_gui_publish( fd_snapct_tile_t * ctx,
392 : fd_stem_context_t * stem,
393 : char const * path,
394 0 : int is_full ) {
395 : /* The messages below cannot be obtained directly from metrics. */
396 0 : fd_snapct_update_t * out = fd_chunk_to_laddr( ctx->out_gui.mem, ctx->out_gui.chunk );
397 0 : FD_TEST( fd_cstr_printf_check( out->read_path, PATH_MAX, NULL, "%s", path ) );
398 0 : out->is_download = 0;
399 0 : out->type = fd_int_if( is_full, FD_SNAPCT_SNAPSHOT_TYPE_FULL, FD_SNAPCT_SNAPSHOT_TYPE_INCREMENTAL );
400 0 : fd_stem_publish( stem, ctx->out_gui.idx, 0UL, ctx->out_gui.chunk, sizeof(fd_snapct_update_t) , 0UL, 0UL, 0UL );
401 0 : ctx->out_gui.chunk = fd_dcache_compact_next( ctx->out_gui.chunk, sizeof(fd_snapct_update_t), ctx->out_gui.chunk0, ctx->out_gui.wmark );
402 0 : }
403 :
404 : static void
405 0 : predict_incremental( fd_snapct_tile_t * ctx ) {
406 0 : if( FD_UNLIKELY( !ctx->load_incremental || ctx->predicted_incremental.committed ) ) return;
407 0 : if( FD_UNLIKELY( ctx->predicted_incremental.full_slot==FD_SSPEER_SLOT_UNKNOWN ) ) return;
408 :
409 0 : fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) );
410 :
411 0 : if( FD_LIKELY( best.addr.l ) ) {
412 0 : ctx->predicted_incremental.peer = best;
413 0 : if( FD_UNLIKELY( ctx->predicted_incremental.slot!=best.incr_slot ) ) {
414 0 : ctx->predicted_incremental.slot = best.incr_slot;
415 0 : ctx->predicted_incremental.pending = 1;
416 0 : }
417 0 : }
418 0 : }
419 :
420 : /* best_incremental_peer picks the peer to download the incremental
421 : from. A server that made a new full while our full loaded no
422 : longer offers an incremental on ours, but the newest one we saw is
423 : still on its disk. */
424 :
425 : static fd_sspeer_t
426 0 : best_incremental_peer( fd_snapct_tile_t * ctx ) {
427 0 : fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 1, ctx->predicted_incremental.full_slot, wfs_floor( ctx ) );
428 0 : if( FD_UNLIKELY( !best.addr.l &&
429 0 : ctx->predicted_incremental.peer.addr.l &&
430 0 : ctx->predicted_incremental.peer.full_slot==ctx->predicted_incremental.full_slot &&
431 0 : !fd_ssping_is_invalidated( ctx->ssping, ctx->predicted_incremental.peer.addr ) ) ) {
432 0 : best = ctx->predicted_incremental.peer;
433 0 : }
434 0 : return best;
435 0 : }
436 :
437 : /* Before NEXT, full_only decides whether to boot from the full alone:
438 : that happens only when no gossip peer is allowed as a snapshot
439 : source, no configured server has an incremental for it, and none is
440 : on disk for that full. Once the full is committed with NEXT, an
441 : incremental is required, since the decision cannot be reverted (no
442 : control message can finish the load with the full alone). */
443 :
444 : static int
445 0 : full_only( fd_snapct_tile_t * ctx ) {
446 0 : if( !ctx->load_incremental || ctx->gossip_enabled ) return 0;
447 0 : if( wfs_short_of_target( ctx, ctx->predicted_incremental.full_slot ) ) return 0;
448 0 : if( ctx->local_in.full_snapshot_slot==ctx->predicted_incremental.full_slot &&
449 0 : ctx->local_in.incremental_snapshot_slot!=ULONG_MAX ) return 0;
450 0 : return !best_incremental_peer( ctx ).addr.l;
451 0 : }
452 :
453 : static void
454 : on_resolve( void * _ctx,
455 : fd_sspeer_key_t const * key,
456 : fd_ip4_port_t addr,
457 : ulong full_slot,
458 : ulong incr_slot,
459 : uchar full_hash[ FD_HASH_FOOTPRINT ],
460 0 : uchar incr_hash[ FD_HASH_FOOTPRINT ] ) {
461 0 : fd_snapct_tile_t * ctx = (fd_snapct_tile_t *)_ctx;
462 :
463 0 : if( FD_UNLIKELY( full_slot!=FD_SSPEER_SLOT_UNKNOWN && full_slot>=FD_SSPEER_PLAUSIBLE_MAX_SLOT ) ) return;
464 0 : if( FD_UNLIKELY( incr_slot!=FD_SSPEER_SLOT_UNKNOWN && incr_slot>=FD_SSPEER_PLAUSIBLE_MAX_SLOT ) ) return;
465 :
466 : /* Do not update peers that have been permanently blacklisted. */
467 0 : if( FD_UNLIKELY( key && blacklist_map_ele_query( ctx->blacklist_map, key, NULL, ctx->blacklist_pool ) ) ) return;
468 : /* Do not re-add peers whose addr is temporarily banned by ssping. */
469 0 : if( FD_UNLIKELY( fd_ssping_is_invalidated( ctx->ssping, addr ) ) ) return;
470 :
471 : /* add() handles both new and existing peers, so peers removed from
472 : the selector during a previous blacklist, timeout, or failed
473 : re-resolve are re-added with the freshly resolved data. */
474 0 : ulong score = fd_sspeer_selector_add( ctx->selector, key, addr, FD_SSPEER_LATENCY_UNKNOWN,
475 0 : full_slot, incr_slot, full_hash, incr_hash );
476 0 : if( FD_UNLIKELY( score==FD_SSPEER_SCORE_INVALID ) ) {
477 0 : if( FD_UNLIKELY( key==NULL ) ) {
478 0 : FD_LOG_DEBUG(( "selector add on resolve failed for NULL peer key" ));
479 0 : } else {
480 0 : if( FD_UNLIKELY( !key->is_url ) ) {
481 0 : FD_BASE58_ENCODE_32_BYTES( key->pubkey->key, pubkey_b58 );
482 0 : FD_LOG_DEBUG(( "selector add on resolve failed for peer with pubkey %s", pubkey_b58 ));
483 0 : } else {
484 0 : FD_LOG_DEBUG(( "selector add on resolve failed for peer %s with addr " FD_IP4_ADDR_FMT ":%hu", key->url.hostname,
485 0 : FD_IP4_ADDR_FMT_ARGS( key->url.resolved_addr.addr ), fd_ushort_bswap( key->url.resolved_addr.port ) ));
486 0 : }
487 0 : }
488 0 : }
489 0 : fd_sspeer_selector_process_cluster_slot( ctx->selector );
490 0 : predict_incremental( ctx );
491 0 : }
492 :
493 : static void
494 : on_ping( void * _ctx,
495 : fd_ip4_port_t addr,
496 0 : ulong latency ) {
497 0 : fd_snapct_tile_t * ctx = (fd_snapct_tile_t *)_ctx;
498 :
499 0 : ulong cnt = fd_sspeer_selector_update_on_ping( ctx->selector, addr, latency );
500 0 : if( FD_UNLIKELY( !cnt ) ) {
501 : /* The update may fail in normal operation, e.g. after a peer has
502 : been removed from the selector. The log level is set to a
503 : minimum accordingly. */
504 0 : FD_LOG_DEBUG(( "selector update on ping did not find address " FD_IP4_ADDR_FMT ":%hu",
505 0 : FD_IP4_ADDR_FMT_ARGS( addr.addr ), fd_ushort_bswap( addr.port ) ));
506 0 : }
507 0 : predict_incremental( ctx );
508 0 : }
509 :
510 : static void
511 : on_snapshot_hash( fd_snapct_tile_t * ctx,
512 : fd_sspeer_key_t const * key,
513 : fd_ip4_port_t addr,
514 0 : fd_gossip_update_message_t const * msg ) {
515 0 : ulong full_slot = msg->snapshot_hashes->full_slot;
516 0 : ulong incr_slot = FD_SSPEER_SLOT_UNKNOWN;
517 0 : uchar const * incr_hash = NULL;
518 :
519 0 : for( ulong i=0UL; i<msg->snapshot_hashes->incremental_len; i++ ) {
520 0 : if( FD_LIKELY( !incr_hash || msg->snapshot_hashes->incremental[ i ].slot>incr_slot ) ) {
521 0 : incr_slot = msg->snapshot_hashes->incremental[ i ].slot;
522 0 : incr_hash = msg->snapshot_hashes->incremental[ i ].hash;
523 0 : }
524 0 : }
525 :
526 0 : if( FD_UNLIKELY( full_slot>=FD_SSPEER_PLAUSIBLE_MAX_SLOT ) ) return;
527 0 : if( FD_UNLIKELY( incr_slot!=FD_SSPEER_SLOT_UNKNOWN && incr_slot>=FD_SSPEER_PLAUSIBLE_MAX_SLOT ) ) return;
528 :
529 0 : if( FD_UNLIKELY( !addr.l ) ) {
530 : /* A peer that does not advertise an rpc_addr cannot be added to
531 : the selector: if previously added, remove it. The remove
532 : operation becomes a no-op if the peer is not found. */
533 0 : fd_sspeer_selector_remove( ctx->selector, key );
534 0 : return;
535 0 : }
536 : /* Do not re-add peers that have been permanently blacklisted. */
537 0 : if( FD_UNLIKELY( blacklist_map_ele_query( ctx->blacklist_map, key, NULL, ctx->blacklist_pool ) ) ) return;
538 : /* Do not re-add peers whose addr is temporarily banned by ssping. */
539 0 : if( FD_UNLIKELY( fd_ssping_is_invalidated( ctx->ssping, addr ) ) ) return;
540 : /* The add may fail due to capacity/pool exhaustion. The cluster
541 : slot is recomputed from tracked peers only, so a failed add will
542 : not influence the cluster slot. */
543 0 : fd_sspeer_selector_add( ctx->selector, key, addr, FD_SSPEER_LATENCY_UNKNOWN,
544 0 : full_slot, incr_slot,
545 0 : msg->snapshot_hashes->full_hash, incr_hash );
546 0 : fd_sspeer_selector_process_cluster_slot( ctx->selector );
547 0 : predict_incremental( ctx );
548 0 : }
549 :
550 : static void
551 : send_expected_slot( fd_snapct_tile_t * ctx,
552 : fd_stem_context_t * stem,
553 0 : ulong slot ) {
554 0 : uint tsorig; uint tspub;
555 0 : fd_ssmsg_slot_to_frag( slot, &tsorig, &tspub );
556 0 : fd_stem_publish( stem, ctx->out_rp.idx, FD_SSMSG_EXPECTED_SLOT, 0UL, 0UL, 0UL, tsorig, tspub );
557 0 : }
558 :
559 : /* snapshot_pool_select picks a snapshot file to overwrite with newly
560 : downloaded data. pool[return value] gives the inode to recycle. */
561 :
562 : static uint
563 : snapshot_pool_select( fd_backup_inode_t const * pool,
564 : uint slot0,
565 0 : uint slot1 ) {
566 0 : uint selected = UINT_MAX;
567 0 : ulong oldest = ULONG_MAX;
568 0 : for( uint i=slot0; i<slot1; i++ ) {
569 0 : if( FD_UNLIKELY( pool[ i ].full_slot==ULONG_MAX ) ) return i;
570 0 : ulong slot = pool[ i ].incr_slot==ULONG_MAX ? pool[ i ].full_slot : pool[ i ].incr_slot;
571 0 : if( slot<oldest ) {
572 0 : selected = i;
573 0 : oldest = slot;
574 0 : }
575 0 : }
576 0 : return selected;
577 0 : }
578 :
579 : /* snapshot_output_prepare recycles a 'partial' snapshot file or
580 : overwrites an older snapshot. */
581 :
582 : static void
583 : snapshot_output_prepare( fd_snapct_tile_t * ctx,
584 0 : int full ) {
585 0 : int fd = full ? ctx->local_out.full_snapshot_fd : ctx->local_out.incremental_snapshot_fd;
586 0 : char * name = full ? ctx->local_out.full_snapshot_name : ctx->local_out.incremental_snapshot_name;
587 :
588 0 : FD_TEST( ctx->local_out.dir_fd!=-1 && fd!=-1 );
589 0 : FD_TEST( fd>=FD_SNAP_FD( 0U ) && fd<FD_SNAP_FD( FD_SNAP_MAX ) );
590 :
591 0 : char partial_name[ FD_SNAP_NAME_MAX ];
592 0 : fd_snap_pool_partial_name( partial_name, (uint)(fd-FD_SNAP_FD( 0U )) );
593 0 : if( strcmp( name, partial_name ) ) {
594 0 : char const * local_path = full ? ctx->local_in.full_snapshot_path : ctx->local_in.incremental_snapshot_path;
595 0 : char const * local_name = strrchr( local_path, '/' );
596 0 : local_name = local_name ? local_name+1 : local_path;
597 0 : if( !strcmp( name, local_name ) ) {
598 0 : if( full ) ctx->local_in.full_snapshot_slot = ULONG_MAX;
599 0 : else ctx->local_in.incremental_snapshot_slot = ULONG_MAX;
600 0 : }
601 :
602 0 : if( FD_UNLIKELY( -1==syscall( SYS_renameat2, ctx->local_out.dir_fd, name,
603 0 : ctx->local_out.dir_fd, partial_name, 0U ) ) )
604 0 : FD_LOG_ERR(( "renameat2(%s, %s) failed (%i-%s)", name, partial_name, errno, fd_io_strerror( errno ) ));
605 0 : fd_cstr_ncpy( name, partial_name, FD_SNAP_NAME_MAX );
606 0 : }
607 :
608 0 : if( FD_UNLIKELY( -1==ftruncate( fd, 0UL ) ) )
609 0 : FD_LOG_ERR(( "ftruncate(%s) failed (%i-%s)", name, errno, fd_io_strerror( errno ) ));
610 0 : if( FD_UNLIKELY( -1==lseek( fd, 0L, SEEK_SET ) ) )
611 0 : FD_LOG_ERR(( "lseek(%s) failed (%i-%s)", name, errno, fd_io_strerror( errno ) ));
612 0 : }
613 :
614 : static void
615 0 : rename_full_snapshot( fd_snapct_tile_t * ctx ) {
616 0 : FD_TEST( -1!=ctx->local_out.dir_fd );
617 :
618 0 : if( FD_LIKELY( -1!=ctx->local_out.full_snapshot_fd && ctx->http_full_snapshot_name[ 0 ]!='\0' ) ) {
619 0 : int err = (int)syscall( SYS_renameat2, ctx->local_out.dir_fd, ctx->local_out.full_snapshot_name,
620 0 : ctx->local_out.dir_fd, ctx->http_full_snapshot_name, RENAME_NOREPLACE );
621 0 : if( FD_UNLIKELY( err && errno!=EEXIST ) )
622 0 : FD_LOG_ERR(( "renameat2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
623 0 : if( FD_LIKELY( !err ) )
624 0 : fd_cstr_ncpy( ctx->local_out.full_snapshot_name, ctx->http_full_snapshot_name, FD_SNAP_NAME_MAX );
625 0 : }
626 0 : }
627 :
628 : static void
629 0 : rename_incr_snapshot( fd_snapct_tile_t * ctx ) {
630 0 : FD_TEST( -1!=ctx->local_out.dir_fd );
631 :
632 0 : if( FD_LIKELY( -1!=ctx->local_out.incremental_snapshot_fd && ctx->http_incr_snapshot_name[ 0 ]!='\0' ) ) {
633 0 : int err = (int)syscall( SYS_renameat2, ctx->local_out.dir_fd, ctx->local_out.incremental_snapshot_name,
634 0 : ctx->local_out.dir_fd, ctx->http_incr_snapshot_name, RENAME_NOREPLACE );
635 0 : if( FD_UNLIKELY( err && errno!=EEXIST ) )
636 0 : FD_LOG_ERR(( "renameat2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
637 0 : if( FD_LIKELY( !err ) )
638 0 : fd_cstr_ncpy( ctx->local_out.incremental_snapshot_name, ctx->http_incr_snapshot_name, FD_SNAP_NAME_MAX );
639 0 : }
640 0 : }
641 :
642 : static ulong
643 : rlimit_file_cnt( fd_topo_t const * topo FD_PARAM_UNUSED,
644 0 : fd_topo_tile_t const * tile ) {
645 0 : ulong cnt = 1UL + /* stderr */
646 0 : 1UL + /* logfile */
647 0 : 1UL; /* boot control pipe */
648 0 : if( download_enabled( tile ) ) {
649 0 : cnt += FD_SSPING_FD_CNT + /* ssping sockets */
650 0 : 2UL + /* dirfd + full snapshot pool fd */
651 0 : tile->snapct.sources.servers_cnt; /* http resolver peer full sockets */
652 0 : if( snapct_needs_incr( tile ) ) {
653 0 : cnt += 1UL + /* incremental snapshot pool fd */
654 0 : tile->snapct.sources.servers_cnt; /* http resolver peer incr sockets */
655 0 : }
656 0 : }
657 0 : return cnt;
658 0 : }
659 :
660 : static ulong
661 : populate_allowed_seccomp( fd_topo_t const * topo,
662 : fd_topo_tile_t const * tile,
663 : ulong out_cnt,
664 0 : struct sock_filter * out ) {
665 :
666 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
667 :
668 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
669 0 : fd_snapct_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapct_tile_t), sizeof(fd_snapct_tile_t) );
670 :
671 0 : int min_ping_fd = INT_MAX;
672 0 : int max_ping_fd = 0;
673 0 : if( download_enabled( tile ) ) {
674 0 : min_ping_fd = FD_SSPING_FD_MIN;
675 0 : max_ping_fd = FD_SSPING_FD_MIN + (int)FD_SSPING_FD_CNT - 1;
676 0 : }
677 :
678 0 : uint epoll_inner_fd = (uint)FD_WAKER_INNER_FD( tile->waker_client_idx );
679 0 : uint epoll_outer_fd = (uint)FD_WAKER_OUTER_FD;
680 0 : populate_sock_filter_policy_fd_snapct_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->local_out.dir_fd, (uint)ctx->local_out.full_snapshot_fd, (uint)ctx->local_out.incremental_snapshot_fd, (uint)min_ping_fd, (uint)max_ping_fd, (uint)ctx->netdb_fds->etc_hosts, (uint)ctx->netdb_fds->etc_resolv_conf, epoll_inner_fd, epoll_outer_fd );
681 0 : return sock_filter_policy_fd_snapct_tile_instr_cnt;
682 0 : }
683 :
684 : static ulong
685 : populate_allowed_fds( fd_topo_t const * topo,
686 : fd_topo_tile_t const * tile,
687 : ulong out_fds_cnt,
688 0 : int * out_fds ) {
689 0 : if( FD_UNLIKELY( out_fds_cnt<9UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu is too small", out_fds_cnt ));
690 :
691 0 : ulong out_cnt = 0;
692 0 : out_fds[ out_cnt++ ] = 2UL; /* stderr */
693 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) ) {
694 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
695 0 : }
696 0 : out_fds[ out_cnt++ ] = FD_WAKER_OUTER_FD; /* waker outer epoll fd (rearm) */
697 0 : out_fds[ out_cnt++ ] = FD_WAKER_INNER_FD( tile->waker_client_idx ); /* waker inner epoll fd */
698 :
699 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
700 :
701 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
702 0 : fd_snapct_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapct_tile_t), sizeof(fd_snapct_tile_t) );
703 0 : if( FD_LIKELY( -1!=ctx->local_out.dir_fd ) ) out_fds[ out_cnt++ ] = ctx->local_out.dir_fd;
704 0 : if( FD_LIKELY( -1!=ctx->local_out.full_snapshot_fd ) ) out_fds[ out_cnt++ ] = ctx->local_out.full_snapshot_fd;
705 0 : if( FD_LIKELY( -1!=ctx->local_out.incremental_snapshot_fd ) ) out_fds[ out_cnt++ ] = ctx->local_out.incremental_snapshot_fd;
706 0 : if( FD_LIKELY( -1!=ctx->netdb_fds->etc_hosts ) ) out_fds[ out_cnt++ ] = ctx->netdb_fds->etc_hosts;
707 0 : if( FD_LIKELY( -1!=ctx->netdb_fds->etc_resolv_conf ) ) out_fds[ out_cnt++ ] = ctx->netdb_fds->etc_resolv_conf;
708 0 : if( FD_LIKELY( download_enabled( tile ) ) ) {
709 0 : if( FD_UNLIKELY( out_cnt+FD_SSPING_FD_CNT > out_fds_cnt ) ) {
710 0 : FD_LOG_ERR(( "out_fds_cnt %lu must be at least %lu", out_fds_cnt, out_cnt + FD_SSPING_FD_CNT ));
711 0 : }
712 0 : for( int i=FD_SSPING_FD_MIN; i<FD_SSPING_FD_MIN+(int)FD_SSPING_FD_CNT; i++ ) {
713 0 : out_fds[ out_cnt++ ] = i;
714 0 : }
715 0 : }
716 :
717 0 : return out_cnt;
718 0 : }
719 :
720 : static void
721 : init_load( fd_snapct_tile_t * ctx,
722 : fd_stem_context_t * stem,
723 : int full,
724 0 : int file ) {
725 0 : ctx->snapshot_start_timestamp_ns = fd_clock_tile_now( ctx->clock );
726 0 : fd_ssctrl_init_t * out = fd_chunk_to_laddr( ctx->out_ld.mem, ctx->out_ld.chunk );
727 0 : out->file = file;
728 0 : out->zstd = !file || (full ? ctx->local_in.full_snapshot_zstd : ctx->local_in.incremental_snapshot_zstd);
729 0 : if( file ) {
730 0 : out->slot = full ? ctx->local_in.full_snapshot_slot : ctx->local_in.incremental_snapshot_slot;
731 0 : if( full ) fd_memcpy( out->snapshot_hash, ctx->local_in.full_snapshot_hash, FD_HASH_FOOTPRINT );
732 0 : else fd_memcpy( out->snapshot_hash, ctx->local_in.incremental_snapshot_hash, FD_HASH_FOOTPRINT );
733 0 : } else {
734 0 : out->slot = full ? ctx->predicted_incremental.full_slot : ctx->predicted_incremental.slot;
735 0 : if( full ) fd_memcpy( out->snapshot_hash, ctx->peer.full_hash, FD_HASH_FOOTPRINT );
736 0 : else fd_memcpy( out->snapshot_hash, ctx->peer.incr_hash, FD_HASH_FOOTPRINT );
737 0 : }
738 :
739 0 : out->is_redirect = !file && full;
740 :
741 0 : if( file ) out->file_sz = full ? ctx->local_in.full_snapshot_size : ctx->local_in.incremental_snapshot_size;
742 0 : else out->file_sz = 0UL;
743 :
744 0 : fd_stem_publish( stem, ctx->out_ld.idx, full ? FD_SNAPSHOT_MSG_CTRL_INIT_FULL : FD_SNAPSHOT_MSG_CTRL_INIT_INCR, ctx->out_ld.chunk, sizeof(fd_ssctrl_init_t), 0UL, 0UL, 0UL );
745 0 : ctx->out_ld.chunk = fd_dcache_compact_next( ctx->out_ld.chunk, sizeof(fd_ssctrl_init_t), ctx->out_ld.chunk0, ctx->out_ld.wmark );
746 0 : ctx->flush_ack = 0;
747 0 : ctx->start_sent = 0;
748 0 : ctx->load_complete = 0;
749 :
750 : /* The full learns its name from the redirect, the incremental gets
751 : its name at START, and a local file has none. Clear stale names
752 : before the rename functions run. */
753 0 : if( !full ) fd_cstr_fini( ctx->http_incr_snapshot_name );
754 0 : else if( file ) fd_cstr_fini( ctx->http_full_snapshot_name );
755 0 : else FD_TEST( fd_cstr_printf_check( ctx->http_full_snapshot_name, PATH_MAX, NULL, "snapshot.tar.bz2" ) );
756 :
757 0 : if( !file ) snapshot_output_prepare( ctx, full );
758 :
759 0 : if( !full ) {
760 0 : if( file ) ctx->predicted_incremental.slot = ctx->local_in.incremental_snapshot_slot;
761 0 : ctx->predicted_incremental.committed = 1;
762 0 : ctx->predicted_incremental.pending = 1;
763 0 : }
764 0 : }
765 :
766 : static void
767 : log_download( fd_snapct_tile_t * ctx,
768 : int full,
769 0 : int is_https ) {
770 0 : fd_sspeer_t const * peer = &ctx->peer;
771 0 : char const * what = full ? "full" : "incremental";
772 0 : ulong slot = full ? ctx->predicted_incremental.full_slot : ctx->predicted_incremental.slot;
773 :
774 0 : if( peer->key.is_url ) {
775 0 : char const * scheme = is_https ? "https" : "http";
776 0 : if( peer->key.url.hostname[ 0 ] ) {
777 0 : FD_LOG_NOTICE(( "downloading %s snapshot from %s%s://%s:%hu%s",
778 0 : what, fd_log_style_bold(), scheme, peer->key.url.hostname, fd_ushort_bswap( peer->addr.port ), fd_log_style_normal() ));
779 0 : FD_LOG_INFO(( "downloading %s snapshot at slot %lu from configured server %s://%s:%hu at " FD_IP4_ADDR_FMT,
780 0 : what, slot, scheme, peer->key.url.hostname, fd_ushort_bswap( peer->addr.port ), FD_IP4_ADDR_FMT_ARGS( peer->addr.addr ) ));
781 0 : } else {
782 0 : FD_LOG_NOTICE(( "downloading %s snapshot from %s%s://" FD_IP4_ADDR_FMT ":%hu%s",
783 0 : what, fd_log_style_bold(), scheme, FD_IP4_ADDR_FMT_ARGS( peer->addr.addr ), fd_ushort_bswap( peer->addr.port ), fd_log_style_normal() ));
784 0 : FD_LOG_INFO(( "downloading %s snapshot at slot %lu from configured server %s://" FD_IP4_ADDR_FMT ":%hu",
785 0 : what, slot, scheme, FD_IP4_ADDR_FMT_ARGS( peer->addr.addr ), fd_ushort_bswap( peer->addr.port ) ));
786 0 : }
787 0 : return;
788 0 : }
789 :
790 0 : int is_entrypoint = 0;
791 0 : for( ulong i=0UL; i<ctx->resolved_entrypoints_cnt; i++ ) {
792 0 : if( FD_UNLIKELY( ctx->resolved_entrypoints[ i ].addr==peer->addr.addr ) ) { is_entrypoint = 1; break; }
793 0 : }
794 0 : char const * kind = ctx->config.sources.gossip.allow_any ? "untrusted gossip peer" : "trusted gossip peer";
795 0 : if( FD_UNLIKELY( is_entrypoint ) ) kind = "entrypoint gossip peer";
796 :
797 0 : FD_BASE58_ENCODE_32_BYTES( peer->key.pubkey->uc, pubkey_b58 );
798 0 : FD_LOG_NOTICE(( "downloading %s snapshot from %s %s%s%s",
799 0 : what, kind, fd_log_style_bold(), pubkey_b58, fd_log_style_normal() ));
800 0 : FD_LOG_INFO(( "downloading %s snapshot at slot %lu from %s %s at " FD_IP4_ADDR_FMT ":%hu",
801 0 : what, slot, kind, pubkey_b58,
802 0 : FD_IP4_ADDR_FMT_ARGS( peer->addr.addr ), fd_ushort_bswap( peer->addr.port ) ));
803 0 : }
804 :
805 : static void
806 : log_completion( fd_snapct_tile_t * ctx,
807 0 : int full ) {
808 0 : double elapsed = (double)(fd_clock_tile_now( ctx->clock ) - ctx->snapshot_start_timestamp_ns) / 1e9;
809 0 : if( full ) FD_LOG_INFO(( "full snapshot load completed in %.3f seconds", elapsed ));
810 0 : else FD_LOG_INFO(( "incremental snapshot load completed in %.3f seconds", elapsed ));
811 0 : }
812 :
813 : /* Blacklist the current peer: remove it from the selector, invalidate
814 : it in ssping, and add it to the dedicated blacklist map. Skips the
815 : insert if the peer is already blacklisted (dedup). Warns if the
816 : pool is full (the ssping ban still provides temporary protection).
817 : A server the operator listed is only removed from the selector; the
818 : resolver offers it again on its next check. */
819 : static void
820 0 : blacklist_peer( fd_snapct_tile_t * ctx ) {
821 0 : fd_sspeer_selector_remove_by_addr( ctx->selector, ctx->peer.addr );
822 0 : fd_sspeer_selector_process_cluster_slot( ctx->selector );
823 :
824 0 : if( ctx->predicted_incremental.peer.addr.l==ctx->peer.addr.l ||
825 0 : fd_sspeer_key_eq( &ctx->predicted_incremental.peer.key, &ctx->peer.key ) ) {
826 0 : ctx->predicted_incremental.peer.addr.l = 0UL;
827 0 : }
828 :
829 0 : if( ctx->peer.key.is_url ) return;
830 :
831 0 : fd_ssping_invalidate( ctx->ssping, ctx->peer.addr, fd_clock_tile_now( ctx->clock ) );
832 :
833 0 : if( FD_UNLIKELY( blacklist_map_ele_query( ctx->blacklist_map, &ctx->peer.key, NULL, ctx->blacklist_pool ) ) ) return;
834 0 : if( FD_LIKELY( blacklist_pool_free( ctx->blacklist_pool ) ) ) {
835 0 : fd_sspeer_blacklist_entry_t * bl = blacklist_pool_ele_acquire( ctx->blacklist_pool );
836 0 : bl->key = ctx->peer.key;
837 0 : blacklist_map_ele_insert( ctx->blacklist_map, bl, ctx->blacklist_pool );
838 0 : FD_BASE58_ENCODE_32_BYTES( ctx->peer.key.pubkey->uc, pubkey_b58 );
839 0 : FD_LOG_WARNING(( "permanently blacklisted peer identity %s", pubkey_b58 ));
840 0 : } else {
841 0 : FD_LOG_WARNING(( "blacklist pool full, peer banned via ssping only" ));
842 0 : }
843 0 : }
844 :
845 0 : #define DNS_RETRY_NANOS (15L*1000L*1000L*1000L)
846 0 : #define DNS_REQ_ID_ENTRYPOINT (0x100UL) /* req_id: server idx, or this bit + entrypoint idx */
847 :
848 : static void
849 : dns_queue( fd_snapct_tile_t * ctx,
850 0 : long now ) {
851 0 : for( ulong i=0UL; i<ctx->config.sources.servers_cnt; i++ ) {
852 0 : if( FD_LIKELY( ctx->dns_servers[ i ].resolved || !ctx->dns_servers[ i ].retry_nanos || ctx->dns_servers[ i ].retry_nanos>now ) ) continue;
853 0 : if( FD_UNLIKELY( fd_adns_resolve( ctx->adns, ctx->dns_servers[ i ].hostname, i ) ) ) break;
854 0 : ctx->dns_servers[ i ].retry_nanos = 0L; /* in flight */
855 0 : }
856 0 : for( ulong i=0UL; i<ctx->config.entrypoints_cnt; i++ ) {
857 0 : if( FD_LIKELY( ctx->dns_entrypoints[ i ].resolved || !ctx->dns_entrypoints[ i ].retry_nanos || ctx->dns_entrypoints[ i ].retry_nanos>now ) ) continue;
858 0 : if( FD_UNLIKELY( fd_adns_resolve( ctx->adns, ctx->dns_entrypoints[ i ].hostname, DNS_REQ_ID_ENTRYPOINT|i ) ) ) break;
859 0 : ctx->dns_entrypoints[ i ].retry_nanos = 0L;
860 0 : }
861 0 : }
862 :
863 : static void
864 : dns_advance( fd_snapct_tile_t * ctx,
865 0 : long now ) {
866 0 : dns_queue( ctx, now );
867 :
868 0 : fd_adns_result_t res[ 1 ];
869 0 : while( fd_adns_advance( ctx->adns, now, res ) ) {
870 0 : if( FD_UNLIKELY( res->req_id & DNS_REQ_ID_ENTRYPOINT ) ) {
871 0 : ulong i = res->req_id & ~DNS_REQ_ID_ENTRYPOINT;
872 0 : if( FD_UNLIKELY( res->err ) ) {
873 0 : FD_LOG_WARNING(( "could not resolve [gossip.entrypoints] entry \"%s\" (%s), retrying", ctx->config.entrypoints[ i ], fd_gai_strerror( res->err ) ));
874 0 : ctx->dns_entrypoints[ i ].retry_nanos = now+DNS_RETRY_NANOS;
875 0 : continue;
876 0 : }
877 0 : ctx->dns_entrypoints[ i ].resolved = 1;
878 0 : ctx->resolved_entrypoints[ ctx->resolved_entrypoints_cnt++ ] = (fd_ip4_port_t){ .addr=res->addrs[ 0 ], .port=ctx->dns_entrypoints[ i ].port };
879 0 : continue;
880 0 : }
881 :
882 0 : ulong i = res->req_id;
883 0 : if( FD_UNLIKELY( res->err ) ) {
884 0 : FD_LOG_WARNING(( "could not resolve [snapshots.sources.servers] entry \"%s\" (%s), retrying", ctx->config.sources.servers[ i ], fd_gai_strerror( res->err ) ));
885 0 : ctx->dns_servers[ i ].retry_nanos = now+DNS_RETRY_NANOS;
886 0 : continue;
887 0 : }
888 0 : ctx->dns_servers[ i ].resolved = 1;
889 :
890 0 : ulong addr_cnt = fd_ulong_min( res->addr_cnt, FD_TOPO_MAX_RESOLVED_ADDRS );
891 0 : for( ulong j=0UL; j<addr_cnt; j++ ) {
892 0 : ulong k = ctx->resolved_servers_cnt++;
893 0 : ctx->resolved_servers[ k ].addr = (fd_ip4_port_t){ .addr=res->addrs[ j ], .port=ctx->dns_servers[ i ].port };
894 0 : ctx->resolved_servers[ k ].is_https = ctx->dns_servers[ i ].is_https;
895 0 : fd_cstr_ncpy( ctx->resolved_servers[ k ].hostname, ctx->dns_servers[ i ].hostname, sizeof(ctx->resolved_servers[ k ].hostname) );
896 :
897 : /* The peer needs to be added to resolver and to selector.
898 : Only if this succeeds, add the peer to ssping list. */
899 0 : if( FD_LIKELY( !fd_http_resolver_add( ctx->ssresolver,
900 0 : ctx->resolved_servers[ k ].addr,
901 0 : ctx->resolved_servers[ k ].hostname,
902 0 : ctx->resolved_servers[ k ].is_https,
903 0 : ctx->selector ) ) ) {
904 0 : fd_ssping_add( ctx->ssping, ctx->resolved_servers[ k ].addr );
905 0 : }
906 0 : }
907 0 : }
908 0 : }
909 :
910 : static void
911 0 : during_housekeeping( fd_snapct_tile_t * ctx ) {
912 0 : if( FD_UNLIKELY( fd_clock_tile_recal_due( ctx->clock ) ) ) fd_clock_tile_recal( ctx->clock );
913 0 : }
914 :
915 : static long
916 0 : io_due( fd_snapct_tile_t * ctx ) {
917 0 : long next = LONG_MAX;
918 0 : if( FD_LIKELY( ctx->adns ) ) {
919 0 : next = fd_long_min( next, fd_adns_next_deadline( ctx->adns ) );
920 0 : for( ulong i=0UL; i<ctx->config.sources.servers_cnt; i++ )
921 0 : if( !ctx->dns_servers[ i ].resolved && ctx->dns_servers[ i ].retry_nanos ) next = fd_long_min( next, ctx->dns_servers[ i ].retry_nanos );
922 0 : for( ulong i=0UL; i<ctx->config.entrypoints_cnt; i++ )
923 0 : if( !ctx->dns_entrypoints[ i ].resolved && ctx->dns_entrypoints[ i ].retry_nanos ) next = fd_long_min( next, ctx->dns_entrypoints[ i ].retry_nanos );
924 0 : }
925 0 : if( FD_LIKELY( ctx->ssping ) ) next = fd_long_min( next, fd_ssping_next_deadline( ctx->ssping ) );
926 0 : if( FD_LIKELY( ctx->ssresolver ) ) next = fd_long_min( next, fd_http_resolver_next_deadline( ctx->ssresolver ) );
927 0 : return next;
928 0 : }
929 :
930 : static long
931 0 : next_deadline( fd_snapct_tile_t * ctx ) {
932 0 : long next = ctx->io_due_nanos;
933 0 : if( FD_UNLIKELY( ctx->state==FD_SNAPCT_STATE_WAITING_FOR_PEERS ||
934 0 : ctx->state==FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL ||
935 0 : ctx->state==FD_SNAPCT_STATE_COLLECTING_PEERS ||
936 0 : ctx->state==FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL ) ) next = fd_long_min( next, ctx->deadline_nanos );
937 0 : if( FD_UNLIKELY( next==LONG_MAX ) ) return LONG_MAX;
938 0 : if( FD_UNLIKELY( next<=0L ) ) return 0L;
939 0 : return fd_clock_tile_wallclock_to_tickcount( ctx->clock, next );
940 0 : }
941 :
942 : static void
943 : after_credit( fd_snapct_tile_t * ctx,
944 : fd_stem_context_t * stem,
945 : int * opt_poll_in FD_PARAM_UNUSED,
946 0 : int * charge_busy FD_PARAM_UNUSED ) {
947 0 : long now = fd_clock_tile_now( ctx->clock );
948 :
949 0 : int fired = fd_fseq_query( ctx->waker_fseq )==1UL;
950 0 : if( FD_UNLIKELY( fired || now>=ctx->io_due_nanos ) ) {
951 0 : if( FD_LIKELY( fired ) ) fd_fseq_update( ctx->waker_fseq, 0UL );
952 0 : if( FD_LIKELY( ctx->adns ) ) dns_advance( ctx, now );
953 0 : if( FD_LIKELY( ctx->ssping ) ) fd_ssping_advance( ctx->ssping, now, ctx->selector );
954 0 : if( FD_LIKELY( ctx->ssresolver ) ) fd_http_resolver_advance( ctx->ssresolver, now, ctx->selector );
955 0 : if( FD_LIKELY( fired ) ) fd_waker_client_rearm( ctx->waker_client_idx );
956 0 : }
957 0 : ctx->io_due_nanos = io_due( ctx ); /* frags may have queued peers to ping or resolve */
958 :
959 : /* Advances above may remove peers, making cluster_slot dirty.
960 : Recompute so best() calls below use up-to-date scores.
961 : No-op when the dirty flag is not set internally. */
962 0 : fd_sspeer_selector_process_cluster_slot( ctx->selector );
963 :
964 : /* send an expected slot message as the predicted incremental
965 : could have changed as a result of the pinger, resolver, or from
966 : processing gossip frags in gossip_frag. */
967 0 : if( FD_LIKELY( ctx->predicted_incremental.pending ) ) {
968 0 : send_expected_slot( ctx, stem, ctx->predicted_incremental.slot );
969 0 : ctx->predicted_incremental.pending = 0;
970 0 : }
971 :
972 : /* Hold snapld until every feedback path acknowledges INIT, so no
973 : DATA reaches a tile before its INIT completes. */
974 0 : if( FD_UNLIKELY( !ctx->start_sent && !ctx->malformed &&
975 0 : ctx->flush_ack==ctx->flush_ack_cnt &&
976 0 : (ctx->state==FD_SNAPCT_STATE_READING_FULL_FILE ||
977 0 : ctx->state==FD_SNAPCT_STATE_READING_FULL_HTTP ||
978 0 : ctx->state==FD_SNAPCT_STATE_READING_INCREMENTAL_FILE ||
979 0 : ctx->state==FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP) ) ) {
980 0 : int file = ctx->state==FD_SNAPCT_STATE_READING_FULL_FILE || ctx->state==FD_SNAPCT_STATE_READING_INCREMENTAL_FILE;
981 0 : int full = ctx->state==FD_SNAPCT_STATE_READING_FULL_FILE || ctx->state==FD_SNAPCT_STATE_READING_FULL_HTTP;
982 0 : if( !file ) {
983 0 : fd_ssctrl_start_t * out = fd_chunk_to_laddr( ctx->out_ld.mem, ctx->out_ld.chunk );
984 0 : out->addr = ctx->peer.addr;
985 0 : if( full ) {
986 0 : FD_TEST( fd_cstr_printf_check( out->path, PATH_MAX, &out->path_len, "/snapshot.tar.bz2" ) );
987 0 : } else {
988 0 : char hash_b58[ FD_BASE58_ENCODED_32_SZ ];
989 0 : fd_base58_encode_32( ctx->peer.incr_hash, NULL, hash_b58 );
990 0 : FD_TEST( fd_cstr_printf_check( ctx->http_incr_snapshot_name, PATH_MAX, NULL, "incremental-snapshot-%lu-%lu-%s.tar.zst",
991 0 : ctx->predicted_incremental.full_slot, ctx->predicted_incremental.slot, hash_b58 ) );
992 0 : FD_TEST( fd_cstr_printf_check( out->path, PATH_MAX, &out->path_len, "/%s", ctx->http_incr_snapshot_name ) );
993 0 : }
994 :
995 : /* Peers outside the config list use HTTP without a hostname. */
996 0 : out->is_https = 0;
997 0 : out->hostname[0] = '\0';
998 0 : for( ulong i=0UL; i<ctx->resolved_servers_cnt; i++ ) {
999 0 : if( FD_UNLIKELY( ctx->peer.key.is_url &&
1000 0 : ctx->peer.key.url.resolved_addr.l==ctx->resolved_servers[ i ].addr.l &&
1001 0 : !strcmp( ctx->peer.key.url.hostname, ctx->resolved_servers[ i ].hostname ) ) ) {
1002 0 : fd_cstr_ncpy( out->hostname, ctx->resolved_servers[ i ].hostname, sizeof(out->hostname) );
1003 0 : out->is_https = ctx->resolved_servers[ i ].is_https;
1004 0 : break;
1005 0 : }
1006 0 : }
1007 0 : log_download( ctx, full, out->is_https );
1008 0 : }
1009 0 : ulong sz = file ? 0UL : sizeof(fd_ssctrl_start_t);
1010 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_START, ctx->out_ld.chunk, sz, 0UL, 0UL, 0UL );
1011 0 : ctx->out_ld.chunk = fd_dcache_compact_next( ctx->out_ld.chunk, sz, ctx->out_ld.chunk0, ctx->out_ld.wmark );
1012 0 : ctx->start_sent = 1;
1013 0 : }
1014 :
1015 : /* Note: All state transitions should occur within this switch
1016 : statement to make it easier to reason about the state management. */
1017 :
1018 0 : switch ( ctx->state ) {
1019 :
1020 : /* ============================================================== */
1021 0 : case FD_SNAPCT_STATE_INIT: {
1022 0 : if( FD_UNLIKELY( !ctx->download_enabled ) ) {
1023 : /* Download is disabled, so the incremental on disk is the only
1024 : thing that can lift a full short of the target. */
1025 0 : if( FD_UNLIKELY( !ctx->load_incremental &&
1026 0 : wfs_short_of_target( ctx, ctx->local_in.full_snapshot_slot ) ) ) {
1027 0 : FD_LOG_NOTICE(( "WFS target slot %lu ahead of local full snapshot slot %lu; "
1028 0 : "auto-enabling incremental snapshot load",
1029 0 : wfs_floor( ctx ), ctx->local_in.full_snapshot_slot ));
1030 0 : ctx->load_incremental = 1;
1031 0 : }
1032 :
1033 0 : if( FD_UNLIKELY( ctx->load_incremental && ctx->local_in.incremental_snapshot_slot==ULONG_MAX ) ) {
1034 0 : FD_LOG_WARNING(( "incremental snapshots were requested (via config or WFS auto-enable) "
1035 0 : "but no incremental snapshot exists on disk and snapshot download is not enabled. "
1036 0 : "skipping incremental snapshot load." ));
1037 0 : ctx->load_incremental = 0;
1038 0 : }
1039 :
1040 0 : ulong local_slot = (ctx->load_incremental && ctx->local_in.incremental_snapshot_slot!=ULONG_MAX)
1041 0 : ? ctx->local_in.incremental_snapshot_slot
1042 0 : : ctx->local_in.full_snapshot_slot;
1043 :
1044 : /* local_slot is the boot slot: with download disabled no later
1045 : stage can lift it, so fail now rather than after both loads. */
1046 0 : if( FD_UNLIKELY( wfs_short_of_target( ctx, local_slot ) ) ) {
1047 0 : FD_LOG_ERR(( "local snapshots reach slot %lu, behind [consensus.wait_for_supermajority_at_slot] %lu. "
1048 0 : "place a snapshot reaching it in the snapshot directory, or enable a download source "
1049 0 : "via [snapshots.sources.gossip] or [snapshots.sources.servers].",
1050 0 : local_slot, wfs_floor( ctx ) ));
1051 0 : }
1052 :
1053 0 : send_expected_slot( ctx, stem, local_slot );
1054 0 : FD_LOG_NOTICE(( "reading full snapshot from file %s%s%s", fd_log_style_dim(), ctx->local_in.full_snapshot_path, fd_log_style_normal() ));
1055 0 : FD_LOG_INFO(( "reading full snapshot at slot %lu from local file `%s`", ctx->local_in.full_snapshot_slot, ctx->local_in.full_snapshot_path ));
1056 0 : ctx->predicted_incremental.full_slot = ctx->local_in.full_snapshot_slot;
1057 0 : set_state( ctx, FD_SNAPCT_STATE_READING_FULL_FILE );
1058 0 : init_load( ctx, stem, 1, 1 );
1059 0 : break;
1060 0 : }
1061 0 : ctx->deadline_nanos = now+ctx->config.wait_for_peers_timeout_nanos;
1062 0 : set_state( ctx, FD_SNAPCT_STATE_WAITING_FOR_PEERS );
1063 0 : break;
1064 0 : }
1065 :
1066 : /* ============================================================== */
1067 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS: {
1068 0 : if( FD_UNLIKELY( now>ctx->deadline_nanos ) ) FD_LOG_ERR(( "timed out waiting for peers." ));
1069 :
1070 : /* Existence gate only at this point, so no floor. */
1071 0 : fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 0, FD_SSPEER_SLOT_UNKNOWN, 0UL );
1072 0 : if( FD_LIKELY( best.addr.l ) ) {
1073 0 : set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS );
1074 0 : ctx->deadline_nanos = now+FD_SNAPCT_COLLECTING_PEERS_TIMEOUT;
1075 0 : }
1076 0 : break;
1077 0 : }
1078 :
1079 : /* ============================================================== */
1080 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL: {
1081 0 : if( FD_UNLIKELY( now>ctx->deadline_nanos ) ) {
1082 0 : if( FD_UNLIKELY( wfs_short_of_target( ctx, ctx->predicted_incremental.full_slot ) ) ) {
1083 0 : FD_LOG_ERR(( "timed out waiting for an incremental snapshot to lift full snapshot slot %lu to "
1084 0 : "[consensus.wait_for_supermajority_at_slot] %lu.",
1085 0 : ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ));
1086 0 : } else {
1087 0 : FD_LOG_ERR(( "timed out waiting for incremental snapshot peers." ));
1088 0 : }
1089 0 : }
1090 :
1091 0 : FD_TEST( ctx->predicted_incremental.full_slot!=FD_SSPEER_SLOT_UNKNOWN );
1092 0 : fd_sspeer_t best = best_incremental_peer( ctx );
1093 0 : if( FD_LIKELY( best.addr.l ) ) {
1094 0 : set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL );
1095 0 : ctx->deadline_nanos = now;
1096 0 : }
1097 0 : break;
1098 0 : }
1099 :
1100 : /* ============================================================== */
1101 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS: {
1102 0 : if( FD_UNLIKELY( !ctx->gossip.saturated && now<ctx->deadline_nanos ) ) break;
1103 :
1104 : /* Prefer a peer reaching the floor, otherwise the best available. */
1105 0 : fd_sspeer_t best = fd_sspeer_selector_best( ctx->selector, 0, FD_SSPEER_SLOT_UNKNOWN, wfs_floor( ctx ) );
1106 0 : if( FD_UNLIKELY( !best.addr.l ) ) best = fd_sspeer_selector_best( ctx->selector, 0, FD_SSPEER_SLOT_UNKNOWN, 0UL );
1107 0 : if( FD_UNLIKELY( !best.addr.l ) ) {
1108 0 : if( !ctx->download_enabled ) {
1109 0 : FD_LOG_ERR(( "no peers are available and no snapshot download source is configured. aborting." ));
1110 0 : }
1111 0 : ctx->deadline_nanos = now + ctx->config.wait_for_peers_timeout_nanos;
1112 0 : set_state( ctx, FD_SNAPCT_STATE_WAITING_FOR_PEERS );
1113 0 : break;
1114 0 : }
1115 :
1116 0 : fd_sscluster_slot_t cluster = fd_sspeer_selector_cluster_slot( ctx->selector );
1117 :
1118 : /* A failed load returns here to pick another candidate. Whether
1119 : an incremental is loaded is a property of the candidate, so
1120 : start from the config rather than the last pass's verdict. */
1121 :
1122 0 : ctx->load_incremental = ctx->config.incremental_snapshots;
1123 :
1124 : /* Weigh the full snapshots we could actually use, not
1125 : cluster.full: it can sit past the target while the best full
1126 : snapshot we could load falls short. */
1127 :
1128 0 : if( !ctx->load_incremental &&
1129 0 : ( wfs_short_of_target( ctx, ctx->local_in.full_snapshot_slot ) ||
1130 0 : wfs_short_of_target( ctx, best.full_slot ) ) ) {
1131 0 : FD_LOG_NOTICE(( "WFS target slot %lu ahead of the best full snapshot (local %lu, downloadable %lu); "
1132 0 : "auto-enabling incremental snapshot download",
1133 0 : wfs_floor( ctx ), ctx->local_in.full_snapshot_slot, best.full_slot ));
1134 0 : ctx->load_incremental = 1;
1135 0 : }
1136 :
1137 : /* Determine the best effective slot achievable using the local
1138 : full snapshot. When incrementals are disabled, the effective
1139 : slot is the full snapshot slot itself. When enabled, it is the
1140 : best incremental we can pair with the local full (either from
1141 : a local file or downloaded from a peer). */
1142 :
1143 0 : ulong local_effective_slot = ULONG_MAX;
1144 0 : int local_has_incr = 0;
1145 0 : int local_incr_on_disk = 0;
1146 0 : if( FD_LIKELY( ctx->local_in.full_snapshot_slot!=ULONG_MAX ) ) {
1147 0 : if( FD_LIKELY( ctx->load_incremental ) ) {
1148 : /* Age the local incremental against the best one that pairs
1149 : with our full. */
1150 0 : fd_sspeer_t best_incr = fd_sspeer_selector_best( ctx->selector, 1, ctx->local_in.full_snapshot_slot, wfs_floor( ctx ) );
1151 0 : ulong fetch_incr = best_incr.addr.l ? best_incr.incr_slot : ULONG_MAX;
1152 0 : ulong local_incr = ctx->local_in.incremental_snapshot_slot;
1153 :
1154 0 : if( FD_LIKELY( incr_use_local( ctx, local_incr, fetch_incr ) ) ) {
1155 0 : local_effective_slot = local_incr;
1156 0 : local_has_incr = 1;
1157 0 : local_incr_on_disk = 1;
1158 0 : } else if( FD_LIKELY( fetch_incr!=ULONG_MAX ) ) {
1159 0 : local_effective_slot = fetch_incr;
1160 0 : local_has_incr = 1;
1161 0 : } else {
1162 : /* No incremental reaches past our full, from disk or from
1163 : a peer. The full alone still reaches its own slot. */
1164 0 : local_effective_slot = ctx->local_in.full_snapshot_slot;
1165 0 : }
1166 0 : } else {
1167 0 : local_effective_slot = ctx->local_in.full_snapshot_slot;
1168 0 : }
1169 0 : }
1170 :
1171 : /* The slot reached by replacing the local full with a downloaded
1172 : one. The local candidate may download an incremental too, so
1173 : this is not local versus remote, it is which full to use. */
1174 :
1175 0 : ulong download_effective_slot = best.full_slot;
1176 0 : int download_has_incr = 0;
1177 0 : if( ctx->load_incremental ) {
1178 0 : fd_sspeer_t best_incr = fd_sspeer_selector_best( ctx->selector, 1, best.full_slot, wfs_floor( ctx ) );
1179 0 : if( FD_LIKELY( best_incr.addr.l ) ) {
1180 0 : download_effective_slot = best_incr.incr_slot;
1181 0 : download_has_incr = 1;
1182 0 : }
1183 0 : }
1184 :
1185 : /* Objective: keep the local full while its effective slot is
1186 : within max_local_full_effective_age of what replacing it
1187 : would reach. Not of the cluster slot: that is a max over
1188 : what peers advertise, which no peer need serve and any peer
1189 : can inflate. This applies to every boot, not just WFS. */
1190 :
1191 0 : int local_within_age = local_effective_slot!=ULONG_MAX &&
1192 0 : local_effective_slot>=fd_ulong_sat_sub( download_effective_slot, ctx->config.sources.max_local_full_effective_age );
1193 0 : int can_use_local_full = local_within_age;
1194 :
1195 : /* No converse rule is needed: a local snapshot that clears the
1196 : floor is necessarily ahead of a download that does not, so
1197 : the age objective already keeps it. */
1198 :
1199 0 : if( FD_UNLIKELY( can_use_local_full &&
1200 0 : wfs_short_of_target( ctx, local_effective_slot ) &&
1201 0 : !wfs_short_of_target( ctx, download_effective_slot ) ) ) {
1202 0 : FD_LOG_NOTICE(( "local snapshot at effective slot %lu is short of "
1203 0 : "[consensus.wait_for_supermajority_at_slot] %lu, downloading slot %lu instead",
1204 0 : local_effective_slot, wfs_floor( ctx ), download_effective_slot ));
1205 0 : can_use_local_full = 0;
1206 0 : }
1207 :
1208 0 : if( FD_LIKELY( can_use_local_full ) ) {
1209 0 : if( FD_UNLIKELY( ctx->load_incremental && !local_has_incr ) ) {
1210 0 : if( FD_UNLIKELY( wfs_short_of_target( ctx, ctx->local_in.full_snapshot_slot ) ) ) {
1211 0 : FD_LOG_NOTICE(( "no incremental snapshot lifts the local full snapshot at slot %lu to "
1212 0 : "[consensus.wait_for_supermajority_at_slot] %lu yet; keeping the incremental "
1213 0 : "load to wait for a peer that bridges the gap",
1214 0 : ctx->local_in.full_snapshot_slot, wfs_floor( ctx ) ));
1215 0 : } else if( FD_UNLIKELY( !ctx->config.incremental_snapshots ) ) {
1216 0 : FD_LOG_INFO(( "local full snapshot at slot %lu already reaches the WFS target, loading it alone",
1217 0 : ctx->local_in.full_snapshot_slot ));
1218 0 : ctx->load_incremental = 0;
1219 0 : } else {
1220 0 : FD_LOG_WARNING(( "no incremental snapshot is available for the local full snapshot at slot %lu, "
1221 0 : "loading the full snapshot only.", ctx->local_in.full_snapshot_slot ));
1222 0 : ctx->load_incremental = 0;
1223 0 : }
1224 0 : }
1225 :
1226 0 : if( FD_UNLIKELY( !local_incr_on_disk && local_has_incr ) ) ctx->predicted_incremental.slot = local_effective_slot;
1227 :
1228 0 : send_expected_slot( ctx, stem, local_effective_slot );
1229 :
1230 0 : FD_LOG_NOTICE(( "reading full snapshot from file %s%s%s", fd_log_style_dim(), ctx->local_in.full_snapshot_path, fd_log_style_normal() ));
1231 0 : FD_LOG_INFO(( "reading full snapshot at slot %lu (cluster full %lu incremental %lu) from local file `%s`",
1232 0 : ctx->local_in.full_snapshot_slot, cluster.full, cluster.incremental, ctx->local_in.full_snapshot_path ));
1233 0 : ctx->predicted_incremental.full_slot = ctx->local_in.full_snapshot_slot;
1234 0 : if( !local_incr_on_disk ) predict_incremental( ctx );
1235 0 : set_state( ctx, FD_SNAPCT_STATE_READING_FULL_FILE );
1236 0 : init_load( ctx, stem, 1, 1 );
1237 0 : } else {
1238 0 : if( FD_LIKELY( ctx->local_in.full_snapshot_slot!=ULONG_MAX ) ) {
1239 0 : if( local_effective_slot==ULONG_MAX ) {
1240 0 : FD_LOG_NOTICE(( "local full snapshot at slot %lu cannot be used because no matching incremental snapshot "
1241 0 : "could be found (cluster incremental %lu), downloading instead",
1242 0 : ctx->local_in.full_snapshot_slot, cluster.incremental ));
1243 0 : } else if( FD_LIKELY( !local_within_age ) ) {
1244 0 : FD_LOG_NOTICE(( "local full snapshot at slot %lu (effective slot %lu) is too old for downloadable slot %lu max age %u, downloading instead",
1245 0 : ctx->local_in.full_snapshot_slot, local_effective_slot, download_effective_slot, ctx->config.sources.max_local_full_effective_age ));
1246 0 : }
1247 0 : } else {
1248 0 : FD_LOG_INFO(( "no local snapshot available, downloading from peer" ));
1249 0 : }
1250 :
1251 0 : if( FD_LIKELY( download_has_incr ) ) {
1252 0 : ctx->predicted_incremental.slot = download_effective_slot;
1253 0 : } else if( FD_UNLIKELY( ctx->load_incremental ) ) {
1254 0 : if( FD_UNLIKELY( wfs_short_of_target( ctx, best.full_slot ) ) ) {
1255 0 : FD_LOG_NOTICE(( "no incremental snapshot lifts the full snapshot at slot %lu to "
1256 0 : "[consensus.wait_for_supermajority_at_slot] %lu yet; keeping the incremental "
1257 0 : "load to wait for a peer that bridges the gap",
1258 0 : best.full_slot, wfs_floor( ctx ) ));
1259 0 : } else if( FD_UNLIKELY( !ctx->config.incremental_snapshots ) ) {
1260 0 : FD_LOG_INFO(( "full snapshot at slot %lu already reaches the WFS target, downloading it alone",
1261 0 : best.full_slot ));
1262 0 : ctx->load_incremental = 0;
1263 0 : } else {
1264 0 : FD_LOG_WARNING(( "no incremental snapshot is available for the full snapshot at slot %lu, "
1265 0 : "downloading the full snapshot only.", best.full_slot ));
1266 0 : ctx->load_incremental = 0;
1267 0 : }
1268 0 : }
1269 0 : send_expected_slot( ctx, stem, download_effective_slot );
1270 :
1271 0 : ctx->peer = best;
1272 0 : set_state( ctx, FD_SNAPCT_STATE_READING_FULL_HTTP );
1273 0 : ctx->predicted_incremental.full_slot = best.full_slot;
1274 0 : predict_incremental( ctx );
1275 0 : init_load( ctx, stem, 1, 0 );
1276 0 : }
1277 0 : break;
1278 0 : }
1279 :
1280 : /* ============================================================== */
1281 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL: {
1282 0 : if( FD_UNLIKELY( now<ctx->deadline_nanos ) ) break;
1283 :
1284 0 : fd_sspeer_t best = best_incremental_peer( ctx );
1285 0 : if( FD_UNLIKELY( !best.addr.l ) ) {
1286 0 : if( !ctx->download_enabled ) {
1287 0 : if( FD_UNLIKELY( wfs_short_of_target( ctx, ctx->predicted_incremental.full_slot ) ) ) {
1288 0 : FD_LOG_ERR(( "no incremental snapshot peer lifts full snapshot slot %lu to "
1289 0 : "[consensus.wait_for_supermajority_at_slot] %lu, and no snapshot download "
1290 0 : "source is configured. aborting.",
1291 0 : ctx->predicted_incremental.full_slot, wfs_floor( ctx ) ));
1292 0 : } else {
1293 0 : FD_LOG_ERR(( "no incremental snapshot peers are available and no snapshot download source is configured. aborting." ));
1294 0 : }
1295 0 : }
1296 0 : ctx->deadline_nanos = now + ctx->config.wait_for_peers_timeout_nanos;
1297 0 : set_state( ctx, FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL );
1298 0 : break;
1299 0 : }
1300 :
1301 : /* decide whether to use the local incremental snapshot if one
1302 : exists and is not too old, otherwise download a new incremental
1303 : snapshot. The one on disk is built on the local full, so it is
1304 : only a candidate when that is the full we loaded. */
1305 0 : ulong local_slot = ctx->local_in.full_snapshot_slot==ctx->predicted_incremental.full_slot
1306 0 : ? ctx->local_in.incremental_snapshot_slot
1307 0 : : ULONG_MAX;
1308 0 : if( FD_LIKELY( incr_use_local( ctx, local_slot, best.incr_slot ) ) ) {
1309 0 : FD_LOG_NOTICE(( "reading incremental snapshot from file %s%s%s", fd_log_style_dim(), ctx->local_in.incremental_snapshot_path, fd_log_style_normal() ));
1310 0 : FD_LOG_INFO(( "reading incremental snapshot at slot %lu from local file `%s`", ctx->local_in.incremental_snapshot_slot, ctx->local_in.incremental_snapshot_path ));
1311 0 : set_state( ctx, FD_SNAPCT_STATE_READING_INCREMENTAL_FILE );
1312 0 : init_load( ctx, stem, 0, 1 );
1313 0 : } else {
1314 0 : ctx->predicted_incremental.slot = best.incr_slot;
1315 0 : ctx->peer = best;
1316 0 : set_state( ctx, FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP );
1317 0 : init_load( ctx, stem, 0, 0 );
1318 0 : }
1319 0 : break;
1320 0 : }
1321 :
1322 : /* ============================================================== */
1323 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
1324 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1325 0 : ctx->malformed = 0;
1326 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1327 0 : ctx->flush_ack = 0;
1328 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET );
1329 0 : FD_LOG_WARNING(( "failed to load incremental snapshot at slot %lu from local file `%s`. "
1330 0 : "if the file is damaged, remove it, or later boots may select it again.",
1331 0 : ctx->local_in.incremental_snapshot_slot, ctx->local_in.incremental_snapshot_path ));
1332 0 : break;
1333 0 : }
1334 :
1335 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1336 :
1337 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE );
1338 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_DONE, 0UL, 0UL, 0UL, 0UL, 0UL );
1339 0 : ctx->flush_ack = 0;
1340 0 : break;
1341 :
1342 : /* ============================================================== */
1343 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
1344 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1345 0 : ctx->malformed = 0;
1346 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1347 0 : ctx->flush_ack = 0;
1348 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET );
1349 0 : FD_LOG_WARNING(( "failed to load incremental snapshot at slot %lu from local file `%s`. "
1350 0 : "if the file is damaged, remove it, or later boots may select it again.",
1351 0 : ctx->local_in.incremental_snapshot_slot, ctx->local_in.incremental_snapshot_path ));
1352 0 : break;
1353 0 : }
1354 :
1355 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1356 :
1357 0 : log_completion( ctx, 0/*incr*/ );
1358 0 : set_state( ctx, FD_SNAPCT_STATE_SHUTDOWN );
1359 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_SHUTDOWN, 0UL, 0UL, 0UL, 0UL, 0UL );
1360 0 : break;
1361 :
1362 : /* ============================================================== */
1363 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
1364 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1365 0 : ctx->malformed = 0;
1366 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1367 0 : ctx->flush_ack = 0;
1368 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET );
1369 0 : FD_LOG_WARNING(( "failed to load incremental snapshot at slot %lu from http://" FD_IP4_ADDR_FMT ":%hu/%s. "
1370 0 : "blacklisting peer due to download failure.",
1371 0 : ctx->predicted_incremental.slot,
1372 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ), ctx->http_incr_snapshot_name ));
1373 0 : blacklist_peer( ctx );
1374 0 : break;
1375 0 : }
1376 :
1377 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1378 :
1379 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE );
1380 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_DONE, 0UL, 0UL, 0UL, 0UL, 0UL );
1381 0 : ctx->flush_ack = 0;
1382 0 : break;
1383 :
1384 : /* ============================================================== */
1385 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
1386 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1387 0 : ctx->malformed = 0;
1388 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1389 0 : ctx->flush_ack = 0;
1390 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET );
1391 0 : FD_LOG_WARNING(( "failed to load incremental snapshot at slot %lu from http://" FD_IP4_ADDR_FMT ":%hu/%s. "
1392 0 : "blacklisting peer due to download failure.",
1393 0 : ctx->predicted_incremental.slot,
1394 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ), ctx->http_incr_snapshot_name ));
1395 0 : blacklist_peer( ctx );
1396 0 : break;
1397 0 : }
1398 :
1399 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1400 :
1401 0 : log_completion( ctx, 0/*incr*/ );
1402 0 : set_state( ctx, FD_SNAPCT_STATE_SHUTDOWN );
1403 0 : rename_incr_snapshot( ctx );
1404 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_SHUTDOWN, 0UL, 0UL, 0UL, 0UL, 0UL );
1405 0 : break;
1406 :
1407 : /* ============================================================== */
1408 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
1409 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1410 0 : ctx->malformed = 0;
1411 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1412 0 : ctx->flush_ack = 0;
1413 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET );
1414 0 : FD_LOG_WARNING(( "failed to load full snapshot at slot %lu from local file `%s`",
1415 0 : ctx->local_in.full_snapshot_slot, ctx->local_in.full_snapshot_path ));
1416 0 : break;
1417 0 : }
1418 :
1419 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1420 :
1421 0 : if( FD_UNLIKELY( full_only( ctx ) ) ) {
1422 0 : FD_LOG_WARNING(( "no snapshot server offers an incremental snapshot for the full snapshot at slot %lu, loading the full snapshot only.",
1423 0 : ctx->predicted_incremental.full_slot ));
1424 0 : ctx->load_incremental = 0;
1425 0 : send_expected_slot( ctx, stem, ctx->predicted_incremental.full_slot );
1426 0 : }
1427 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE );
1428 0 : fd_stem_publish( stem, ctx->out_ld.idx, ctx->load_incremental ? FD_SNAPSHOT_MSG_CTRL_NEXT : FD_SNAPSHOT_MSG_CTRL_DONE, 0UL, 0UL, 0UL, 0UL, 0UL );
1429 0 : ctx->flush_ack = 0;
1430 0 : break;
1431 :
1432 : /* ============================================================== */
1433 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE: {
1434 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1435 0 : ctx->malformed = 0;
1436 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1437 0 : ctx->flush_ack = 0;
1438 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET );
1439 0 : FD_LOG_WARNING(( "failed to load full snapshot at slot %lu from local file `%s`",
1440 0 : ctx->local_in.full_snapshot_slot, ctx->local_in.full_snapshot_path ));
1441 0 : break;
1442 0 : }
1443 :
1444 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1445 0 : log_completion( ctx, 1/*full*/ );
1446 0 : if( FD_LIKELY( !ctx->load_incremental ) ) {
1447 0 : set_state( ctx, FD_SNAPCT_STATE_SHUTDOWN );
1448 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_SHUTDOWN, 0UL, 0UL, 0UL, 0UL, 0UL );
1449 0 : break;
1450 0 : }
1451 :
1452 : /* Weigh the local incremental against the best downloadable one
1453 : that pairs with the full we just loaded. With download
1454 : disabled there is no alternative to weigh it against. */
1455 :
1456 0 : ulong fetch_incr = ULONG_MAX;
1457 0 : if( FD_LIKELY( ctx->download_enabled ) ) {
1458 0 : fd_sspeer_t best_incr = best_incremental_peer( ctx );
1459 0 : if( FD_LIKELY( best_incr.addr.l ) ) fetch_incr = best_incr.incr_slot;
1460 0 : }
1461 :
1462 : /* No peer yet is no reason to wait: the pair already cleared the
1463 : age budget at selection. */
1464 :
1465 0 : if( FD_LIKELY( incr_use_local( ctx, ctx->local_in.incremental_snapshot_slot, fetch_incr ) ) ) {
1466 0 : FD_LOG_NOTICE(( "reading incremental snapshot from file %s%s%s", fd_log_style_dim(), ctx->local_in.incremental_snapshot_path, fd_log_style_normal() ));
1467 0 : FD_LOG_INFO(( "reading incremental snapshot at slot %lu from local file `%s`", ctx->local_in.incremental_snapshot_slot, ctx->local_in.incremental_snapshot_path ));
1468 0 : set_state( ctx, FD_SNAPCT_STATE_READING_INCREMENTAL_FILE );
1469 0 : init_load( ctx, stem, 0, 1 );
1470 0 : break;
1471 0 : }
1472 :
1473 0 : set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL );
1474 0 : ctx->deadline_nanos = 0L;
1475 0 : break;
1476 0 : }
1477 :
1478 : /* ============================================================== */
1479 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
1480 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1481 0 : ctx->malformed = 0;
1482 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1483 0 : ctx->flush_ack = 0;
1484 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET );
1485 0 : FD_LOG_WARNING(( "failed to load full snapshot at slot %lu from http://" FD_IP4_ADDR_FMT ":%hu/%s. "
1486 0 : "blacklisting peer due to download failure.",
1487 0 : ctx->predicted_incremental.full_slot,
1488 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ), ctx->http_full_snapshot_name ));
1489 0 : blacklist_peer( ctx );
1490 0 : break;
1491 0 : }
1492 :
1493 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1494 :
1495 0 : if( FD_UNLIKELY( full_only( ctx ) ) ) {
1496 0 : FD_LOG_WARNING(( "no snapshot server offers an incremental snapshot for the full snapshot at slot %lu, loading the full snapshot only.",
1497 0 : ctx->predicted_incremental.full_slot ));
1498 0 : ctx->load_incremental = 0;
1499 0 : send_expected_slot( ctx, stem, ctx->predicted_incremental.full_slot );
1500 0 : }
1501 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE );
1502 0 : fd_stem_publish( stem, ctx->out_ld.idx, ctx->load_incremental ? FD_SNAPSHOT_MSG_CTRL_NEXT : FD_SNAPSHOT_MSG_CTRL_DONE, 0UL, 0UL, 0UL, 0UL, 0UL );
1503 0 : ctx->flush_ack = 0;
1504 0 : break;
1505 :
1506 : /* ============================================================== */
1507 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE:
1508 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1509 0 : ctx->malformed = 0;
1510 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1511 0 : ctx->flush_ack = 0;
1512 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET );
1513 0 : FD_LOG_WARNING(( "failed to load full snapshot at slot %lu from http://" FD_IP4_ADDR_FMT ":%hu/%s. "
1514 0 : "blacklisting peer due to download failure.",
1515 0 : ctx->predicted_incremental.full_slot,
1516 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ), ctx->http_full_snapshot_name ));
1517 0 : blacklist_peer( ctx );
1518 0 : break;
1519 0 : }
1520 :
1521 0 : if( ctx->flush_ack < ctx->flush_ack_cnt ) break;
1522 :
1523 0 : rename_full_snapshot( ctx );
1524 :
1525 0 : log_completion( ctx, 1/*full*/ );
1526 0 : if( FD_LIKELY( !ctx->load_incremental ) ) {
1527 0 : set_state( ctx, FD_SNAPCT_STATE_SHUTDOWN );
1528 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_SHUTDOWN, 0UL, 0UL, 0UL, 0UL, 0UL );
1529 0 : break;
1530 0 : }
1531 :
1532 : /* Get the best incremental peer to download from */
1533 0 : fd_sspeer_t best = best_incremental_peer( ctx );
1534 0 : if( FD_UNLIKELY( !best.addr.l ) ) {
1535 0 : ctx->deadline_nanos = now;
1536 0 : set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL );
1537 0 : break;
1538 0 : }
1539 :
1540 0 : ctx->predicted_incremental.slot = best.incr_slot;
1541 0 : ctx->peer = best;
1542 0 : set_state( ctx, FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP );
1543 0 : init_load( ctx, stem, 0, 0 );
1544 0 : break;
1545 :
1546 : /* ============================================================== */
1547 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET:
1548 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET:
1549 0 : if( FD_UNLIKELY( ctx->flush_ack<ctx->flush_ack_cnt ) ) break;
1550 :
1551 0 : if( ctx->metrics.full.num_retries>=ctx->config.max_retry_abort ) {
1552 0 : FD_LOG_ERR(( "hit retry limit of %u for full snapshot, aborting", ctx->config.max_retry_abort ));
1553 0 : }
1554 :
1555 0 : ctx->metrics.full.num_retries++;
1556 0 : FD_LOG_NOTICE(( "retrying full snapshot download (attempt %u/%u)",
1557 0 : ctx->metrics.full.num_retries, ctx->config.max_retry_abort ));
1558 :
1559 0 : ctx->metrics.full.bytes_read = 0UL;
1560 0 : ctx->metrics.full.bytes_written = 0UL;
1561 0 : ctx->metrics.full.bytes_total = 0UL;
1562 :
1563 0 : ctx->metrics.incremental.bytes_read = 0UL;
1564 0 : ctx->metrics.incremental.bytes_written = 0UL;
1565 0 : ctx->metrics.incremental.bytes_total = 0UL;
1566 :
1567 0 : if( !ctx->download_enabled ) {
1568 : /* if we are unable to download new snapshots and unable to load
1569 : our local snapshot, we must shutdown the validator. */
1570 0 : FD_LOG_ERR(( "unable to load local snapshot %s and no snapshot peers were configured. aborting.", ctx->local_in.full_snapshot_path ));
1571 0 : } else {
1572 0 : if( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ) ctx->local_in.full_snapshot_slot = ULONG_MAX;
1573 0 : set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS );
1574 0 : ctx->deadline_nanos = 0L;
1575 0 : }
1576 0 : break;
1577 :
1578 : /* ============================================================== */
1579 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET:
1580 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET:
1581 : /* Once the full is committed with NEXT, an incremental is
1582 : required. The incremental load resets the full's status cache
1583 : and may already have replaced its manifest in replay, so
1584 : reverting would mean restoring both. It would also require
1585 : the incremental's partial account state to be rolled back.
1586 : Booting from the full alone may never catch up either. Every
1587 : path from here loads an incremental or aborts. */
1588 0 : if( FD_UNLIKELY( ctx->flush_ack<ctx->flush_ack_cnt ) ) break;
1589 :
1590 0 : if( ctx->metrics.incremental.num_retries>=ctx->config.max_retry_abort ) {
1591 0 : FD_LOG_ERR(( "hit retry limit of %u for incremental snapshot, aborting", ctx->config.max_retry_abort ));
1592 0 : }
1593 :
1594 0 : ctx->metrics.incremental.num_retries++;
1595 0 : FD_LOG_NOTICE(( "retrying incremental snapshot download (attempt %u/%u)",
1596 0 : ctx->metrics.incremental.num_retries, ctx->config.max_retry_abort ));
1597 :
1598 0 : ctx->metrics.incremental.bytes_read = 0UL;
1599 0 : ctx->metrics.incremental.bytes_written = 0UL;
1600 0 : ctx->metrics.incremental.bytes_total = 0UL;
1601 :
1602 0 : if( !ctx->download_enabled ) {
1603 : /* if we are unable to download new snapshots and unable to load
1604 : our local snapshot, we must shutdown the validator. */
1605 0 : FD_LOG_ERR(( "unable to load local snapshot %s and no snapshot peers were configured. aborting. "
1606 0 : "if the file is damaged, remove it before restarting, or the next boot will select it again.",
1607 0 : ctx->local_in.incremental_snapshot_path ));
1608 0 : } else {
1609 0 : if( ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET ) ctx->local_in.incremental_snapshot_slot = ULONG_MAX;
1610 0 : ctx->predicted_incremental.committed = 0; /* the source is undecided again */
1611 0 : set_state( ctx, FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL );
1612 0 : ctx->deadline_nanos = 0L;
1613 0 : }
1614 0 : break;
1615 :
1616 : /* ============================================================== */
1617 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
1618 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1619 0 : ctx->malformed = 0;
1620 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1621 0 : ctx->flush_ack = 0;
1622 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET );
1623 0 : FD_LOG_WARNING(( "failed to load full snapshot at slot %lu from local file `%s`",
1624 0 : ctx->local_in.full_snapshot_slot, ctx->local_in.full_snapshot_path ));
1625 0 : break;
1626 0 : }
1627 0 : if( FD_UNLIKELY( ctx->flush_ack < ctx->flush_ack_cnt ) ) break;
1628 0 : if( FD_UNLIKELY( ctx->load_complete ) ) {
1629 0 : ctx->load_complete = 0;
1630 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FINI, 0UL, 0UL, 0UL, 0UL, 0UL );
1631 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI );
1632 0 : ctx->flush_ack = 0;
1633 0 : }
1634 0 : break;
1635 :
1636 : /* ============================================================== */
1637 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
1638 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1639 0 : ctx->malformed = 0;
1640 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1641 0 : ctx->flush_ack = 0;
1642 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET );
1643 0 : FD_LOG_WARNING(( "failed to load incremental snapshot at slot %lu from local file `%s`. "
1644 0 : "if the file is damaged, remove it, or later boots may select it again.",
1645 0 : ctx->local_in.incremental_snapshot_slot, ctx->local_in.incremental_snapshot_path ));
1646 0 : break;
1647 0 : }
1648 0 : if( FD_UNLIKELY( ctx->flush_ack < ctx->flush_ack_cnt ) ) break;
1649 0 : if( FD_UNLIKELY( ctx->load_complete ) ) {
1650 0 : ctx->load_complete = 0;
1651 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FINI, 0UL, 0UL, 0UL, 0UL, 0UL );
1652 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI );
1653 0 : ctx->flush_ack = 0;
1654 0 : }
1655 0 : break;
1656 :
1657 : /* ============================================================== */
1658 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
1659 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1660 0 : ctx->malformed = 0;
1661 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1662 0 : ctx->flush_ack = 0;
1663 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET );
1664 0 : FD_LOG_WARNING(( "failed to load full snapshot at slot %lu from http://" FD_IP4_ADDR_FMT ":%hu/%s. "
1665 0 : "blacklisting peer due to download failure",
1666 0 : ctx->predicted_incremental.full_slot,
1667 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ), ctx->http_full_snapshot_name ));
1668 0 : blacklist_peer( ctx );
1669 0 : break;
1670 0 : }
1671 0 : if( FD_UNLIKELY( ctx->flush_ack < ctx->flush_ack_cnt ) ) break;
1672 0 : if( FD_UNLIKELY( ctx->load_complete ) ) {
1673 0 : ctx->load_complete = 0;
1674 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FINI, 0UL, 0UL, 0UL, 0UL, 0UL );
1675 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI );
1676 0 : ctx->flush_ack = 0;
1677 0 : }
1678 0 : break;
1679 :
1680 : /* ============================================================== */
1681 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
1682 0 : if( FD_UNLIKELY( ctx->malformed ) ) {
1683 0 : ctx->malformed = 0;
1684 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FAIL, 0UL, 0UL, 0UL, 0UL, 0UL );
1685 0 : ctx->flush_ack = 0;
1686 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET );
1687 0 : FD_LOG_WARNING(( "failed to load incremental snapshot at slot %lu from http://" FD_IP4_ADDR_FMT ":%hu/%s. "
1688 0 : "blacklisting peer due to download failure",
1689 0 : ctx->predicted_incremental.slot,
1690 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ), ctx->http_incr_snapshot_name ));
1691 0 : blacklist_peer( ctx );
1692 0 : break;
1693 0 : }
1694 0 : if( FD_UNLIKELY( ctx->flush_ack < ctx->flush_ack_cnt ) ) break;
1695 0 : if( FD_UNLIKELY( ctx->load_complete ) ) {
1696 0 : ctx->load_complete = 0;
1697 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_FINI, 0UL, 0UL, 0UL, 0UL, 0UL );
1698 0 : set_state( ctx, FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI );
1699 0 : ctx->flush_ack = 0;
1700 0 : }
1701 0 : break;
1702 :
1703 : /* ============================================================== */
1704 0 : case FD_SNAPCT_STATE_SHUTDOWN:
1705 : /* Transitioning to the shutdown state indicates snapshot load is
1706 : completed without errors. Otherwise, snapct would have aborted
1707 : earlier. */
1708 0 : break;
1709 :
1710 : /* ============================================================== */
1711 0 : default: FD_LOG_ERR(( "unexpected state %s", fd_snapct_state_str( ctx->state ) ));
1712 0 : }
1713 0 : }
1714 :
1715 : static void
1716 : gossip_frag( fd_snapct_tile_t * ctx,
1717 : ulong sig,
1718 : ulong sz FD_PARAM_UNUSED,
1719 0 : ulong chunk ) {
1720 0 : FD_TEST( ctx->gossip_enabled );
1721 :
1722 0 : if( FD_UNLIKELY( sig==FD_GOSSIP_UPDATE_TAG_PEER_SATURATED ) ) {
1723 0 : ctx->gossip.saturated = 1;
1724 0 : return;
1725 0 : }
1726 :
1727 0 : if( !( sig==FD_GOSSIP_UPDATE_TAG_CONTACT_INFO ||
1728 0 : sig==FD_GOSSIP_UPDATE_TAG_CONTACT_INFO_REMOVE ||
1729 0 : sig==FD_GOSSIP_UPDATE_TAG_SNAPSHOT_HASHES ) ) return;
1730 :
1731 0 : fd_gossip_update_message_t const * msg = fd_chunk_to_laddr_const( ctx->gossip_in_mem, chunk );
1732 0 : switch( msg->tag ) {
1733 0 : case FD_GOSSIP_UPDATE_TAG_CONTACT_INFO: {
1734 0 : FD_TEST( msg->contact_info->idx<GOSSIP_PEERS_MAX );
1735 0 : fd_pubkey_t const * pubkey = (fd_pubkey_t const *)msg->origin;
1736 0 : gossip_ci_entry_t * entry = ctx->gossip.ci_table + msg->contact_info->idx;
1737 0 : if( FD_UNLIKELY( !fd_pubkey_eq( &entry->pubkey, pubkey ) ) ) {
1738 : /* Initialize the new gossip entry, which may or may not be allowed */
1739 0 : FD_TEST( fd_pubkey_check_zero( &entry->pubkey ) );
1740 0 : entry->pubkey = *pubkey;
1741 0 : entry->rpc_addr.l = 0UL;
1742 0 : if( ctx->config.sources.gossip.allow_any ) {
1743 0 : entry->allowed = 1;
1744 0 : for( ulong i=0UL; i<ctx->config.sources.gossip.block_list_cnt; i++ ) {
1745 0 : if( fd_pubkey_eq( pubkey, &ctx->config.sources.gossip.block_list[ i ] ) ) {
1746 0 : entry->allowed = 0;
1747 0 : break;
1748 0 : }
1749 0 : }
1750 0 : } else {
1751 0 : entry->allowed = 0;
1752 0 : for( ulong i=0UL; i<ctx->config.sources.gossip.allow_list_cnt; i++ ) {
1753 0 : if( fd_pubkey_eq( pubkey, &ctx->config.sources.gossip.allow_list[ i ] ) ) {
1754 0 : entry->allowed = 1;
1755 0 : break;
1756 0 : }
1757 0 : }
1758 0 : }
1759 0 : FD_TEST( ULONG_MAX==gossip_ci_map_idx_query_const( ctx->gossip.ci_map, pubkey, ULONG_MAX, ctx->gossip.ci_table ) );
1760 0 : if( entry->allowed ) {
1761 0 : gossip_ci_map_idx_insert( ctx->gossip.ci_map, msg->contact_info->idx, ctx->gossip.ci_table );
1762 0 : ctx->gossip.allowed_cnt++;
1763 : /* Allow-list shortcut: if an explicit allow list is
1764 : configured and all expected peers have arrived, declare
1765 : saturation immediately without waiting for the gossip
1766 : tile's general saturation signal. */
1767 0 : if( FD_UNLIKELY( !ctx->config.sources.gossip.allow_any &&
1768 0 : ctx->config.sources.gossip.allow_list_cnt>0UL &&
1769 0 : ctx->gossip.allowed_cnt==ctx->config.sources.gossip.allow_list_cnt ) ) {
1770 0 : FD_LOG_NOTICE(( "all %lu allowed gossip peers discovered", ctx->config.sources.gossip.allow_list_cnt ));
1771 0 : ctx->gossip.saturated = 1;
1772 0 : }
1773 0 : }
1774 0 : }
1775 0 : if( !entry->allowed ) break;
1776 : /* Maybe update the RPC address of a new or existing allowed gossip peer */
1777 0 : fd_ip4_port_t cur_addr = entry->rpc_addr;
1778 0 : fd_ip4_port_t new_addr;
1779 0 : new_addr.addr = msg->contact_info->value->sockets[ FD_GOSSIP_CONTACT_INFO_SOCKET_RPC ].is_ipv6 ? 0 : msg->contact_info->value->sockets[ FD_GOSSIP_CONTACT_INFO_SOCKET_RPC ].ip4;
1780 0 : new_addr.port = msg->contact_info->value->sockets[ FD_GOSSIP_CONTACT_INFO_SOCKET_RPC ].port;
1781 : /* Sanitize non-public/multicast/broadcast addresses to zero so
1782 : they are never stored in entry->rpc_addr or handed to
1783 : downstream code paths (e.g. on_snapshot_hash). Normalizing
1784 : before comparing against cur_addr avoids repeated selector
1785 : removals and log warnings on every gossip refresh. */
1786 0 : uint unfiltered_addr = new_addr.addr;
1787 0 : if( FD_UNLIKELY( !fd_ip4_addr_is_public( new_addr.addr ) ||
1788 0 : fd_ip4_addr_is_mcast( new_addr.addr ) ||
1789 0 : fd_ip4_addr_is_bcast( new_addr.addr ) ) ) {
1790 0 : new_addr.l = 0;
1791 0 : }
1792 :
1793 0 : if( FD_UNLIKELY( new_addr.l!=cur_addr.l ) ) {
1794 0 : fd_sspeer_key_t entry_key = {0};
1795 0 : *entry_key.pubkey = entry->pubkey;
1796 0 : entry_key.is_url = 0;
1797 0 : if( FD_LIKELY( !!new_addr.l ) ) {
1798 : /* Do not re-add peers that have been permanently blacklisted
1799 : or whose new address is temporarily banned by ssping.
1800 : Bookkeeping (ssping cleanup, rpc_addr update) still runs
1801 : so the CI table and ssping pool stay consistent. */
1802 0 : if( FD_LIKELY( !blacklist_map_ele_query( ctx->blacklist_map, &entry_key, NULL, ctx->blacklist_pool ) &&
1803 0 : !fd_ssping_is_invalidated( ctx->ssping, new_addr ) ) ) {
1804 0 : if( FD_LIKELY( FD_SSPEER_SCORE_INVALID!=fd_sspeer_selector_add( ctx->selector, &entry_key, new_addr,
1805 0 : FD_SSPEER_LATENCY_UNKNOWN, FD_SSPEER_SLOT_UNKNOWN,
1806 0 : FD_SSPEER_SLOT_UNKNOWN, NULL, NULL ) ) ) {
1807 0 : fd_ssping_add( ctx->ssping, new_addr );
1808 0 : }
1809 0 : } else {
1810 : /* Evict stale old address from selector to stay in sync
1811 : with the CI table update below. Increment ssping
1812 : refcnt for new_addr to balance the fd_ssping_remove
1813 : on the next address change or peer removal. */
1814 0 : fd_sspeer_selector_remove( ctx->selector, &entry_key );
1815 0 : fd_ssping_add( ctx->ssping, new_addr );
1816 0 : }
1817 0 : } else {
1818 0 : if( FD_UNLIKELY( !!unfiltered_addr ) ) {
1819 0 : FD_LOG_WARNING(( "filtered non-public RPC address " FD_IP4_ADDR_FMT " from gossip peer", FD_IP4_ADDR_FMT_ARGS( unfiltered_addr ) ));
1820 0 : }
1821 0 : fd_sspeer_selector_remove( ctx->selector, &entry_key );
1822 0 : }
1823 0 : if( FD_LIKELY( !!cur_addr.l ) ) {
1824 0 : fd_ssping_remove( ctx->ssping, cur_addr );
1825 0 : }
1826 0 : entry->rpc_addr = new_addr;
1827 0 : if( !ctx->config.sources.gossip.allow_any ) {
1828 0 : FD_BASE58_ENCODE_32_BYTES( pubkey->uc, pubkey_b58 );
1829 0 : if( FD_LIKELY( !!new_addr.l ) ) {
1830 0 : FD_LOG_NOTICE(( "allowed gossip peer added with public key `%s` and RPC address `" FD_IP4_ADDR_FMT ":%hu`",
1831 0 : pubkey_b58, FD_IP4_ADDR_FMT_ARGS( new_addr.addr ), fd_ushort_bswap( new_addr.port ) ));
1832 0 : } else {
1833 0 : FD_LOG_WARNING(( "allowed gossip peer with public key `%s` does not advertise an RPC address", pubkey_b58 ));
1834 0 : }
1835 0 : }
1836 0 : }
1837 0 : break;
1838 0 : }
1839 0 : case FD_GOSSIP_UPDATE_TAG_CONTACT_INFO_REMOVE: {
1840 0 : FD_TEST( msg->contact_info_remove->idx<GOSSIP_PEERS_MAX );
1841 0 : gossip_ci_entry_t * entry = ctx->gossip.ci_table + msg->contact_info_remove->idx;
1842 0 : ulong rem_idx = gossip_ci_map_idx_remove( ctx->gossip.ci_map, &entry->pubkey, ULONG_MAX, ctx->gossip.ci_table );
1843 0 : if( rem_idx!=ULONG_MAX ) {
1844 0 : FD_TEST( entry->allowed && rem_idx==msg->contact_info_remove->idx );
1845 0 : ctx->gossip.allowed_cnt--;
1846 0 : fd_ip4_port_t addr = entry->rpc_addr;
1847 0 : if( FD_LIKELY( !!addr.l ) ) {
1848 0 : fd_ssping_remove( ctx->ssping, addr );
1849 0 : fd_sspeer_key_t entry_key = {0};
1850 0 : *entry_key.pubkey = entry->pubkey;
1851 0 : entry_key.is_url = 0;
1852 0 : fd_sspeer_selector_remove( ctx->selector, &entry_key );
1853 0 : }
1854 0 : if( !ctx->config.sources.gossip.allow_any ) {
1855 0 : FD_BASE58_ENCODE_32_BYTES( entry->pubkey.uc, pubkey_b58 );
1856 0 : FD_LOG_WARNING(( "allowed gossip peer removed with public key `%s` and RPC address `" FD_IP4_ADDR_FMT ":%hu`",
1857 0 : pubkey_b58, FD_IP4_ADDR_FMT_ARGS( addr.addr ), fd_ushort_bswap( addr.port ) ));
1858 0 : }
1859 0 : }
1860 0 : fd_memset( entry, 0, sizeof(*entry) );
1861 0 : break;
1862 0 : }
1863 0 : case FD_GOSSIP_UPDATE_TAG_SNAPSHOT_HASHES: {
1864 0 : ulong idx = gossip_ci_map_idx_query_const( ctx->gossip.ci_map, (fd_pubkey_t const *)msg->origin, ULONG_MAX, ctx->gossip.ci_table );
1865 0 : if( FD_LIKELY( idx!=ULONG_MAX ) ) {
1866 0 : gossip_ci_entry_t * entry = ctx->gossip.ci_table + idx;
1867 0 : FD_TEST( entry->allowed );
1868 0 : fd_sspeer_key_t entry_key = {0};
1869 0 : *entry_key.pubkey = entry->pubkey;
1870 0 : entry_key.is_url = 0;
1871 0 : on_snapshot_hash( ctx, &entry_key, entry->rpc_addr, msg );
1872 0 : }
1873 0 : break;
1874 0 : }
1875 0 : default:
1876 0 : FD_LOG_ERR(( "snapct: unexpected gossip tag %u", (uint)msg->tag ));
1877 0 : break;
1878 0 : }
1879 0 : }
1880 :
1881 : /* Validate and handle a pipeline control ack for INIT_{FULL,INCR},
1882 : NEXT, DONE, and FINI. Returns 0 on success, and -1 if the sig was
1883 : unrecognized or the state was unexpected. */
1884 : static int
1885 : process_ctrl_ack( fd_snapct_tile_t * ctx,
1886 0 : ulong sig ) {
1887 0 : switch( sig ) {
1888 0 : case FD_SNAPSHOT_MSG_CTRL_INIT_FULL:
1889 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_READING_FULL_HTTP ||
1890 0 : ctx->state==FD_SNAPCT_STATE_READING_FULL_FILE ) ) {
1891 0 : ctx->flush_ack++;
1892 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
1893 0 : } else if( FD_UNLIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET ||
1894 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ) ) {
1895 : /* Safe to ignore -- stale ack from before RESET. */
1896 0 : } else return -1;
1897 0 : return 0;
1898 :
1899 0 : case FD_SNAPSHOT_MSG_CTRL_INIT_INCR:
1900 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP ||
1901 0 : ctx->state==FD_SNAPCT_STATE_READING_INCREMENTAL_FILE ) ) {
1902 0 : ctx->flush_ack++;
1903 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
1904 0 : } else if( FD_UNLIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET ||
1905 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET ) ) {
1906 : /* Safe to ignore -- stale ack from before RESET. */
1907 0 : } else return -1;
1908 0 : return 0;
1909 :
1910 0 : case FD_SNAPSHOT_MSG_CTRL_NEXT:
1911 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE ||
1912 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE ) ) {
1913 0 : ctx->flush_ack++;
1914 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
1915 0 : } else if( FD_UNLIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET ||
1916 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ) ) {
1917 : /* Safe to ignore -- stale ack from before RESET. */
1918 0 : } else return -1;
1919 0 : return 0;
1920 :
1921 0 : case FD_SNAPSHOT_MSG_CTRL_DONE:
1922 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE ||
1923 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE ||
1924 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE ||
1925 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE ) ) {
1926 0 : ctx->flush_ack++;
1927 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
1928 0 : } else if( FD_UNLIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET ||
1929 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ||
1930 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET ||
1931 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET ) ) {
1932 : /* Safe to ignore -- stale ack from before RESET. */
1933 0 : } else return -1;
1934 0 : return 0;
1935 :
1936 0 : case FD_SNAPSHOT_MSG_CTRL_FINI:
1937 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI ||
1938 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI ||
1939 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI ||
1940 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI ) ) {
1941 0 : ctx->flush_ack++;
1942 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
1943 0 : } else if( FD_UNLIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET ||
1944 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ||
1945 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET ||
1946 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET ) ) {
1947 : /* Safe to ignore -- stale ack from before RESET. */
1948 0 : } else return -1;
1949 0 : return 0;
1950 :
1951 0 : default:
1952 0 : return -1;
1953 0 : }
1954 0 : }
1955 :
1956 : static void
1957 : snapld_frag( fd_snapct_tile_t * ctx,
1958 : ulong sig,
1959 : ulong sz,
1960 : ulong chunk,
1961 0 : fd_stem_context_t * stem ) {
1962 0 : if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_META ) ) {
1963 : /* Before snapld starts sending down data fragments, it first sends
1964 : a metadata message containing the total size of the snapshot.
1965 : Both file and HTTP paths send this message. */
1966 0 : int full, file;
1967 0 : switch( ctx->state ) {
1968 0 : case FD_SNAPCT_STATE_READING_FULL_FILE: full = 1; file = 1; break;
1969 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE: full = 0; file = 1; break;
1970 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP: full = 1; file = 0; break;
1971 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP: full = 0; file = 0; break;
1972 :
1973 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET:
1974 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET:
1975 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET:
1976 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET:
1977 0 : return; /* Ignore */
1978 0 : default: FD_LOG_ERR(( "invalid meta frag in state %s", fd_snapct_state_str( ctx->state ) ));
1979 0 : }
1980 :
1981 0 : FD_TEST( sz==sizeof(fd_ssctrl_meta_t) );
1982 0 : fd_ssctrl_meta_t const * meta = fd_chunk_to_laddr_const( ctx->snapld_in_mem, chunk );
1983 :
1984 0 : if( FD_UNLIKELY( meta->total_sz==0UL ) ) {
1985 0 : if( FD_UNLIKELY( !ctx->malformed ) ) {
1986 0 : FD_LOG_WARNING(( "received zero-length metadata for %s snapshot, marking malformed", full ? "full" : "incremental" ));
1987 0 : ctx->malformed = 1;
1988 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL );
1989 0 : }
1990 0 : return;
1991 0 : }
1992 :
1993 0 : if( full ) ctx->metrics.full.bytes_total = meta->total_sz;
1994 0 : else ctx->metrics.incremental.bytes_total = meta->total_sz;
1995 :
1996 : /* Publish snapshot path to GUI. For file loads, use the local
1997 : path directly. For HTTP downloads, construct the full URL from
1998 : the resolved snapshot name. */
1999 0 : if( file ) {
2000 0 : if( FD_LIKELY( !!ctx->out_gui.mem ) ) {
2001 0 : snapshot_path_gui_publish( ctx, stem, full ? ctx->local_in.full_snapshot_path : ctx->local_in.incremental_snapshot_path, full );
2002 0 : }
2003 0 : } else {
2004 0 : if( full ) fd_cstr_ncpy( ctx->http_full_snapshot_name, meta->resolved_name, PATH_MAX );
2005 :
2006 : /* The server sent a newer full than the one we picked. The
2007 : incremental has to be built on the full we actually load. */
2008 0 : if( FD_UNLIKELY( full && meta->resolved_slot!=ULONG_MAX && meta->resolved_slot!=ctx->predicted_incremental.full_slot ) ) {
2009 0 : FD_LOG_NOTICE(( "full snapshot resolved to slot %lu instead of %lu, pairing the incremental snapshot with it",
2010 0 : meta->resolved_slot, ctx->predicted_incremental.full_slot ));
2011 0 : ctx->predicted_incremental.full_slot = meta->resolved_slot;
2012 0 : ctx->predicted_incremental.slot = FD_SSPEER_SLOT_UNKNOWN;
2013 0 : predict_incremental( ctx );
2014 0 : if( ctx->predicted_incremental.slot==FD_SSPEER_SLOT_UNKNOWN ) send_expected_slot( ctx, stem, meta->resolved_slot );
2015 0 : }
2016 :
2017 0 : if( FD_LIKELY( !!ctx->out_gui.mem ) ) {
2018 0 : char snapshot_path[ PATH_MAX+30UL ]; /* 30 is fd_cstr_nlen( "https://255.255.255.255:65536/", ULONG_MAX ) */
2019 0 : FD_TEST( fd_cstr_printf_check( snapshot_path, sizeof(snapshot_path), NULL, "http://" FD_IP4_ADDR_FMT ":%hu/%s",
2020 0 : FD_IP4_ADDR_FMT_ARGS( ctx->peer.addr.addr ), fd_ushort_bswap( ctx->peer.addr.port ),
2021 0 : full ? ctx->http_full_snapshot_name : ctx->http_incr_snapshot_name ) );
2022 0 : snapshot_path_gui_publish( ctx, stem, snapshot_path, full );
2023 0 : }
2024 0 : }
2025 :
2026 0 : return;
2027 0 : }
2028 0 : if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_CTRL_FAIL ) ) {
2029 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET ||
2030 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ||
2031 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET ||
2032 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET ) ) {
2033 0 : ctx->flush_ack++;
2034 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
2035 0 : } else {
2036 0 : FD_LOG_ERR(( "unexpected control frag %lu (%s) in state %d (%s)", sig, fd_ssctrl_msg_ctrl_str( sig ), ctx->state, fd_snapct_state_str( ctx->state ) ));
2037 0 : }
2038 0 : return;
2039 0 : }
2040 0 : if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_CTRL_ERROR ) ) {
2041 : /* CTRL_ERROR message directly from snapld can be snapld-generated
2042 : or snapct-generated-and-forwarded. */
2043 0 : switch( ctx->state ) {
2044 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
2045 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
2046 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE:
2047 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
2048 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
2049 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
2050 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
2051 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
2052 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE:
2053 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
2054 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
2055 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
2056 0 : FD_LOG_WARNING(( "received error from snapld in state %d (%s)",
2057 0 : ctx->state, fd_snapct_state_str( ctx->state ) ));
2058 0 : ctx->malformed = 1;
2059 0 : break;
2060 0 : default:
2061 0 : break;
2062 0 : }
2063 0 : return;
2064 0 : }
2065 0 : if( FD_UNLIKELY( sig==FD_SNAPSHOT_MSG_LOAD_COMPLETE ) ) {
2066 0 : int full = 0;
2067 0 : switch( ctx->state ) {
2068 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
2069 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP: full = 1; break;
2070 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
2071 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP: full = 0; break;
2072 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET:
2073 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET:
2074 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET:
2075 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET:
2076 0 : return; /* Ignore during reset. */
2077 0 : default:
2078 0 : FD_LOG_ERR(( "invalid load_complete in state %s", fd_snapct_state_str( ctx->state ) ));
2079 0 : return;
2080 0 : }
2081 0 : if( FD_UNLIKELY( ctx->malformed ) ) return;
2082 : /* Validate that all expected bytes were received. */
2083 0 : ulong bytes_read = full ? ctx->metrics.full.bytes_read : ctx->metrics.incremental.bytes_read;
2084 0 : ulong bytes_total = full ? ctx->metrics.full.bytes_total : ctx->metrics.incremental.bytes_total;
2085 0 : if( FD_UNLIKELY( !bytes_total || bytes_read!=bytes_total ) ) {
2086 0 : ctx->malformed = 1;
2087 0 : FD_LOG_WARNING(( "load_complete but bytes_read %lu != bytes_total %lu for %s snapshot",
2088 0 : bytes_read, bytes_total, full ? "full" : "incremental" ));
2089 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL );
2090 0 : return;
2091 0 : }
2092 0 : ctx->load_complete = 1;
2093 0 : return;
2094 0 : }
2095 0 : if( FD_UNLIKELY( sig!=FD_SNAPSHOT_MSG_DATA ) ) {
2096 0 : if( process_ctrl_ack( ctx, sig ) ) {
2097 0 : FD_LOG_ERR(( "unexpected control frag %lu (%s) in state %d (%s)", sig, fd_ssctrl_msg_ctrl_str( sig ), ctx->state, fd_snapct_state_str( ctx->state ) ));
2098 0 : }
2099 0 : return;
2100 0 : }
2101 :
2102 0 : int full, file;
2103 0 : switch( ctx->state ) {
2104 : /* Expected cases, fall through below */
2105 0 : case FD_SNAPCT_STATE_READING_FULL_FILE: full = 1; file = 1; break;
2106 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE: full = 0; file = 1; break;
2107 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP: full = 1; file = 0; break;
2108 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP: full = 0; file = 0; break;
2109 :
2110 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET:
2111 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET:
2112 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET:
2113 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET:
2114 : /* We are waiting for a reset to fully propagate through the
2115 : pipeline, just throw away any trailing data frags. */
2116 0 : return;
2117 :
2118 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
2119 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
2120 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
2121 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
2122 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE:
2123 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
2124 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE:
2125 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
2126 : /* Based on previously received data frags, we expected that the
2127 : current full / incremental snapshot was finished, but then we
2128 : received additional data frags. Unsafe to continue so throw
2129 : away the whole snapshot. Do not publish MSG_CTRL_ERROR here:
2130 : data forwarding is already complete, and the state machine
2131 : will publish MSG_CTRL_FAIL on the next tick. */
2132 0 : if( !ctx->malformed ) {
2133 0 : ctx->malformed = 1;
2134 0 : FD_LOG_WARNING(( "complete snapshot loaded but read %lu extra bytes", sz ));
2135 0 : }
2136 0 : return;
2137 :
2138 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS:
2139 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL:
2140 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS:
2141 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL:
2142 0 : case FD_SNAPCT_STATE_SHUTDOWN:
2143 0 : default:
2144 0 : FD_LOG_ERR(( "invalid data frag in state %s", fd_snapct_state_str( ctx->state ) ));
2145 0 : return;
2146 0 : }
2147 :
2148 0 : if( FD_UNLIKELY( full && ctx->metrics.full.bytes_total==0UL ) ) {
2149 0 : if( !ctx->malformed ) {
2150 0 : ctx->malformed = 1;
2151 0 : FD_LOG_WARNING(( "received data frag for full snapshot with zero bytes_total" ));
2152 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL );
2153 0 : }
2154 0 : return;
2155 0 : }
2156 0 : if( FD_UNLIKELY( !full && ctx->metrics.incremental.bytes_total==0UL ) ) {
2157 0 : if( !ctx->malformed ) {
2158 0 : ctx->malformed = 1;
2159 0 : FD_LOG_WARNING(( "received data frag for incremental snapshot with zero bytes_total" ));
2160 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL );
2161 0 : }
2162 0 : return;
2163 0 : }
2164 :
2165 0 : if( full ) ctx->metrics.full.bytes_read += sz;
2166 0 : else ctx->metrics.incremental.bytes_read += sz;
2167 :
2168 0 : if( !file && -1!=ctx->local_out.dir_fd ) {
2169 0 : uchar const * data = fd_chunk_to_laddr_const( ctx->snapld_in_mem, chunk );
2170 0 : int fd = full ? ctx->local_out.full_snapshot_fd : ctx->local_out.incremental_snapshot_fd;
2171 0 : ulong written_sz = 0;
2172 0 : while( written_sz<sz ) {
2173 0 : long result = write( fd, data+written_sz, sz-written_sz );
2174 0 : if( FD_UNLIKELY( -1==result && errno==EINTR ) ) continue;
2175 0 : if( FD_UNLIKELY( -1==result && errno==ENOSPC ) ) {
2176 0 : FD_LOG_ERR(( "Out of disk space when writing out snapshot data to `%s`", ctx->config.snapshots_path ));
2177 0 : } else if( FD_UNLIKELY( 0L>result ) ) {
2178 0 : FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
2179 0 : } else if( FD_UNLIKELY( 0L==result ) ) {
2180 0 : FD_LOG_ERR(( "write() returned 0 for non-zero count" ));
2181 0 : }
2182 :
2183 0 : written_sz += (ulong)result;
2184 0 : }
2185 0 : if( full ) ctx->metrics.full.bytes_written += sz;
2186 0 : else ctx->metrics.incremental.bytes_written += sz;
2187 0 : }
2188 :
2189 0 : if( FD_UNLIKELY( ( full && ctx->metrics.full.bytes_read > ctx->metrics.full.bytes_total ) ||
2190 0 : (!full && ctx->metrics.incremental.bytes_read > ctx->metrics.incremental.bytes_total ) ) ) {
2191 0 : if( !ctx->malformed ) {
2192 0 : ctx->malformed = 1;
2193 0 : FD_LOG_WARNING(( "expected %s snapshot size of %lu bytes but read %lu bytes",
2194 0 : full ? "full" : "incremental",
2195 0 : full ? ctx->metrics.full.bytes_total : ctx->metrics.incremental.bytes_total,
2196 0 : full ? ctx->metrics.full.bytes_read : ctx->metrics.incremental.bytes_read ));
2197 0 : fd_stem_publish( stem, ctx->out_ld.idx, FD_SNAPSHOT_MSG_CTRL_ERROR, 0UL, 0UL, 0UL, 0UL, 0UL );
2198 0 : }
2199 0 : return;
2200 0 : }
2201 0 : }
2202 :
2203 : static void
2204 : ctrl_ack_frag( fd_snapct_tile_t * ctx,
2205 0 : ulong sig ) {
2206 0 : switch( sig ) {
2207 0 : case FD_SNAPSHOT_MSG_CTRL_FAIL:
2208 0 : if( FD_LIKELY( ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_RESET ||
2209 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_FULL_FILE_RESET ||
2210 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_RESET ||
2211 0 : ctx->state==FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_RESET ) ) {
2212 0 : ctx->flush_ack++;
2213 0 : FD_TEST( ctx->flush_ack <= ctx->flush_ack_cnt );
2214 0 : } else {
2215 0 : FD_LOG_ERR(( "unexpected control frag %lu (%s) in state %d (%s)", sig, fd_ssctrl_msg_ctrl_str( sig ), ctx->state, fd_snapct_state_str( ctx->state ) ));
2216 0 : }
2217 0 : return;
2218 :
2219 0 : case FD_SNAPSHOT_MSG_CTRL_SHUTDOWN:
2220 0 : return;
2221 :
2222 0 : case FD_SNAPSHOT_MSG_CTRL_ERROR:
2223 0 : switch( ctx->state ) {
2224 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
2225 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
2226 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE:
2227 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
2228 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
2229 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
2230 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
2231 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
2232 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE:
2233 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
2234 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
2235 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
2236 : /* Do not publish MSG_CTRL_ERROR: the error originated
2237 : downstream, so re-publishing would be redundant.
2238 : MSG_CTRL_FAIL follows on the next state machine tick. */
2239 0 : FD_LOG_WARNING(( "received error from downstream tile while in state %s",
2240 0 : fd_snapct_state_str( ctx->state ) ));
2241 0 : ctx->malformed = 1;
2242 0 : break;
2243 0 : default:
2244 0 : break;
2245 0 : }
2246 0 : return;
2247 :
2248 0 : default:
2249 0 : break;
2250 0 : }
2251 0 : if( process_ctrl_ack( ctx, sig ) ) {
2252 0 : FD_LOG_ERR(( "unexpected control frag %lu (%s) in state %d (%s)", sig, fd_ssctrl_msg_ctrl_str( sig ), ctx->state, fd_snapct_state_str( ctx->state ) ));
2253 0 : }
2254 0 : }
2255 :
2256 : static int
2257 : returnable_frag( fd_snapct_tile_t * ctx,
2258 : ulong in_idx,
2259 : ulong seq FD_PARAM_UNUSED,
2260 : ulong sig,
2261 : ulong chunk,
2262 : ulong sz,
2263 : ulong ctl FD_PARAM_UNUSED,
2264 : ulong tsorig FD_PARAM_UNUSED,
2265 : ulong tspub FD_PARAM_UNUSED,
2266 0 : fd_stem_context_t * stem ) {
2267 0 : if( FD_LIKELY( ctx->in_kind[ in_idx ]==IN_KIND_GOSSIP ) ) {
2268 0 : gossip_frag( ctx, sig, sz, chunk );
2269 0 : } else if( ctx->in_kind[ in_idx ]==IN_KIND_SNAPLD ) {
2270 0 : snapld_frag( ctx, sig, sz, chunk, stem );
2271 0 : } else if( ctx->in_kind[ in_idx ]==IN_KIND_ACK ) {
2272 0 : ctrl_ack_frag( ctx, sig );
2273 0 : } else FD_LOG_ERR(( "invalid in_kind %lu %u", in_idx, (uint)ctx->in_kind[ in_idx ] ));
2274 0 : return 0;
2275 0 : }
2276 :
2277 : static void
2278 : privileged_init( fd_topo_t const * topo,
2279 0 : fd_topo_tile_t const * tile ) {
2280 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
2281 :
2282 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
2283 0 : fd_snapct_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapct_tile_t), sizeof(fd_snapct_tile_t) );
2284 0 : void * _ssping = FD_SCRATCH_ALLOC_APPEND( l, fd_ssping_align(), fd_ssping_footprint( TOTAL_PEERS_MAX ) );
2285 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(gossip_ci_entry_t), sizeof(gossip_ci_entry_t)*GOSSIP_PEERS_MAX );
2286 0 : FD_SCRATCH_ALLOC_APPEND( l, gossip_ci_map_align(), gossip_ci_map_footprint( gossip_ci_map_chain_cnt_est( GOSSIP_PEERS_MAX ) ) );
2287 0 : void * _ssresolver = FD_SCRATCH_ALLOC_APPEND( l, fd_http_resolver_align(), fd_http_resolver_footprint( SERVER_PEERS_MAX ) );
2288 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_sspeer_selector_align(), fd_sspeer_selector_footprint( TOTAL_PEERS_MAX ) );
2289 0 : FD_SCRATCH_ALLOC_APPEND( l, blacklist_pool_align(), blacklist_pool_footprint( TOTAL_PEERS_MAX ) );
2290 0 : FD_SCRATCH_ALLOC_APPEND( l, blacklist_map_align(), blacklist_map_footprint( blacklist_map_chain_cnt_est( TOTAL_PEERS_MAX ) ) );
2291 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_adns_align(), fd_adns_footprint( ADNS_REQS_MAX ) );
2292 :
2293 0 : FD_TEST( fd_rng_secure( &ctx->ssping_seed, sizeof(ctx->ssping_seed) ) );
2294 0 : FD_TEST( fd_rng_secure( &ctx->gossip_ci_seed, sizeof(ctx->gossip_ci_seed) ) );
2295 0 : FD_TEST( fd_rng_secure( &ctx->selector_seed, sizeof(ctx->selector_seed) ) );
2296 0 : FD_TEST( fd_rng_secure( &ctx->blacklist_seed, sizeof(ctx->blacklist_seed) ) );
2297 :
2298 : /* The resolver only needs a trust store if some server is https. */
2299 0 : int any_https = 0;
2300 0 : for( ulong i=0UL; i<tile->snapct.sources.servers_cnt; i++ ) {
2301 0 : char hostname[ FD_FQDN_BUF_MAX ];
2302 0 : ushort port;
2303 0 : int is_https;
2304 0 : fd_dns_peer_parse( tile->snapct.sources.servers[ i ], "snapshots.sources.servers", hostname, &port, &is_https );
2305 0 : any_https |= is_https;
2306 0 : }
2307 :
2308 0 : ctx->ssping = NULL;
2309 0 : int epoll_fd = FD_WAKER_INNER_FD( tile->waker_client_idx );
2310 0 : if( FD_LIKELY( download_enabled( tile ) ) ) ctx->ssping = fd_ssping_join( fd_ssping_new( _ssping, TOTAL_PEERS_MAX, ctx->ssping_seed, on_ping, ctx, epoll_fd ) );
2311 0 : if( FD_LIKELY( tile->snapct.sources.servers_cnt ) ) ctx->ssresolver = fd_http_resolver_join( fd_http_resolver_new( _ssresolver, SERVER_PEERS_MAX, snapct_needs_incr( tile ), any_https, on_resolve, ctx, epoll_fd ) );
2312 0 : else ctx->ssresolver = NULL;
2313 :
2314 0 : ctx->netdb_fds->etc_hosts = -1;
2315 0 : ctx->netdb_fds->etc_resolv_conf = -1;
2316 0 : if( FD_LIKELY( download_enabled( tile ) ) ) FD_TEST( fd_netdb_open_fds( ctx->netdb_fds ) );
2317 0 : ctx->resolved_entrypoints_cnt = 0UL;
2318 0 : ctx->resolved_servers_cnt = 0UL;
2319 :
2320 0 : fd_snap_pool_layout_t layout = fd_snap_pool_layout(
2321 0 : tile->snapct.max_full_snapshots_to_keep,
2322 0 : tile->snapct.max_incremental_snapshots_to_keep,
2323 0 : snapct_needs_incr( tile ),
2324 0 : download_enabled( tile ) );
2325 0 : uint snap_full_max = (uint)layout.full_max;
2326 0 : uint snap_incr_max = (uint)layout.incr_max;
2327 0 : uint retained_snap_max = (uint)layout.retained_max;
2328 0 : uint scratch_full_cnt = (uint)layout.scratch_full;
2329 0 : uint scratch_incr_cnt = (uint)layout.scratch_incr;
2330 0 : uint snap_max = (uint)layout.max;
2331 :
2332 0 : ulong full_slot = ULONG_MAX;
2333 0 : ulong incremental_slot = ULONG_MAX;
2334 0 : int full_is_zstd = 0;
2335 0 : int incremental_is_zstd = 0;
2336 0 : char full_path[ PATH_MAX ] = {0};
2337 0 : char incremental_path[ PATH_MAX ] = {0};
2338 0 : uchar full_snapshot_hash[ FD_HASH_FOOTPRINT ] = {0};
2339 0 : uchar incremental_snapshot_hash[ FD_HASH_FOOTPRINT ] = {0};
2340 0 : int found = 0==fd_ssarchive_latest_best( tile->snapct.snapshots_path,
2341 0 : tile->snapct.incremental_snapshots,
2342 0 : fd_wfs_configured( tile->snapct.wfs_slot,
2343 0 : tile->snapct.wfs_hash_is_zero,
2344 0 : (ulong)tile->snapct.wfs_shred_version )
2345 0 : ? tile->snapct.wfs_slot : 0UL,
2346 0 : &full_slot,
2347 0 : &incremental_slot,
2348 0 : full_path,
2349 0 : incremental_path,
2350 0 : &full_is_zstd,
2351 0 : &incremental_is_zstd,
2352 0 : full_snapshot_hash,
2353 0 : incremental_snapshot_hash );
2354 :
2355 0 : if( FD_UNLIKELY( !found ) ) {
2356 0 : if( FD_UNLIKELY( !download_enabled( tile ) ) ) {
2357 0 : FD_LOG_ERR(( "No snapshots found in `%s` and no download sources are enabled. "
2358 0 : "Please enable downloading via [snapshots.sources.gossip] or [snapshots.sources.servers] and restart.", tile->snapct.snapshots_path ));
2359 0 : }
2360 0 : ctx->local_in.full_snapshot_slot = ULONG_MAX;
2361 0 : ctx->local_in.incremental_snapshot_slot = ULONG_MAX;
2362 0 : ctx->local_in.full_snapshot_size = 0UL;
2363 0 : ctx->local_in.incremental_snapshot_size = 0UL;
2364 0 : ctx->local_in.full_snapshot_zstd = 0;
2365 0 : ctx->local_in.incremental_snapshot_zstd = 0;
2366 0 : fd_cstr_fini( ctx->local_in.full_snapshot_path );
2367 0 : fd_cstr_fini( ctx->local_in.incremental_snapshot_path );
2368 0 : fd_memset( ctx->local_in.full_snapshot_hash, 0, FD_HASH_FOOTPRINT );
2369 0 : fd_memset( ctx->local_in.incremental_snapshot_hash, 0, FD_HASH_FOOTPRINT );
2370 0 : } else {
2371 0 : FD_TEST( full_slot!=ULONG_MAX );
2372 :
2373 0 : ctx->local_in.full_snapshot_slot = full_slot;
2374 0 : ctx->local_in.incremental_snapshot_slot = incremental_slot;
2375 0 : ctx->local_in.full_snapshot_zstd = full_is_zstd;
2376 0 : ctx->local_in.incremental_snapshot_zstd = incremental_is_zstd;
2377 :
2378 0 : fd_cstr_ncpy( ctx->local_in.full_snapshot_path, full_path, PATH_MAX );
2379 0 : fd_memcpy( ctx->local_in.full_snapshot_hash, full_snapshot_hash, FD_HASH_FOOTPRINT );
2380 0 : struct stat full_stat;
2381 0 : if( FD_UNLIKELY( -1==stat( ctx->local_in.full_snapshot_path, &full_stat ) ) ) FD_LOG_ERR(( "stat() failed `%s` (%i-%s)", full_path, errno, fd_io_strerror( errno ) ));
2382 0 : if( FD_UNLIKELY( !S_ISREG( full_stat.st_mode ) ) ) FD_LOG_ERR(( "full snapshot path `%s` is not a regular file", full_path ));
2383 0 : ctx->local_in.full_snapshot_size = (ulong)full_stat.st_size;
2384 :
2385 0 : if( FD_LIKELY( incremental_slot!=ULONG_MAX ) ) {
2386 0 : fd_cstr_ncpy( ctx->local_in.incremental_snapshot_path, incremental_path, PATH_MAX );
2387 0 : fd_memcpy( ctx->local_in.incremental_snapshot_hash, incremental_snapshot_hash, FD_HASH_FOOTPRINT );
2388 0 : struct stat incremental_stat;
2389 0 : if( FD_UNLIKELY( -1==stat( ctx->local_in.incremental_snapshot_path, &incremental_stat ) ) ) FD_LOG_ERR(( "stat() failed `%s` (%i-%s)", incremental_path, errno, fd_io_strerror( errno ) ));
2390 0 : if( FD_UNLIKELY( !S_ISREG( incremental_stat.st_mode ) ) ) FD_LOG_ERR(( "incremental snapshot path `%s` is not a regular file", incremental_path ));
2391 0 : ctx->local_in.incremental_snapshot_size = (ulong)incremental_stat.st_size;
2392 0 : } else {
2393 0 : ctx->local_in.incremental_snapshot_size = 0UL;
2394 0 : fd_cstr_fini( ctx->local_in.incremental_snapshot_path );
2395 0 : }
2396 0 : }
2397 :
2398 0 : ctx->local_out.dir_fd = -1;
2399 0 : ctx->local_out.full_snapshot_fd = -1;
2400 0 : ctx->local_out.incremental_snapshot_fd = -1;
2401 0 : fd_cstr_fini( ctx->local_out.full_snapshot_name );
2402 0 : fd_cstr_fini( ctx->local_out.incremental_snapshot_name );
2403 0 : if( FD_LIKELY( download_enabled( tile ) ) ) {
2404 0 : ctx->local_out.dir_fd = open( tile->snapct.snapshots_path, O_RDONLY|O_DIRECTORY|O_CLOEXEC );
2405 0 : if( FD_UNLIKELY( -1==ctx->local_out.dir_fd ) )
2406 0 : FD_LOG_ERR(( "open(%s) failed (%i-%s)", tile->snapct.snapshots_path, errno, fd_io_strerror( errno ) ));
2407 :
2408 0 : FD_TEST( snap_max && snap_max<=FD_SNAP_MAX );
2409 0 : fd_backup_inode_t pool[ FD_SNAP_MAX ] = {0};
2410 0 : if( FD_UNLIKELY( snap_max<FD_SNAP_MAX && -1!=fcntl( FD_SNAP_FD( snap_max ), F_GETFD ) ) ) {
2411 0 : FD_LOG_ERR(( "inherited snapshot pool is larger than the expected %u slots", snap_max ));
2412 0 : }
2413 0 : fd_snap_pool_recover( ctx->local_out.dir_fd, tile->snapct.snapshots_path, pool, snap_max );
2414 0 : if( FD_LIKELY( snap_full_max ) ) {
2415 0 : uint full_idx = snapshot_pool_select( pool, 0U, snap_full_max );
2416 0 : FD_TEST( full_idx!=UINT_MAX );
2417 0 : ctx->local_out.full_snapshot_fd = FD_SNAP_FD( full_idx );
2418 0 : fd_cstr_ncpy( ctx->local_out.full_snapshot_name, pool[ full_idx ].name, FD_SNAP_NAME_MAX );
2419 0 : } else {
2420 0 : uint full_idx = retained_snap_max; /* scratch full slot */
2421 0 : FD_TEST( scratch_full_cnt && full_idx<snap_max );
2422 0 : ctx->local_out.full_snapshot_fd = FD_SNAP_FD( full_idx );
2423 0 : fd_cstr_ncpy( ctx->local_out.full_snapshot_name, pool[ full_idx ].name, FD_SNAP_NAME_MAX );
2424 0 : }
2425 :
2426 0 : int need_incr_fd = snapct_needs_incr( tile );
2427 0 : if( FD_LIKELY( need_incr_fd && snap_incr_max ) ) {
2428 0 : uint incr_idx = snapshot_pool_select( pool, snap_full_max, retained_snap_max );
2429 0 : FD_TEST( incr_idx!=UINT_MAX );
2430 0 : ctx->local_out.incremental_snapshot_fd = FD_SNAP_FD( incr_idx );
2431 0 : fd_cstr_ncpy( ctx->local_out.incremental_snapshot_name, pool[ incr_idx ].name, FD_SNAP_NAME_MAX );
2432 0 : } else if( FD_LIKELY( need_incr_fd ) ) {
2433 0 : uint incr_idx = retained_snap_max + scratch_full_cnt; /* scratch incremental slot */
2434 0 : FD_TEST( scratch_incr_cnt && incr_idx<snap_max );
2435 0 : ctx->local_out.incremental_snapshot_fd = FD_SNAP_FD( incr_idx );
2436 0 : fd_cstr_ncpy( ctx->local_out.incremental_snapshot_name, pool[ incr_idx ].name, FD_SNAP_NAME_MAX );
2437 0 : }
2438 :
2439 0 : FD_TEST( ctx->local_out.full_snapshot_fd!=ctx->local_out.incremental_snapshot_fd );
2440 0 : }
2441 :
2442 0 : if( FD_LIKELY( fd_sandbox_gettid()==fd_sandbox_getpid() ) ) {
2443 0 : for( uint i=0U; i<snap_max; i++ ) {
2444 0 : int fd = FD_SNAP_FD( i );
2445 0 : if( fd==ctx->local_out.full_snapshot_fd || fd==ctx->local_out.incremental_snapshot_fd ) continue;
2446 0 : if( FD_UNLIKELY( close( fd ) ) )
2447 0 : FD_LOG_ERR(( "close(snapshot pool fd %d) failed (%i-%s)", fd, errno, fd_io_strerror( errno ) ));
2448 0 : }
2449 0 : }
2450 0 : }
2451 :
2452 : static inline fd_snapct_out_link_t
2453 : out1( fd_topo_t const * topo,
2454 : fd_topo_tile_t const * tile,
2455 0 : char const * name ) {
2456 0 : ulong idx = ULONG_MAX;
2457 :
2458 0 : for( ulong i=0UL; i<tile->out_cnt; i++ ) {
2459 0 : fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ i ] ];
2460 0 : if( !strcmp( link->name, name ) ) {
2461 0 : if( FD_UNLIKELY( idx!=ULONG_MAX ) ) FD_LOG_ERR(( "tile %s:%lu had multiple output links named %s but expected one", tile->name, tile->kind_id, name ));
2462 0 : idx = i;
2463 0 : }
2464 0 : }
2465 :
2466 0 : if( FD_UNLIKELY( idx==ULONG_MAX ) ) return (fd_snapct_out_link_t){ .idx = ULONG_MAX, .mem = NULL, .chunk0 = 0, .wmark = 0, .chunk = 0, .mtu = 0 };
2467 :
2468 0 : ulong mtu = topo->links[ tile->out_link_id[ idx ] ].mtu;
2469 0 : if( FD_UNLIKELY( mtu==0UL ) ) return (fd_snapct_out_link_t){ .idx = idx, .mem = NULL, .chunk0 = ULONG_MAX, .wmark = ULONG_MAX, .chunk = ULONG_MAX, .mtu = mtu };
2470 :
2471 0 : void * mem = topo->workspaces[ topo->objs[ topo->links[ tile->out_link_id[ idx ] ].dcache_obj_id ].wksp_id ].wksp;
2472 0 : ulong chunk0 = fd_dcache_compact_chunk0( mem, topo->links[ tile->out_link_id[ idx ] ].dcache );
2473 0 : ulong wmark = fd_dcache_compact_wmark ( mem, topo->links[ tile->out_link_id[ idx ] ].dcache, mtu );
2474 0 : return (fd_snapct_out_link_t){ .idx = idx, .mem = mem, .chunk0 = chunk0, .wmark = wmark, .chunk = chunk0, .mtu = mtu };
2475 0 : }
2476 :
2477 : static void
2478 : unprivileged_init( fd_topo_t const * topo,
2479 0 : fd_topo_tile_t const * tile ) {
2480 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
2481 :
2482 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
2483 0 : fd_snapct_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_snapct_tile_t), sizeof(fd_snapct_tile_t) );
2484 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_ssping_align(), fd_ssping_footprint( TOTAL_PEERS_MAX ) );
2485 0 : void * _ci_table = FD_SCRATCH_ALLOC_APPEND( l, alignof(gossip_ci_entry_t), sizeof(gossip_ci_entry_t) * GOSSIP_PEERS_MAX );
2486 0 : void * _ci_map = FD_SCRATCH_ALLOC_APPEND( l, gossip_ci_map_align(), gossip_ci_map_footprint( gossip_ci_map_chain_cnt_est( GOSSIP_PEERS_MAX ) ) );
2487 0 : FD_SCRATCH_ALLOC_APPEND( l, fd_http_resolver_align(), fd_http_resolver_footprint( SERVER_PEERS_MAX ) );
2488 0 : void * _selector = FD_SCRATCH_ALLOC_APPEND( l, fd_sspeer_selector_align(), fd_sspeer_selector_footprint( TOTAL_PEERS_MAX ) );
2489 0 : void * _bl_pool = FD_SCRATCH_ALLOC_APPEND( l, blacklist_pool_align(), blacklist_pool_footprint( TOTAL_PEERS_MAX ) );
2490 0 : void * _bl_map = FD_SCRATCH_ALLOC_APPEND( l, blacklist_map_align(), blacklist_map_footprint( blacklist_map_chain_cnt_est( TOTAL_PEERS_MAX ) ) );
2491 0 : void * _adns = FD_SCRATCH_ALLOC_APPEND( l, fd_adns_align(), fd_adns_footprint( ADNS_REQS_MAX ) );
2492 :
2493 0 : ctx->config = tile->snapct;
2494 0 : ctx->gossip_enabled = gossip_enabled( tile );
2495 0 : ctx->download_enabled = download_enabled( tile );
2496 0 : ctx->wfs_is_configured = fd_wfs_configured( ctx->config.wfs_slot, ctx->config.wfs_hash_is_zero, (ulong)ctx->config.wfs_shred_version );
2497 0 : ctx->load_incremental = ctx->config.incremental_snapshots;
2498 :
2499 0 : ctx->waker_client_idx = tile->waker_client_idx;
2500 0 : FD_TEST( ctx->waker_client_idx!=ULONG_MAX );
2501 0 : ctx->waker_fseq = fd_fseq_join( fd_topo_obj_laddr( topo, tile->waker_fseq_obj_id ) );
2502 0 : FD_TEST( ctx->waker_fseq );
2503 0 : fd_clock_tile_init( ctx->clock );
2504 :
2505 0 : ctx->io_due_nanos = 0L;
2506 :
2507 0 : ctx->adns = NULL;
2508 0 : if( FD_LIKELY( ctx->download_enabled ) ) {
2509 0 : ctx->adns = fd_adns_join( fd_adns_new( _adns, ADNS_REQS_MAX, FD_WAKER_INNER_FD( tile->waker_client_idx ) ) );
2510 0 : FD_TEST( ctx->adns );
2511 0 : for( ulong i=0UL; i<ctx->config.sources.servers_cnt; i++ ) {
2512 0 : fd_dns_peer_parse( ctx->config.sources.servers[ i ], "snapshots.sources.servers", ctx->dns_servers[ i ].hostname, &ctx->dns_servers[ i ].port, &ctx->dns_servers[ i ].is_https );
2513 0 : ctx->dns_servers[ i ].resolved = 0;
2514 0 : FD_TEST( !fd_adns_resolve( ctx->adns, ctx->dns_servers[ i ].hostname, i ) );
2515 0 : ctx->dns_servers[ i ].retry_nanos = 0L; /* in flight */
2516 0 : }
2517 0 : for( ulong i=0UL; i<ctx->config.entrypoints_cnt; i++ ) {
2518 0 : fd_dns_peer_parse( ctx->config.entrypoints[ i ], "gossip.entrypoints", ctx->dns_entrypoints[ i ].hostname, &ctx->dns_entrypoints[ i ].port, NULL );
2519 0 : ctx->dns_entrypoints[ i ].resolved = 0;
2520 0 : FD_TEST( !fd_adns_resolve( ctx->adns, ctx->dns_entrypoints[ i ].hostname, DNS_REQ_ID_ENTRYPOINT|i ) );
2521 0 : ctx->dns_entrypoints[ i ].retry_nanos = 0L;
2522 0 : }
2523 0 : }
2524 :
2525 0 : ctx->selector = fd_sspeer_selector_join( fd_sspeer_selector_new( _selector, TOTAL_PEERS_MAX, ctx->selector_seed ) );
2526 0 : ctx->blacklist_pool = blacklist_pool_join( blacklist_pool_new( _bl_pool, TOTAL_PEERS_MAX ) );
2527 0 : ctx->blacklist_map = blacklist_map_join( blacklist_map_new( _bl_map, blacklist_map_chain_cnt_est( TOTAL_PEERS_MAX ), ctx->blacklist_seed ) );
2528 :
2529 0 : if( FD_UNLIKELY( !ctx->config.incremental_snapshots ) ) {
2530 0 : if( ctx->wfs_is_configured ) {
2531 0 : FD_LOG_NOTICE(( "incremental snapshots disabled via [snapshots.incremental_snapshots] "
2532 0 : "but may be auto-enabled to reach WFS target slot %lu",
2533 0 : ctx->config.wfs_slot ));
2534 0 : } else {
2535 0 : FD_LOG_WARNING(( "incremental snapshots disabled via [snapshots.incremental_snapshots]." ));
2536 0 : }
2537 0 : }
2538 :
2539 0 : ctx->state = FD_SNAPCT_STATE_INIT;
2540 0 : ctx->malformed = 0;
2541 0 : ctx->load_complete = 0;
2542 0 : FD_CHECK_ERR( ctx->config.wait_for_peers_timeout_nanos>0L, "snapct wait_for_peers_timeout_nanos must be positive" );
2543 0 : ctx->deadline_nanos = fd_clock_tile_now( ctx->clock ) + ctx->config.wait_for_peers_timeout_nanos;
2544 0 : ctx->flush_ack = 0;
2545 0 : ctx->flush_ack_cnt = 0;
2546 0 : ctx->peer.addr.l = 0UL;
2547 :
2548 0 : fd_memset( ctx->http_full_snapshot_name, 0, PATH_MAX );
2549 0 : fd_memset( ctx->http_incr_snapshot_name, 0, PATH_MAX );
2550 :
2551 0 : ctx->gossip_in_mem = NULL;
2552 0 : int has_snapld_dc = 0, ack_cnt = 0;
2553 0 : FD_TEST( tile->in_cnt<=FD_TOPO_MAX_TILE_IN_LINKS );
2554 0 : for( ulong i=0UL; i<(tile->in_cnt); i++ ) {
2555 0 : fd_topo_link_t const * in_link = &topo->links[ tile->in_link_id[ i ] ];
2556 0 : if( 0==strcmp( in_link->name, "gossip_ciaddr" ) ) {
2557 0 : ctx->in_kind[ i ] = IN_KIND_GOSSIP;
2558 0 : ctx->gossip_in_mem = topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ].wksp;
2559 0 : } else if( 0==strcmp( in_link->name, "snapld_dc" ) ) {
2560 0 : ctx->in_kind[ i ] = IN_KIND_SNAPLD;
2561 0 : ctx->snapld_in_mem = topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ].wksp;
2562 0 : FD_TEST( !has_snapld_dc );
2563 0 : has_snapld_dc = 1;
2564 0 : } else if( 0==strcmp( in_link->name, "snapin_ct" ) ) {
2565 0 : ctx->in_kind[ i ] = IN_KIND_ACK;
2566 0 : ack_cnt++;
2567 0 : }
2568 0 : }
2569 0 : FD_TEST( has_snapld_dc && ack_cnt>0 );
2570 0 : ctx->flush_ack_cnt = ack_cnt + 1; /* +1 for snapld (acks via snapld_dc) */
2571 0 : FD_TEST( ctx->gossip_enabled==(ctx->gossip_in_mem!=NULL) );
2572 :
2573 0 : ctx->predicted_incremental.full_slot = FD_SSPEER_SLOT_UNKNOWN;
2574 0 : ctx->predicted_incremental.slot = FD_SSPEER_SLOT_UNKNOWN;
2575 0 : ctx->predicted_incremental.pending = 0;
2576 0 : ctx->predicted_incremental.committed = 0;
2577 0 : fd_memset( &ctx->predicted_incremental.peer, 0, sizeof(fd_sspeer_t) );
2578 :
2579 0 : fd_memset( &ctx->metrics, 0, sizeof(ctx->metrics) );
2580 :
2581 0 : fd_memset( _ci_table, 0, sizeof(gossip_ci_entry_t) * GOSSIP_PEERS_MAX );
2582 0 : ctx->gossip.ci_table = _ci_table;
2583 0 : ctx->gossip.ci_map = gossip_ci_map_join( gossip_ci_map_new( _ci_map, gossip_ci_map_chain_cnt_est( GOSSIP_PEERS_MAX ), ctx->gossip_ci_seed ) );
2584 0 : ctx->gossip.allowed_cnt = 0UL;
2585 0 : ctx->gossip.saturated = !ctx->gossip_enabled;
2586 :
2587 0 : if( FD_UNLIKELY( tile->out_cnt<2UL || tile->out_cnt>3UL ) ) FD_LOG_ERR(( "tile `" NAME "` has %lu outs, expected 2-3", tile->out_cnt ));
2588 0 : ctx->out_ld = out1( topo, tile, "snapct_ld" );
2589 0 : ctx->out_gui = out1( topo, tile, "snapct_gui" );
2590 0 : ctx->out_rp = out1( topo, tile, "snapct_repr" );
2591 0 : }
2592 :
2593 0 : #define STEM_BURST 6UL
2594 :
2595 0 : #define STEM_LAZY (128L*3000L)
2596 :
2597 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_snapct_tile_t
2598 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_snapct_tile_t)
2599 :
2600 0 : #define STEM_CALLBACK_SHOULD_SHUTDOWN should_shutdown
2601 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
2602 0 : #define STEM_CALLBACK_NEXT_DEADLINE next_deadline
2603 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
2604 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
2605 0 : #define STEM_CALLBACK_RETURNABLE_FRAG returnable_frag
2606 :
2607 : #include "../../disco/stem/fd_stem.c"
2608 :
2609 :
2610 : #ifndef FD_TILE_TEST
2611 : static ulong
2612 0 : max_event_sz( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
2613 0 : return sizeof(fd_event_metrics_trigger_t);
2614 0 : }
2615 :
2616 : fd_topo_run_tile_t fd_tile_snapct = {
2617 : .name = NAME,
2618 : .max_event_sz = max_event_sz,
2619 : .rlimit_file_cnt_fn = rlimit_file_cnt,
2620 : .populate_allowed_seccomp = populate_allowed_seccomp,
2621 : .populate_allowed_fds = populate_allowed_fds,
2622 : .scratch_align = scratch_align,
2623 : .scratch_footprint = scratch_footprint,
2624 : .privileged_init = privileged_init,
2625 : .unprivileged_init = unprivileged_init,
2626 : .run = stem_run,
2627 : .keep_host_networking = 1,
2628 : .allow_connect = 1,
2629 : .allow_renameat = 1,
2630 : };
2631 : #endif
2632 :
2633 : #undef NAME
|
