Line data Source code
1 : #define _GNU_SOURCE
2 : #include "../../shared/fd_config.h"
3 : #include "../../shared/fd_bootinfo.h"
4 : #include "../../shared/fd_action.h"
5 : #include "../../platform/fd_sys_util.h"
6 : #include "../../../disco/topo/fd_topo.h"
7 : #include "../../../disco/wait_info/fd_wait_info.h"
8 : #include "../../../util/pod/fd_pod.h"
9 : #include "../../../util/rng/fd_rng.h"
10 : #include "generated/wait_seccomp.h"
11 :
12 : #include <errno.h>
13 : #include <poll.h>
14 : #include <signal.h>
15 : #include <unistd.h>
16 : #include <sys/resource.h>
17 : #include <sys/syscall.h>
18 : #include <linux/capability.h>
19 :
20 : /* ANSI escape codes for TUI rendering. */
21 :
22 0 : #define ANSI_RESET "\033[0m"
23 0 : #define ANSI_DIM "\033[2m"
24 0 : #define ANSI_RED "\033[31m"
25 0 : #define ANSI_GREEN "\033[32m"
26 : #define ANSI_CLEARLN "\033[K"
27 :
28 : /* Check result codes. */
29 :
30 0 : #define CHECK_FAIL 0
31 0 : #define CHECK_PASS 1
32 0 : #define CHECK_SKIP 2
33 :
34 : /* Exit 0 once safe, 128+signum if a signal arrives first (the shell
35 : convention, as in run.c), and 1 for any other failure. A stop
36 : script also needs to tell "already stopped" from "not safe yet". */
37 :
38 0 : #define WAIT_EXIT_GONE 2 /* the validator exited while we were attached */
39 :
40 : /* Log at the ERR level without exiting, so the exit code stays ours.
41 : Mirrors the macro run.c defines for the same reason. */
42 :
43 0 : #define FD_LOG_ERR_NOEXIT(a) do { \
44 0 : long _fd_log_msg_now = fd_log_wallclock(); \
45 0 : fd_log_private_1( 4, _fd_log_msg_now, __FILE__, __LINE__, __func__, \
46 0 : fd_log_private_0 a ); \
47 0 : } while(0)
48 :
49 : /* frame_buf accumulates a frame for one atomic terminal write,
50 : frame_len its used bytes (following watch.c pattern). */
51 :
52 : static char frame_buf[ 8192 ];
53 : static ulong frame_len;
54 :
55 : /* g_tty is 1 when the status panel should be rendered. Resolved
56 : before entering the sandbox: isatty() issues ioctl(TCGETS), which
57 : the seccomp policy does not allow. The panel redraws in place, so
58 : it is only rendered with a tty and without --silent; elsewhere the
59 : log carries the outcome. */
60 :
61 : static int g_tty;
62 :
63 0 : #define PRINT(...) do { \
64 0 : ulong _print_len; \
65 0 : if( FD_UNLIKELY( !fd_cstr_printf_check( frame_buf+frame_len, \
66 0 : sizeof(frame_buf)-frame_len, &_print_len, \
67 0 : __VA_ARGS__ ) ) ) \
68 0 : FD_LOG_ERR(( "frame buffer overflow (%lu/%lu used)", frame_len, sizeof(frame_buf) )); \
69 0 : frame_len += _print_len; \
70 0 : } while(0)
71 :
72 : static void
73 0 : flush_frame( void ) {
74 0 : ulong written = 0UL;
75 0 : while( written<frame_len ) {
76 0 : long w = write( STDOUT_FILENO, frame_buf+written, frame_len-written );
77 0 : if( FD_UNLIKELY( -1L==w && (errno==EAGAIN || errno==EINTR) ) ) continue;
78 0 : else if( FD_UNLIKELY( -1L==w ) ) break;
79 0 : else if( FD_UNLIKELY( 0L==w ) ) break;
80 0 : written += (ulong)w;
81 0 : }
82 0 : frame_len = 0UL;
83 0 : }
84 :
85 : /* Number of lines rendered per frame. Must equal the number of
86 : render_check_line calls in render_status (currently 4: health,
87 : leader gap, snapshot, delinquent) plus 1 status line. */
88 :
89 : #define WAIT_CHECK_CNT 4UL
90 : #define LINES_PER_FRAME (WAIT_CHECK_CNT + 1UL)
91 :
92 : /* Aggregated check results passed to render_status. */
93 :
94 : struct wait_check {
95 : int result;
96 : char desc[ 256 ];
97 : };
98 :
99 : typedef struct wait_check wait_check_t;
100 :
101 : /* g_last_state holds the checks as of the last poll, so an exit that
102 : never found a safe window can say what it was waiting on. Formatted
103 : every poll but only ever printed once, on the way out. */
104 :
105 : static char g_last_state[ 1024 ] = "no checks completed";
106 :
107 : static void
108 : render_check_line( char const * label,
109 : int result,
110 0 : char const * desc ) {
111 0 : char const * icon;
112 0 : char const * desc_pre;
113 0 : char const * desc_post;
114 :
115 0 : switch( result ) {
116 0 : case CHECK_PASS:
117 0 : icon = ANSI_GREEN "\xe2\x9c\x93" ANSI_RESET; /* checkmark */
118 0 : desc_pre = "";
119 0 : desc_post = "";
120 0 : break;
121 0 : case CHECK_FAIL:
122 0 : icon = ANSI_RED "\xe2\x9c\x97" ANSI_RESET; /* x mark */
123 0 : desc_pre = "";
124 0 : desc_post = "";
125 0 : break;
126 0 : default: /* CHECK_SKIP */
127 0 : icon = ANSI_DIM "-" ANSI_RESET;
128 0 : desc_pre = ANSI_DIM;
129 0 : desc_post = ANSI_RESET;
130 0 : break;
131 0 : }
132 :
133 0 : PRINT( " %-13s%s %s%s%s" ANSI_CLEARLN "\n", label, icon, desc_pre, desc, desc_post );
134 0 : }
135 :
136 : static void
137 : render_status( wait_check_t const * health,
138 : wait_check_t const * leader,
139 : wait_check_t const * snap,
140 : wait_check_t const * delinquent,
141 : int all_pass,
142 : long elapsed_s,
143 0 : int first_frame ) {
144 0 : if( FD_UNLIKELY( !g_tty ) ) return;
145 :
146 0 : frame_len = 0UL;
147 :
148 : /* Hide cursor, disable line wrap, reposition if not first frame. */
149 :
150 0 : PRINT( "\033[?25l\033[?7l" );
151 0 : if( FD_UNLIKELY( !first_frame ) ) {
152 0 : PRINT( "\033[%luA\r", LINES_PER_FRAME );
153 0 : }
154 :
155 0 : render_check_line( "health", health->result, health->desc );
156 0 : render_check_line( "leader gap", leader->result, leader->desc );
157 0 : render_check_line( "snapshot", snap->result, snap->desc );
158 0 : render_check_line( "delinquent", delinquent->result, delinquent->desc );
159 :
160 : /* The passing frame omits the status line; success goes to the log,
161 : not stdout. Nothing is drawn after it, so the shorter frame is
162 : never repositioned over. */
163 :
164 0 : if( FD_LIKELY( !all_pass ) ) {
165 0 : PRINT( " waiting... (%lds elapsed)" ANSI_CLEARLN "\n", elapsed_s );
166 0 : }
167 :
168 : /* Clear any leftover lines below and restore cursor. */
169 :
170 0 : PRINT( "\033[0J\033[?7h\033[?25h" );
171 :
172 0 : flush_frame();
173 0 : }
174 :
175 : static void
176 0 : restore_cursor( void ) {
177 0 : if( FD_UNLIKELY( !g_tty ) ) return;
178 0 : frame_len = 0UL;
179 0 : PRINT( "\033[?7h\033[?25h" );
180 0 : flush_frame();
181 0 : }
182 :
183 : /* g_running clears and g_signal records the number when a signal
184 : arrives; sig_atomic_t guarantees atomic access from a handler. */
185 : static volatile sig_atomic_t g_running = 1;
186 : static volatile sig_atomic_t g_signal;
187 :
188 : static void
189 0 : signal_handler( int sig ) {
190 0 : g_signal = sig;
191 0 : g_running = 0;
192 0 : }
193 :
194 : /* exit_interrupted reports a signal death the way a shell does. */
195 :
196 : static void __attribute__((noreturn))
197 0 : exit_interrupted( void ) {
198 0 : restore_cursor();
199 0 : FD_LOG_ERR_NOEXIT(( "interrupted before finding a safe window (%s)", g_last_state ));
200 0 : fd_sys_util_exit_group( 128+(int)g_signal );
201 0 : }
202 :
203 : /* wait_for_safe_window polls the running validator until all safety
204 : checks pass. Never returns: exits 0 once safe, 128+signum if a
205 : signal arrives first, WAIT_EXIT_GONE if the validator exits, and 1
206 : on any other failure.
207 :
208 : The caller must have already: joined the replay workspace,
209 : called fd_topo_workspace_fill, installed signal handlers, and
210 : joined rng. pidfd is a file descriptor obtained from pidfd_open()
211 : before entering the sandbox; it refers to the exact process
212 : incarnation and is immune to PID reuse. target_pid is retained for
213 : log messages only. */
214 :
215 : static void
216 : wait_for_safe_window( fd_wait_info_box_t const * wait_info_box,
217 : ulong min_idle_slots,
218 : ulong min_idle_ns,
219 : ulong max_delinquent_stake_pct,
220 : int skip_health,
221 : int skip_snapshot,
222 : pid_t target_pid,
223 : int pidfd,
224 0 : fd_rng_t * rng ) {
225 :
226 : /* Block until replay publishes. seq==0 is the zeroed box, which
227 : would otherwise read as a valid snapshot in which nothing is
228 : happening. seq only advances, so this is a one time gate. */
229 :
230 0 : fd_wait_info_t init_info;
231 0 : for(;;) {
232 0 : ulong seq = fd_wait_info_seq( wait_info_box );
233 0 : if( FD_LIKELY( seq && fd_wait_info_try_read( &init_info, wait_info_box ) ) ) break;
234 0 : if( FD_UNLIKELY( !g_running ) ) exit_interrupted();
235 :
236 0 : struct pollfd pfd = { .fd = pidfd, .events = POLLIN };
237 0 : struct timespec zero_ts = { .tv_sec = 0, .tv_nsec = 0 };
238 0 : int pret = (int)ppoll( &pfd, 1, &zero_ts, NULL );
239 0 : if( FD_UNLIKELY( pret>0 ) ) {
240 0 : FD_LOG_ERR_NOEXIT(( "validator (pid %d) exited before publishing", target_pid ));
241 0 : fd_sys_util_exit_group( WAIT_EXIT_GONE );
242 0 : }
243 0 : if( FD_UNLIKELY( pret<0 && errno!=EINTR ) )
244 0 : FD_LOG_ERR(( "ppoll(pidfd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
245 0 : fd_sys_util_nanosleep( 0U, 1000000U ); /* 1 ms; do not spin on the sequence */
246 0 : }
247 :
248 0 : int snapshots_disabled = !init_info.snap_full_interval_blocks &&
249 0 : !init_info.snap_incr_interval_blocks;
250 0 : int full_configured = init_info.snap_full_interval_blocks>0UL;
251 0 : int incremental_configured = !skip_snapshot && init_info.snap_incr_interval_blocks>0UL;
252 :
253 0 : ulong snap_incr_baseline = init_info.snap_produced_incr_cnt;
254 :
255 0 : uint const poll_interval_secs = 1U;
256 0 : long start_time = fd_log_wallclock();
257 0 : int first_frame = 1;
258 :
259 0 : while( FD_LIKELY( g_running ) ) {
260 :
261 : /* 0. Verify the validator is still alive (pidfd becomes readable
262 : when the process exits). */
263 :
264 0 : struct pollfd pfd = { .fd = pidfd, .events = POLLIN };
265 0 : struct timespec zero_ts = { .tv_sec = 0, .tv_nsec = 0 };
266 0 : int pret = (int)ppoll( &pfd, 1, &zero_ts, NULL );
267 0 : if( FD_UNLIKELY( pret>0 ) ) {
268 0 : restore_cursor();
269 0 : FD_LOG_ERR_NOEXIT(( "validator (pid %d) exited while waiting for safe window (%s)",
270 0 : target_pid, g_last_state ));
271 0 : fd_sys_util_exit_group( WAIT_EXIT_GONE );
272 0 : } else if( FD_UNLIKELY( pret<0 && errno!=EINTR ) ) {
273 0 : restore_cursor();
274 0 : FD_LOG_ERR(( "ppoll(pidfd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
275 0 : }
276 :
277 : /* Read a consistent snapshot of the wait_info data from the
278 : replay tile via the sequence. If a write is in progress,
279 : skip this iteration and recheck liveness next loop. */
280 :
281 0 : fd_wait_info_t info;
282 0 : if( FD_UNLIKELY( !fd_wait_info_try_read( &info, wait_info_box ) ) ) {
283 0 : FD_SPIN_PAUSE();
284 0 : fd_sys_util_nanosleep( 0U, (uint)fd_rng_ulong_roll( rng, 1000000UL ) ); /* 0-1ms jitter to break synchronization */
285 0 : if( !fd_wait_info_try_read( &info, wait_info_box ) ) {
286 0 : fd_sys_util_nanosleep( poll_interval_secs, 0U );
287 0 : continue;
288 0 : }
289 0 : }
290 :
291 : /* 1. Health check: replay tile caught_up flag. */
292 :
293 0 : wait_check_t health = { .result = CHECK_FAIL };
294 :
295 0 : if( FD_UNLIKELY( skip_health ) ) {
296 0 : health.result = CHECK_SKIP;
297 0 : fd_cstr_printf_check( health.desc, sizeof(health.desc), NULL, "skipped" );
298 0 : } else if( FD_LIKELY( info.caught_up ) ) {
299 0 : health.result = CHECK_PASS;
300 0 : fd_cstr_printf_check( health.desc, sizeof(health.desc), NULL, "caught up" );
301 0 : } else {
302 0 : health.result = CHECK_FAIL;
303 0 : fd_cstr_printf_check( health.desc, sizeof(health.desc), NULL, "not caught up" );
304 0 : }
305 :
306 : /* 2. Leader-slot idle gap check (intervening slots between the gap
307 : anchor and next_leader_slot, exclusive of both). The anchor
308 : takes the tip because the leader slot arrives on cluster time
309 : (health bounds that lag at 12 slots, --skip-health-check does
310 : not), and leader_highwater_slot because reset_slot lags it after
311 : we lead, which would count slots we led as idle. */
312 :
313 0 : wait_check_t leader = { .result = CHECK_FAIL };
314 :
315 0 : ulong reset_slot = info.reset_slot;
316 0 : ulong gap_anchor = fd_ulong_max( fd_ulong_max( info.reset_slot, info.tip_slot ), info.leader_highwater_slot );
317 0 : ulong next_leader_slot = info.next_leader_slot;
318 0 : ulong leader_slot_val = info.leader_slot;
319 0 : ulong epoch_end_slot = info.epoch_end_slot;
320 0 : ulong slots_per_epoch = info.slots_per_epoch;
321 0 : ulong ns_per_slot_val = info.ns_per_slot;
322 :
323 0 : ulong idle_gap = 0UL;
324 :
325 : /* When --min-idle-seconds is used, convert the time threshold to
326 : slots using the live ns_per_slot from the replay tile. */
327 :
328 0 : ulong effective_min_idle_slots = min_idle_slots;
329 0 : if( FD_UNLIKELY( min_idle_ns ) ) {
330 0 : if( FD_UNLIKELY( !ns_per_slot_val ) ) {
331 0 : leader.result = CHECK_FAIL;
332 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "waiting for data" );
333 0 : goto post_leader_check;
334 0 : }
335 0 : effective_min_idle_slots = fd_ulong_sat_add( min_idle_ns, ns_per_slot_val - 1UL ) / ns_per_slot_val;
336 0 : }
337 :
338 0 : if( FD_UNLIKELY( slots_per_epoch && effective_min_idle_slots>=slots_per_epoch ) ) {
339 0 : restore_cursor();
340 0 : FD_LOG_ERR(( "requested idle gap of %lu slots meets or exceeds epoch length of %lu slots; will never be satisfied", effective_min_idle_slots, slots_per_epoch ));
341 0 : }
342 :
343 0 : if( FD_UNLIKELY( effective_min_idle_slots && (reset_slot==0UL || epoch_end_slot==0UL) ) ) {
344 : /* reset_slot==0 or epoch_end_slot==0 means the replay tile
345 : hasn't published valid data yet. Block until the replay
346 : tile is initialized to avoid trusting stale values. */
347 0 : leader.result = CHECK_FAIL;
348 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "waiting for data" );
349 0 : } else if( FD_UNLIKELY( effective_min_idle_slots && epoch_end_slot<reset_slot ) ) {
350 : /* epoch_end_slot is behind reset_slot; the validator crossed an
351 : epoch boundary but epoch_end_slot hasn't been refreshed yet.
352 : The leader schedule for the new epoch may not be loaded, so
353 : the leader gap check cannot be trusted. */
354 0 : leader.result = CHECK_FAIL;
355 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "epoch boundary crossed, waiting for update" );
356 0 : } else if( FD_UNLIKELY( leader_slot_val ) ) {
357 0 : leader.result = CHECK_FAIL;
358 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "leading slot %lu", leader_slot_val );
359 0 : } else if( FD_UNLIKELY( effective_min_idle_slots && !info.next_leader_known ) ) {
360 : /* Cannot see the schedule; do not pass the gap check on silence.
361 : With no gap required there is nothing to check, so a schedule
362 : we cannot see must not block. */
363 0 : leader.result = CHECK_FAIL;
364 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "waiting for leader schedule" );
365 0 : } else if( FD_UNLIKELY( next_leader_slot==0UL ) ) {
366 : /* reset_slot>0 but next_leader_slot==0 means the replay tile
367 : mapped ULONG_MAX to 0; no upcoming leader slots (e.g. no
368 : stake or past all leader slots in the epoch). */
369 0 : leader.result = CHECK_SKIP;
370 0 : idle_gap = ULONG_MAX;
371 0 : } else {
372 0 : leader.result = CHECK_SKIP;
373 : /* Count only the intervening slots (exclusive of both
374 : reset_slot and next_leader_slot). Saturate because
375 : next_leader_slot can lag behind reset_slot briefly
376 : after a leader slot ends (stale value). */
377 0 : idle_gap = fd_ulong_sat_sub( next_leader_slot, gap_anchor + 1UL );
378 0 : }
379 :
380 0 : if( FD_LIKELY( leader.result!=CHECK_FAIL ) ) {
381 : /* Epoch boundary guard: if we are too close to the end of the
382 : epoch, the next epoch's leader schedule may not be loaded yet,
383 : which could cause a false positive on the leader gap check.
384 : Mirrors the Agave guard:
385 : https://github.com/anza-xyz/agave/blob/v4.2.1/validator/src/commands/wait_for_restart_window/mod.rs#L290-L293 */
386 :
387 0 : ulong epoch_boundary_slots = fd_ulong_sat_sub( epoch_end_slot, gap_anchor );
388 :
389 0 : if( FD_UNLIKELY( epoch_boundary_slots<effective_min_idle_slots ) ) {
390 0 : leader.result = CHECK_FAIL;
391 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "epoch boundary in %lu slots (need %lu)", epoch_boundary_slots, effective_min_idle_slots );
392 0 : } else if( FD_UNLIKELY( !effective_min_idle_slots ) ) {
393 0 : leader.result = CHECK_PASS;
394 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "not leading (gap check disabled)" );
395 0 : } else if( FD_LIKELY( idle_gap>=effective_min_idle_slots ) ) {
396 0 : leader.result = CHECK_PASS;
397 0 : if( FD_UNLIKELY( idle_gap==ULONG_MAX ) ) {
398 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "idle (no upcoming leader slots)" );
399 0 : } else {
400 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "idle %lu slots (need %lu)", idle_gap, effective_min_idle_slots );
401 0 : }
402 0 : } else {
403 0 : leader.result = CHECK_FAIL;
404 0 : fd_cstr_printf_check( leader.desc, sizeof(leader.desc), NULL, "next leader in %lu slots (need %lu)", idle_gap, effective_min_idle_slots );
405 0 : }
406 0 : }
407 :
408 0 : post_leader_check:;
409 : /* 3. Snapshot check: no snapshot currently being written, and if
410 : incremental snapshots are configured, an incremental that is
411 : newer than the latest full must exist. */
412 :
413 0 : wait_check_t snap = { .result = CHECK_FAIL };
414 :
415 0 : if( FD_UNLIKELY( skip_snapshot ) ) {
416 0 : snap.result = CHECK_SKIP;
417 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "skipped" );
418 0 : } else {
419 0 : int active = info.snap_active;
420 0 : ulong finished_full = info.snap_finished_full;
421 0 : ulong finished_incr = info.snap_finished_incr;
422 :
423 0 : if( FD_UNLIKELY( active ) ) {
424 0 : snap.result = CHECK_FAIL;
425 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "snapshot in progress" );
426 0 : } else if( FD_UNLIKELY( snapshots_disabled ) ) {
427 : /* No periodic schedule: intervals off, or no snapmk tile. A
428 : manual snapshot is still possible, which the active check
429 : above covers. */
430 0 : snap.result = CHECK_SKIP;
431 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "skipped (periodic disabled)" );
432 0 : } else if( FD_UNLIKELY( full_configured && !finished_full ) ) {
433 0 : snap.result = CHECK_FAIL;
434 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "waiting for first full snapshot" );
435 0 : } else if( FD_UNLIKELY( incremental_configured && finished_incr<=finished_full ) ) {
436 0 : snap.result = CHECK_FAIL;
437 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "waiting for incremental (full at slot %lu)", finished_full );
438 0 : } else if( FD_UNLIKELY( incremental_configured && info.snap_produced_incr_cnt<=snap_incr_baseline ) ) {
439 0 : snap.result = CHECK_FAIL;
440 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "waiting for a fresh incremental (last at slot %lu)", finished_incr );
441 0 : } else {
442 0 : snap.result = CHECK_PASS;
443 0 : fd_cstr_printf_check( snap.desc, sizeof(snap.desc), NULL, "idle" );
444 0 : }
445 0 : }
446 :
447 : /* 4. Delinquent stake check. */
448 :
449 0 : wait_check_t delinquent = { .result = CHECK_FAIL };
450 :
451 0 : if( FD_UNLIKELY( max_delinquent_stake_pct>=100UL ) ) {
452 0 : delinquent.result = CHECK_SKIP;
453 0 : fd_cstr_printf_check( delinquent.desc, sizeof(delinquent.desc), NULL, "skipped" );
454 0 : } else if( FD_UNLIKELY( !info.delinquent_known ) ) {
455 : /* Block rather than skip; covers the Alpenglow warmup window. */
456 0 : delinquent.result = CHECK_FAIL;
457 0 : fd_cstr_printf_check( delinquent.desc, sizeof(delinquent.desc), NULL, "waiting for stake data" );
458 0 : } else if( FD_UNLIKELY( reset_slot==0UL ) ) {
459 0 : delinquent.result = CHECK_FAIL;
460 0 : fd_cstr_printf_check( delinquent.desc, sizeof(delinquent.desc), NULL, "waiting for data" );
461 0 : } else {
462 0 : ulong delinquent_lamports = info.delinquent_stake_lamports;
463 0 : ulong total_lamports = info.cluster_active_stake_lamports;
464 :
465 : /* d > p*(T/100) implies d*100 > p*T. floor(T/100) makes the
466 : threshold at most 99 lamports stricter (safe direction). */
467 :
468 0 : ulong t100 = total_lamports / 100UL;
469 :
470 0 : if( FD_UNLIKELY( !t100 ) ) {
471 0 : delinquent.result = CHECK_FAIL;
472 0 : fd_cstr_printf_check( delinquent.desc, sizeof(delinquent.desc), NULL, "waiting for stake data" );
473 0 : } else {
474 : /* The shown percentage is rounded up, hence <=, which keeps
475 : it within max exactly when the check passes. Comparison
476 : is exact. */
477 0 : int over = delinquent_lamports > max_delinquent_stake_pct * t100;
478 0 : ulong pct = (delinquent_lamports + t100 - 1UL) / t100;
479 0 : pct = fd_ulong_min( pct, 100UL );
480 0 : delinquent.result = over ? CHECK_FAIL : CHECK_PASS;
481 0 : fd_cstr_printf_check( delinquent.desc, sizeof(delinquent.desc), NULL, "%s%lu%% delinquent (max %lu%%)",
482 0 : pct ? "\xe2\x89\xa4" : "", pct, max_delinquent_stake_pct );
483 0 : }
484 0 : }
485 :
486 0 : int all_pass = ( health.result!=CHECK_FAIL ) &&
487 0 : ( leader.result!=CHECK_FAIL ) &&
488 0 : ( snap.result!=CHECK_FAIL ) &&
489 0 : ( delinquent.result!=CHECK_FAIL );
490 :
491 0 : long elapsed_s = (fd_log_wallclock() - start_time) / (1000L * 1000L * 1000L);
492 :
493 : /* Record before the g_running check below, so an exit that never
494 : found a safe window can report what it was waiting on. */
495 :
496 0 : fd_cstr_printf_check( g_last_state, sizeof(g_last_state), NULL,
497 0 : "health: %s; leader gap: %s; snapshot: %s; delinquent: %s",
498 0 : health.desc, leader.desc, snap.desc, delinquent.desc );
499 :
500 0 : if( FD_UNLIKELY( !g_running ) ) break;
501 :
502 0 : render_status( &health, &leader, &snap, &delinquent,
503 0 : all_pass, elapsed_s, first_frame );
504 0 : first_frame = 0;
505 :
506 0 : if( FD_UNLIKELY( all_pass ) ) {
507 0 : FD_LOG_NOTICE(( "success at slot %lu (%s)", reset_slot, g_last_state ));
508 0 : fd_sys_util_exit_group( 0 );
509 0 : }
510 :
511 0 : fd_sys_util_nanosleep( poll_interval_secs, 0U );
512 0 : }
513 :
514 : /* Interrupted by signal; restore cursor and exit. */
515 :
516 0 : exit_interrupted();
517 0 : }
518 :
519 : static void
520 : wait_cmd_perm( args_t * args FD_PARAM_UNUSED,
521 : fd_cap_chk_t * chk,
522 0 : config_t const * config ) {
523 : /* config here is the default topology, not the running validator's;
524 : only wait_cmd_fn knows which workspace it ends up joining. Size
525 : the limit to the whole topology. */
526 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
527 :
528 0 : fd_cap_chk_raise_rlimit( chk, "wait", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
529 :
530 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
531 0 : fd_cap_chk_cap( chk, "wait", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
532 0 : if( FD_LIKELY( getuid() != config->uid ) )
533 0 : fd_cap_chk_cap( chk, "wait", CAP_SETUID, "call `setresuid(2)` to switch uid to the sandbox user" );
534 0 : if( FD_LIKELY( getgid() != config->gid ) )
535 0 : fd_cap_chk_cap( chk, "wait", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
536 0 : }
537 :
538 : static void
539 : wait_cmd_fn( args_t * args,
540 0 : config_t * config ) {
541 0 : int skip_health = args->wait.skip_health_check;
542 0 : int skip_snapshot = args->wait.skip_snapshot_check;
543 :
544 : /* Attach to the running validator's config (once). */
545 :
546 0 : fd_bootinfo_adopt_named( config, args->wait.name );
547 :
548 : /* Read the validator PID before sandboxing (requires filesystem). */
549 :
550 0 : char bootinfo_path[ PATH_MAX ];
551 0 : FD_TEST( fd_cstr_printf_check( bootinfo_path, sizeof(bootinfo_path), NULL, "%s/%s.bootinfo",
552 0 : config->hugetlbfs.mount_path, config->name ) );
553 :
554 0 : fd_bootinfo_t info;
555 0 : if( FD_UNLIKELY( -1==fd_bootinfo_path_read( bootinfo_path, &info ) ) )
556 0 : FD_LOG_ERR(( "failed to read bootinfo at %s", bootinfo_path ));
557 :
558 0 : if( FD_UNLIKELY( !fd_bootinfo_live( &info ) ) ) {
559 0 : FD_LOG_ERR_NOEXIT(( "validator is no longer running (pid %lu)", info.pid ));
560 0 : fd_sys_util_exit_group( WAIT_EXIT_GONE );
561 0 : }
562 :
563 0 : pid_t target_pid = (pid_t)info.pid;
564 0 : if( FD_UNLIKELY( target_pid<=0 || (ulong)target_pid!=info.pid ) )
565 0 : FD_LOG_ERR(( "invalid validator pid %lu", info.pid ));
566 :
567 : /* Open a pidfd for the target process before entering the sandbox.
568 : The pidfd is immune to PID reuse; all liveness probes inside the
569 : sandbox use this fd instead of kill(). */
570 :
571 0 : int pidfd = (int)syscall( SYS_pidfd_open, target_pid, 0U );
572 0 : if( FD_UNLIKELY( pidfd<0 ) )
573 0 : FD_LOG_ERR(( "pidfd_open(%d) failed (%i-%s); kernel 5.3+ required",
574 0 : target_pid, errno, fd_io_strerror( errno ) ));
575 :
576 : /* Re-verify the validator is still running with the same target_pid.
577 : Closes the TOCTOU window between the initial fd_bootinfo_live
578 : check and pidfd_open; if the validator restarted with a new PID,
579 : bootinfo now names a different process than the pidfd refers to. */
580 :
581 0 : if( FD_UNLIKELY( -1==fd_bootinfo_path_read( bootinfo_path, &info ) ||
582 0 : info.pid!=(ulong)target_pid || !fd_bootinfo_live( &info ) ) ) {
583 0 : FD_LOG_ERR_NOEXIT(( "validator (pid %d) exited during wait setup", target_pid ));
584 0 : fd_sys_util_exit_group( WAIT_EXIT_GONE );
585 0 : }
586 :
587 : /* Verify topology layout and join the replay workspace before
588 : sandboxing (requires filesystem access and mlock). */
589 :
590 0 : fd_bootinfo_check_layout( config );
591 :
592 0 : ulong wksp_id = fd_topo_find_wksp( &config->topo, "replay" );
593 0 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "no replay workspace found in topology" ));
594 :
595 0 : fd_topo_wksp_t * replay_wksp = &config->topo.workspaces[ wksp_id ];
596 0 : fd_topo_join_workspace( &config->topo, replay_wksp, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
597 :
598 : /* Install signal handlers before entering the sandbox. */
599 :
600 0 : struct sigaction sa = { .sa_handler = signal_handler };
601 0 : if( FD_UNLIKELY( sigaction( SIGTERM, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGTERM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
602 0 : if( FD_UNLIKELY( sigaction( SIGINT, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGINT) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
603 :
604 : /* fd_log colorize already honors NO_COLOR/TERM/--log-colorize, but is
605 : keyed to stderr, so additionally require stdout to be a tty. */
606 :
607 0 : g_tty = !args->wait.silent && fd_log_colorize() && isatty( STDOUT_FILENO );
608 :
609 : /* Close stdin; the wait command never reads from it, and the
610 : seccomp policy does not allow read(). */
611 :
612 0 : if( FD_LIKELY( pidfd!=STDIN_FILENO ) ) close( STDIN_FILENO );
613 :
614 : /* Enter the seccomp sandbox. */
615 :
616 0 : int allow_fds[ 4 ];
617 0 : ulong allow_fds_cnt = 0;
618 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
619 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
620 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
621 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd();
622 0 : allow_fds[ allow_fds_cnt++ ] = pidfd;
623 :
624 0 : struct sock_filter seccomp_filter[ 128UL ];
625 0 : populate_sock_filter_policy_wait( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd() );
626 :
627 0 : if( FD_LIKELY( config->development.sandbox ) ) {
628 0 : fd_sandbox_enter( config->uid,
629 0 : config->gid,
630 0 : 0,
631 0 : 0,
632 0 : 0,
633 0 : 1, /* Keep controlling terminal for Ctrl+C */
634 0 : 0,
635 0 : allow_fds_cnt, /* ppoll checks nfds>RLIMIT_NOFILE */
636 0 : 0UL,
637 0 : 0UL,
638 0 : 0UL,
639 0 : allow_fds_cnt,
640 0 : allow_fds,
641 0 : sock_filter_policy_wait_instr_cnt,
642 0 : seccomp_filter );
643 0 : } else {
644 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
645 0 : }
646 :
647 : /* Resolve pointers in mmap'd workspace memory (safe in sandbox). */
648 :
649 0 : fd_topo_workspace_fill( &config->topo, replay_wksp );
650 :
651 0 : ulong wait_info_obj_id = fd_pod_query_ulong( config->topo.props, "wait_info", ULONG_MAX );
652 0 : if( FD_UNLIKELY( wait_info_obj_id==ULONG_MAX ) ) FD_LOG_ERR(( "wait_info object not found in topology" ));
653 0 : fd_wait_info_box_t * wait_info_box = fd_wait_info_box_join( fd_topo_obj_laddr( &config->topo, wait_info_obj_id ) );
654 0 : if( FD_UNLIKELY( !wait_info_box ) ) FD_LOG_ERR(( "failed to join wait_info object" ));
655 :
656 0 : ulong min_idle_slots = args->wait.min_idle_slots;
657 0 : ulong min_idle_ns = args->wait.min_idle_ns;
658 0 : ulong max_delinquent_stake_pct = args->wait.max_delinquent_stake_pct;
659 :
660 0 : if( FD_UNLIKELY( min_idle_ns ) ) {
661 0 : ulong min_idle_s = min_idle_ns / 1000000000UL;
662 0 : FD_LOG_NOTICE(( "waiting for a safe window "
663 0 : "(min-idle-seconds=%lus, max-delinquent-stake=%lu%%, skip-health=%s, skip-snapshot=%s)",
664 0 : min_idle_s,
665 0 : max_delinquent_stake_pct,
666 0 : skip_health ? "yes" : "no",
667 0 : skip_snapshot ? "yes" : "no" ));
668 0 : } else {
669 0 : FD_LOG_NOTICE(( "waiting for a safe window "
670 0 : "(min-idle-slots=%lu, max-delinquent-stake=%lu%%, skip-health=%s, skip-snapshot=%s)",
671 0 : min_idle_slots,
672 0 : max_delinquent_stake_pct,
673 0 : skip_health ? "yes" : "no",
674 0 : skip_snapshot ? "yes" : "no" ));
675 0 : }
676 :
677 0 : fd_rng_t _rng[1];
678 0 : fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, (uint)fd_log_wallclock(), 0UL ) );
679 0 : FD_TEST( rng );
680 :
681 0 : wait_for_safe_window( wait_info_box, min_idle_slots, min_idle_ns, max_delinquent_stake_pct,
682 0 : skip_health, skip_snapshot, target_pid, pidfd, rng );
683 0 : }
684 :
685 : static void
686 : wait_cmd_args( int * pargc,
687 : char *** pargv,
688 0 : args_t * args ) {
689 0 : ulong min_idle_slots_raw = fd_env_strip_cmdline_ulong( pargc, pargv, "--min-idle-slots", NULL, ULONG_MAX );
690 0 : ulong min_idle_s = fd_env_strip_cmdline_ulong( pargc, pargv, "--min-idle-seconds", NULL, ULONG_MAX );
691 :
692 0 : if( FD_UNLIKELY( min_idle_slots_raw!=ULONG_MAX && min_idle_s!=ULONG_MAX ) ) {
693 0 : FD_LOG_ERR(( "--min-idle-slots and --min-idle-seconds are mutually exclusive" ));
694 0 : }
695 :
696 0 : if( FD_UNLIKELY( min_idle_s!=ULONG_MAX ) ) {
697 : /* Convert seconds to nanoseconds. */
698 0 : if( FD_UNLIKELY( min_idle_s > ULONG_MAX/1000000000UL ) )
699 0 : FD_LOG_ERR(( "--min-idle-seconds value too large" ));
700 0 : if( FD_UNLIKELY( !min_idle_s ) )
701 0 : FD_LOG_NOTICE(( "--min-idle-seconds 0 disables the leader gap and epoch boundary checks (still blocks during active block production)" ));
702 0 : args->wait.min_idle_ns = min_idle_s * 1000000000UL;
703 0 : args->wait.min_idle_slots = 0UL; /* will be computed at runtime */
704 0 : } else {
705 0 : args->wait.min_idle_ns = 0UL;
706 0 : args->wait.min_idle_slots = ( min_idle_slots_raw==ULONG_MAX ) ? 1500UL : min_idle_slots_raw;
707 0 : if( FD_UNLIKELY( !args->wait.min_idle_slots ) )
708 0 : FD_LOG_NOTICE(( "--min-idle-slots 0 disables the leader gap and epoch boundary checks (still blocks during active block production)" ));
709 0 : }
710 :
711 0 : ulong max_delinquent = fd_env_strip_cmdline_ulong( pargc, pargv, "--max-delinquent-stake", NULL, 5UL );
712 0 : if( FD_UNLIKELY( max_delinquent>100UL ) )
713 0 : FD_LOG_ERR(( "--max-delinquent-stake must be in range [0,100]" ));
714 0 : args->wait.max_delinquent_stake_pct = max_delinquent;
715 0 : if( FD_UNLIKELY( max_delinquent>=100UL ) )
716 0 : FD_LOG_NOTICE(( "--max-delinquent-stake %lu disables the delinquent stake check", max_delinquent ));
717 :
718 0 : args->wait.skip_health_check = fd_env_strip_cmdline_contains( pargc, pargv, "--skip-health-check" );
719 0 : args->wait.skip_snapshot_check = fd_env_strip_cmdline_contains( pargc, pargv, "--skip-snapshot-check" );
720 0 : args->wait.silent = fd_env_strip_cmdline_contains( pargc, pargv, "--silent" );
721 0 : args->wait.name[ 0 ] = '\0';
722 0 : int has_name_flag = 0;
723 0 : for( int i = 0; i < *pargc; i++ ) {
724 0 : if( !strcmp( (*pargv)[ i ], "--name" ) ) { has_name_flag = 1; break; }
725 0 : }
726 0 : char const * name = fd_env_strip_cmdline_cstr( pargc, pargv, "--name", NULL, NULL );
727 0 : if( FD_UNLIKELY( !name && has_name_flag ) )
728 0 : FD_LOG_ERR(( "--name requires a value" ));
729 0 : if( FD_UNLIKELY( name ) ) {
730 0 : if( FD_UNLIKELY( name[ 0 ]=='\0' ) )
731 0 : FD_LOG_ERR(( "--name must not be empty" ));
732 0 : if( FD_UNLIKELY( strlen( name )>=sizeof(args->wait.name) ) )
733 0 : FD_LOG_ERR(( "--name too long (max %lu characters)", sizeof(args->wait.name)-1UL ));
734 0 : fd_cstr_ncpy( args->wait.name, name, sizeof(args->wait.name) );
735 0 : }
736 0 : }
737 :
738 : static void
739 0 : wait_args_help( fd_action_help_t * help ) {
740 0 : fd_action_help_arg( help, "--min-idle-slots", "<slots>", "Minimum number of idle slots required before the next leader\n"
741 0 : "slot. Default: 1500. 0 disables the gap check" );
742 0 : fd_action_help_arg( help, "--min-idle-seconds", "<seconds>", "Minimum idle time in seconds before the next leader slot.\n"
743 0 : "Converted to slots using the live slot duration.\n"
744 0 : "Mutually exclusive with --min-idle-slots.\n"
745 0 : "0 disables the gap check" );
746 0 : fd_action_help_arg( help, "--max-delinquent-stake", "<percent>", "Maximum integer percentage of delinquent stake allowed,\n"
747 0 : "in range [0-100]. Default: 5.\n"
748 0 : "100 disables the check" );
749 0 : fd_action_help_arg( help, "--skip-health-check", NULL, "Skip the health check (replay caught-up status)" );
750 0 : fd_action_help_arg( help, "--skip-snapshot-check", NULL, "Skip the snapshot check" );
751 0 : fd_action_help_arg( help, "--silent", NULL, "Do not print the live status panel" );
752 0 : fd_action_help_arg( help, "--name", "<name>", "Name of the validator instance to attach to, if more than one is\n"
753 0 : "running on this host" );
754 0 : }
755 :
756 : action_t fd_action_wait = {
757 : .name = "wait",
758 : .args = wait_cmd_args,
759 : .fn = wait_cmd_fn,
760 : .require_config = 0, /* fd_bootinfo_adopt_named replaces config with the running validator's published config */
761 : .perm = wait_cmd_perm,
762 : .description = "Wait for a safe window to stop the validator.",
763 : .detail = "Attaches to a running validator and polls every second until\n"
764 : "all of the following are true:\n"
765 : "\n"
766 : " - The validator is caught up (replay tile caught_up flag)\n"
767 : " - There are enough idle slots before the next leader slot\n"
768 : " - No snapshot is currently being written. When periodic\n"
769 : " snapshot production is enabled, a full snapshot exists on\n"
770 : " disk and, if incremental snapshots are configured, a fresh\n"
771 : " incremental has been produced since the command started\n"
772 : " - Cluster delinquent stake is below the threshold\n"
773 : "\n"
774 : "The validator keeps running either way. Exit codes:\n"
775 : "\n"
776 : " 0 a safe window was found\n"
777 : " 2 the validator exited while waiting\n"
778 : " 128+sig interrupted by a signal (130 SIGINT, 143 SIGTERM)\n"
779 : " 1 any other failure",
780 : .usage = "wait [OPTIONS]",
781 : .args_help = wait_args_help,
782 : };
|
