Line data Source code
1 : #include "watch.h"
2 : #include "generated/watch_seccomp.h"
3 :
4 : #include "../../fd_bootinfo.h"
5 :
6 : #include "../../../../discof/backup/fd_snapmk_tile.h"
7 : #include "../../../../discof/restore/fd_snapct_tile.h"
8 : #include "../../../../discof/gossip/fd_gossip_tile.h"
9 : #include "../../../../disco/events/fd_event_client.h"
10 : #include "../../../../disco/metrics/fd_metrics.h"
11 : #include "../../../../disco/node_info/fd_node_info.h"
12 : #include "../../../../disco/genesis/fd_genesis_cluster.h"
13 : #include "../../../../util/pod/fd_pod.h"
14 : #include "../../../../util/tile/fd_tile.h"
15 :
16 : #include <errno.h>
17 : #include <sys/select.h>
18 : #include <time.h>
19 : #include <unistd.h>
20 : #include <sys/resource.h>
21 : #include <linux/capability.h>
22 :
23 : #include "../../../platform/fd_sys_util.h"
24 :
25 : void
26 : watch_cmd_perm( args_t * args FD_PARAM_UNUSED,
27 : fd_cap_chk_t * chk,
28 0 : config_t const * config ) {
29 0 : ulong mlock_limit = fd_topo_mlock( &config->topo );
30 :
31 0 : fd_cap_chk_raise_rlimit( chk, "watch", RLIMIT_MEMLOCK, mlock_limit, "call `rlimit(2)` to increase `RLIMIT_MEMLOCK` so all memory can be locked with `mlock(2)`" );
32 :
33 0 : if( fd_sandbox_requires_cap_sys_admin( config->uid, config->gid ) )
34 0 : fd_cap_chk_cap( chk, "watch", CAP_SYS_ADMIN, "call `unshare(2)` with `CLONE_NEWUSER` to sandbox the process in a user namespace" );
35 0 : if( FD_LIKELY( getuid() != config->uid ) )
36 0 : fd_cap_chk_cap( chk, "watch", CAP_SETUID, "call `setresuid(2)` to switch uid to the sanbox user" );
37 0 : if( FD_LIKELY( getgid() != config->gid ) )
38 0 : fd_cap_chk_cap( chk, "watch", CAP_SETGID, "call `setresgid(2)` to switch gid to the sandbox user" );
39 0 : }
40 :
41 :
42 : static ulong lines_printed;
43 : static int ended_on_newline = 1;
44 :
45 : static char frame_buf[ 65536UL ];
46 : static ulong frame_len;
47 :
48 : /* Show all category detail rows, not just the primary row of each.
49 : Set from --full. */
50 : static int watch_full = 0;
51 :
52 : static void
53 0 : flush_frame( void ) {
54 0 : ulong written = 0UL;
55 0 : while( written<frame_len ) {
56 0 : long w = write( STDOUT_FILENO, frame_buf+written, frame_len-written );
57 0 : if( FD_UNLIKELY( -1==w && errno==EAGAIN ) ) continue;
58 0 : else if( FD_UNLIKELY( -1==w ) ) FD_LOG_ERR(( "write() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
59 0 : else if( FD_UNLIKELY( 0==w ) ) break;
60 0 : written += (ulong)w;
61 0 : }
62 0 : frame_len = 0UL;
63 0 : }
64 :
65 : static int
66 0 : drain( int fd ) {
67 0 : int needs_reprint = 0;
68 :
69 0 : while( 1 ) {
70 0 : uchar buf[ 16384UL ];
71 0 : long result = read( fd, buf, sizeof(buf) );
72 0 : if( FD_UNLIKELY( -1==result && errno==EAGAIN ) ) break;
73 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "read() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
74 0 : else if( FD_UNLIKELY( !result ) ) fd_sys_util_exit_group( 0 ); /* writer gone: validator exited */
75 :
76 0 : if( FD_LIKELY( !needs_reprint ) ) {
77 : /* Buffer the erase sequence and first log chunk together so the
78 : terminal never renders a blank frame between erase and content. */
79 0 : frame_len = 0UL;
80 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
81 0 : FD_TEST( fd_cstr_printf_check( frame_buf, sizeof(frame_buf), &frame_len, "\033[%luA\033[%luM\033[1A\033[0J", lines_printed, lines_printed ) );
82 0 : } else {
83 0 : FD_TEST( fd_cstr_printf_check( frame_buf, sizeof(frame_buf), &frame_len, "\033[%luA\033[%luM\033[0J", lines_printed, lines_printed ) );
84 0 : }
85 0 : }
86 0 : FD_TEST( frame_len+(ulong)result<=sizeof(frame_buf) );
87 0 : fd_memcpy( frame_buf+frame_len, buf, (ulong)result );
88 0 : frame_len += (ulong)result;
89 0 : flush_frame();
90 0 : needs_reprint = 1;
91 :
92 0 : ended_on_newline = buf[ (ulong)result-1UL ]=='\n';
93 0 : }
94 :
95 0 : return needs_reprint;
96 0 : }
97 :
98 : static char *
99 : fmt_bytes( char * buf,
100 : ulong buf_sz,
101 0 : long bytes ) {
102 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
103 0 : if( FD_LIKELY( 8L*bytes<1000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%ld bit/s", 8L*bytes ) );
104 0 : else if( FD_LIKELY( 8L*bytes<1000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Kbit/s", (double)(8L*bytes)/1000.0 ) );
105 0 : else if( FD_LIKELY( 8L*bytes<1000000000L ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Mbit/s", (double)(8L*bytes)/1000000.0 ) );
106 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f Gbit/s", (double)(8L*bytes)/1000000000.0 ) );
107 :
108 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%12s", tmp ) );
109 0 : return buf;
110 0 : }
111 :
112 : static char *
113 : fmt_count( char * buf,
114 : ulong buf_sz,
115 0 : ulong count ) {
116 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
117 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
118 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
119 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
120 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f G", (double)count/1000000000.0 ) );
121 :
122 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%7s", tmp ) );
123 0 : return buf;
124 0 : }
125 :
126 : static char *
127 : fmt_countf( char * buf,
128 : ulong buf_sz,
129 0 : double count ) {
130 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
131 0 : if( FD_LIKELY( count<1000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
132 0 : else if( FD_LIKELY( count<1000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f K", (double)count/1000.0 ) );
133 0 : else if( FD_LIKELY( count<1000000000UL ) ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f M", (double)count/1000000.0 ) );
134 0 : else memcpy( tmp, "-", 2UL );
135 :
136 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%7s", tmp ) );
137 0 : return buf;
138 0 : }
139 :
140 : static char *
141 : fmt_count_tight( char * buf,
142 : ulong buf_sz,
143 0 : ulong count ) {
144 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
145 0 : if( count<1000UL ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%lu", count ) );
146 0 : else if( count<1000000UL ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fK", (double)count/1e3 ) );
147 0 : else if( count<1000000000UL ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fM", (double)count/1e6 ) );
148 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fG", (double)count/1e9 ) );
149 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%6s", tmp ) );
150 0 : return buf;
151 0 : }
152 :
153 : static char *
154 : fmt_countf_tight( char * buf,
155 : ulong buf_sz,
156 0 : double count ) {
157 0 : char * tmp = fd_alloca_check( 1UL, buf_sz );
158 0 : if( count<1000.0 ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1f", count ) );
159 0 : else if( count<1000000.0 ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fK", count/1e3 ) );
160 0 : else if( count<1000000000.0 ) FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fM", count/1e6 ) );
161 0 : else FD_TEST( fd_cstr_printf_check( tmp, buf_sz, NULL, "%.1fG", count/1e9 ) );
162 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%6s", tmp ) );
163 0 : return buf;
164 0 : }
165 :
166 : static long
167 : diff_link( config_t const * config,
168 : char const * link_name,
169 : ulong const * prev_link,
170 : ulong const * cur_link,
171 0 : ulong idx ) {
172 0 : long result = 0L;
173 :
174 0 : ulong seen[ (FD_TOPO_MAX_LINKS+63UL)/64UL ] = {0};
175 0 : ulong overall_polled_idx = 0UL;
176 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
177 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
178 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
179 0 : fd_topo_link_t const * link = &config->topo.links[ tile->in_link_id[ j ] ];
180 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
181 :
182 0 : if( FD_LIKELY( !strcmp( link->name, link_name ) ) ) {
183 0 : ulong id = tile->in_link_id[ j ];
184 0 : if( FD_LIKELY( !( seen[ id>>6 ] & (1UL<<(id&63UL)) ) ) ) {
185 0 : seen[ id>>6 ] |= 1UL<<(id&63UL);
186 0 : result += (long)cur_link[ overall_polled_idx*8UL+idx ]-(long)prev_link[ overall_polled_idx*8UL+idx ];
187 0 : }
188 0 : }
189 :
190 0 : overall_polled_idx++;
191 0 : }
192 0 : }
193 0 : return result;
194 0 : }
195 :
196 : /* diff_tile returns the positive increase between the last and current
197 : value of an increasing counter. Clamps wraparound to zero (this
198 : happens to gauge-based counters, e.g. "number of events in an epoch") */
199 :
200 : static ulong
201 : diff_tile( config_t const * config,
202 : char const * tile_name,
203 : ulong const * prev_tile,
204 : ulong const * cur_tile,
205 0 : ulong idx ) {
206 0 : ulong result = 0UL;
207 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
208 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
209 0 : if( FD_UNLIKELY( strcmp( tile->name, tile_name ) ) ) continue;
210 0 : ulong prev = prev_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
211 0 : ulong cur = cur_tile[ i*FD_METRICS_TOTAL_SZ+idx ];
212 0 : result += cur>=prev ? cur-prev : 0UL;
213 0 : }
214 0 : return result;
215 0 : }
216 :
217 : static ulong
218 0 : total_crds( ulong const * metrics ) {
219 0 : ulong sum = 0UL;
220 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_CRDS_VALUE_TYPE_CNT; i++ ) {
221 0 : sum += metrics[ MIDX( GAUGE, GOSSIP, CRDS_USED_CONTACT_INFO_V1 )+i ];
222 0 : }
223 0 : return sum;
224 0 : }
225 :
226 : /* diff_tile_regime is diff_tile summed over a TileRegime group mask */
227 :
228 : static ulong
229 : diff_tile_regime( config_t const * config,
230 : char const * tile_name,
231 : ulong const * prev_tile,
232 : ulong const * cur_tile,
233 0 : ulong mask ) {
234 0 : ulong result = 0UL;
235 0 : for( ; mask; mask=fd_ulong_pop_lsb( mask ) ) {
236 0 : result += diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+(ulong)fd_ulong_find_lsb( mask ) );
237 0 : }
238 0 : return result;
239 0 : }
240 :
241 : static ulong
242 0 : total_regime( ulong const * metrics ) {
243 0 : ulong sum = 0UL;
244 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) {
245 0 : sum += metrics[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
246 0 : }
247 0 : return sum;
248 0 : }
249 :
250 : /* Bench */
251 : static ulong tps_sent_samples_idx = 0UL;
252 : static ulong tps_sent_samples[ 200UL ];
253 : /* Replay */
254 : static ulong cups_samples_idx = 0UL;
255 : static ulong cups_samples[ 100UL ];
256 : static ulong sps_samples_idx = 0UL;
257 : static ulong sps_samples[ 200UL ];
258 : static ulong tps_samples_idx = 0UL;
259 : static ulong tps_samples[ 200UL ];
260 : /* Snapshot */
261 : static ulong snapshot_rx_idx = 0UL;
262 : static ulong snapshot_rx_samples[ 100UL ];
263 : static long snap_cur_nanos = 0L;
264 : static long snap_prev_nanos = 0L;
265 : static ulong snap_dt_idx = 0UL;
266 : static long snap_dt_samples[ 200UL ];
267 :
268 : static long
269 0 : wallclock_monotonic( void ) {
270 0 : struct timespec ts;
271 0 : clock_gettime( CLOCK_MONOTONIC, &ts );
272 0 : return (long)ts.tv_sec*(long)1e9+ts.tv_nsec;
273 0 : }
274 :
275 : static double
276 0 : window_seconds( ulong num ) {
277 0 : ulong len = sizeof(snap_dt_samples)/sizeof(snap_dt_samples[0]);
278 0 : ulong cnt = fd_ulong_min( snap_dt_idx, fd_ulong_min( num, len ) );
279 0 : long sum = 0L;
280 0 : for( ulong i=1UL; i<=cnt; i++ ) sum += snap_dt_samples[ (snap_dt_idx-i)%len ];
281 0 : return (double)fd_long_max( sum, 1L )/1e9;
282 0 : }
283 :
284 : static ulong snapshot_acc_idx = 0UL;
285 : static ulong snapshot_acc_samples[ 100UL ];
286 : static ulong snapshot_wr_idx = 0UL;
287 : static ulong snapshot_wr_samples[ 100UL ];
288 : /* Backup */
289 : static ulong backup_acc_idx = 0UL;
290 : static ulong backup_acc_samples[ 100UL ];
291 : static ulong backup_read_idx = 0UL;
292 : static ulong backup_read_samples[ 100UL ];
293 : static ulong backup_comp_idx = 0UL;
294 : static ulong backup_comp_samples[ 100UL ];
295 : static ulong backup_upload_idx = 0UL;
296 : static ulong backup_upload_samples[ 100UL ];
297 : /* Event */
298 : static ulong events_sent_samples_idx = 0UL;
299 : static ulong events_sent_samples[ 100UL ];
300 : static ulong events_acked_samples_idx = 0UL;
301 : static ulong events_acked_samples[ 100UL ];
302 : static ulong event_bytes_written_samples_idx = 0UL;
303 : static ulong event_bytes_written_samples[ 100UL ];
304 : static ulong event_bytes_read_samples_idx = 0UL;
305 : static ulong event_bytes_read_samples[ 100UL ];
306 :
307 : /* Peer discovery follows logistic growth so the per-capita rate
308 : (dn/dt)/n is linear in n and hits zero at n=N_total. A streaming
309 : least-squares of per-capita rate vs count over 150ms buckets projects
310 : N_total.
311 :
312 : The displayed bar is n/N_total slew-limited to 60%/s (the raw
313 : projection can jump when the fit locks on) and capped at 90%. The
314 : final 10% is walked by the table-quiet gap against the tile's
315 : 500ms saturation deadline, so full coincides exactly with the
316 : tile's PEER_SATURATED condition, which is mirrored here. */
317 :
318 : static long gossip_bucket_nanos = 0L;
319 : static ulong gossip_bucket_peers = 0UL;
320 : static double gossip_sx = 0.0, gossip_sy = 0.0, gossip_sxx = 0.0, gossip_sxy = 0.0, gossip_sw = 0.0;
321 : static ulong gossip_peer_hwm = 0UL;
322 : static long gossip_peer_hwm_nanos = 0L;
323 : static long gossip_disp_nanos = 0L;
324 : static double gossip_disp_pct = 0.0;
325 : static double gossip_conv_pct = 0.0;
326 : static int gossip_saturated = 0;
327 :
328 0 : #define GOSSIP_SAT_QUIET_SECS (0.5) /* FD_GOSSIP_PEER_SAT_QUIET_NS */
329 0 : #define GOSSIP_BAR_SLEW (60.0) /* %/s */
330 0 : #define GOSSIP_BAR_FIT_CAP (90.0) /* % from the fit, rest from the quiet gap */
331 :
332 : static void
333 : sample_gossip( config_t const * config,
334 : ulong const * cur_tile,
335 0 : long now ) {
336 0 : ulong tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
337 0 : if( FD_UNLIKELY( tile_idx==ULONG_MAX || gossip_saturated ) ) return;
338 :
339 0 : ulong const * t = &cur_tile[ tile_idx*FD_METRICS_TOTAL_SZ ];
340 0 : ulong peers = t[ MIDX( GAUGE, GOSSIP, CRDS_PEER_COUNT_STAKED ) ]+t[ MIDX( GAUGE, GOSSIP, CRDS_PEER_COUNT_UNSTAKED ) ];
341 :
342 0 : if( FD_UNLIKELY( peers>gossip_peer_hwm ) ) {
343 0 : gossip_peer_hwm = peers;
344 0 : gossip_peer_hwm_nanos = now;
345 0 : } else if( FD_UNLIKELY( peers>1UL && gossip_peer_hwm_nanos &&
346 0 : (double)(now-gossip_peer_hwm_nanos)/1e9>GOSSIP_SAT_QUIET_SECS ) ) {
347 : /* Mirrors the tile's PEER_SATURATED condition. */
348 0 : gossip_saturated = 1;
349 0 : gossip_conv_pct = 100.0;
350 0 : return;
351 0 : }
352 :
353 : /* Feed the regression once discovery is underway (the entrypoint
354 : alone doesn't count). */
355 0 : if( FD_LIKELY( peers>=10UL ) ) {
356 0 : if( FD_UNLIKELY( !gossip_bucket_nanos ) ) {
357 0 : gossip_bucket_nanos = now;
358 0 : gossip_bucket_peers = peers;
359 0 : } else if( FD_UNLIKELY( (double)(now-gossip_bucket_nanos)/1e9>=0.15 ) ) {
360 0 : double dt = (double)(now-gossip_bucket_nanos)/1e9;
361 0 : double x = 0.5*(double)(peers+gossip_bucket_peers);
362 0 : double y = ((double)peers-(double)gossip_bucket_peers)/dt/x; /* per-capita rate */
363 0 : gossip_sx += x; gossip_sy += y; gossip_sxx += x*x; gossip_sxy += x*y; gossip_sw += 1.0;
364 0 : gossip_bucket_nanos = now;
365 0 : gossip_bucket_peers = peers;
366 0 : }
367 0 : }
368 :
369 0 : double target = 0.0;
370 0 : if( FD_LIKELY( gossip_sw>=4.0 ) ) {
371 0 : double den = gossip_sw*gossip_sxx-gossip_sx*gossip_sx;
372 0 : if( FD_LIKELY( den>1e-9 ) ) {
373 0 : double slope = (gossip_sw*gossip_sxy-gossip_sx*gossip_sy)/den;
374 0 : double intercept = (gossip_sy-slope*gossip_sx)/gossip_sw;
375 0 : if( FD_LIKELY( slope<-1e-12 && intercept>0.0 ) ) {
376 0 : double total = -intercept/slope; /* n where per-capita rate hits zero */
377 0 : target = fd_double_if( total>0.0, 100.0*(double)peers/total, 0.0 );
378 0 : target = fd_double_if( target>GOSSIP_BAR_FIT_CAP, GOSSIP_BAR_FIT_CAP, target );
379 0 : }
380 0 : }
381 0 : }
382 :
383 0 : if( FD_UNLIKELY( !gossip_disp_nanos ) ) gossip_disp_nanos = now;
384 0 : double step = GOSSIP_BAR_SLEW*(double)(now-gossip_disp_nanos)/1e9;
385 0 : gossip_disp_nanos = now;
386 0 : if( FD_LIKELY( target>gossip_disp_pct ) )
387 0 : gossip_disp_pct = fd_double_if( gossip_disp_pct+step>target, target, gossip_disp_pct+step );
388 :
389 0 : double gap = 0.0;
390 0 : if( FD_LIKELY( peers>1UL && gossip_peer_hwm_nanos ) )
391 0 : gap = (double)(now-gossip_peer_hwm_nanos)/1e9/GOSSIP_SAT_QUIET_SECS;
392 :
393 0 : double pct = gossip_disp_pct+(100.0-GOSSIP_BAR_FIT_CAP)*gap;
394 0 : pct = fd_double_if( pct>99.0, 99.0, pct );
395 0 : gossip_conv_pct = fd_double_if( pct>gossip_conv_pct, pct, gossip_conv_pct );
396 0 : }
397 :
398 : /* Accounts */
399 : static ulong accdb_samples_idx = 0UL;
400 : static ulong accdb_acquired_samples[ 200UL ];
401 : static ulong accdb_writable_samples[ 200UL ];
402 : static ulong accdb_missed_samples [ 200UL ];
403 : static ulong accdb_evicted_samples [ 200UL ];
404 : static ulong accdb_waited_samples [ 200UL ];
405 : static ulong accdb_bytes_rd_samples[ 200UL ];
406 : static ulong accdb_bytes_wr_samples[ 200UL ];
407 : static ulong accdb_bytes_cp_samples[ 200UL ];
408 : static ulong accdb_bytes_pe_samples[ 200UL ];
409 : static ulong accdb_evicted_class_samples[ 8UL ][ 200UL ];
410 : static ulong accdb_preevicted_samples[ 200UL ];
411 : static ulong accdb_preevicted_class_samples[ 8UL ][ 200UL ];
412 : static ulong accdb_committed_new_class_samples[ 8UL ][ 200UL ];
413 : static ulong accdb_committed_overwrite_class_samples[ 8UL ][ 200UL ];
414 : /* Repair server */
415 : static ulong shreds_stored_samples_idx = 0UL;
416 : static ulong shreds_stored_sample[ 200UL ] ;
417 : static ulong rserve_rps_valid_samples_idx = 0UL;
418 : static ulong rserve_rps_valid_samples[ 100UL ];
419 : static ulong rserve_rps_invalid_samples_idx = 0UL;
420 : static ulong rserve_rps_invalid_samples[ 100UL ];
421 : static ulong rserve_rps_miss_samples[ 100UL ];
422 : static ulong rserve_rps_sigvfy_samples[ 100UL ];
423 : static ulong rserve_rps_stale_samples[ 100UL ];
424 : static ulong rserve_rps_other_rej_samples[ 100UL ];
425 :
426 0 : #define RESET "\033[0m"
427 : #define BOLD "\033[1m"
428 : #define DIM "\033[2m"
429 :
430 0 : #define RED "\033[31m"
431 0 : #define GREEN "\033[32m"
432 : #define ORANGE "\033[38;5;208m"
433 0 : #define YELLOW "\033[33m"
434 : #define BLUE "\033[34m"
435 : #define MAGENTA "\033[35m"
436 : #define CYAN "\033[36m"
437 :
438 : #define BGREEN "\033[92m"
439 : #define BYELLOW "\033[93m"
440 :
441 : /* Cluster accents matching the GUI palette. */
442 0 : #define CLUS_MAINNET "\033[38;5;43m" /* #1CE7C2 */
443 0 : #define CLUS_TESTNET "\033[38;5;178m" /* #E7B81C */
444 0 : #define CLUS_DEVNET "\033[38;5;166m" /* #E7601C */
445 0 : #define CLUS_PYTHNET "\033[38;5;129m" /* #9D1CE7 */
446 0 : #define CLUS_PYTHTEST "\033[38;5;162m" /* #E71C88 */
447 0 : #define CLUS_UNKNOWN "\033[38;5;245m" /* #898989 */
448 :
449 : #define CLEARLN "\033[K"
450 :
451 : /* Row grammar: an accent tinted glyph + bold colored section label
452 : in a fixed 12 char column, a dim divider, then dim lowercase keys
453 : with plain values so the data carries the visual weight. ROWH()
454 : opens a section row (pass the label pre-padded to 12 chars), ROWC
455 : opens a continuation row aligned under the divider, K() emits one
456 : key, U() a dim unit suffix. Icons are single cell glyphs from the
457 : geometric shapes block so every row aligns in every font. */
458 : #define ROWH( icon, color, label ) color icon " " BOLD label RESET DIM "│" RESET
459 : #define ROWC " " DIM "│" RESET
460 : #define K( key ) " " DIM key " " RESET
461 : #define U( unit ) DIM unit RESET
462 :
463 : /* Severity color for utilization style percentages: quiet when calm,
464 : loud only when it matters. */
465 : static char const *
466 0 : sev_color( double pct ) {
467 0 : if( FD_UNLIKELY( pct>=85.0 ) ) return RED;
468 0 : if( FD_UNLIKELY( pct>=50.0 ) ) return YELLOW;
469 0 : return "";
470 0 : }
471 :
472 : /* Unicode block progress bar, filled green with a dim tail. */
473 : static char *
474 : fmt_bar( char * buf,
475 : ulong buf_sz,
476 : double pct,
477 0 : ulong width ) {
478 0 : pct = fd_double_if( pct<0.0, 0.0, fd_double_if( pct>100.0, 100.0, pct ) );
479 0 : ulong filled = (ulong)( pct*(double)width/100.0+0.5 );
480 0 : ulong len = 0UL, l;
481 0 : FD_TEST( fd_cstr_printf_check( buf, buf_sz, &len, GREEN ) );
482 0 : for( ulong i=0UL; i<filled; i++ ) { FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, "█" ) ); len += l; }
483 0 : FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, DIM ) ); len += l;
484 0 : for( ulong i=filled; i<width; i++ ) { FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, "░" ) ); len += l; }
485 0 : FD_TEST( fd_cstr_printf_check( buf+len, buf_sz-len, &l, RESET ) );
486 0 : return buf;
487 0 : }
488 :
489 : #define BAR( pct, width ) (__extension__({ \
490 : fmt_bar( fd_alloca_check( 1UL, 256UL ), 256UL, pct, width ); \
491 : }))
492 :
493 0 : #define PRINT(...) do { \
494 0 : ulong _len; \
495 0 : FD_TEST( fd_cstr_printf_check( frame_buf+frame_len, sizeof(frame_buf)-frame_len, &_len, __VA_ARGS__ ) ); \
496 0 : frame_len += _len; \
497 0 : } while(0)
498 :
499 0 : #define SNAP_DT_NS() (fd_long_max( snap_cur_nanos-snap_prev_nanos, 1L ))
500 :
501 : #define DIFF_LINK_BYTES( link_name, metric_type, metric_subtype, metric ) (__extension__({ \
502 : long bytes = diff_link( config, link_name, prev_link, cur_link, MIDX( metric_type, metric_subtype, metric ) ); \
503 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)((double)bytes*1e9/(double)SNAP_DT_NS()) ); \
504 : }))
505 :
506 0 : #define DIFF_BYTES( tile_name, metric_type, metric_subtype, metric ) (__extension__({ \
507 0 : ulong bytes = diff_tile( config, tile_name, prev_tile, cur_tile, MIDX( metric_type, metric_subtype, metric ) ); \
508 0 : fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)((double)bytes*1e9/(double)SNAP_DT_NS()) ); \
509 0 : }))
510 :
511 0 : #define COUNT( count ) (__extension__({ \
512 0 : fmt_count( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
513 0 : }))
514 :
515 0 : #define COUNTF( count ) (__extension__({ \
516 0 : fmt_countf( fd_alloca_check( 1UL, 64UL ), 64UL, count ); \
517 0 : }))
518 :
519 0 : #define COUNT_T( count ) (__extension__({ \
520 0 : fmt_count_tight( fd_alloca_check( 1UL, 32UL ), 32UL, count ); \
521 0 : }))
522 :
523 0 : #define COUNTF_T( count ) (__extension__({ \
524 0 : fmt_countf_tight( fd_alloca_check( 1UL, 32UL ), 32UL, count ); \
525 0 : }))
526 :
527 : static int
528 : write_bench( config_t const * config,
529 : ulong const * cur_tile,
530 0 : ulong const * prev_tile ) {
531 0 : if( FD_UNLIKELY( fd_topo_find_tile( &config->topo, "benchs", 0UL )==ULONG_MAX ) ) return 0;
532 :
533 0 : ulong tps_sum = 0UL;
534 0 : ulong num_tps_samples = fd_ulong_min( tps_sent_samples_idx, sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0]));
535 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_sent_samples[ i ];
536 0 : char * tps_str = COUNTF( (double)tps_sum/window_seconds( num_tps_samples ) );
537 :
538 0 : PRINT( ROWH( "◎", BGREEN, "bench " )
539 0 : K( "tps" ) "%s"
540 0 : K( "benchg" ), tps_str );
541 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
542 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchg" ) ) ) continue;
543 :
544 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
545 0 : double backp_pct = 100.0*(double)diff_tile_regime( config, "benchg", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)total_ticks;
546 0 : double idle_pct = 100.0*(double)diff_tile_regime( config, "benchg", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)total_ticks;
547 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
548 :
549 0 : PRINT( " %s%.1f" U( "%%" ) RESET, sev_color( busy_pct ), busy_pct );
550 0 : }
551 :
552 0 : PRINT( K( "benchs" ) );
553 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
554 0 : if( FD_LIKELY( strcmp( config->topo.tiles[ i ].name, "benchs" ) ) ) continue;
555 :
556 0 : ulong total_ticks = total_regime( &cur_tile[ i*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ i*FD_METRICS_TOTAL_SZ ] );
557 0 : double backp_pct = 100.0*(double)diff_tile_regime( config, "benchs", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)total_ticks;
558 0 : double idle_pct = 100.0*(double)diff_tile_regime( config, "benchs", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)total_ticks;
559 0 : double busy_pct = 100.0 - idle_pct - backp_pct;
560 :
561 0 : PRINT( " %s%.1f" U( "%%" ) RESET, sev_color( busy_pct ), busy_pct );
562 0 : }
563 :
564 0 : PRINT( CLEARLN "\n" );
565 0 : return 1;
566 0 : }
567 :
568 : static void
569 : write_backtest( config_t const * config,
570 0 : ulong const * cur_tile ) {
571 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
572 0 : ulong start_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, START_SLOT ) ];
573 0 : ulong final_slot = cur_tile[ backt_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, BACKT, END_SLOT ) ];
574 :
575 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
576 0 : ulong current_slot = cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT ) ];
577 0 : current_slot = current_slot ? current_slot : start_slot;
578 :
579 0 : ulong completed_slots = current_slot-start_slot;
580 :
581 0 : if( FD_UNLIKELY( !final_slot ) ) { /* unbounded */
582 0 : PRINT( ROWH( "◇", BGREEN, "backtest " )
583 0 : K( "slots" ) "%lu" U( " / ?" ) CLEARLN "\n", completed_slots );
584 0 : return;
585 0 : }
586 :
587 0 : ulong total_slots = final_slot-start_slot;
588 0 : double progress = total_slots ? 100.0 * (double)completed_slots / (double)total_slots : 100.0;
589 0 : PRINT( ROWH( "◇", BGREEN, "backtest " )
590 0 : " %s " BOLD "%5.1f" RESET U( "%%" )
591 0 : K( "slots" ) "%lu" U( " /" ) " %lu" CLEARLN "\n",
592 0 : BAR( progress, 20UL ), progress, completed_slots, total_slots );
593 0 : }
594 :
595 : static void
596 : write_snapshots( config_t const * config,
597 : ulong const * cur_tile,
598 0 : ulong const * prev_tile ) {
599 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
600 0 : ulong snapdc_tile_cnt = fd_topo_tile_name_cnt( &config->topo, "snapdc" );
601 0 : ulong snapin_tile_cnt = fd_topo_tile_name_cnt( &config->topo, "snapin" );
602 0 : ulong state = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, LOAD_STATE ) ];
603 :
604 0 : double progress = 0.0;
605 0 : int incremental = 0;
606 0 : switch( state ) {
607 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS:
608 0 : case FD_SNAPCT_STATE_WAITING_FOR_PEERS_INCREMENTAL:
609 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS:
610 0 : case FD_SNAPCT_STATE_COLLECTING_PEERS_INCREMENTAL:
611 0 : break;
612 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_FILE:
613 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_FINI:
614 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_FILE_DONE:
615 0 : case FD_SNAPCT_STATE_READING_INCREMENTAL_HTTP:
616 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_FINI:
617 0 : case FD_SNAPCT_STATE_FLUSHING_INCREMENTAL_HTTP_DONE:
618 0 : incremental = 1;
619 0 : __attribute__((fallthrough));
620 0 : case FD_SNAPCT_STATE_READING_FULL_FILE:
621 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_FINI:
622 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_FILE_DONE:
623 0 : case FD_SNAPCT_STATE_READING_FULL_HTTP:
624 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_FINI:
625 0 : case FD_SNAPCT_STATE_FLUSHING_FULL_HTTP_DONE: {
626 : /* Progress is the compressed-equivalent of bytes fully applied
627 : (parse+write consumption, converted back through the decompress
628 : ratio), same as snapshot-load, rather than bytes merely
629 : downloaded by snapct. */
630 0 : ulong consumed, dc_in, dc_out, size_bytes;
631 0 : if( FD_UNLIKELY( incremental ) ) {
632 0 : consumed = ULONG_MAX;
633 0 : for( ulong i=0UL; i<snapin_tile_cnt; i++ ) {
634 0 : ulong snapin_idx = fd_topo_find_tile( &config->topo, "snapin", i );
635 0 : consumed = fd_ulong_min( consumed, cur_tile[ snapin_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPIN, PARSE_PROGRESS_BYTES_INCREMENTAL ) ] );
636 0 : }
637 0 : dc_in = 0UL;
638 0 : dc_out = 0UL;
639 0 : for( ulong i=0UL; i<snapdc_tile_cnt; i++ ) {
640 0 : ulong snapdc_idx = fd_topo_find_tile( &config->topo, "snapdc", i );
641 0 : dc_in += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, COMPRESSED_PROGRESS_BYTES_INCREMENTAL ) ];
642 0 : dc_out += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, DECOMPRESSED_PROGRESS_BYTES_INCREMENTAL ) ];
643 0 : }
644 0 : size_bytes = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, READ_EXPECTED_BYTES_INCREMENTAL ) ];
645 0 : } else {
646 0 : consumed = ULONG_MAX;
647 0 : for( ulong i=0UL; i<snapin_tile_cnt; i++ ) {
648 0 : ulong snapin_idx = fd_topo_find_tile( &config->topo, "snapin", i );
649 0 : consumed = fd_ulong_min( consumed, cur_tile[ snapin_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPIN, PARSE_PROGRESS_BYTES_FULL ) ] );
650 0 : }
651 0 : dc_in = 0UL;
652 0 : dc_out = 0UL;
653 0 : for( ulong i=0UL; i<snapdc_tile_cnt; i++ ) {
654 0 : ulong snapdc_idx = fd_topo_find_tile( &config->topo, "snapdc", i );
655 0 : dc_in += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, COMPRESSED_PROGRESS_BYTES_FULL ) ];
656 0 : dc_out += cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPDC, DECOMPRESSED_PROGRESS_BYTES_FULL ) ];
657 0 : }
658 0 : size_bytes = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, READ_EXPECTED_BYTES_FULL ) ];
659 0 : }
660 0 : double done_comp = dc_out ? (double)dc_in*( (double)consumed/(double)dc_out ) : 0.0;
661 0 : if( FD_LIKELY( size_bytes ) ) progress = fd_double_if( done_comp>(double)size_bytes, 100.0, 100.0*done_comp/(double)size_bytes );
662 0 : break;
663 0 : }
664 0 : case FD_SNAPCT_STATE_SHUTDOWN:
665 0 : progress = 100.0;
666 0 : break;
667 0 : }
668 :
669 0 : ulong snap_rx_sum = 0UL;
670 0 : ulong num_snap_rx_samples = fd_ulong_min( snapshot_rx_idx, sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0]) );
671 0 : for( ulong i=0UL; i<num_snap_rx_samples; i++ ) snap_rx_sum += snapshot_rx_samples[ i ];
672 0 : double megabytes_per_second = 0.0;
673 0 : if( FD_LIKELY( num_snap_rx_samples ) ) megabytes_per_second = (double)snap_rx_sum/window_seconds( num_snap_rx_samples )/1e6;
674 :
675 0 : ulong accounts_sum = 0UL;
676 0 : ulong num_accounts_samples = fd_ulong_min( snapshot_acc_idx, sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0]) );
677 0 : for( ulong i=0UL; i<num_accounts_samples; i++ ) accounts_sum += snapshot_acc_samples[ i ];
678 0 : double million_accounts_per_second = 0.0;
679 0 : if( FD_LIKELY( num_accounts_samples ) ) million_accounts_per_second = (double)accounts_sum/window_seconds( num_accounts_samples )/1e6;
680 :
681 0 : ulong snap_wr_sum = 0UL;
682 0 : ulong num_snap_wr_samples = fd_ulong_min( snapshot_wr_idx, sizeof(snapshot_wr_samples)/sizeof(snapshot_wr_samples[0]) );
683 0 : for( ulong i=0UL; i<num_snap_wr_samples; i++ ) snap_wr_sum += snapshot_wr_samples[ i ];
684 0 : double wr_megabytes_per_second = 0.0;
685 0 : if( FD_LIKELY( num_snap_wr_samples ) ) wr_megabytes_per_second = (double)snap_wr_sum/window_seconds( num_snap_wr_samples )/1e6;
686 :
687 0 : ulong snapct_total_ticks = total_regime( &cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapct_idx*FD_METRICS_TOTAL_SZ ] );
688 0 : ulong snapld_total_ticks = total_regime( &cur_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ fd_topo_find_tile( &config->topo, "snapld", 0UL )*FD_METRICS_TOTAL_SZ ] );
689 0 : ulong snapdc_total_ticks = 0UL;
690 0 : for( ulong i=0UL; i<snapdc_tile_cnt; i++ ) {
691 0 : ulong snapdc_idx = fd_topo_find_tile( &config->topo, "snapdc", i );
692 0 : snapdc_total_ticks += total_regime( &cur_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ snapdc_idx*FD_METRICS_TOTAL_SZ ] );
693 0 : }
694 0 : ulong snapin_total_ticks = 0UL;
695 0 : for( ulong i=0UL; i<snapin_tile_cnt; i++ ) {
696 0 : ulong idx = fd_topo_find_tile( &config->topo, "snapin", i );
697 0 : snapin_total_ticks += total_regime( &cur_tile[ idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ idx*FD_METRICS_TOTAL_SZ ] );
698 0 : }
699 0 : snapct_total_ticks = fd_ulong_max( snapct_total_ticks, 1UL );
700 0 : snapld_total_ticks = fd_ulong_max( snapld_total_ticks, 1UL );
701 0 : snapdc_total_ticks = fd_ulong_max( snapdc_total_ticks, 1UL );
702 0 : snapin_total_ticks = fd_ulong_max( snapin_total_ticks, 1UL );
703 :
704 0 : double snapct_backp_pct = 100.0*(double)diff_tile_regime( config, "snapct", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)snapct_total_ticks;
705 0 : double snapld_backp_pct = 100.0*(double)diff_tile_regime( config, "snapld", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)snapld_total_ticks;
706 0 : double snapdc_backp_pct = 100.0*(double)diff_tile_regime( config, "snapdc", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)snapdc_total_ticks;
707 0 : double snapin_backp_pct = 100.0*(double)diff_tile_regime( config, "snapin", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)snapin_total_ticks;
708 :
709 0 : double snapct_idle_pct = 100.0*(double)diff_tile_regime( config, "snapct", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)snapct_total_ticks;
710 0 : double snapld_idle_pct = 100.0*(double)diff_tile_regime( config, "snapld", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)snapld_total_ticks;
711 0 : double snapdc_idle_pct = 100.0*(double)diff_tile_regime( config, "snapdc", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)snapdc_total_ticks;
712 0 : double snapin_idle_pct = 100.0*(double)diff_tile_regime( config, "snapin", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)snapin_total_ticks;
713 :
714 0 : double busy [ 4 ] = { 100.0-snapct_idle_pct-snapct_backp_pct,
715 0 : 100.0-snapld_idle_pct-snapld_backp_pct,
716 0 : 100.0-snapdc_idle_pct-snapdc_backp_pct,
717 0 : 100.0-snapin_idle_pct-snapin_backp_pct };
718 0 : double backp[ 4 ] = { snapct_backp_pct, snapld_backp_pct, snapdc_backp_pct, snapin_backp_pct };
719 0 : char const * stage[ 4 ] = { "ct", "ld", "dc", "in" };
720 :
721 0 : PRINT( ROWH( "◐", BYELLOW, "snapshot " )
722 0 : " %s " BOLD "%5.1f" RESET U( "%%" )
723 0 : K( "state" ) "%s"
724 0 : K( "rx" ) "%3.0f" U( " MB/s" )
725 0 : K( "wr" ) "%3.0f" U( " MB/s" )
726 0 : K( "acc" ) "%3.1f" U( " M/s" )
727 0 : K( "busy" ),
728 0 : BAR( progress, 20UL ),
729 0 : progress,
730 0 : fd_snapct_state_str( (int)state ),
731 0 : megabytes_per_second,
732 0 : wr_megabytes_per_second,
733 0 : million_accounts_per_second );
734 0 : for( ulong i=0UL; i<4UL; i++ )
735 0 : PRINT( " " U( "%s" ) " %s%3.0f" U( "%%" ) RESET, stage[ i ], sev_color( busy[ i ] ), busy[ i ] );
736 0 : PRINT( K( "backp" ) );
737 0 : for( ulong i=0UL; i<4UL; i++ )
738 0 : PRINT( " " U( "%s" ) " %s%3.0f" U( "%%" ) RESET, stage[ i ], sev_color( backp[ i ] ), backp[ i ] );
739 0 : PRINT( CLEARLN "\n" );
740 0 : }
741 :
742 : static long
743 : diff_tile_idx( ulong const * prev_tile,
744 : ulong const * cur_tile,
745 : ulong tile_idx,
746 0 : ulong metric_off ) {
747 0 : return (long)cur_tile [ tile_idx*FD_METRICS_TOTAL_SZ+metric_off ] -
748 0 : (long)prev_tile[ tile_idx*FD_METRICS_TOTAL_SZ+metric_off ];
749 0 : }
750 :
751 : static void
752 : accdb_per_tile_offsets( char const * name,
753 0 : ulong * offs /* 11 entries: acquired, writable, missed, waited, rd, wr, cp, evicted_class_base, preevicted_class_base, committed_new_class_base, committed_overwrite_class_base */ ) {
754 0 : if( !strcmp( name, "execle" ) ) {
755 0 : offs[0] =MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_READONLY_ACQUIRED ); offs[1]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_WRITABLE_ACQUIRED);
756 0 : offs[2] =MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_CACHE_MISSED ); offs[3]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_WAITED );
757 0 : offs[4] =MIDX(COUNTER,EXECLE,ACCDB_DISK_READ_BYTES ); offs[5]=MIDX(COUNTER,EXECLE,ACCDB_DISK_WRITTEN_BYTES );
758 0 : offs[6] =MIDX(COUNTER,EXECLE,ACCDB_COPIED_BYTES ); offs[7]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_EVICTED );
759 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_NEW_COMMITTED );
760 0 : offs[10]=MIDX(COUNTER,EXECLE,ACCDB_ACCOUNT_OVERWRITE_COMMITTED);
761 0 : } else if( !strcmp( name, "execrp" ) ) {
762 0 : offs[0] =MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_READONLY_ACQUIRED ); offs[1]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_WRITABLE_ACQUIRED);
763 0 : offs[2] =MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_CACHE_MISSED ); offs[3]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_WAITED );
764 0 : offs[4] =MIDX(COUNTER,EXECRP,ACCDB_DISK_READ_BYTES ); offs[5]=MIDX(COUNTER,EXECRP,ACCDB_DISK_WRITTEN_BYTES );
765 0 : offs[6] =MIDX(COUNTER,EXECRP,ACCDB_COPIED_BYTES ); offs[7]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_EVICTED );
766 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_NEW_COMMITTED );
767 0 : offs[10]=MIDX(COUNTER,EXECRP,ACCDB_ACCOUNT_OVERWRITE_COMMITTED);
768 0 : } else if( !strcmp( name, "replay" ) ) {
769 0 : offs[0] =MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_READONLY_ACQUIRED ); offs[1]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_WRITABLE_ACQUIRED);
770 0 : offs[2] =MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_CACHE_MISSED ); offs[3]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_WAITED );
771 0 : offs[4] =MIDX(COUNTER,REPLAY,ACCDB_DISK_READ_BYTES ); offs[5]=MIDX(COUNTER,REPLAY,ACCDB_DISK_WRITTEN_BYTES );
772 0 : offs[6] =MIDX(COUNTER,REPLAY,ACCDB_COPIED_BYTES ); offs[7]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_EVICTED );
773 0 : offs[8] =ULONG_MAX ; offs[9]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_NEW_COMMITTED );
774 0 : offs[10]=MIDX(COUNTER,REPLAY,ACCDB_ACCOUNT_OVERWRITE_COMMITTED);
775 0 : } else if( !strcmp( name, "accdb" ) ) {
776 : /* The accdb tile only runs background work (compact, preevict,
777 : advance_root, purge); it never acquires/releases. Sentinel
778 : everything that comes from the acquire/release path. */
779 0 : offs[0] =ULONG_MAX ; offs[1]=ULONG_MAX ;
780 0 : offs[2] =ULONG_MAX ; offs[3]=ULONG_MAX ;
781 0 : offs[4] =MIDX(COUNTER,ACCDB,DISK_READ_BYTES ); offs[5]=MIDX(COUNTER,ACCDB,DISK_WRITTEN_BYTES );
782 0 : offs[6] =ULONG_MAX ; offs[7]=ULONG_MAX ;
783 0 : offs[8] =MIDX(COUNTER,ACCDB,ACCOUNT_PREEVICTED ); offs[9]=ULONG_MAX ;
784 0 : offs[10]=ULONG_MAX;
785 0 : } else if( !strcmp( name, "rpc" ) ) {
786 : /* RPC is a read-only accdb consumer. It only emits the subset
787 : of counters that fd_accdb_read_one_nocache touches; everything
788 : else is sentinel and skipped by sample_accdb. */
789 0 : offs[0] =MIDX(COUNTER,RPC,ACCDB_ACCOUNT_READONLY_ACQUIRED); offs[1]=ULONG_MAX;
790 0 : offs[2] =MIDX(COUNTER,RPC,ACCDB_ACCOUNT_CACHE_MISSED); offs[3]=MIDX(COUNTER,RPC,ACCDB_ACCOUNT_WAITED);
791 0 : offs[4] =MIDX(COUNTER,RPC,ACCDB_DISK_READ_BYTES ); offs[5]=ULONG_MAX;
792 0 : offs[6] =MIDX(COUNTER,RPC,ACCDB_COPIED_BYTES ); offs[7]=ULONG_MAX;
793 0 : offs[8] =ULONG_MAX; offs[9]=ULONG_MAX;
794 0 : offs[10]=ULONG_MAX;
795 0 : } else if( !strcmp( name, "resolv" ) ) {
796 : /* Resolv is a read-only accdb consumer (address lookup table
797 : reads on the receive path). Same RO subset as RPC. */
798 0 : offs[0] =MIDX(COUNTER,RESOLV,ACCDB_ACCOUNT_READONLY_ACQUIRED ); offs[1]=ULONG_MAX;
799 0 : offs[2] =MIDX(COUNTER,RESOLV,ACCDB_ACCOUNT_CACHE_MISSED); offs[3]=MIDX(COUNTER,RESOLV,ACCDB_ACCOUNT_WAITED);
800 0 : offs[4] =MIDX(COUNTER,RESOLV,ACCDB_DISK_READ_BYTES ); offs[5]=ULONG_MAX;
801 0 : offs[6] =MIDX(COUNTER,RESOLV,ACCDB_COPIED_BYTES ); offs[7]=ULONG_MAX;
802 0 : offs[8] =ULONG_MAX; offs[9]=ULONG_MAX;
803 0 : offs[10]=ULONG_MAX;
804 0 : } else {
805 0 : for( ulong i=0UL; i<11UL; i++ ) offs[i] = ULONG_MAX;
806 0 : }
807 0 : }
808 :
809 : static void
810 : sample_accdb( config_t const * config,
811 : ulong const * prev_tile,
812 0 : ulong const * cur_tile ) {
813 0 : long acquired = 0L, writable = 0L, missed = 0L, evicted = 0L, waited = 0L;
814 0 : long bytes_rd = 0L, bytes_wr = 0L, bytes_cp = 0L, bytes_pe = 0L;
815 0 : long preevicted = 0L;
816 0 : long evicted_class[ 8 ] = {0};
817 0 : long preevicted_class[ 8 ] = {0};
818 0 : long committed_new_class[ 8 ] = {0};
819 0 : long committed_overwrite_class[ 8 ] = {0};
820 :
821 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
822 0 : ulong offs[11];
823 0 : accdb_per_tile_offsets( config->topo.tiles[ i ].name, offs );
824 0 : if( offs[0]!=ULONG_MAX ) {
825 0 : for( ulong c=0UL; c<8UL; c++ ) acquired += diff_tile_idx( prev_tile, cur_tile, i, offs[0] + c );
826 0 : }
827 0 : if( offs[1]!=ULONG_MAX ) {
828 0 : for( ulong c=0UL; c<8UL; c++ ) writable += diff_tile_idx( prev_tile, cur_tile, i, offs[1] + c );
829 0 : }
830 0 : if( offs[3]!=ULONG_MAX ) waited += diff_tile_idx( prev_tile, cur_tile, i, offs[3] );
831 0 : if( offs[4]!=ULONG_MAX ) bytes_rd += diff_tile_idx( prev_tile, cur_tile, i, offs[4] );
832 0 : if( offs[6]!=ULONG_MAX ) bytes_cp += diff_tile_idx( prev_tile, cur_tile, i, offs[6] );
833 0 : if( offs[5]!=ULONG_MAX ) {
834 0 : long this_wr = diff_tile_idx( prev_tile, cur_tile, i, offs[5] );
835 0 : if( !strcmp( config->topo.tiles[ i ].name, "accdb" ) ) bytes_pe += this_wr;
836 0 : else bytes_wr += this_wr;
837 0 : }
838 0 : for( ulong c=0UL; c<8UL; c++ ) {
839 0 : if( offs[2]!=ULONG_MAX ) missed += diff_tile_idx( prev_tile, cur_tile, i, offs[2] + c );
840 0 : if( offs[7]!=ULONG_MAX ) {
841 0 : long d = diff_tile_idx( prev_tile, cur_tile, i, offs[7] + c );
842 0 : evicted_class[ c ] += d;
843 0 : evicted += d;
844 0 : }
845 0 : if( offs[8]!=ULONG_MAX ) {
846 0 : long d = diff_tile_idx( prev_tile, cur_tile, i, offs[8] + c );
847 0 : preevicted_class[ c ] += d;
848 0 : preevicted += d;
849 0 : }
850 0 : if( offs[9] !=ULONG_MAX ) committed_new_class [ c ] += diff_tile_idx( prev_tile, cur_tile, i, offs[9] + c );
851 0 : if( offs[10]!=ULONG_MAX ) committed_overwrite_class[ c ] += diff_tile_idx( prev_tile, cur_tile, i, offs[10] + c );
852 0 : }
853 0 : }
854 :
855 0 : ulong slot = accdb_samples_idx % (sizeof(accdb_acquired_samples)/sizeof(accdb_acquired_samples[0]));
856 0 : accdb_acquired_samples[ slot ] = (ulong)acquired;
857 0 : accdb_writable_samples[ slot ] = (ulong)writable;
858 0 : accdb_missed_samples [ slot ] = (ulong)missed;
859 0 : accdb_evicted_samples [ slot ] = (ulong)evicted;
860 0 : accdb_waited_samples [ slot ] = (ulong)waited;
861 0 : accdb_bytes_rd_samples[ slot ] = (ulong)bytes_rd;
862 0 : accdb_bytes_wr_samples[ slot ] = (ulong)bytes_wr;
863 0 : accdb_bytes_cp_samples[ slot ] = (ulong)bytes_cp;
864 0 : accdb_bytes_pe_samples[ slot ] = (ulong)bytes_pe;
865 0 : accdb_preevicted_samples[ slot ] = (ulong)preevicted;
866 0 : for( ulong c=0UL; c<8UL; c++ ) {
867 0 : accdb_evicted_class_samples [ c ][ slot ] = (ulong)evicted_class [ c ];
868 0 : accdb_preevicted_class_samples [ c ][ slot ] = (ulong)preevicted_class [ c ];
869 0 : accdb_committed_new_class_samples [ c ][ slot ] = (ulong)committed_new_class [ c ];
870 0 : accdb_committed_overwrite_class_samples[ c ][ slot ] = (ulong)committed_overwrite_class[ c ];
871 0 : }
872 0 : accdb_samples_idx++;
873 0 : }
874 :
875 : static uint
876 : write_accdb( config_t const * config,
877 : ulong const * cur_tile,
878 0 : ulong const * prev_tile ) {
879 0 : ulong accdb_tile_idx = fd_topo_find_tile( &config->topo, "accdb", 0UL );
880 0 : if( accdb_tile_idx==ULONG_MAX ) return 0U;
881 :
882 0 : ulong const * t = cur_tile + accdb_tile_idx*FD_METRICS_TOTAL_SZ;
883 :
884 0 : ulong accdb_total_ticks = total_regime( &cur_tile[ accdb_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ accdb_tile_idx*FD_METRICS_TOTAL_SZ ] );
885 0 : accdb_total_ticks = fd_ulong_max( accdb_total_ticks, 1UL );
886 0 : double accdb_backp_pct = 100.0*(double)diff_tile_regime( config, "accdb", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)accdb_total_ticks;
887 0 : double accdb_idle_pct = 100.0*(double)diff_tile_regime( config, "accdb", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)accdb_total_ticks;
888 0 : double accdb_busy_pct = 100.0 - accdb_backp_pct - accdb_idle_pct;
889 :
890 0 : ulong acct_cnt = t[ MIDX( GAUGE, ACCDB, ACCOUNT_USED ) ];
891 0 : ulong acct_cap = t[ MIDX( GAUGE, ACCDB, ACCOUNT_CAPACITY ) ];
892 0 : ulong used_bytes = t[ MIDX( GAUGE, ACCDB, DISK_USED_BYTES ) ];
893 0 : ulong current_bytes = t[ MIDX( GAUGE, ACCDB, DISK_CONSUMED_BYTES ) ];
894 0 : ulong alloc_bytes = t[ MIDX( GAUGE, ACCDB, DISK_ALLOCATED_BYTES ) ];
895 0 : ulong in_compaction = t[ MIDX( GAUGE, ACCDB, IS_COMPACTING ) ];
896 0 : ulong compact_req = t[ MIDX( COUNTER, ACCDB, COMPACTION_REQUESTED ) ];
897 0 : ulong compact_done = t[ MIDX( COUNTER, ACCDB, COMPACTION_COMPLETED ) ];
898 :
899 0 : ulong frag_bytes = current_bytes>used_bytes ? current_bytes-used_bytes : 0UL;
900 0 : double data_gb = (double)alloc_bytes/1e9;
901 0 : double live_gb = (double)used_bytes/1e9;
902 0 : double frag_gb = (double)frag_bytes/1e9;
903 0 : double frag_pct = current_bytes ? 100.0*(double)frag_bytes/(double)current_bytes : 0.0;
904 0 : double index_pct = acct_cap ? 100.0*(double)acct_cnt/(double)acct_cap : 0.0;
905 :
906 0 : PRINT( ROWH( "■", GREEN, "accounts " )
907 0 : K( "cache" ) "%lu" U( " GiB" )
908 0 : K( "disk" ) "%.1f" U( " GB" )
909 0 : K( "live" ) "%.1f" U( " GB" )
910 0 : K( "frag" ) "%.1f" U( " GB" ) " %s%4.1f" U( "%%" ) RESET
911 0 : K( "index" ) "%s%4.1f" U( "%%" ) RESET " " U( "%.1fM/%.1fM" )
912 0 : K( "compact" ) "%s " U( "%lu/%lu" )
913 0 : K( "busy" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
914 0 : config->firedancer.accounts.cache_size_gib,
915 0 : data_gb, live_gb,
916 0 : frag_gb, sev_color( frag_pct ), frag_pct,
917 0 : sev_color( index_pct ), index_pct, (double)acct_cnt/1e6, (double)acct_cap/1e6,
918 0 : in_compaction ? YELLOW "running" RESET : DIM "idle" RESET, compact_done, compact_req,
919 0 : sev_color( accdb_busy_pct ), accdb_busy_pct );
920 :
921 0 : ulong const cap = sizeof(accdb_acquired_samples)/sizeof(accdb_acquired_samples[0]);
922 0 : ulong n = fd_ulong_min( accdb_samples_idx, cap );
923 0 : if( !n ) n = 1UL;
924 :
925 0 : ulong sum_acq = 0UL, sum_wr = 0UL, sum_miss = 0UL, sum_evict = 0UL, sum_wait = 0UL;
926 0 : ulong sum_brd = 0UL, sum_bwr = 0UL, sum_bcp = 0UL, sum_bpe = 0UL;
927 0 : ulong sum_pre = 0UL;
928 0 : for( ulong i=0UL; i<n; i++ ) {
929 0 : sum_acq += accdb_acquired_samples[ i ];
930 0 : sum_wr += accdb_writable_samples[ i ];
931 0 : sum_miss += accdb_missed_samples [ i ];
932 0 : sum_evict += accdb_evicted_samples [ i ];
933 0 : sum_wait += accdb_waited_samples [ i ];
934 0 : sum_brd += accdb_bytes_rd_samples[ i ];
935 0 : sum_bwr += accdb_bytes_wr_samples[ i ];
936 0 : sum_bcp += accdb_bytes_cp_samples[ i ];
937 0 : sum_bpe += accdb_bytes_pe_samples[ i ];
938 0 : sum_pre += accdb_preevicted_samples[ i ];
939 0 : }
940 :
941 0 : double acquired = (double)sum_acq/window_seconds( n );
942 0 : double writable = (double)sum_wr/window_seconds( n );
943 0 : double missed = (double)sum_miss/window_seconds( n );
944 0 : double evicted = (double)sum_evict/window_seconds( n );
945 0 : double waited = (double)sum_wait/window_seconds( n );
946 0 : double bytes_rd = (double)sum_brd/window_seconds( n );
947 0 : double bytes_wr = (double)sum_bwr/window_seconds( n );
948 0 : double bytes_cp = (double)sum_bcp/window_seconds( n );
949 0 : double bytes_pe = (double)sum_bpe/window_seconds( n );
950 0 : double preevicted = (double)sum_pre/window_seconds( n );
951 :
952 0 : double hit_pct = acquired>0.0 ? 100.0*(acquired-missed)/acquired : 0.0;
953 :
954 0 : char * read_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_rd );
955 0 : char * write_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_wr );
956 0 : char * copy_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_cp );
957 0 : char * preevict_str= fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, (long)bytes_pe );
958 0 : char * acq_str = COUNTF( acquired );
959 0 : char * wr_str = COUNTF( writable );
960 0 : char * miss_str = COUNTF( missed );
961 0 : char * evict_str = COUNTF( evicted );
962 0 : char * pre_str = COUNTF( preevicted );
963 0 : char * wait_str = COUNTF( waited );
964 :
965 0 : if( FD_LIKELY( !watch_full ) ) return 1;
966 :
967 0 : PRINT( ROWC
968 0 : K( "acquire" ) "%s" U( "/s" ) " " U( "(%s wr/s)" )
969 0 : K( "hit" ) "%5.1f" U( "%%" )
970 0 : K( "miss" ) "%s" U( "/s" )
971 0 : K( "evict" ) "%s" U( "/s (+%s/s)" )
972 0 : K( "wait" ) "%s" U( "/s" )
973 0 : K( "io" ) "%s" U( " rd" ) " %s" U( " wr-acq" ) " %s" U( " wr-pe" ) " %s" U( " cp" ) CLEARLN "\n",
974 0 : acq_str, wr_str, hit_pct, miss_str, evict_str, pre_str, wait_str,
975 0 : read_str, write_str, preevict_str, copy_str );
976 :
977 0 : char * evict_class_str[ 8 ];
978 0 : char * preevict_class_str[ 8 ];
979 0 : char * commit_new_class_str[ 8 ];
980 0 : char * commit_overwrite_class_str[ 8 ];
981 0 : for( ulong c=0UL; c<8UL; c++ ) {
982 0 : ulong sum_c = 0UL, sum_pc = 0UL, sum_cn = 0UL, sum_co = 0UL;
983 0 : for( ulong i=0UL; i<n; i++ ) {
984 0 : sum_c += accdb_evicted_class_samples [ c ][ i ];
985 0 : sum_pc += accdb_preevicted_class_samples [ c ][ i ];
986 0 : sum_cn += accdb_committed_new_class_samples [ c ][ i ];
987 0 : sum_co += accdb_committed_overwrite_class_samples[ c ][ i ];
988 0 : }
989 0 : evict_class_str [ c ] = COUNTF_T( (double)sum_c/window_seconds( n ) );
990 0 : preevict_class_str [ c ] = COUNTF_T( (double)sum_pc/window_seconds( n ) );
991 0 : commit_new_class_str [ c ] = COUNTF_T( (double)sum_cn/window_seconds( n ) );
992 0 : commit_overwrite_class_str[ c ] = COUNTF_T( (double)sum_co/window_seconds( n ) );
993 0 : }
994 :
995 : /* Per size class grid: dim header row, then one row per family
996 : with every cell right aligned in a fixed 19 char column. */
997 0 : static char const * class_lbl[ 8 ] = { "128b", "512b", "2k", "8k", "32k", "128k", "1m", "10m" };
998 :
999 0 : PRINT( ROWC K( "class " ) );
1000 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( DIM "%21s" RESET, class_lbl[ c ] );
1001 0 : PRINT( CLEARLN "\n" );
1002 :
1003 0 : PRINT( ROWC K( "evict/s " ) );
1004 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( " %6s" DIM "+" RESET "%6s", evict_class_str[ c ], preevict_class_str[ c ] );
1005 0 : PRINT( CLEARLN "\n" );
1006 :
1007 0 : PRINT( ROWC K( "commit/s" ) );
1008 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( " %6s" DIM "=" RESET "%6s", commit_new_class_str[ c ], commit_overwrite_class_str[ c ] );
1009 0 : PRINT( CLEARLN "\n" );
1010 :
1011 0 : ulong cache_used_off = MIDX( GAUGE, ACCDB, CACHE_CLASS_USED );
1012 0 : ulong cache_max_off = MIDX( GAUGE, ACCDB, CACHE_CLASS_CAPACITY );
1013 0 : char * cache_used_str[ 8 ];
1014 0 : char * cache_max_str [ 8 ];
1015 0 : double cache_pct [ 8 ];
1016 0 : for( ulong c=0UL; c<8UL; c++ ) {
1017 0 : ulong used = t[ cache_used_off + c ];
1018 0 : ulong max = t[ cache_max_off + c ];
1019 0 : cache_used_str[ c ] = COUNT_T( used );
1020 0 : cache_max_str [ c ] = COUNT_T( max );
1021 0 : cache_pct [ c ] = max ? 100.0*(double)used/(double)max : 0.0;
1022 0 : }
1023 :
1024 0 : PRINT( ROWC K( "cache " ) );
1025 0 : for( ulong c=0UL; c<8UL; c++ ) {
1026 : /* COUNT_T strings are already 6 wide: cell = 1+5+1+1+6+1+6 = 21. */
1027 0 : PRINT( " %s%5.1f" U( "%%" ) RESET " %s" DIM "/" RESET "%s",
1028 0 : sev_color( cache_pct[ c ] ), cache_pct[ c ],
1029 0 : cache_used_str[ c ], cache_max_str[ c ] );
1030 0 : }
1031 0 : PRINT( CLEARLN "\n" );
1032 :
1033 0 : ulong cache_resv_off = MIDX( GAUGE, ACCDB, CACHE_CLASS_RESERVED );
1034 0 : ulong cache_tracked_off = MIDX( GAUGE, ACCDB, IS_CACHE_CLASS_RESERVATION_TRACKED );
1035 0 : char * cache_resv_str[ 8 ];
1036 0 : for( ulong c=0UL; c<8UL; c++ ) {
1037 0 : if( !t[ cache_tracked_off + c ] ) cache_resv_str[ c ] = " off ";
1038 0 : else cache_resv_str[ c ] = COUNT_T( t[ cache_resv_off + c ] );
1039 0 : }
1040 :
1041 0 : PRINT( ROWC K( "reserved" ) );
1042 0 : for( ulong c=0UL; c<8UL; c++ ) PRINT( " %6s", cache_resv_str[ c ] );
1043 0 : PRINT( CLEARLN "\n" );
1044 0 : return 7;
1045 0 : }
1046 :
1047 : static uint
1048 : write_wfs( config_t const * config,
1049 0 : ulong const * cur_tile ) {
1050 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
1051 0 : if( FD_UNLIKELY( gossip_tile_idx==ULONG_MAX ) ) return 0U;
1052 :
1053 0 : int wfs_state = (int)cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STATE ) ];
1054 0 : if( FD_LIKELY( wfs_state==FD_GOSSIP_WFS_STATE_DONE ) ) return 0U;
1055 :
1056 0 : char const * state_str;
1057 0 : switch( wfs_state ) {
1058 0 : case FD_GOSSIP_WFS_STATE_INIT: state_str = "loading snapshot"; break;
1059 0 : case FD_GOSSIP_WFS_STATE_WAIT: state_str = "waiting"; break;
1060 0 : case FD_GOSSIP_WFS_STATE_PUBLISH: state_str = "starting"; break;
1061 0 : default: return 0U;
1062 0 : }
1063 :
1064 0 : ulong _stake_online = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKE_ONLINE_LAMPORTS ) ];
1065 0 : ulong _stake_total = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKE_TRACKED_LAMPORTS ) ];
1066 0 : ulong peers_online = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKED_PEER_ONLINE ) ];
1067 0 : ulong peers_total = cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, GOSSIP, WAIT_FOR_SUPERMAJORITY_STAKED_PEER_COUNT ) ];
1068 :
1069 0 : ulong ipecho_tile_idx = fd_topo_find_tile( &config->topo, "ipecho", 0UL );
1070 0 : ulong shred_ver = 0UL;
1071 0 : if( FD_LIKELY( ipecho_tile_idx!=ULONG_MAX ) ) shred_ver = cur_tile[ ipecho_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, IPECHO, SHRED_VERSION ) ];
1072 :
1073 0 : double stake_pct = _stake_total>0UL ? 100.0*(double)_stake_online/(double)_stake_total : 0.0;
1074 0 : double stake_div = (_stake_total<(ulong)1e14) ? 1e9 : 1e15;
1075 0 : char const * stake_unit = (_stake_total<(ulong)1e14) ? " SOL" : "M";
1076 0 : double stake_online = (double)_stake_online / stake_div;
1077 0 : double stake_total = (double)_stake_total / stake_div;
1078 :
1079 0 : PRINT( ROWH( "○", YELLOW, "cluster boot" )
1080 0 : K( "state" ) "%s"
1081 0 : K( "stake" ) "%s " BOLD "%3.0f" RESET U( "%%" ) " " U( "(%.1f%s / %.1f%s)" )
1082 0 : K( "shred ver" ) "%lu"
1083 0 : K( "peers" ) "%lu" U( " online " ) "%lu" U( " offline" )
1084 0 : K( "bank hash" ) "%s" CLEARLN "\n",
1085 0 : state_str,
1086 0 : BAR( stake_pct, 10UL ),
1087 0 : stake_pct,
1088 0 : stake_online,
1089 0 : stake_unit,
1090 0 : stake_total,
1091 0 : stake_unit,
1092 0 : shred_ver,
1093 0 : peers_online,
1094 0 : peers_total>peers_online ? peers_total-peers_online : 0UL,
1095 0 : config->firedancer.consensus.wait_for_supermajority_with_bank_hash );
1096 0 : return 1U;
1097 0 : }
1098 :
1099 : static uint
1100 : write_gossip( config_t const * config,
1101 : ulong const * cur_tile,
1102 : ulong const * prev_tile,
1103 : ulong const * cur_link,
1104 0 : ulong const * prev_link ) {
1105 0 : ulong gossip_tile_idx = fd_topo_find_tile( &config->topo, "gossip", 0UL );
1106 0 : if( gossip_tile_idx==ULONG_MAX ) return 0U;
1107 0 : ulong const * t = &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ];
1108 0 : char * contact_info = COUNT( t[ MIDX( GAUGE, GOSSIP, CRDS_USED_CONTACT_INFO_V2 ) ] );
1109 :
1110 0 : ulong gossip_total_ticks = total_regime( &cur_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gossip_tile_idx*FD_METRICS_TOTAL_SZ ] );
1111 0 : gossip_total_ticks = fd_ulong_max( gossip_total_ticks, 1UL );
1112 0 : double gossip_backp_pct = 100.0*(double)diff_tile_regime( config, "gossip", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)gossip_total_ticks;
1113 0 : double gossip_idle_pct = 100.0*(double)diff_tile_regime( config, "gossip", prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)gossip_total_ticks;
1114 0 : double gossip_busy_pct = 100.0 - gossip_backp_pct - gossip_idle_pct;
1115 :
1116 0 : PRINT( ROWH( "●", BLUE, "gossip " )
1117 0 : K( "rx" ) "%s"
1118 0 : K( "tx" ) "%s"
1119 0 : K( "crds" ) "%s"
1120 0 : K( "peers" ) "%s ",
1121 0 : DIFF_LINK_BYTES( "net_gossvf", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1122 0 : DIFF_LINK_BYTES( "gossip_net", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1123 0 : COUNT( total_crds( t ) ),
1124 0 : contact_info );
1125 0 : if( FD_UNLIKELY( gossip_saturated ) ) PRINT( GREEN "✓" RESET DIM " saturated" RESET );
1126 0 : else PRINT( "%s " DIM "%3.0f%%" RESET, BAR( gossip_conv_pct, 8UL ), gossip_conv_pct );
1127 0 : PRINT( K( "busy" ) "%s%3.0f" U( "%%" ) RESET
1128 0 : K( "backp" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
1129 0 : sev_color( gossip_busy_pct ), gossip_busy_pct,
1130 0 : sev_color( gossip_backp_pct ), gossip_backp_pct );
1131 0 : return 1U;
1132 0 : }
1133 :
1134 : static uint
1135 : write_repair( config_t const * config,
1136 : ulong const * cur_tile,
1137 : ulong const * cur_link,
1138 0 : ulong const * prev_link ) {
1139 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "rotor", 0UL );
1140 0 : char const * repair_label = "rotor ";
1141 0 : if( repair_tile_idx==ULONG_MAX ) return 0U;
1142 0 : ulong highest_off = MIDX( GAUGE, ROTOR, HIGHEST_DELIVERED_SLOT );
1143 0 : ulong current_off = MIDX( GAUGE, ROTOR, HIGHEST_RECEIVED_SLOT );
1144 0 : ulong repair_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+highest_off ];
1145 0 : ulong turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+current_off ];
1146 0 : turbine_slot = fd_ulong_max( turbine_slot, cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, ROTOR, CATCHUP_SLOT ) ] );
1147 0 : long repair_lag = (long)repair_slot-(long)turbine_slot;
1148 0 : PRINT( ROWH( "▲", RED, "%s" )
1149 0 : K( "rx" ) "%s"
1150 0 : K( "tx" ) "%s"
1151 0 : K( "repair slot" ) "%lu %s%+03ld" RESET
1152 0 : K( "turbine slot" ) "%lu" CLEARLN "\n",
1153 0 : repair_label,
1154 0 : DIFF_LINK_BYTES( "net_repair", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1155 0 : DIFF_LINK_BYTES( "repair_net", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1156 0 : repair_slot,
1157 0 : repair_lag<-4L ? RED : DIM,
1158 0 : repair_lag,
1159 0 : turbine_slot );
1160 0 : return 1U;
1161 0 : }
1162 :
1163 : static uint
1164 : write_rserve( config_t const * config,
1165 : ulong const * cur_tile,
1166 : ulong const * cur_link,
1167 0 : ulong const * prev_link ) {
1168 0 : ulong rserve_tile_idx = fd_topo_find_tile( &config->topo, "rserve", 0UL );
1169 0 : if( rserve_tile_idx==ULONG_MAX ) return 0UL;
1170 :
1171 0 : ulong const * t = cur_tile + rserve_tile_idx*FD_METRICS_TOTAL_SZ;
1172 :
1173 0 : ulong shreds_stored_sum = 0UL;
1174 0 : ulong num_stored_shreds = fd_ulong_min( shreds_stored_samples_idx, sizeof(shreds_stored_sample)/sizeof(shreds_stored_sample[0]));
1175 0 : for( ulong i=0UL; i<num_stored_shreds; i++ ) shreds_stored_sum += shreds_stored_sample[ i ];
1176 0 : char * shreds_stored = COUNTF( (double)shreds_stored_sum/window_seconds( num_stored_shreds ) );
1177 :
1178 0 : ulong shreds_cur = t[ MIDX( GAUGE, RSERVE, SHRED_USED ) ];
1179 0 : ulong shreds_max = t[ MIDX( GAUGE, RSERVE, SHRED_CAPACITY ) ];
1180 :
1181 0 : ulong valid_sum = 0UL;
1182 0 : ulong num_valid_samples = fd_ulong_min( rserve_rps_valid_samples_idx, sizeof(rserve_rps_valid_samples)/sizeof(rserve_rps_valid_samples[0]) );
1183 0 : for( ulong i=0UL; i<num_valid_samples; i++ ) valid_sum += rserve_rps_valid_samples[ i ];
1184 0 : char * valid_str = COUNTF( (double)valid_sum/window_seconds( num_valid_samples ) );
1185 :
1186 0 : ulong invalid_sum = 0UL;
1187 0 : ulong num_invalid_samples = fd_ulong_min( rserve_rps_invalid_samples_idx, sizeof(rserve_rps_invalid_samples)/sizeof(rserve_rps_invalid_samples[0]) );
1188 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) invalid_sum += rserve_rps_invalid_samples[ i ];
1189 0 : char * invalid_str = COUNTF( (double)invalid_sum/window_seconds( num_invalid_samples ) );
1190 :
1191 0 : ulong num_total_samples = fd_ulong_max( num_valid_samples, 1UL );
1192 0 : char * total_str = COUNTF( (double)(valid_sum+invalid_sum)/window_seconds( num_total_samples ) );
1193 :
1194 0 : ulong n_inv = fd_ulong_max( num_invalid_samples, 1UL );
1195 :
1196 0 : ulong miss_sum = 0UL;
1197 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) miss_sum += rserve_rps_miss_samples[ i ];
1198 0 : char * miss_str = COUNTF( (double)miss_sum/window_seconds( n_inv ) );
1199 :
1200 0 : ulong sigvfy_sum = 0UL;
1201 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) sigvfy_sum += rserve_rps_sigvfy_samples[ i ];
1202 0 : char * sigvfy_str = COUNTF( (double)sigvfy_sum/window_seconds( n_inv ) );
1203 :
1204 0 : ulong stale_sum = 0UL;
1205 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) stale_sum += rserve_rps_stale_samples[ i ];
1206 0 : char * stale_str = COUNTF( (double)stale_sum/window_seconds( n_inv ) );
1207 :
1208 0 : ulong other_sum = 0UL;
1209 0 : for( ulong i=0UL; i<num_invalid_samples; i++ ) other_sum += rserve_rps_other_rej_samples[ i ];
1210 0 : char * other_str = COUNTF( (double)other_sum/window_seconds( n_inv ) );
1211 :
1212 0 : ulong disk_used = t[ MIDX( GAUGE, RSERVE, DISK_USED_BYTES ) ];
1213 0 : ulong disk_alloc = t[ MIDX( GAUGE, RSERVE, DISK_PEAK_BYTES ) ];
1214 0 : double disk_gb = (double)disk_alloc/1e9;
1215 0 : double disk_pct = disk_alloc ? 100.0*(double)disk_used/(double)disk_alloc : 0.0;
1216 :
1217 0 : char * shreds_cur_str = COUNT( shreds_cur );
1218 0 : char * shreds_max_str = COUNT( shreds_max );
1219 :
1220 0 : PRINT( ROWH( "□", GREEN, "rserve " )
1221 0 : K( "rx" ) "%s"
1222 0 : K( "tx" ) "%s"
1223 0 : K( "shreds" ) "%s" U( "/s" ) " " U( "(%s / %s)" )
1224 0 : K( "disk" ) "%.1f" U( " GB" ) " %s%4.1f" U( "%%" ) RESET,
1225 0 : DIFF_LINK_BYTES( "net_rserve", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1226 0 : DIFF_LINK_BYTES( "rserve_net", COUNTER, LINK, FRAG_CONSUMED_BYTES ),
1227 0 : shreds_stored, shreds_cur_str, shreds_max_str,
1228 0 : disk_gb, sev_color( disk_pct ), disk_pct );
1229 0 : PRINT( K( "rps" ) "%s" U( "/s" ) " " U( "(%s valid, %s invalid)" ) CLEARLN "\n",
1230 0 : total_str, valid_str, invalid_str );
1231 :
1232 0 : if( FD_LIKELY( !watch_full ) ) return 1U;
1233 :
1234 0 : PRINT( ROWC
1235 0 : K( "miss" ) "%s" U( "/s" )
1236 0 : K( "sigvfy" ) "%s" U( "/s" )
1237 0 : K( "stale" ) "%s" U( "/s" )
1238 0 : K( "other" ) "%s" U( "/s" ) CLEARLN "\n",
1239 0 : miss_str, sigvfy_str, stale_str, other_str );
1240 0 : return 2U;
1241 0 : }
1242 :
1243 : static uint
1244 : write_replay( config_t const * config,
1245 0 : ulong const * cur_tile ) {
1246 0 : ulong repair_tile_idx = fd_topo_find_tile( &config->topo, "rotor", 0UL );
1247 0 : ulong replay_tile_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
1248 0 : if( replay_tile_idx==ULONG_MAX ) return 0U;
1249 :
1250 0 : ulong reset_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ];
1251 0 : ulong next_leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, NEXT_LEADER_SLOT ) ];
1252 0 : ulong leader_slot = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, LEADER_SLOT ) ];
1253 0 : char * next_leader_slot_str = fd_alloca_check( 1UL, 64UL );
1254 :
1255 0 : ulong turbine_slot;
1256 0 : if( repair_tile_idx!=ULONG_MAX ) {
1257 0 : turbine_slot = cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, ROTOR, HIGHEST_RECEIVED_SLOT ) ];
1258 0 : turbine_slot = fd_ulong_max( turbine_slot, cur_tile[ repair_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, ROTOR, CATCHUP_SLOT ) ] );
1259 0 : } else {
1260 0 : turbine_slot = reset_slot;
1261 0 : }
1262 :
1263 0 : ulong slot_in_seconds = (ulong)((double)(next_leader_slot-reset_slot)*0.4);
1264 0 : if( FD_UNLIKELY( leader_slot ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "now" ) );
1265 0 : else if( FD_LIKELY( next_leader_slot>0UL ) ) FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "%lum %lus", slot_in_seconds/60UL, slot_in_seconds%60UL ) );
1266 0 : else FD_TEST( fd_cstr_printf_check( next_leader_slot_str, 64UL, NULL, "never" ) );
1267 :
1268 0 : ulong root_distance = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, ROOT_SLOT_DISTANCE ) ];
1269 0 : ulong live_banks = cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, BANK_USED ) ];
1270 :
1271 0 : ulong sps_sum = 0UL;
1272 0 : ulong num_sps_samples = fd_ulong_min( sps_samples_idx, sizeof(sps_samples)/sizeof(sps_samples[0]));
1273 0 : for( ulong i=0UL; i<num_sps_samples; i++ ) sps_sum += sps_samples[ i ];
1274 0 : char * sps_str = COUNTF( (double)sps_sum/window_seconds( num_sps_samples ) );
1275 :
1276 0 : ulong tps_sum = 0UL;
1277 0 : ulong num_tps_samples = fd_ulong_min( tps_samples_idx, sizeof(tps_samples)/sizeof(tps_samples[0]));
1278 0 : for( ulong i=0UL; i<num_tps_samples; i++ ) tps_sum += tps_samples[ i ];
1279 0 : char * tps_str = COUNTF( (double)tps_sum/window_seconds( num_tps_samples ) );
1280 :
1281 0 : ulong cups_sum = 0UL;
1282 0 : ulong num_cups_samples = fd_ulong_min( cups_samples_idx, sizeof(cups_samples)/sizeof(cups_samples[0]));
1283 0 : for( ulong i=0UL; i<num_cups_samples; i++ ) cups_sum += cups_samples[ i ];
1284 0 : char * mcups_str = COUNTF( (double)cups_sum/window_seconds( num_cups_samples ) );
1285 :
1286 0 : long replay_lag = (long)reset_slot-(long)turbine_slot;
1287 0 : PRINT( ROWH( "▶", MAGENTA, "replay " )
1288 0 : K( "slot" ) BOLD "%lu" RESET " %s%+03ld" RESET
1289 0 : K( "cu/s" ) "%s"
1290 0 : K( "tps" ) "%s"
1291 0 : K( "sps" ) "%s"
1292 0 : K( "leader in" ) "%s"
1293 0 : K( "root dist" ) "%lu"
1294 0 : K( "banks" ) "%2lu" CLEARLN "\n",
1295 0 : reset_slot,
1296 0 : replay_lag<-4L ? RED : DIM,
1297 0 : replay_lag,
1298 0 : mcups_str,
1299 0 : tps_str,
1300 0 : sps_str,
1301 0 : next_leader_slot_str,
1302 0 : root_distance,
1303 0 : live_banks );
1304 0 : return 1U;
1305 0 : }
1306 :
1307 : #define VOTOR_RATE( metric ) (__extension__({ \
1308 : ulong cnt = diff_tile( config, "votor", prev_tile, cur_tile, MIDX( COUNTER, VOTOR, metric ) ); \
1309 : COUNTF( (double)cnt*1e9/(double)SNAP_DT_NS() ); \
1310 : }))
1311 :
1312 : static uint
1313 : write_votor( config_t const * config,
1314 : ulong const * cur_tile,
1315 0 : ulong const * prev_tile ) {
1316 0 : ulong votor_tile_idx = fd_topo_find_tile( &config->topo, "votor", 0UL );
1317 0 : if( votor_tile_idx==ULONG_MAX ) return 0U;
1318 0 : ulong const * t = &cur_tile[ votor_tile_idx*FD_METRICS_TOTAL_SZ ];
1319 :
1320 0 : ulong replay_tile_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
1321 0 : ulong reset_slot = replay_tile_idx!=ULONG_MAX ? cur_tile[ replay_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, REPLAY, RESET_SLOT ) ] : 0UL;
1322 :
1323 0 : ulong finalized_slot = t[ MIDX( GAUGE, VOTOR, HIGHEST_FINAL_CERT_SLOT ) ];
1324 0 : ulong slots_used = t[ MIDX( GAUGE, VOTOR, VOTING_SLOT_USED ) ];
1325 0 : ulong slots_max = slots_used+t[ MIDX( GAUGE, VOTOR, VOTING_SLOT_FREE ) ];
1326 0 : ulong peers_out = t[ MIDX( GAUGE, VOTOR, CLIENT_CONN_USED )+FD_METRICS_ENUM_QUIC_CONN_STATE_V_ACTIVE_IDX ];
1327 0 : ulong peers_in = t[ MIDX( GAUGE, VOTOR, SERVER_CONN_USED )+FD_METRICS_ENUM_QUIC_CONN_STATE_V_ACTIVE_IDX ];
1328 0 : long rank = (long)t[ MIDX( GAUGE, VOTOR, RANK ) ];
1329 0 : double slots_pct = slots_max ? 100.0*(double)slots_used/(double)slots_max : 0.0;
1330 0 : long final_lag = (long)finalized_slot-(long)reset_slot;
1331 :
1332 0 : PRINT( ROWH( "◇", CYAN, "votor " )
1333 0 : K( "final" ) BOLD "%lu" RESET " %s%+03ld" RESET
1334 0 : K( "slots" ) "%s%lu" RESET U( "/%lu" ),
1335 0 : finalized_slot,
1336 0 : final_lag<-32L ? RED : DIM,
1337 0 : final_lag,
1338 0 : sev_color( slots_pct ), slots_used, slots_max );
1339 0 : if( FD_UNLIKELY( rank<0L ) ) {
1340 0 : PRINT( " " YELLOW "unstaked" RESET );
1341 0 : } else {
1342 0 : PRINT( K( "votes" ) "%s" U( "/s" )
1343 0 : K( "certs" ) "%s" U( "/s" )
1344 0 : K( "peers" ) "%lu" U( "out" ) " %lu" U( "in" )
1345 0 : K( "rank" ) "%ld",
1346 0 : VOTOR_RATE( VOTE_RX_SUCCESS ),
1347 0 : VOTOR_RATE( CERT_RX_SUCCESS ),
1348 0 : peers_out, peers_in,
1349 0 : rank );
1350 0 : }
1351 0 : PRINT( CLEARLN "\n" );
1352 0 : return 1U;
1353 0 : }
1354 :
1355 : #undef VOTOR_RATE
1356 :
1357 : static uint
1358 : write_gui( config_t const * config,
1359 : ulong const * cur_tile,
1360 0 : ulong const * prev_tile ) {
1361 0 : char const * gui_name = "gui";
1362 0 : ulong gui_tile_idx = fd_topo_find_tile( &config->topo, gui_name, 0UL );
1363 0 : if( FD_UNLIKELY( gui_tile_idx==ULONG_MAX ) ) return 0U;
1364 :
1365 0 : ulong off_conn_active, off_websocket_conn_active, off_websocket_frame_tx, off_websocket_frame_rx;
1366 0 : char * bytes_read_s;
1367 0 : char * bytes_written_s;
1368 0 : off_conn_active = MIDX( GAUGE, GUI, CONN_OPEN );
1369 0 : off_websocket_conn_active = MIDX( GAUGE, GUI, WEBSOCKET_CONN_OPEN );
1370 0 : off_websocket_frame_tx = MIDX( COUNTER, GUI, WEBSOCKET_FRAME_TX );
1371 0 : off_websocket_frame_rx = MIDX( COUNTER, GUI, WEBSOCKET_FRAME_RX );
1372 :
1373 0 : bytes_read_s = DIFF_BYTES( gui_name, COUNTER, GUI, CONN_RX_BYTES );
1374 0 : bytes_written_s = DIFF_BYTES( gui_name, COUNTER, GUI, CONN_TX_BYTES );
1375 :
1376 0 : ulong connection_count = cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+off_conn_active ]+cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ+off_websocket_conn_active ];
1377 0 : ulong gui_total_ticks = total_regime( &cur_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] )-total_regime( &prev_tile[ gui_tile_idx*FD_METRICS_TOTAL_SZ ] );
1378 0 : gui_total_ticks = fd_ulong_max( gui_total_ticks, 1UL );
1379 0 : double gui_backp_pct = 100.0*(double)diff_tile_regime( config, gui_name, prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_BACKPRESSURE_MASK )/(double)gui_total_ticks;
1380 0 : double gui_idle_pct = 100.0*(double)diff_tile_regime( config, gui_name, prev_tile, cur_tile, FD_METRICS_ENUM_TILE_REGIME_G_IDLE_MASK )/(double)gui_total_ticks;
1381 0 : double gui_busy_pct = 100.0 - gui_backp_pct - gui_idle_pct;
1382 :
1383 0 : ulong sent_frame_count = diff_tile( config, gui_name, prev_tile, cur_tile, off_websocket_frame_tx );
1384 0 : char * sent_frame_count_s = COUNT( sent_frame_count );
1385 0 : ulong received_frame_count = diff_tile( config, gui_name, prev_tile, cur_tile, off_websocket_frame_rx );
1386 :
1387 0 : PRINT( ROWH( "◉", CYAN, "gui " )
1388 0 : K( "conns" ) "%lu"
1389 0 : K( "frames" ) "%s" U( " in" ) " %s" U( " out" )
1390 0 : K( "bw" ) "%s" U( " in" ) " %s" U( " out" )
1391 0 : K( "busy" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
1392 0 : connection_count,
1393 0 : COUNT( received_frame_count ),
1394 0 : sent_frame_count_s,
1395 0 : bytes_read_s,
1396 0 : bytes_written_s,
1397 0 : sev_color( gui_busy_pct ), gui_busy_pct );
1398 0 : return 1U;
1399 0 : }
1400 :
1401 : static uint
1402 : write_event( config_t const * config,
1403 0 : ulong const * cur_tile ) {
1404 0 : ulong event_tile_idx = fd_topo_find_tile( &config->topo, "event", 0UL );
1405 0 : if( event_tile_idx==ULONG_MAX ) return 0U;
1406 :
1407 0 : ulong connection_state = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, CONN_STATE ) ];
1408 0 : char const * connection_state_str;
1409 0 : switch( connection_state ) {
1410 0 : case FD_EVENT_CLIENT_STATE_DISCONNECTED: connection_state_str = RED "disconnected" RESET; break;
1411 0 : case FD_EVENT_CLIENT_STATE_CONNECTING: connection_state_str = YELLOW "connecting" RESET; break;
1412 0 : case FD_EVENT_CLIENT_STATE_AUTHENTICATING: connection_state_str = YELLOW "authenticating" RESET; break;
1413 0 : case FD_EVENT_CLIENT_STATE_CONNECTED: connection_state_str = GREEN "connected" RESET; break;
1414 0 : default: connection_state_str = RED "unknown" RESET; break;
1415 0 : }
1416 :
1417 0 : ulong event_queue_count = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_USED ) ];
1418 0 : ulong event_queue_unsent = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_UNSENT_COUNT ) ];
1419 0 : ulong event_queue_unacked = event_queue_count>event_queue_unsent ? event_queue_count-event_queue_unsent : 0UL;
1420 0 : ulong event_last_acked_id = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, LAST_ACKED_NONCE ) ];
1421 0 : ulong event_queue_drops = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( COUNTER, EVENT, QUEUE_EVICTED ) ];
1422 0 : ulong event_queue_bytes_used = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_USED_BYTES ) ];
1423 0 : ulong event_queue_bytes_capacity = cur_tile[ event_tile_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, EVENT, QUEUE_CAPACITY_BYTES ) ];
1424 :
1425 0 : double event_queue_pct_full = event_queue_bytes_capacity>0UL ? 100.0*(double)event_queue_bytes_used/(double)event_queue_bytes_capacity : 0.0;
1426 :
1427 0 : ulong events_sent_sum = 0UL;
1428 0 : ulong num_events_sent_samples = fd_ulong_min( events_sent_samples_idx, sizeof(events_sent_samples)/sizeof(events_sent_samples[0]));
1429 0 : for( ulong i=0UL; i<num_events_sent_samples; i++ ) events_sent_sum += events_sent_samples[ i ];
1430 0 : char * events_sent_str = COUNTF( (double)events_sent_sum/window_seconds( num_events_sent_samples ) );
1431 :
1432 0 : ulong events_acked_sum = 0UL;
1433 0 : ulong num_events_acked_samples = fd_ulong_min( events_acked_samples_idx, sizeof(events_acked_samples)/sizeof(events_acked_samples[0]));
1434 0 : for( ulong i=0UL; i<num_events_acked_samples; i++ ) events_acked_sum += events_acked_samples[ i ];
1435 0 : char * events_acked_str = COUNTF( (double)events_acked_sum/window_seconds( num_events_acked_samples ) );
1436 :
1437 0 : ulong bytes_written_sum = 0UL;
1438 0 : ulong num_bytes_written_samples = fd_ulong_min( event_bytes_written_samples_idx, sizeof(event_bytes_written_samples)/sizeof(event_bytes_written_samples[0]));
1439 0 : for( ulong i=0UL; i<num_bytes_written_samples; i++ ) bytes_written_sum += event_bytes_written_samples[ i ];
1440 0 : long bytes_written_per_sec = (long)((double)bytes_written_sum/window_seconds( num_bytes_written_samples ));
1441 0 : char * bytes_written_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes_written_per_sec );
1442 :
1443 0 : ulong bytes_read_sum = 0UL;
1444 0 : ulong num_bytes_read_samples = fd_ulong_min( event_bytes_read_samples_idx, sizeof(event_bytes_read_samples)/sizeof(event_bytes_read_samples[0]));
1445 0 : for( ulong i=0UL; i<num_bytes_read_samples; i++ ) bytes_read_sum += event_bytes_read_samples[ i ];
1446 0 : long bytes_read_per_sec = (long)((double)bytes_read_sum/window_seconds( num_bytes_read_samples ));
1447 0 : char * bytes_read_str = fmt_bytes( fd_alloca_check( 1UL, 64UL ), 64UL, bytes_read_per_sec );
1448 :
1449 0 : char * event_queue_unacked_s = COUNT( event_queue_unacked );
1450 0 : char * event_queue_unsent_s = COUNT( event_queue_unsent );
1451 0 : char * event_last_acked_id_s = COUNT( event_last_acked_id );
1452 :
1453 0 : PRINT( ROWH( "◆", YELLOW, "event " )
1454 0 : K( "state" ) "%s"
1455 0 : K( "unacked" ) "%s"
1456 0 : K( "queue" ) "%s"
1457 0 : K( "sent" ) "%s" U( "/s" )
1458 0 : K( "acked" ) "%s" U( "/s" )
1459 0 : K( "last ack" ) "%s"
1460 0 : K( "bw" ) "%s" U( " in" ) " %s" U( " out" )
1461 0 : K( "drops" ) "%s"
1462 0 : K( "full" ) "%s%3.0f" U( "%%" ) RESET CLEARLN "\n",
1463 0 : connection_state_str,
1464 0 : event_queue_unacked_s,
1465 0 : event_queue_unsent_s,
1466 0 : events_sent_str,
1467 0 : events_acked_str,
1468 0 : event_last_acked_id_s,
1469 0 : bytes_read_str,
1470 0 : bytes_written_str,
1471 0 : COUNT( event_queue_drops ),
1472 0 : sev_color( event_queue_pct_full ), event_queue_pct_full );
1473 0 : return 1U;
1474 0 : }
1475 :
1476 : /* Pad a value to a fixed display width so fields that fill in after
1477 : boot (identity, vote, shred version, ...) never shift the columns
1478 : to their right on redraw. Unknown values render as a dim "-". */
1479 : static char *
1480 : fmt_field( char * buf,
1481 : ulong buf_sz,
1482 : char const * val,
1483 : int known,
1484 0 : int width ) {
1485 0 : if( FD_LIKELY( known ) ) FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, "%-*s", width, val ) );
1486 0 : else FD_TEST( fd_cstr_printf_check( buf, buf_sz, NULL, DIM "%-*s" RESET, width, "-" ) );
1487 0 : return buf;
1488 0 : }
1489 :
1490 : static char *
1491 : base58_short( char out[ static 16 ],
1492 0 : uchar const key[ static 32 ] ) {
1493 0 : char enc[ FD_BASE58_ENCODED_32_SZ ];
1494 0 : ulong len;
1495 0 : fd_base58_encode_32( key, &len, enc );
1496 0 : char * p = fd_cstr_init( out );
1497 0 : p = fd_cstr_append_text( p, enc, 6 ),
1498 0 : p = fd_cstr_append_text( p, "…", sizeof("…")-1 ),
1499 0 : p = fd_cstr_append_text( p, enc+len-6, 6 );
1500 0 : fd_cstr_fini( p );
1501 0 : return out;
1502 0 : }
1503 :
1504 : static void
1505 : fmt_balance( char out[ static 64 ],
1506 0 : ulong lamports ) {
1507 0 : ulong maj = lamports / 1000000000UL;
1508 0 : ulong min = lamports % 1000000000UL;
1509 0 : fd_cstr_printf_check( out, 64UL, NULL, "%lu.%09lu", maj, min );
1510 0 : }
1511 :
1512 : static uint
1513 : write_node_info( config_t const * config,
1514 : ulong const * cur_tile,
1515 0 : fd_node_info_t const * node_info ) {
1516 :
1517 0 : char identity_b58[ FD_BASE58_ENCODED_32_SZ ];
1518 0 : int has_identity = node_info && !fd_pubkey_check_zero( &node_info->identity );
1519 0 : if( FD_LIKELY( has_identity ) ) fd_base58_encode_32( node_info->identity.key, NULL, identity_b58 );
1520 :
1521 0 : char vote_acc_b58[ FD_BASE58_ENCODED_32_SZ ];
1522 0 : int has_vote_acc = node_info && !fd_pubkey_check_zero( &node_info->vote_account );
1523 0 : if( FD_LIKELY( has_vote_acc ) ) fd_base58_encode_32( node_info->vote_account.key, NULL, vote_acc_b58 );
1524 :
1525 0 : char shred_ver[ 16 ];
1526 0 : int has_shred_ver = 1;
1527 0 : ulong ipecho_idx = fd_topo_find_tile( &config->topo, "ipecho", 0UL );
1528 0 : ushort shred_version = 0;
1529 0 : if( FD_LIKELY( ipecho_idx!=ULONG_MAX ) ) shred_version = (ushort)cur_tile[ ipecho_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, IPECHO, SHRED_VERSION ) ];
1530 0 : if( shred_version ) {
1531 0 : fd_cstr_printf_check( shred_ver, sizeof(shred_ver), NULL, "%hu", shred_version );
1532 0 : } else if( config->consensus.expected_shred_version ) {
1533 0 : fd_cstr_printf_check( shred_ver, sizeof(shred_ver), NULL, "(%hu)", config->consensus.expected_shred_version );
1534 0 : } else {
1535 0 : has_shred_ver = 0;
1536 0 : }
1537 :
1538 0 : char genesis_hash_b58[ FD_BASE58_ENCODED_32_SZ ] = {0};
1539 0 : char genesis_short[ 16 ] = {0};
1540 0 : int has_genesis_b58 = 0;
1541 0 : if( FD_LIKELY( node_info && !fd_pubkey_check_zero( &node_info->genesis_hash ) ) ) {
1542 0 : fd_base58_encode_32( node_info->genesis_hash.key, NULL, genesis_hash_b58 );
1543 0 : base58_short( genesis_short, node_info->genesis_hash.key );
1544 0 : has_genesis_b58 = 1;
1545 0 : }
1546 :
1547 0 : char const * cluster_str = "";
1548 0 : char const * cluster_color = "";
1549 0 : if( has_genesis_b58 ) {
1550 0 : ulong cluster_id = fd_genesis_cluster_identify( genesis_hash_b58 );
1551 0 : cluster_str = cluster_id==FD_CLUSTER_MAINNET_BETA ? "mainnet" : fd_genesis_cluster_name( cluster_id );
1552 0 : switch( cluster_id ) {
1553 0 : case FD_CLUSTER_MAINNET_BETA: cluster_color = CLUS_MAINNET; break;
1554 0 : case FD_CLUSTER_TESTNET: cluster_color = CLUS_TESTNET; break;
1555 0 : case FD_CLUSTER_DEVNET: cluster_color = CLUS_DEVNET; break;
1556 0 : case FD_CLUSTER_PYTHNET: cluster_color = CLUS_PYTHNET; break;
1557 0 : case FD_CLUSTER_PYTHTEST: cluster_color = CLUS_PYTHTEST; break;
1558 0 : default: cluster_color = CLUS_UNKNOWN; break;
1559 0 : }
1560 0 : }
1561 :
1562 0 : char uptime_str[ 32UL ];
1563 0 : int has_uptime = 1;
1564 0 : long now = fd_log_wallclock();
1565 0 : if( FD_LIKELY( config->boot_timestamp_nanos>0L && now>config->boot_timestamp_nanos ) ) {
1566 0 : ulong elapsed_s = (ulong)( (now - config->boot_timestamp_nanos) / (long)1e9 );
1567 0 : ulong days = elapsed_s / 86400UL;
1568 0 : ulong hours = (elapsed_s % 86400UL) / 3600UL;
1569 0 : ulong mins = (elapsed_s % 3600UL) / 60UL;
1570 0 : ulong secs = elapsed_s % 60UL;
1571 0 : if( days ) fd_cstr_printf_check( uptime_str, sizeof(uptime_str), NULL, "%lud %luh %lum", days, hours, mins );
1572 0 : else if( hours ) fd_cstr_printf_check( uptime_str, sizeof(uptime_str), NULL, "%luh %lum %lus", hours, mins, secs );
1573 0 : else fd_cstr_printf_check( uptime_str, sizeof(uptime_str), NULL, "%lum %lus", mins, secs );
1574 0 : } else {
1575 0 : has_uptime = 0;
1576 0 : }
1577 :
1578 0 : ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
1579 0 : if( FD_UNLIKELY( replay_idx==ULONG_MAX ) ) return 0U;
1580 0 : ulong const * replay_metrics = &cur_tile[ replay_idx*FD_METRICS_TOTAL_SZ ];
1581 0 : ulong identity_balance = replay_metrics[ MIDX( GAUGE, REPLAY, IDENTITY_BALANCE_LAMPORTS ) ];
1582 0 : ulong stake_amount = replay_metrics[ MIDX( GAUGE, REPLAY, ACTIVE_STAKE_LAMPORTS ) ];
1583 0 : ulong epoch_credits = replay_metrics[ MIDX( GAUGE, REPLAY, EPOCH_CREDITS ) ];
1584 0 : ulong tot_stake = replay_metrics[ MIDX( GAUGE, REPLAY, CLUSTER_ACTIVE_STAKE_LAMPORTS ) ];
1585 0 : char identity_balance_str[ 64 ]; fmt_balance( identity_balance_str, identity_balance );
1586 0 : char stake_amount_str [ 64 ]; fmt_balance( stake_amount_str, stake_amount );
1587 0 : double stake_percent = 0.0;
1588 0 : if( tot_stake>0UL ) stake_percent = 100.0*(double)stake_amount/(double)tot_stake;
1589 :
1590 : /* Fixed width slots (base58 keys are at most 44 chars) so fields
1591 : that appear over time never shift their neighbors. */
1592 0 : char id_field [ 64 ]; fmt_field( id_field, sizeof(id_field), identity_b58, has_identity, 44 );
1593 0 : char vote_field[ 64 ]; fmt_field( vote_field, sizeof(vote_field), vote_acc_b58, has_vote_acc, 44 );
1594 0 : char clus_field[ 32 ]; fmt_field( clus_field, sizeof(clus_field), cluster_str, has_genesis_b58, 8 );
1595 0 : char up_field [ 64 ]; fmt_field( up_field, sizeof(up_field), uptime_str, has_uptime, 12 );
1596 0 : char sv_field [ 32 ]; fmt_field( sv_field, sizeof(sv_field), shred_ver, has_shred_ver, 7 );
1597 :
1598 0 : PRINT( ROWH( "◈", ORANGE, "node " )
1599 0 : K( "cluster" ) "%s" BOLD "%s" RESET
1600 0 : K( "id" ) BOLD "%s" RESET
1601 0 : K( "vote" ) "%s"
1602 0 : K( "uptime" ) "%s"
1603 0 : K( "shred" ) "%s"
1604 0 : K( "genesis" ) "%s",
1605 0 : cluster_color,
1606 0 : clus_field,
1607 0 : id_field,
1608 0 : vote_field,
1609 0 : up_field,
1610 0 : sv_field,
1611 0 : has_genesis_b58 ? genesis_short : DIM "-" RESET );
1612 0 : PRINT( CLEARLN "\n" );
1613 :
1614 0 : if( FD_LIKELY( !watch_full ) ) return 1U;
1615 :
1616 0 : PRINT( ROWC
1617 0 : K( "balance" ) "%s" U( " SOL" )
1618 0 : K( "stake" ) "%s" U( " SOL" ) " " U( "(%.2f%%)" )
1619 0 : K( "credits" ) "%lu" CLEARLN "\n",
1620 0 : identity_balance_str, stake_amount_str, stake_percent, epoch_credits );
1621 :
1622 0 : return 2U;
1623 0 : }
1624 :
1625 : static uint
1626 : write_backup( config_t const * config,
1627 0 : ulong const * cur_tile ) {
1628 0 : ulong snapmk_idx = fd_topo_find_tile( &config->topo, "snapmk", 0UL );
1629 0 : ulong snapsv_idx = fd_topo_find_tile( &config->topo, "snapsv", 0UL );
1630 0 : ulong conns = snapsv_idx==ULONG_MAX ? 0UL : cur_tile[ snapsv_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPSV, CONN_OPEN ) ];
1631 0 : ulong state = snapmk_idx==ULONG_MAX ? SNAPMK_STATE_IDLE : cur_tile[ snapmk_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPMK, JOB_STATE ) ];
1632 0 : int active = state>=SNAPMK_STATE_START && state<=SNAPMK_STATE_FAIL;
1633 0 : int server_on = conns>0UL;
1634 0 : if( FD_UNLIKELY( !active && !server_on ) ) return 0U;
1635 :
1636 0 : ulong upload_sum = 0UL;
1637 0 : ulong upload_cnt = fd_ulong_min( backup_upload_idx, sizeof(backup_upload_samples)/sizeof(backup_upload_samples[0]) );
1638 0 : for( ulong i=0UL; i<upload_cnt; i++ ) upload_sum += backup_upload_samples[ i ];
1639 0 : double upload_mbps = upload_cnt ? 8.0*(double)upload_sum/window_seconds( upload_cnt )/1e6 : 0.0;
1640 :
1641 0 : PRINT( ROWH( "□", BLUE, "backup " ) );
1642 0 : if( FD_LIKELY( server_on ) ) {
1643 0 : PRINT( K( "conns" ) "%3lu" K( "upload" ) "%4.0f" U( " Mbit/s" ), conns, upload_mbps );
1644 0 : }
1645 0 : if( FD_UNLIKELY( !active ) ) {
1646 0 : PRINT( CLEARLN "\n" );
1647 0 : return 1U;
1648 0 : }
1649 :
1650 0 : ulong snapmk_off = snapmk_idx*FD_METRICS_TOTAL_SZ;
1651 0 : ulong full_slot = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, LAST_STARTED_SLOT_FULL ) ];
1652 0 : ulong incr_slot = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, LAST_STARTED_SLOT_INCREMENTAL ) ];
1653 0 : int incremental = incr_slot>full_slot;
1654 :
1655 0 : double progress = 0.0;
1656 0 : if( FD_LIKELY( !incremental && state==SNAPMK_STATE_ACCDB_DISK ) ) {
1657 0 : ulong snaprd_idx = fd_topo_find_tile( &config->topo, "snaprd", 0UL );
1658 0 : if( FD_LIKELY( snaprd_idx!=ULONG_MAX ) ) {
1659 0 : ulong snaprd_off = snaprd_idx*FD_METRICS_TOTAL_SZ;
1660 0 : ulong done = cur_tile[ snaprd_off+MIDX( GAUGE, SNAPRD, EXPORT_PROGRESS_BYTES ) ];
1661 0 : ulong total = cur_tile[ snaprd_off+MIDX( GAUGE, SNAPRD, EXPORT_EXPECTED_BYTES ) ];
1662 0 : if( FD_LIKELY( total ) ) progress = 100.0*(double)fd_ulong_min( done, total )/(double)total;
1663 0 : }
1664 0 : } else if( FD_LIKELY( !incremental && state>SNAPMK_STATE_ACCDB_DISK && state<=SNAPMK_STATE_DONE ) ) {
1665 0 : progress = 100.0;
1666 0 : } else if( FD_UNLIKELY( incremental && state==SNAPMK_STATE_ACCDB_DELTA ) ) {
1667 0 : ulong done = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, INCREMENTAL_ACCOUNT_PROGRESS ) ];
1668 0 : ulong total = cur_tile[ snapmk_off+MIDX( GAUGE, SNAPMK, INCREMENTAL_ACCOUNT_EXPECTED ) ];
1669 0 : progress = total ? 100.0*(double)fd_ulong_min( done, total )/(double)total : 100.0;
1670 0 : } else if( FD_UNLIKELY( incremental && state>SNAPMK_STATE_ACCDB_DELTA && state<=SNAPMK_STATE_DONE ) ) {
1671 0 : progress = 100.0;
1672 0 : }
1673 :
1674 0 : ulong acc_sum = 0UL;
1675 0 : ulong acc_cnt = fd_ulong_min( backup_acc_idx, sizeof(backup_acc_samples)/sizeof(backup_acc_samples[0]) );
1676 0 : for( ulong i=0UL; i<acc_cnt; i++ ) acc_sum += backup_acc_samples[ i ];
1677 :
1678 0 : ulong read_sum = 0UL;
1679 0 : ulong read_cnt = fd_ulong_min( backup_read_idx, sizeof(backup_read_samples)/sizeof(backup_read_samples[0]) );
1680 0 : for( ulong i=0UL; i<read_cnt; i++ ) read_sum += backup_read_samples[ i ];
1681 :
1682 0 : ulong comp_sum = 0UL;
1683 0 : ulong comp_cnt = fd_ulong_min( backup_comp_idx, sizeof(backup_comp_samples)/sizeof(backup_comp_samples[0]) );
1684 0 : for( ulong i=0UL; i<comp_cnt; i++ ) comp_sum += backup_comp_samples[ i ];
1685 :
1686 0 : double accounts_per_second = acc_cnt ? (double)acc_sum/window_seconds( acc_cnt ) : 0.0;
1687 0 : double read_mbps = read_cnt ? (double)read_sum/window_seconds( read_cnt )/1e6 : 0.0;
1688 0 : double comp_mbps = comp_cnt ? (double)comp_sum/window_seconds( comp_cnt )/1e6 : 0.0;
1689 0 : char * accounts_rate = COUNTF_T( accounts_per_second );
1690 :
1691 0 : PRINT( " %s " BOLD "%5.1f" RESET U( "%%" )
1692 0 : K( "type" ) "%-4s"
1693 0 : K( "slot" ) "%10lu"
1694 0 : K( "acc" ) "%s" U( "/s" )
1695 0 : K( "read" ) "%4.0f" U( " MB/s" )
1696 0 : K( "comp" ) "%4.0f" U( " MB/s" ) CLEARLN "\n",
1697 0 : BAR( progress, 20UL ), progress,
1698 0 : incremental ? "incr" : "full",
1699 0 : incremental ? incr_slot : full_slot,
1700 0 : accounts_rate,
1701 0 : read_mbps,
1702 0 : comp_mbps );
1703 0 : return 1U;
1704 0 : }
1705 :
1706 : static void
1707 : write_summary( config_t const * config,
1708 : fd_node_info_box_t const * shinfo,
1709 : ulong const * cur_tile,
1710 : ulong const * prev_tile,
1711 : ulong const * cur_link,
1712 0 : ulong const * prev_link ) {
1713 0 : (void)config;
1714 0 : (void)prev_tile;
1715 0 : (void)cur_tile;
1716 :
1717 0 : if( FD_UNLIKELY( !ended_on_newline ) ) PRINT( "\n" );
1718 0 : PRINT( "\033[?7l" ); /* disable autowrap mode */
1719 0 : lines_printed = 0UL;
1720 :
1721 0 : ulong snapct_idx = fd_topo_find_tile( &config->topo, "snapct", 0UL );
1722 0 : int shutdown = 1;
1723 0 : if( FD_LIKELY( snapct_idx!=ULONG_MAX ) ) shutdown = cur_tile[ snapct_idx*FD_METRICS_TOTAL_SZ+MIDX( GAUGE, SNAPCT, LOAD_STATE ) ]==FD_SNAPCT_STATE_SHUTDOWN;
1724 :
1725 0 : static long snap_shutdown_time = 0L;
1726 0 : if( FD_UNLIKELY( !snap_shutdown_time && !shutdown ) ) snap_shutdown_time = 1L; /* Was not shutdown on boot */
1727 0 : if( FD_UNLIKELY( !snap_shutdown_time && shutdown ) ) snap_shutdown_time = 2L; /* Was shutdown on boot */
1728 0 : if( FD_UNLIKELY( snap_shutdown_time==1L && shutdown ) ) snap_shutdown_time = fd_log_wallclock();
1729 :
1730 0 : fd_node_info_t node_info[1] = {0};
1731 0 : if( FD_LIKELY( shinfo ) ) fd_node_info_read( node_info, shinfo );
1732 0 : lines_printed += write_node_info( config, cur_tile, node_info );
1733 :
1734 0 : if( FD_UNLIKELY( write_bench( config, cur_tile, prev_tile ) ) ) lines_printed++;
1735 :
1736 0 : ulong backt_idx = fd_topo_find_tile( &config->topo, "backt", 0UL );
1737 0 : if( FD_UNLIKELY( backt_idx!=ULONG_MAX ) ) {
1738 0 : lines_printed++;
1739 0 : write_backtest( config, cur_tile );
1740 0 : }
1741 :
1742 0 : long now = fd_log_wallclock();
1743 0 : if( FD_UNLIKELY( snap_shutdown_time==1L || now<snap_shutdown_time+(long)2e9 ) ) {
1744 0 : lines_printed++;
1745 0 : write_snapshots( config, cur_tile, prev_tile );
1746 0 : }
1747 :
1748 0 : lines_printed += write_accdb( config, cur_tile, prev_tile );
1749 0 : lines_printed += write_wfs( config, cur_tile );
1750 0 : lines_printed += write_gossip( config, cur_tile, prev_tile, cur_link, prev_link );
1751 0 : lines_printed += write_repair( config, cur_tile, cur_link, prev_link );
1752 0 : lines_printed += write_rserve( config, cur_tile, cur_link, prev_link );
1753 0 : lines_printed += write_replay( config, cur_tile );
1754 0 : lines_printed += write_votor( config, cur_tile, prev_tile );
1755 0 : lines_printed += write_gui( config, cur_tile, prev_tile );
1756 0 : lines_printed += write_event( config, cur_tile );
1757 0 : lines_printed += write_backup( config, cur_tile );
1758 :
1759 0 : PRINT( "\033[?7h" ); /* enable autowrap mode */
1760 0 : }
1761 :
1762 : static void
1763 : snap_tiles( fd_topo_t const * topo,
1764 0 : ulong * tiles ) {
1765 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
1766 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
1767 0 : volatile ulong const * metrics = fd_metrics_tile( tile->metrics );
1768 0 : FD_TEST( metrics );
1769 0 : for( ulong j=0UL; j<FD_METRICS_TOTAL_SZ/8UL; j++ ) tiles[ i*FD_METRICS_TOTAL_SZ+j ] = metrics[ j ];
1770 0 : }
1771 0 : }
1772 :
1773 : static void
1774 : snap_links( fd_topo_t const * topo,
1775 0 : ulong * links ) {
1776 0 : ulong overall_polled_idx = 0UL;
1777 :
1778 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
1779 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
1780 :
1781 0 : ulong polled_in_idx = 0UL;
1782 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
1783 0 : if( FD_UNLIKELY( !tile->in_link_poll[ j ] ) ) continue;
1784 :
1785 0 : volatile ulong const * metrics = fd_metrics_link_in( tile->metrics, polled_in_idx );
1786 0 : FD_TEST( metrics );
1787 0 : for( ulong k=0UL; k<FD_METRICS_ALL_LINK_IN_TOTAL; k++ ) links[ overall_polled_idx*8UL+k ] = metrics[ k ];
1788 0 : polled_in_idx++;
1789 0 : overall_polled_idx++;
1790 0 : }
1791 0 : }
1792 0 : }
1793 :
1794 : static ulong tiles[ 2UL*FD_TILE_MAX*FD_METRICS_TOTAL_SZ ];
1795 : static ulong links[ 2UL*4096UL*8UL*FD_METRICS_ALL_LINK_IN_TOTAL ];
1796 :
1797 : static void
1798 : run( config_t const * config,
1799 0 : int drain_output_fd ) {
1800 0 : (void)config;
1801 0 : (void)drain_output_fd;
1802 :
1803 0 : ulong node_info_obj_id = fd_pod_query_ulong( config->topo.props, "node_info", ULONG_MAX );
1804 0 : fd_node_info_box_t * node_info = node_info_obj_id==ULONG_MAX ? NULL :
1805 0 : fd_node_info_box_join( fd_topo_obj_laddr( &config->topo, node_info_obj_id ) );
1806 0 : FD_TEST( node_info_obj_id==ULONG_MAX || node_info );
1807 :
1808 0 : ulong tile_cnt = config->topo.tile_cnt;
1809 :
1810 0 : ulong cons_cnt = 0UL;
1811 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
1812 0 : for( ulong j=0UL; j<config->topo.tiles[ i ].in_cnt; j++ ) {
1813 0 : if( FD_UNLIKELY( config->topo.tiles[ i ].in_link_poll[ j ] ) ) cons_cnt++;
1814 0 : }
1815 0 : }
1816 :
1817 0 : FD_TEST( tile_cnt<=FD_TILE_MAX );
1818 0 : FD_TEST( cons_cnt<=4096UL );
1819 :
1820 0 : snap_tiles( &config->topo, tiles );
1821 0 : fd_memcpy( tiles+tile_cnt*FD_METRICS_TOTAL_SZ, tiles, tile_cnt*FD_METRICS_TOTAL_SZ*sizeof(ulong) );
1822 :
1823 0 : snap_links( &config->topo, links );
1824 0 : fd_memcpy( links+(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links, cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL*sizeof(ulong) );
1825 :
1826 0 : snap_cur_nanos = wallclock_monotonic();
1827 0 : snap_prev_nanos = snap_cur_nanos-(long)1e7;
1828 :
1829 0 : ulong last_snap = 1UL;
1830 :
1831 :
1832 0 : frame_len = 0UL;
1833 0 : write_summary( config, node_info, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1834 0 : flush_frame();
1835 :
1836 0 : long next = fd_log_wallclock()+(long)1e7;
1837 0 : for(;;) {
1838 0 : long timeout_ns = next-fd_log_wallclock();
1839 0 : if( FD_LIKELY( timeout_ns>0L ) ) {
1840 0 : fd_set rfds;
1841 0 : FD_ZERO( &rfds );
1842 0 : if( FD_LIKELY( drain_output_fd>=0 ) ) FD_SET( drain_output_fd, &rfds );
1843 0 : struct timeval tv = { .tv_sec = timeout_ns/(long)1e9, .tv_usec = (timeout_ns%(long)1e9)/1000L+1L };
1844 0 : if( FD_UNLIKELY( -1==select( drain_output_fd>=0 ? drain_output_fd+1 : 0, drain_output_fd>=0 ? &rfds : NULL, NULL, NULL, &tv ) && errno!=EINTR ) )
1845 0 : FD_LOG_ERR(( "select() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1846 0 : }
1847 0 : if( FD_UNLIKELY( drain_output_fd>=0 ) ) {
1848 0 : if( FD_UNLIKELY( drain( drain_output_fd ) ) ) {
1849 0 : frame_len = 0UL;
1850 0 : write_summary( config, node_info, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1851 0 : flush_frame();
1852 0 : }
1853 0 : }
1854 :
1855 0 : long now = fd_log_wallclock();
1856 0 : if( FD_UNLIKELY( now>=next ) ) {
1857 0 : last_snap = 1UL-last_snap;
1858 0 : snap_tiles( &config->topo, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
1859 0 : snap_links( &config->topo, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1860 0 : snap_prev_nanos = snap_cur_nanos;
1861 0 : snap_cur_nanos = wallclock_monotonic();
1862 0 : snap_dt_samples[ (snap_dt_idx++)%(sizeof(snap_dt_samples)/sizeof(snap_dt_samples[0])) ] = snap_cur_nanos-snap_prev_nanos;
1863 :
1864 : /* Bench */
1865 0 : tps_sent_samples[ tps_sent_samples_idx%(sizeof(tps_sent_samples)/sizeof(tps_sent_samples[0])) ] = diff_tile( config, "benchs", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, BENCHS, TXN_TX ) );
1866 0 : tps_sent_samples_idx++;
1867 :
1868 : /* Replay */
1869 0 : sps_samples[ sps_samples_idx%(sizeof(sps_samples)/sizeof(sps_samples[0])) ] = diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, REPLAY, BANK_FROZEN ) );
1870 0 : sps_samples_idx++;
1871 0 : tps_samples[ tps_samples_idx%(sizeof(tps_samples)/sizeof(tps_samples[0])) ] = diff_tile( config, "replay", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, REPLAY, CHAIN_TXN_HEIGHT ) );
1872 0 : tps_samples_idx++;
1873 0 : cups_samples[ cups_samples_idx%(sizeof(cups_samples)/sizeof(cups_samples[0])) ] =
1874 0 : diff_tile( config, "execrp", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EXECRP, CU_CONSUMED ) ) +
1875 0 : diff_tile( config, "execle", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EXECLE, CU_CONSUMED ) );
1876 0 : cups_samples_idx++;
1877 :
1878 : /* Snapshot */
1879 0 : snapshot_rx_samples[ snapshot_rx_idx%(sizeof(snapshot_rx_samples)/sizeof(snapshot_rx_samples[0])) ] = diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, READ_PROGRESS_BYTES_FULL ) ) +
1880 0 : diff_tile( config, "snapct", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPCT, READ_PROGRESS_BYTES_INCREMENTAL ) );
1881 0 : snapshot_rx_idx++;
1882 0 : snapshot_acc_samples[ snapshot_acc_idx%(sizeof(snapshot_acc_samples)/sizeof(snapshot_acc_samples[0])) ] = diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, SNAPIN, ACCOUNT_PROGRESS_NEW ) );
1883 0 : snapshot_acc_idx++;
1884 0 : snapshot_wr_samples[ snapshot_wr_idx%(sizeof(snapshot_wr_samples)/sizeof(snapshot_wr_samples[0])) ] = diff_tile( config, "snapin", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPIN, DISK_WRITTEN_BYTES ) );
1885 0 : snapshot_wr_idx++;
1886 :
1887 : /* Backup */
1888 0 : backup_acc_samples[ backup_acc_idx%(sizeof(backup_acc_samples)/sizeof(backup_acc_samples[0])) ] = diff_tile( config, "snapzp", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPZP, ACCOUNT_COMPRESSED ) );
1889 0 : backup_acc_idx++;
1890 0 : backup_read_samples[ backup_read_idx%(sizeof(backup_read_samples)/sizeof(backup_read_samples[0])) ] = diff_tile( config, "snaprd", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPRD, DISK_READ_BYTES ) );
1891 0 : backup_read_idx++;
1892 0 : backup_comp_samples[ backup_comp_idx%(sizeof(backup_comp_samples)/sizeof(backup_comp_samples[0])) ] = diff_tile( config, "snapmk", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPMK, COMPRESS_CONSUMED_BYTES ) ) +
1893 0 : diff_tile( config, "snapzp", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPZP, COMPRESS_CONSUMED_BYTES ) );
1894 0 : backup_comp_idx++;
1895 0 : backup_upload_samples[ backup_upload_idx%(sizeof(backup_upload_samples)/sizeof(backup_upload_samples[0])) ] = diff_tile( config, "snapsv", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, SNAPSV, HTTP_TX_BYTES ) );
1896 0 : backup_upload_idx++;
1897 :
1898 : /* Events */
1899 0 : events_sent_samples[ events_sent_samples_idx%(sizeof(events_sent_samples)/sizeof(events_sent_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, EVENT_TX ) );
1900 0 : events_sent_samples_idx++;
1901 0 : events_acked_samples[ events_acked_samples_idx%(sizeof(events_acked_samples)/sizeof(events_acked_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, ACK_RX ) );
1902 0 : events_acked_samples_idx++;
1903 0 : event_bytes_written_samples[ event_bytes_written_samples_idx%(sizeof(event_bytes_written_samples)/sizeof(event_bytes_written_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, STREAM_TX_BYTES ) );
1904 0 : event_bytes_written_samples_idx++;
1905 0 : event_bytes_read_samples[ event_bytes_read_samples_idx%(sizeof(event_bytes_read_samples)/sizeof(event_bytes_read_samples[0])) ] = diff_tile( config, "event", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, EVENT, STREAM_RX_BYTES ) );
1906 0 : event_bytes_read_samples_idx++;
1907 :
1908 : /* Accounts */
1909 0 : sample_accdb( config, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ );
1910 :
1911 : /* Gossip */
1912 0 : sample_gossip( config, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, now );
1913 :
1914 : /* Repair server */
1915 0 : shreds_stored_sample[ shreds_stored_samples_idx%(sizeof(shreds_stored_sample)/sizeof(shreds_stored_sample[0])) ] = diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( GAUGE, RSERVE, SHRED_USED ) );
1916 0 : shreds_stored_samples_idx++;
1917 :
1918 0 : rserve_rps_valid_samples[ rserve_rps_valid_samples_idx%(sizeof(rserve_rps_valid_samples)/sizeof(rserve_rps_valid_samples[0])) ] =
1919 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, RESPONSE_TX_PING ) ) +
1920 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, RESPONSE_TX_WINDOW_INDEX ) ) +
1921 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, RESPONSE_TX_HIGHEST_WINDOW_INDEX ) ) +
1922 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, RESPONSE_TX_ORPHAN ) );
1923 0 : rserve_rps_valid_samples_idx++;
1924 :
1925 0 : rserve_rps_invalid_samples[ rserve_rps_invalid_samples_idx%(sizeof(rserve_rps_invalid_samples)/sizeof(rserve_rps_invalid_samples[0])) ] = 0UL;
1926 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_CNT; r++ ) {
1927 0 : rserve_rps_invalid_samples[ rserve_rps_invalid_samples_idx%(sizeof(rserve_rps_invalid_samples)/sizeof(rserve_rps_invalid_samples[0])) ] +=
1928 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, REQUEST_UNANSWERED )+r );
1929 0 : }
1930 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_REQUEST_DROP_REASON_CNT; r++ ) {
1931 0 : rserve_rps_invalid_samples[ rserve_rps_invalid_samples_idx%(sizeof(rserve_rps_invalid_samples)/sizeof(rserve_rps_invalid_samples[0])) ] +=
1932 0 : diff_tile( config, "rserve", tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+r );
1933 0 : }
1934 0 : {
1935 0 : ulong slot = rserve_rps_invalid_samples_idx % (sizeof(rserve_rps_miss_samples)/sizeof(rserve_rps_miss_samples[0]));
1936 0 : ulong const * prev = tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ;
1937 0 : ulong const * cur = tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ;
1938 :
1939 0 : rserve_rps_miss_samples[ slot ] = 0UL;
1940 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_CNT; r++ ) rserve_rps_miss_samples[ slot ] += diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_UNANSWERED )+r );
1941 :
1942 0 : rserve_rps_sigvfy_samples[ slot ] = diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_SIGNATURE_FAILED_IDX );
1943 :
1944 0 : rserve_rps_stale_samples[ slot ] = diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_TIMESTAMP_SKEW_IDX );
1945 :
1946 0 : rserve_rps_other_rej_samples[ slot ] =
1947 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_FROM_SELF_IDX ) +
1948 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_PONG_TOKEN_INVALID_IDX ) +
1949 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_WRONG_RECIPIENT_IDX ) +
1950 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_SHRED_INDEX_INVALID_IDX ) +
1951 0 : diff_tile( config, "rserve", prev, cur, MIDX( COUNTER, RSERVE, REQUEST_DROPPED )+FD_METRICS_ENUM_REQUEST_DROP_REASON_V_FEC_SET_INDEX_INVALID_IDX );
1952 0 : }
1953 0 : rserve_rps_invalid_samples_idx++;
1954 :
1955 : /* Move cursor to top of dashboard and overwrite in place.
1956 : All output is buffered and flushed in a single write() so
1957 : the terminal never renders a partially drawn frame. */
1958 0 : frame_len = 0UL;
1959 0 : PRINT( "\033[?25l" ); /* hide cursor during redraw */
1960 0 : if( FD_UNLIKELY( !ended_on_newline ) ) {
1961 0 : PRINT( "\033[%luA\r", lines_printed+1UL );
1962 0 : } else {
1963 0 : PRINT( "\033[%luA\r", lines_printed );
1964 0 : }
1965 0 : write_summary( config, node_info, tiles+last_snap*tile_cnt*FD_METRICS_TOTAL_SZ, tiles+(1UL-last_snap)*tile_cnt*FD_METRICS_TOTAL_SZ, links+last_snap*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL), links+(1UL-last_snap)*(cons_cnt*8UL*FD_METRICS_ALL_LINK_IN_TOTAL) );
1966 0 : PRINT( "\033[0J" ); /* clear any leftover lines below */
1967 0 : PRINT( "\033[?25h" ); /* show cursor */
1968 0 : flush_frame();
1969 0 : next += (long)1e7;
1970 0 : }
1971 0 : }
1972 0 : }
1973 :
1974 : void
1975 : watch_cmd_args( int * pargc,
1976 : char *** pargv,
1977 0 : args_t * args ) {
1978 0 : args->watch.drain_output_fd = -1;
1979 0 : args->watch.full = fd_env_strip_cmdline_contains( pargc, pargv, "--full" );
1980 0 : }
1981 :
1982 : void
1983 : watch_cmd_fn( args_t * args,
1984 0 : config_t * config ) {
1985 : /* Development commands spawn watch internally with the validator's
1986 : own config, only discover when invoked standalone. */
1987 0 : if( FD_LIKELY( args->watch.drain_output_fd==-1 ) ) fd_bootinfo_adopt( config );
1988 :
1989 0 : int allow_fds[ 5 ];
1990 0 : ulong allow_fds_cnt = 0;
1991 0 : allow_fds[ allow_fds_cnt++ ] = 0; /* stdin */
1992 0 : allow_fds[ allow_fds_cnt++ ] = 1; /* stdout */
1993 0 : allow_fds[ allow_fds_cnt++ ] = 2; /* stderr */
1994 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) )
1995 0 : allow_fds[ allow_fds_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
1996 0 : if( FD_UNLIKELY( args->watch.drain_output_fd!=-1 ) )
1997 0 : allow_fds[ allow_fds_cnt++ ] = args->watch.drain_output_fd; /* maybe we are interposing firedancer log output with the monitor */
1998 :
1999 0 : if( FD_LIKELY( args->watch.drain_output_fd==-1 ) ) fd_bootinfo_check_layout( config );
2000 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
2001 :
2002 0 : struct sock_filter seccomp_filter[ 128UL ];
2003 0 : uint drain_output_fd = args->watch.drain_output_fd >= 0 ? (uint)args->watch.drain_output_fd : (uint)-1;
2004 0 : populate_sock_filter_policy_watch( 128UL, seccomp_filter, (uint)fd_log_private_logfile_fd(), drain_output_fd );
2005 :
2006 0 : if( FD_LIKELY( config->development.sandbox ) ) {
2007 0 : fd_sandbox_enter( config->uid,
2008 0 : config->gid,
2009 0 : 0,
2010 0 : 0,
2011 0 : 0,
2012 0 : 1, /* Keep controlling terminal for main so it can receive Ctrl+C */
2013 0 : 0,
2014 0 : 0UL,
2015 0 : 0UL,
2016 0 : 0UL,
2017 0 : 0UL,
2018 0 : allow_fds_cnt,
2019 0 : allow_fds,
2020 0 : sock_filter_policy_watch_instr_cnt,
2021 0 : seccomp_filter );
2022 0 : } else {
2023 0 : fd_sandbox_switch_uid_gid( config->uid, config->gid );
2024 0 : }
2025 :
2026 0 : fd_topo_fill( &config->topo );
2027 :
2028 0 : watch_full = args->watch.full;
2029 0 : run( config, args->watch.drain_output_fd );
2030 0 : }
2031 :
2032 : static void
2033 0 : watch_args_help( fd_action_help_t * help ) {
2034 0 : fd_action_help_arg( help, "--full", NULL, "Show all detail rows for each category, not just the primary row" );
2035 0 : }
2036 :
2037 : action_t fd_action_watch = {
2038 : .name = "watch",
2039 : .args = watch_cmd_args,
2040 : .fn = watch_cmd_fn,
2041 : .require_config = 0,
2042 : .perm = watch_cmd_perm,
2043 : .description = "Watch a locally running Firedancer instance with a terminal GUI",
2044 : .detail = "Connects to a running validator and renders a terminal dashboard of the\n"
2045 : "most important monitoring and operational metrics.",
2046 : .usage = "watch [--full]",
2047 : .args_help = watch_args_help,
2048 : };
|
