Line data Source code
1 : /* The rotor command attaches to a running validator's rotor tile and
2 : prints per-second repair request / response counters and network
3 : drop counters. */
4 :
5 : #include "../../../disco/topo/fd_topob.h"
6 : #include "../../shared/fd_config.h" /* config_t */
7 : #include "../../shared_dev/commands/dev.h" /* dev_cmd_perm */
8 :
9 : #include "../../../disco/metrics/fd_metrics.h"
10 : #include "../../../disco/net/fd_net_tile.h" /* fd_net_tile_name */
11 :
12 : #include <stdio.h>
13 : #include <time.h> /* localtime_r */
14 : #include <unistd.h> /* sleep */
15 :
16 : extern action_t fd_action_rotor;
17 :
18 : /* metrics_src holds the shared-memory metric arrays the command
19 : samples. Link arrays are the consumer-side in-link metrics
20 : (overruns are counted by the consumer). */
21 :
22 : #define ROTOR_METRICS_LINK_MAX (256UL)
23 :
24 : struct metrics_src {
25 : volatile ulong * rotor; /* rotor tile metrics */
26 : volatile ulong * shred[ ROTOR_METRICS_LINK_MAX ]; /* shred tile metrics */
27 : ulong shred_cnt;
28 : volatile ulong * net[ ROTOR_METRICS_LINK_MAX ]; /* net tile metrics */
29 : ulong net_cnt;
30 : int net_kind; /* 0 xdp "net", 1 "mlx5", 2 "sock" */
31 : volatile ulong * net_shred [ ROTOR_METRICS_LINK_MAX ]; /* net->shred links, consumer side (shred tiles) */
32 : ulong net_shred_cnt;
33 : volatile ulong * net_repair[ ROTOR_METRICS_LINK_MAX ]; /* net->rotor links, consumer side (rotor) */
34 : ulong net_repair_cnt;
35 : volatile ulong * repair_net[ ROTOR_METRICS_LINK_MAX ]; /* rotor->net link, consumer side (net tiles) */
36 : ulong repair_net_cnt;
37 : };
38 : typedef struct metrics_src metrics_src_t;
39 :
40 : /* metrics_snap is one sample of the counters we print. */
41 :
42 : struct metrics_snap {
43 : long ts;
44 :
45 : ulong req[ FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_CNT ];
46 : ulong pkt_tx;
47 : ulong rerequest;
48 : ulong meta_failed; /* shred_block_id + fec_root + parent_fec_count verify failures */
49 : ulong shred_old;
50 : ulong resp_cnt; /* responses matched to an inflight request (latency histogram count) */
51 : ulong resp_sum_ns; /* latency histogram sum */
52 :
53 : ulong inflight, slot_current, slot_highest_repaired, slot_last_requested, orphan_last_requested, peers;
54 :
55 : ulong shred_repair_rx, shred_turbine_rx;
56 : ulong shred_okay, shred_dup, shred_bad_slot, shred_ignored, shred_completes;
57 :
58 : struct { ulong overrun, consumed, filtered; } net_shred, net_repair, repair_net;
59 : ulong net_rx, net_rx_drop, net_tx, net_tx_drop;
60 :
61 : /* tile regime nanos: [0] rotor, [1..] shred tiles. Same idle
62 : definition as the GUI: caught-up postfrag (spinning for input),
63 : caught-up sleeping, and backpressure prefrag / sleeping are idle,
64 : everything else is busy. backp = the backpressure share (stalled
65 : on a downstream consumer). */
66 : struct { ulong total, busy, backp; } regime[ 1UL+ROTOR_METRICS_LINK_MAX ];
67 : };
68 : typedef struct metrics_snap metrics_snap_t;
69 :
70 : static void
71 0 : metrics_snap_regime( volatile ulong * m, ulong * total, ulong * busy, ulong * backp ) {
72 0 : ulong r[ FD_METRICS_ENUM_TILE_REGIME_CNT ];
73 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) r[ i ] = m[ MIDX( COUNTER, TILE, REGIME_DURATION_NANOS )+i ];
74 0 : ulong bp = r[ FD_METRICS_ENUM_TILE_REGIME_V_BACKPRESSURE_PREFRAG_IDX ] + r[ FD_METRICS_ENUM_TILE_REGIME_V_BACKPRESSURE_SLEEPING_IDX ];
75 0 : ulong idle = r[ FD_METRICS_ENUM_TILE_REGIME_V_CAUGHT_UP_POSTFRAG_IDX ] + r[ FD_METRICS_ENUM_TILE_REGIME_V_CAUGHT_UP_SLEEPING_IDX ] + bp;
76 0 : *total = 0UL; for( ulong i=0UL; i<FD_METRICS_ENUM_TILE_REGIME_CNT; i++ ) *total += r[ i ];
77 0 : *busy = *total-idle;
78 0 : *backp = bp;
79 0 : }
80 :
81 : static void
82 : metrics_src_link( fd_topo_t * topo,
83 : fd_topo_tile_t * tile,
84 : char const * link_name,
85 : ulong kind_id,
86 : volatile ulong ** out,
87 0 : ulong * cnt ) {
88 0 : ulong in_idx = fd_topo_find_tile_in_link( topo, tile, link_name, kind_id );
89 0 : if( FD_UNLIKELY( in_idx==ULONG_MAX ) ) return; /* not every tile has every link */
90 0 : if( FD_UNLIKELY( *cnt>=ROTOR_METRICS_LINK_MAX ) ) FD_LOG_ERR(( "too many %s links", link_name ));
91 0 : out[ (*cnt)++ ] = fd_metrics_link_in( tile->metrics, in_idx );
92 0 : }
93 :
94 : static void
95 : metrics_src_init( metrics_src_t * src,
96 0 : config_t * config ) {
97 0 : fd_topo_t * topo = &config->topo;
98 0 : memset( src, 0, sizeof(*src) );
99 :
100 : /* Every tile's metrics object lives in the metric_in workspace, so
101 : joining that one and filling it resolves tile->metrics for all. */
102 0 : ulong wksp_id = fd_topo_find_wksp( topo, "metric_in" );
103 0 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "metric_in workspace not found" ));
104 0 : fd_topo_join_workspace( topo, &topo->workspaces[ wksp_id ], FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
105 0 : fd_topo_workspace_fill( topo, &topo->workspaces[ wksp_id ] );
106 :
107 0 : ulong rotor_id = fd_topo_find_tile( topo, "rotor", 0UL );
108 0 : if( FD_UNLIKELY( rotor_id==ULONG_MAX ) ) FD_LOG_ERR(( "rotor tile not found" ));
109 0 : fd_topo_tile_t * rotor_tile = &topo->tiles[ rotor_id ];
110 0 : src->rotor = fd_metrics_tile( rotor_tile->metrics );
111 :
112 0 : char const * net_name = fd_net_tile_name( config->net.provider );
113 0 : src->net_kind = !strcmp( net_name, "net" ) ? 0 : !strcmp( net_name, "mlx5" ) ? 1 : 2;
114 :
115 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
116 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
117 0 : if( !strcmp( tile->name, "shred" ) ) {
118 0 : if( FD_UNLIKELY( src->shred_cnt>=ROTOR_METRICS_LINK_MAX ) ) FD_LOG_ERR(( "too many shred tiles" ));
119 0 : src->shred[ src->shred_cnt++ ] = fd_metrics_tile( tile->metrics );
120 0 : for( ulong j=0UL; j<tile->in_cnt; j++ ) {
121 0 : if( !strcmp( topo->links[ tile->in_link_id[ j ] ].name, "net_shred" ) ) {
122 0 : if( FD_UNLIKELY( src->net_shred_cnt>=ROTOR_METRICS_LINK_MAX ) ) FD_LOG_ERR(( "too many net_shred links" ));
123 0 : src->net_shred[ src->net_shred_cnt++ ] = fd_metrics_link_in( tile->metrics, j );
124 0 : }
125 0 : }
126 0 : } else if( !strcmp( tile->name, net_name ) ) {
127 0 : if( FD_UNLIKELY( src->net_cnt>=ROTOR_METRICS_LINK_MAX ) ) FD_LOG_ERR(( "too many net tiles" ));
128 0 : src->net[ src->net_cnt++ ] = fd_metrics_tile( tile->metrics );
129 0 : metrics_src_link( topo, tile, "repair_net", 0UL, src->repair_net, &src->repair_net_cnt );
130 0 : }
131 0 : }
132 0 : for( ulong j=0UL; j<rotor_tile->in_cnt; j++ ) {
133 0 : if( !strcmp( topo->links[ rotor_tile->in_link_id[ j ] ].name, "net_repair" ) ) {
134 0 : if( FD_UNLIKELY( src->net_repair_cnt>=ROTOR_METRICS_LINK_MAX ) ) FD_LOG_ERR(( "too many net_repair links" ));
135 0 : src->net_repair[ src->net_repair_cnt++ ] = fd_metrics_link_in( rotor_tile->metrics, j );
136 0 : }
137 0 : }
138 0 : }
139 :
140 : static void
141 0 : metrics_snap_links( volatile ulong ** links, ulong cnt, ulong * overrun, ulong * consumed, ulong * filtered ) {
142 0 : *overrun = *consumed = *filtered = 0UL;
143 0 : for( ulong i=0UL; i<cnt; i++ ) {
144 0 : *overrun += links[ i ][ MIDX( COUNTER, LINK, FRAG_READING_OVERRUN ) ] + links[ i ][ MIDX( COUNTER, LINK, FRAG_POLLING_OVERRUN ) ];
145 0 : *consumed += links[ i ][ MIDX( COUNTER, LINK, FRAG_CONSUMED ) ];
146 0 : *filtered += links[ i ][ MIDX( COUNTER, LINK, FRAG_FILTERED ) ];
147 0 : }
148 0 : }
149 :
150 : static void
151 0 : metrics_snap_take( metrics_snap_t * s, metrics_src_t const * src ) {
152 0 : volatile ulong * r = src->rotor;
153 0 : memset( s, 0, sizeof(*s) );
154 0 : s->ts = fd_log_wallclock();
155 :
156 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_CNT; i++ ) s->req[ i ] = r[ MIDX( COUNTER, ROTOR, REQUEST_TX )+i ];
157 0 : s->pkt_tx = r[ MIDX( COUNTER, ROTOR, PKT_TX ) ];
158 : /* The rotor tile does not produce SHRED_REREQUESTED. */
159 0 : s->rerequest = 0UL;
160 0 : s->shred_old = r[ MIDX( COUNTER, ROTOR, SHRED_OLD_IGNORED ) ];
161 0 : s->meta_failed = r[ MIDX( COUNTER, ROTOR, SHRED_RX_UNMATCHED ) ] + r[ MIDX( COUNTER, ROTOR, FEC_SET_ROOT_FAILED ) ] + r[ MIDX( COUNTER, ROTOR, PARENT_FEC_SET_COUNT_FAILED ) ];
162 0 : for( ulong k=0UL; k<FD_HISTF_BUCKET_CNT; k++ ) s->resp_cnt += r[ MIDX( HISTOGRAM, ROTOR, RESPONSE_LATENCY_NANOS )+k ];
163 0 : s->resp_sum_ns = r[ MIDX( HISTOGRAM, ROTOR, RESPONSE_LATENCY_NANOS )+FD_HISTF_BUCKET_CNT ];
164 :
165 0 : s->inflight = r[ MIDX( GAUGE, ROTOR, REQUEST_IN_FLIGHT ) ];
166 0 : s->slot_current = fd_ulong_max( r[ MIDX( GAUGE, ROTOR, HIGHEST_RECEIVED_SLOT ) ], r[ MIDX( GAUGE, ROTOR, CATCHUP_SLOT ) ] );
167 0 : s->slot_highest_repaired = r[ MIDX( GAUGE, ROTOR, HIGHEST_DELIVERED_SLOT ) ];
168 : /* Nor SLOT_LAST_REQUESTED, ORPHAN_LAST_REQUESTED, PEER_USED. */
169 0 : s->slot_last_requested = 0UL;
170 0 : s->orphan_last_requested = 0UL;
171 0 : s->peers = 0UL;
172 :
173 0 : for( ulong i=0UL; i<src->shred_cnt; i++ ) {
174 0 : volatile ulong * m = src->shred[ i ];
175 0 : s->shred_repair_rx += m[ MIDX( COUNTER, SHRED, REPAIR_RX ) ];
176 0 : s->shred_turbine_rx += m[ MIDX( COUNTER, SHRED, TURBINE_RX ) ];
177 0 : s->shred_okay += m[ MIDX( COUNTER, SHRED, PROCESSED_ACCEPTED ) ];
178 0 : s->shred_dup += m[ MIDX( COUNTER, SHRED, PROCESSED_DUPLICATE ) ];
179 0 : s->shred_bad_slot += m[ MIDX( COUNTER, SHRED, PROCESSED_LEADER_UNKNOWN ) ];
180 0 : s->shred_ignored += m[ MIDX( COUNTER, SHRED, PROCESSED_IGNORED ) ];
181 0 : s->shred_completes += m[ MIDX( COUNTER, SHRED, PROCESSED_COMPLETED ) ];
182 0 : }
183 :
184 0 : metrics_snap_regime( src->rotor, &s->regime[ 0 ].total, &s->regime[ 0 ].busy, &s->regime[ 0 ].backp );
185 0 : for( ulong i=0UL; i<src->shred_cnt; i++ ) metrics_snap_regime( src->shred[ i ], &s->regime[ 1UL+i ].total, &s->regime[ 1UL+i ].busy, &s->regime[ 1UL+i ].backp );
186 :
187 0 : metrics_snap_links( (volatile ulong **)src->net_shred, src->net_shred_cnt, &s->net_shred.overrun, &s->net_shred.consumed, &s->net_shred.filtered );
188 0 : metrics_snap_links( (volatile ulong **)src->net_repair, src->net_repair_cnt, &s->net_repair.overrun, &s->net_repair.consumed, &s->net_repair.filtered );
189 0 : metrics_snap_links( (volatile ulong **)src->repair_net, src->repair_net_cnt, &s->repair_net.overrun, &s->repair_net.consumed, &s->repair_net.filtered );
190 :
191 0 : for( ulong i=0UL; i<src->net_cnt; i++ ) {
192 0 : volatile ulong * m = src->net[ i ];
193 0 : switch( src->net_kind ) {
194 0 : case 0: /* xdp */
195 0 : s->net_rx += m[ MIDX( COUNTER, NET, PKT_RX ) ];
196 0 : s->net_tx += m[ MIDX( COUNTER, NET, PKT_TX_SUBMITTED ) ];
197 0 : s->net_rx_drop += m[ MIDX( COUNTER, NET, RX_FILL_RING_STALLED ) ]
198 0 : + m[ MIDX( COUNTER, NET, XDP_RX_RING_FULL_DROPPED ) ] + m[ MIDX( COUNTER, NET, XDP_RX_OTHER_DROPPED ) ];
199 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_PKT_RX_DROP_REASON_CNT; r++ ) s->net_rx_drop += m[ MIDX( COUNTER, NET, PKT_RX_DROPPED )+r ];
200 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_PKT_TX_DROP_REASON_CNT; r++ ) s->net_tx_drop += m[ MIDX( COUNTER, NET, PKT_TX_DROPPED )+r ];
201 0 : break;
202 0 : case 1: /* mlx5 */
203 0 : s->net_rx += m[ MIDX( COUNTER, MLX5, PKT_RX ) ];
204 0 : s->net_tx += m[ MIDX( COUNTER, MLX5, PKT_TX_COMPLETED ) ];
205 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_PKT_RX_DROP_REASON_CNT; r++ ) s->net_rx_drop += m[ MIDX( COUNTER, MLX5, PKT_RX_DROPPED )+r ];
206 0 : for( ulong r=0UL; r<FD_METRICS_ENUM_PKT_TX_DROP_REASON_CNT; r++ ) s->net_tx_drop += m[ MIDX( COUNTER, MLX5, PKT_TX_DROPPED )+r ];
207 0 : break;
208 0 : default: /* sock */
209 0 : s->net_rx += m[ MIDX( COUNTER, SOCK, PKT_RX ) ];
210 0 : s->net_tx += m[ MIDX( COUNTER, SOCK, PKT_TX ) ];
211 0 : s->net_tx_drop += m[ MIDX( COUNTER, SOCK, PKT_TX_FAILED ) ];
212 0 : break;
213 0 : }
214 0 : }
215 0 : }
216 :
217 : /* fmt_num renders v with thousands separators into buf (>=32 bytes)
218 : and returns buf. */
219 :
220 : static char *
221 0 : fmt_num( char * buf, double v ) {
222 0 : char raw[ 32 ];
223 0 : snprintf( raw, sizeof(raw), "%.0f", v<0.0 ? 0.0 : v );
224 0 : ulong len = strlen( raw ), o = 0UL;
225 0 : for( ulong i=0UL; i<len; i++ ) {
226 0 : if( i && (len-i)%3==0 ) buf[ o++ ] = ',';
227 0 : buf[ o++ ] = raw[ i ];
228 0 : }
229 0 : buf[ o ] = '\0';
230 0 : return buf;
231 0 : }
232 :
233 : /* col prints one "label value" cell of fixed width so rows line up. */
234 :
235 : static void
236 0 : col( char const * label, double v ) {
237 0 : char b[ 32 ];
238 0 : printf( "%-12s %12s ", label, fmt_num( b, v ) );
239 0 : }
240 :
241 : static void
242 0 : col_pct( char const * label, double v, double pct ) {
243 0 : char b[ 32 ];
244 0 : printf( "%-12s %12s (%3.0f%%)", label, fmt_num( b, v ), pct );
245 0 : }
246 :
247 : static void
248 : metrics_snap_print( metrics_snap_t const * s,
249 : metrics_snap_t const * p,
250 0 : metrics_src_t const * src ) {
251 0 : struct tm tm; time_t t = (time_t)(s->ts/(long)1e9); localtime_r( &t, &tm );
252 0 : char tbuf[ 16 ]; strftime( tbuf, sizeof(tbuf), "%H:%M:%S", &tm );
253 :
254 : /* Everything below is the cumulative counter value since the tile
255 : booted (gauges are the current value). Only the tile busy line is
256 : computed over the last interval, since a lifetime average would
257 : hide what the tiles are doing right now. */
258 :
259 0 : ulong req_tot = 0UL;
260 0 : for( ulong i=0UL; i<FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_CNT; i++ ) {
261 0 : if( i==FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_PONG_IDX ) continue;
262 0 : req_tot += s->req[ i ];
263 0 : }
264 0 : double resp_pct = req_tot ? 100.0*(double)s->resp_cnt /(double)req_tot : 0.0;
265 0 : double rereq_pct = req_tot ? 100.0*(double)s->rerequest/(double)req_tot : 0.0;
266 0 : double lat_ms = s->resp_cnt ? (double)s->resp_sum_ns/(double)s->resp_cnt/1e6 : 0.0;
267 0 : long gap = (long)s->slot_current-(long)s->slot_highest_repaired;
268 :
269 0 : # define T(field) ((double)s->field)
270 0 : # define Q(idx) ((double)s->req[ idx ])
271 :
272 0 : printf( "==== %s %s\n", tbuf,
273 0 : "======================================================================================================================" );
274 :
275 0 : printf( " slots " );
276 0 : col( "current", (double)s->slot_current );
277 0 : col( "repaired", (double)s->slot_highest_repaired );
278 0 : col( "gap", (double)gap );
279 0 : col( "last_req", (double)s->slot_last_requested );
280 0 : col( "orphan_req", (double)s->orphan_last_requested );
281 0 : printf( "\n state " );
282 0 : col( "inflight", (double)s->inflight );
283 0 : col( "peers", (double)s->peers );
284 0 : printf( "\n\n" );
285 :
286 0 : printf( " requests " );
287 0 : col( "window", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_WINDOW_INDEX_IDX ) );
288 0 : col( "highest", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_HIGHEST_WINDOW_INDEX_IDX ) );
289 0 : col( "orphan", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_ORPHAN_IDX ) );
290 0 : col( "parent_fec", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_PARENT_FEC_SET_COUNT_IDX ) );
291 0 : printf( "\n " );
292 0 : col( "fec_root", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_FEC_SET_ROOT_IDX ) );
293 0 : col( "shred_bid", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_WINDOW_INDEX_FOR_BLOCK_ID_IDX ) );
294 0 : col( "pong", Q( FD_METRICS_ENUM_REPAIR_REQUEST_TYPE_V_PONG_IDX ) );
295 0 : printf( "\n " );
296 0 : col( "total", (double)req_tot );
297 0 : col_pct( "rerequest", T( rerequest ), rereq_pct ); printf( " " );
298 0 : col( "pkt_tx", T( pkt_tx ) );
299 0 : printf( "\n\n" );
300 :
301 0 : printf( " responses " );
302 0 : col_pct( "matched", T( resp_cnt ), resp_pct ); printf( " of requests " );
303 0 : printf( "%-12s %9.1f ms ", "latency", lat_ms );
304 0 : col( "meta_failed", T( meta_failed ) );
305 0 : col( "shred_old", T( shred_old ) );
306 0 : printf( "\n" );
307 :
308 0 : printf( " shred " );
309 0 : col( "repair_rx", T( shred_repair_rx ) );
310 0 : col( "turbine_rx", T( shred_turbine_rx ) );
311 0 : col( "okay", T( shred_okay ) );
312 0 : col( "duplicate", T( shred_dup ) );
313 0 : printf( "\n " );
314 0 : col( "bad_slot", T( shred_bad_slot ) );
315 0 : col( "ignored", T( shred_ignored ) );
316 0 : col( "completes", T( shred_completes ) );
317 0 : printf( "\n\n" );
318 :
319 0 : printf( " drops " );
320 0 : col( "net_shred", T( net_shred.overrun ) );
321 0 : col( "consumed", T( net_shred.consumed ) );
322 0 : col( "filtered", T( net_shred.filtered ) );
323 0 : printf( "(%lu links, overrun = frags skipped)", src->net_shred_cnt );
324 0 : printf( "\n " );
325 0 : col( "net_repair", T( net_repair.overrun ) );
326 0 : col( "consumed", T( net_repair.consumed ) );
327 0 : printf( "\n " );
328 0 : col( "repair_net", T( repair_net.overrun ) );
329 0 : col( "consumed", T( repair_net.consumed ) );
330 0 : printf( "\n " );
331 0 : col( "net_rx", T( net_rx ) );
332 0 : col( "rx_drop", T( net_rx_drop ) );
333 0 : col( "net_tx", T( net_tx ) );
334 0 : col( "tx_drop", T( net_tx_drop ) );
335 0 : printf( "\n\n" );
336 :
337 0 : printf( " tile busy " );
338 0 : for( ulong i=0UL; i<1UL+src->shred_cnt; i++ ) {
339 0 : ulong dtot = s->regime[ i ].total-p->regime[ i ].total;
340 0 : double busy = dtot ? 100.0*(double)(s->regime[ i ].busy -p->regime[ i ].busy )/(double)dtot : 0.0;
341 0 : double backp = dtot ? 100.0*(double)(s->regime[ i ].backp-p->regime[ i ].backp)/(double)dtot : 0.0;
342 0 : char label[ 16 ];
343 0 : if( !i ) snprintf( label, sizeof(label), "rotor" );
344 0 : else snprintf( label, sizeof(label), "shred:%lu", i-1UL );
345 0 : printf( "%-12s %5.1f%% (bp %4.1f%%) ", label, busy, backp );
346 0 : }
347 0 : printf( " (last interval)\n\n" );
348 :
349 0 : # undef T
350 0 : # undef Q
351 0 : fflush( stdout );
352 0 : }
353 :
354 : static void
355 : rotor_metrics_fn( args_t * args,
356 0 : config_t * config ) {
357 0 : metrics_src_t src[1];
358 0 : metrics_src_init( src, config );
359 0 : FD_LOG_NOTICE(( "rotor metrics: %lu shred tiles, %lu %s tiles, %lu net_shred links, %lu net_repair links, %lu repair_net links",
360 0 : src->shred_cnt, src->net_cnt, fd_net_tile_name( config->net.provider ), src->net_shred_cnt, src->net_repair_cnt, src->repair_net_cnt ));
361 :
362 0 : metrics_snap_t prev[1], cur[1];
363 0 : metrics_snap_take( prev, src );
364 0 : for(;;) {
365 0 : sleep( 1 );
366 0 : metrics_snap_take( cur, src );
367 0 : metrics_snap_print( cur, prev, src );
368 0 : *prev = *cur;
369 0 : if( args->rotor.once ) break;
370 0 : }
371 0 : }
372 :
373 : static void
374 : rotor_cmd_args( int * pargc,
375 : char *** pargv,
376 0 : args_t * args ) {
377 0 : args->rotor.once = fd_env_strip_cmdline_contains( pargc, pargv, "--once" );
378 0 : }
379 :
380 : action_t fd_action_rotor = {
381 : .name = "rotor",
382 : .args = rotor_cmd_args,
383 : .fn = rotor_metrics_fn,
384 : .perm = dev_cmd_perm,
385 : .description = "Inspect a running validator's alpenglow rotor tile",
386 : .detail = "Attaches to a running validator's rotor tile and prints once a second\n"
387 : "per-second repair request counters by type, matched responses and\n"
388 : "mean latency, shred tile receive counters, and net / link drop\n"
389 : "counters (net_shred, net_repair, repair_net overruns, net tile rx/tx\n"
390 : "drops).\n"
391 : "\n"
392 : " --once print a single snapshot and exit",
393 : .usage = "rotor [--once]",
394 : };
|
