Line data    Source code

       1             : #include "run/run.h"
       2             : 
       3             : #include "../fd_bootinfo.h"
       4             : #include "../../../disco/metrics/fd_metrics.h"
       5             : #include "../../../disco/metrics/generated/fd_metrics_replay.h"
       6             : 
       7             : void
       8             : ready_cmd_args( int *    pargc,
       9             :                 char *** pargv,
      10           0 :                 args_t * args ) {
      11           0 :   args->ready.ready_slot = fd_env_strip_cmdline_ulong( pargc, pargv, "--ready-slot", NULL, 0UL );
      12           0 : }
      13             : 
      14             : void
      15             : ready_cmd_fn( args_t *   args,
      16           0 :               config_t * config ) {
      17           0 :   fd_bootinfo_adopt( config );
      18           0 :   ulong wksp_id = fd_topo_find_wksp( &config->topo, "metric_in" );
      19           0 :   FD_TEST( wksp_id!=ULONG_MAX );
      20             : 
      21           0 :   fd_bootinfo_check_layout( config );
      22           0 :   fd_topo_join_workspace( &config->topo, &config->topo.workspaces[ wksp_id ], FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
      23           0 :   fd_topo_workspace_fill( &config->topo, &config->topo.workspaces[ wksp_id ] );
      24             : 
      25           0 :   for( ulong i=0; i<config->topo.tile_cnt; i++) {
      26           0 :     fd_topo_tile_t * tile = &config->topo.tiles[i];
      27             : 
      28           0 :     long start = fd_log_wallclock();
      29           0 :     int printed = 0;
      30           0 :     do {
      31           0 :       ulong status = fd_metrics_tile( tile->metrics )[ FD_METRICS_GAUGE_TILE_STATE_OFF ];
      32             : 
      33           0 :       if( FD_LIKELY( status==1UL ) ) break;
      34           0 :       else if( FD_UNLIKELY( tile->allow_shutdown && status==2UL ) ) break;
      35           0 :       else if( FD_UNLIKELY( status ) )
      36           0 :         FD_LOG_ERR(( "status for tile %s:%lu is in bad state %lu", tile->name, tile->kind_id, status ));
      37             : 
      38           0 :       if( FD_UNLIKELY( !printed && (fd_log_wallclock()-start) > 2L*1000*1000*1000L ) ) {
      39           0 :         FD_LOG_NOTICE(( "waiting for tile %s%s:%lu%s to be ready", fd_log_style_bold(), tile->name, tile->kind_id, fd_log_style_normal() ));
      40           0 :         printed = 1;
      41           0 :       }
      42           0 :     } while(1);
      43           0 :   }
      44             : 
      45             :   /* Optionally wait for the replay tile to reach a certain slot */
      46           0 :   if( FD_UNLIKELY( args->ready.ready_slot ) ) {
      47           0 :     ulong replay_idx = fd_topo_find_tile( &config->topo, "replay", 0UL );
      48           0 :     if( FD_UNLIKELY( replay_idx==ULONG_MAX ) ) {
      49           0 :       FD_LOG_ERR(( "--ready-slot specified but no replay tile found" ));
      50           0 :     }
      51           0 :     fd_topo_tile_t * replay_tile = &config->topo.tiles[ replay_idx ];
      52             : 
      53           0 :     long start = fd_log_wallclock();
      54           0 :     int printed = 0;
      55           0 :     do {
      56           0 :       ulong reset_slot = fd_metrics_tile( replay_tile->metrics )[ FD_METRICS_GAUGE_REPLAY_RESET_SLOT_OFF ];
      57             : 
      58           0 :       if( FD_LIKELY( reset_slot>=args->ready.ready_slot ) ) break;
      59             : 
      60           0 :       if( FD_UNLIKELY( !printed && (fd_log_wallclock()-start) > 4e9L ) ) {
      61           0 :         FD_LOG_NOTICE(( "waiting for reset slot to reach %lu %s(currently %lu)%s", args->ready.ready_slot, fd_log_style_dim(), reset_slot, fd_log_style_normal() ));
      62           0 :         printed = 1;
      63           0 :       }
      64           0 :     } while(1);
      65           0 :   }
      66             : 
      67           0 :   fd_topo_leave_workspaces( &config->topo );
      68           0 :   FD_LOG_NOTICE(( "all tiles ready" ));
      69           0 : }
      70             : 
      71             : static void
      72           0 : ready_args_help( fd_action_help_t * help ) {
      73           0 :   fd_action_help_arg( help, "--ready-slot", "<slot>", "After all tiles are ready, also block until the replay tile's reset\n"
      74           0 :                                                      "slot reaches at least <slot> (default 0, which waits for tiles only)" );
      75           0 : }
      76             : 
      77             : action_t fd_action_ready = {
      78             :   .name           = "ready",
      79             :   .args           = ready_cmd_args,
      80             :   .fn             = ready_cmd_fn,
      81             :   .require_config = 0,
      82             :   .perm           = NULL,
      83             :   .description    = "Wait for all tiles to be running",
      84             :   .detail         = "Connects to a running validator and blocks until every tile reports a\n"
      85             :                     "running status, then exits 0.  A tile is a single thread pinned to a CPU\n"
      86             :                     "core that performs one part of the validator's work.  Useful in scripts\n"
      87             :                     "that must wait for the validator to finish booting before proceeding.",
      88             :   .usage          = "ready [OPTIONS]",
      89             :   .args_help      = ready_args_help,
      90             : };