Line data    Source code

       1             : #ifndef HEADER_fd_src_discof_restore_utils_fd_ssctrl_h
       2             : #define HEADER_fd_src_discof_restore_utils_fd_ssctrl_h
       3             : 
       4             : #include "../../../util/net/fd_net_headers.h"
       5             : #include "../../../waltz/fd_fqdn.h"
       6             : #include "../../../flamenco/runtime/fd_runtime_const.h"
       7             : 
       8             : /* The snapshot tiles have a somewhat involved state machine, which is
       9             :    controlled by snapct.  Imagine first the following sequence:
      10             : 
      11             :     1. snapct is reading a full snapshot from the network and sends some
      12             :        data to snapdc to be decompressed.
      13             :     2. snapct hits a network error, and resets the connection to a new
      14             :        peer.
      15             :     3. The decompressor fails on data from the old peer, and sends a
      16             :        malformed message to snapct.
      17             :     4. snapct receives the malformed message, and abandons the new
      18             :        connection, even though it was not malformed.
      19             : 
      20             :    There are basically two ways to prevent this.  Option A is the tiles
      21             :    can pass not just control messages to one another, but also tag them
      22             :    with some xid indicating which "attempt" the control message is for.
      23             : 
      24             :    This is pretty hard to reason about, and the state machine can grow
      25             :    quite complicated.
      26             : 
      27             :    There's an easier way: the tiles just are fully synchronized with
      28             :    snapct.  Whatever "attempt" snapct is on, we ensure all other tiles
      29             :    are on it too.  This means when any tile fails a snapshot, all tiles
      30             :    must fail it and fully flush all frags in the pipeline before snapct
      31             :    can proceed with a new attempt.
      32             : 
      33             :    The control flow then is basically,
      34             : 
      35             :      1. All tiles start in the IDLE state.
      36             :      2. snapct initializes the pipeline by sending an INIT message.
      37             :         Each tile enters the PROCESSING state and then forwards the INIT
      38             :         message down the pipeline.  When snapct receives this INIT
      39             :         message from every path, the entire pipeline is in PROCESSING
      40             :         state.  snapct then sends START with the fetch target to snapld,
      41             :         which begins producing snapshot data.
      42             :      3. Tiles continue to process data / frags as applicable.  If an
      43             :         error occurs, the tile enters the ERROR state and also sends an
      44             :         ERROR message downstream.  All downstream tiles also enter the
      45             :         ERROR state and forward the message.  Note that upstream tiles
      46             :         will not be in an ERROR state and will continue producing frags.
      47             :         When snapct receives the ERROR message, it will send a FAIL
      48             :         message.  snapct then waits for this FAIL message to be
      49             :         propagated through the pipeline and received back.  snapct
      50             :         also guarantees to flush any pending load data in the
      51             :         pipeline.  It then knows that all tiles are synchronized back
      52             :         in an IDLE state and it can try again with a new INIT.
      53             :      4. Once all snapshot data has entered the pipeline, snapct sends
      54             :         a FINI message through the pipeline and waits for it to be
      55             :         received back.  This synchronizes tiles to finish any
      56             :         remaining in-flight work.  It then sends either a NEXT or a
      57             :         DONE message through the pipeline, again waiting for it to be
      58             :         received back.  NEXT means another snapshot follows, whereas
      59             :         DONE means a shutdown will follow.
      60             : 
      61             :    That keeps the tiles in lockstep, and simplifies the state machine
      62             :    to a manageable level.
      63             : 
      64             :    In more detail, all tiles in the feedback loop of the pipeline must
      65             :    comply with the following rules; snapct enforces them; tiles outside
      66             :    of the feedback loop may support a subset of these:
      67             :      - Allowed state transitions on given control message:
      68             :         IDLE       to PROCESSING: on INIT_FULL / INIT_INCR ctrl msg.
      69             :         PROCESSING to FINISHING : on FINI ctrl msg (*).
      70             :         FINISHING  to IDLE      : on NEXT / DONE ctrl msg.
      71             :         IDLE       to SHUTDOWN  : on SHUTDOWN ctrl msg.
      72             :      - Control messages that apply to all states (except SHUTDOWN):
      73             :         On ERROR msg, always transition to ERROR.
      74             :         On FAIL msg, always transition to IDLE.
      75             :      - START is sent only from snapct to snapld.  It is not forwarded
      76             :        through the feedback loop and does not change snapld's state.
      77             :      - When in SHUTDOWN state, no data or control message is allowed.
      78             :      - Error handling:
      79             :         A tile that enters ERROR state on its own must forward an
      80             :         ERROR control message and discard any incoming message.  When
      81             :         in ERROR state, data is discarded, and only a FAIL control
      82             :         message is processed and forwarded (all others are discarded).
      83             :         As a result, only one ERROR message can propagate through the
      84             :         pipeline at any given time.
      85             :      - Holding onto a control message:
      86             :         A tile may hold onto a control message and forward it later
      87             :         after performing some asynchronous routine.  The tile's state
      88             :         transition may also be deferred until that control message is
      89             :         forwarded.
      90             :      - (*) A tile may self-transition from PROCESSING to FINISHING
      91             :         early, if it can detect end-of-stream.  The FINI message is
      92             :         used to synchronize the transition.
      93             : 
      94             :    Each non-ERROR control message is only generated once in snapct and
      95             :    will not be re-sent.  Except for START, the pipeline will be locked
      96             :    on flushing that control message until all tiles forward it on, or an
      97             :    ERROR message is triggered by any of the tiles and forwarded. */
      98             : 
      99             : /* Snapshot data MTU and snapld-to-snapdc link depth. */
     100           0 : #define FD_SNAPSHOT_DATA_DEPTH                 (1024UL)
     101             : 
     102             : /* Every snapin is a reliable consumer on every snapdc lane.  This
     103             :    runway keeps one stalled snapin from blocking all decompressors. */
     104           0 : #define FD_SNAPSHOT_DC_IN_DEPTH                (2048UL)
     105             : 
     106           0 : #define FD_SNAPSHOT_DATA_MTU                   (65408UL)
     107             : 
     108           0 : #define FD_SNAPSHOT_STATE_IDLE                 (0UL) /* Performing no work and should receive no data frags */
     109           0 : #define FD_SNAPSHOT_STATE_PROCESSING           (1UL) /* Performing usual work, no errors / EoF condition encountered */
     110           0 : #define FD_SNAPSHOT_STATE_FINISHING            (2UL) /* Tile has observed EoF, expects no additional data frags */
     111           0 : #define FD_SNAPSHOT_STATE_ERROR                (3UL) /* Some error occurred, will wait for a FAIL command to reset */
     112           0 : #define FD_SNAPSHOT_STATE_SHUTDOWN             (4UL) /* All work finished, tile can perform final cleanup and exit */
     113             : 
     114           0 : #define FD_SNAPSHOT_MSG_DATA                   (0UL) /* Fragment represents some snapshot data */
     115           0 : #define FD_SNAPSHOT_MSG_META                   (1UL) /* Fragment represents a fd_ssctrl_meta_t message */
     116             : 
     117           0 : #define FD_SNAPSHOT_MSG_CTRL_INIT_FULL         (2UL) /* Pipeline should start processing a full snapshot */
     118           0 : #define FD_SNAPSHOT_MSG_CTRL_INIT_INCR         (3UL) /* Pipeline should start processing an incremental snapshot */
     119           0 : #define FD_SNAPSHOT_MSG_CTRL_START             (4UL) /* snapct -> snapld: all INIT paths acknowledged */
     120           0 : #define FD_SNAPSHOT_MSG_CTRL_FAIL              (5UL) /* Current snapshot failed, queue rollback and reset to idle */
     121           0 : #define FD_SNAPSHOT_MSG_CTRL_NEXT              (6UL) /* Current snapshot succeeded, commit work, go idle, and expect another snapshot */
     122           0 : #define FD_SNAPSHOT_MSG_CTRL_DONE              (7UL) /* Current snapshot succeeded, commit work, go idle, and expect shutdown */
     123           0 : #define FD_SNAPSHOT_MSG_CTRL_SHUTDOWN          (8UL) /* Snapshot load successful, no work left to do, perform final cleanup and shut down*/
     124           0 : #define FD_SNAPSHOT_MSG_CTRL_ERROR             (9UL) /* Some tile encountered an error with the current stream */
     125           0 : #define FD_SNAPSHOT_MSG_CTRL_FINI             (10UL) /* Current snapshot has been fully loaded, finish processing */
     126             : 
     127             : /* snapld -> snapct (via snapld_dc) */
     128           0 : #define FD_SNAPSHOT_MSG_LOAD_COMPLETE         (11UL) /* snapld finished reading/downloading all data */
     129             : 
     130             : /* Sigs passed from snapdc to snapin contain the frame index of the
     131             :    frag so snapin knows how to order frames as they arrive. */
     132             : 
     133           0 : #define FD_SNAPDC_SIG_TYPE_BITS (8UL)
     134             : 
     135             : FD_FN_CONST static inline ulong
     136           0 : fd_snapdc_data_sig( ulong frame_idx ) {
     137           0 :   return FD_SNAPSHOT_MSG_DATA | (frame_idx<<FD_SNAPDC_SIG_TYPE_BITS);
     138           0 : }
     139             : 
     140             : FD_FN_CONST static inline ulong
     141           0 : fd_snapdc_sig_type( ulong sig ) {
     142           0 :   return sig & ((1UL<<FD_SNAPDC_SIG_TYPE_BITS)-1UL);
     143           0 : }
     144             : 
     145             : FD_FN_CONST static inline ulong
     146           0 : fd_snapdc_sig_frame( ulong sig ) {
     147           0 :   return sig>>FD_SNAPDC_SIG_TYPE_BITS;
     148           0 : }
     149             : 
     150             : /* Pipeline initialization, forwarded by snapld through snapdc to
     151             :    snapin. */
     152             : typedef struct fd_ssctrl_init {
     153             :   int           file;
     154             :   int           zstd;
     155             :   ulong         slot; /* slot advertised by the snapshot peer */
     156             :   uchar         snapshot_hash[ FD_HASH_FOOTPRINT ]; /* advertised snapshot hash from snapshot file name */
     157             :   int           is_redirect; /* 1 if using well-known redirect path */
     158             :   ulong         file_sz;     /* file size in bytes (file loads only, 0 for HTTP) */
     159             : } fd_ssctrl_init_t;
     160             : 
     161             : /* Fetch target sent only from snapct to snapld after all INIT acks.
     162             :    For file loads, START has no payload. */
     163             : typedef struct fd_ssctrl_start {
     164             :   fd_ip4_port_t addr;
     165             :   int           is_https;
     166             :   ulong         path_len;
     167             :   char          hostname[ FD_FQDN_BUF_MAX ];
     168             :   char          path[ PATH_MAX ];
     169             : } fd_ssctrl_start_t;
     170             : 
     171             : /* The fragment signature selects the payload.  Publish only the
     172             :    selected member's size; the union sizes the shared snapct -> snapld
     173             :    buffer. */
     174             : typedef union fd_ssctrl_msg {
     175             :   fd_ssctrl_init_t  init;
     176             :   fd_ssctrl_start_t start;
     177             : } fd_ssctrl_msg_t;
     178             : 
     179             : /* Sent by snapld to tell snapct metadata about a downloaded snapshot. */
     180             : typedef struct fd_ssctrl_meta {
     181             :   ulong total_sz;
     182             :   ulong resolved_slot;
     183             :   uchar resolved_hash[ FD_HASH_FOOTPRINT ];
     184             :   char  resolved_name[ PATH_MAX ];
     185             : } fd_ssctrl_meta_t;
     186             : 
     187             : struct fd_snapshot_account_hdr {
     188             :   uchar   pubkey[ FD_PUBKEY_FOOTPRINT ];
     189             :   uchar   owner[ FD_PUBKEY_FOOTPRINT ];
     190             :   ulong   lamports;
     191             :   uchar   executable;
     192             :   ulong   data_len;
     193             : };
     194             : typedef struct fd_snapshot_account_hdr fd_snapshot_account_hdr_t;
     195             : 
     196             : /* fd_snapshot_account_hdr_init initializes a fd_snapshot_account_hdr_t struct
     197             :    with the appropriate account metadata fields. */
     198             : static inline void
     199             : fd_snapshot_account_hdr_init( fd_snapshot_account_hdr_t * account,
     200             :                            uchar const                    pubkey[ FD_PUBKEY_FOOTPRINT ],
     201             :                            uchar const                    owner[ FD_PUBKEY_FOOTPRINT ],
     202             :                            ulong                          lamports,
     203             :                            uchar                          executable,
     204           0 :                            ulong                          data_len ) {
     205           0 :   fd_memcpy( account->pubkey, pubkey, FD_PUBKEY_FOOTPRINT );
     206           0 :   fd_memcpy( account->owner,  owner,  FD_PUBKEY_FOOTPRINT );
     207           0 :   account->lamports   = lamports;
     208           0 :   account->executable = executable;
     209           0 :   account->data_len   = data_len;
     210           0 : }
     211             : 
     212             : static inline const char *
     213           0 : fd_ssctrl_state_str( ulong state ) {
     214           0 :   switch( state ) {
     215           0 :     case FD_SNAPSHOT_STATE_IDLE:        return "idle";
     216           0 :     case FD_SNAPSHOT_STATE_PROCESSING:  return "processing";
     217           0 :     case FD_SNAPSHOT_STATE_FINISHING:   return "finishing";
     218           0 :     case FD_SNAPSHOT_STATE_ERROR:       return "error";
     219           0 :     case FD_SNAPSHOT_STATE_SHUTDOWN:    return "shutdown";
     220           0 :     default:                            return "unknown";
     221           0 :   }
     222           0 : }
     223             : 
     224             : static inline const char *
     225           0 : fd_ssctrl_msg_ctrl_str( ulong sig ) {
     226           0 :   switch( fd_snapdc_sig_type( sig ) ) {
     227           0 :     case FD_SNAPSHOT_MSG_DATA:                  return "data";
     228           0 :     case FD_SNAPSHOT_MSG_META:                  return "meta";
     229           0 :     case FD_SNAPSHOT_MSG_CTRL_INIT_FULL:        return "init_full";
     230           0 :     case FD_SNAPSHOT_MSG_CTRL_INIT_INCR:        return "init_incr";
     231           0 :     case FD_SNAPSHOT_MSG_CTRL_START:            return "start";
     232           0 :     case FD_SNAPSHOT_MSG_CTRL_FAIL:             return "fail";
     233           0 :     case FD_SNAPSHOT_MSG_CTRL_NEXT:             return "next";
     234           0 :     case FD_SNAPSHOT_MSG_CTRL_DONE:             return "done";
     235           0 :     case FD_SNAPSHOT_MSG_CTRL_SHUTDOWN:         return "shutdown";
     236           0 :     case FD_SNAPSHOT_MSG_CTRL_ERROR:            return "error";
     237           0 :     case FD_SNAPSHOT_MSG_CTRL_FINI:             return "fini";
     238           0 :     case FD_SNAPSHOT_MSG_LOAD_COMPLETE:         return "load_complete";
     239           0 :     default:                                    return "unknown";
     240           0 :   }
     241           0 : }
     242             : 
     243             : #endif /* HEADER_fd_src_discof_restore_utils_fd_ssctrl_h */