Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_bundle_fd_bundle_tile_private_h
       2             : #define HEADER_fd_src_disco_bundle_fd_bundle_tile_private_h
       3             : 
       4             : #include "fd_bundle_auth.h"
       5             : #include "fd_keepalive.h"
       6             : #include "../fd_clock_tile.h"
       7             : #include "../stem/fd_stem.h"
       8             : #include "../keyguard/fd_keyswitch.h"
       9             : #include "../keyguard/fd_keyguard_client.h"
      10             : #include "../../waltz/grpc/fd_grpc_client.h"
      11             : #include "../../waltz/http/fd_url.h"
      12             : #include "../../waltz/resolv/fd_netdb.h"
      13             : #include "../../waltz/fd_rtt_est.h"
      14             : #include "../../util/hist/fd_histf.h"
      15             : 
      16             : #define FD_BUNDLE_CLIENT_MAX_TXN_PER_BUNDLE (5UL)
      17             : 
      18             : #include "../../waltz/tlsrec/fd_tlsrec.h"
      19             : #include "../../ballet/x509/fd_x509_ca_store.h"
      20             : #include "../../ballet/x509/fd_x509_verify.h"
      21             : 
      22             : struct fd_bundle_out_ctx {
      23             :   ulong       idx;
      24             :   fd_wksp_t * mem;
      25             :   ulong       chunk0;
      26             :   ulong       wmark;
      27             :   ulong       chunk;
      28             : };
      29             : 
      30             : typedef struct fd_bundle_out_ctx fd_bundle_out_ctx_t;
      31             : 
      32             : /* fd_bundle_metrics_t contains private metric counters.  These get
      33             :    published to fd_metrics periodically. */
      34             : 
      35             : struct fd_bundle_metrics {
      36             :   ulong txn_received_cnt;
      37             :   ulong bundle_received_cnt;
      38             :   ulong packet_received_cnt;
      39             :   ulong proto_received_bytes;
      40             :   ulong ping_ack_cnt;
      41             : 
      42             :   ulong decode_fail_cnt;
      43             :   ulong transport_fail_cnt;
      44             :   ulong missing_builder_info_fail_cnt;
      45             :   ulong backpressure_drop_cnt;
      46             : 
      47             :   fd_histf_t msg_rx_delay[1];
      48             : };
      49             : 
      50             : typedef struct fd_bundle_metrics fd_bundle_metrics_t;
      51             : 
      52             : /* fd_bundle_tile_t is the context object provided to callbacks from
      53             :    stem, and contains all state needed to progress the tile. */
      54             : 
      55             : struct fd_bundle_tile {
      56             :   /* Key switch */
      57             :   fd_keyswitch_t * keyswitch;
      58             : 
      59             :   /* Key guard */
      60             :   fd_keyguard_client_t keyguard_client[1];
      61             : 
      62             :   uint is_ssl : 1;
      63             :   int  keylog_fd;
      64             : 
      65             :   /* Native TLS */
      66             :   fd_tls_t           tls[1];
      67             :   fd_chacha_rng_t    tls_rng[1];
      68             :   fd_tlsrec_conn_t   tls_conn[1];
      69             :   fd_x509_ca_store_t ca_store[1];
      70             : 
      71             :   /* Config */
      72             :   char   server_fqdn[ FD_FQDN_BUF_MAX ]; /* cstr */
      73             :   ulong  server_fqdn_len;
      74             :   char   server_sni[ FD_SNI_BUF_MAX ]; /* cstr */
      75             :   ulong  server_sni_len;
      76             :   ushort server_tcp_port;
      77             : 
      78             :   /* Resolver */
      79             :   fd_netdb_fds_t netdb_fds[1];
      80             :   uint server_ip4_addr; /* last DNS lookup result */
      81             : 
      82             :   /* TCP socket */
      83             :   int  tcp_sock;
      84             :   int  so_rcvbuf;
      85             :   uint tcp_sock_connected : 1;
      86             :   uint defer_reset : 1;
      87             :   uint sock_in_epoll : 1;
      88             :   uint epoll_out_armed : 1;
      89             :   long cached_ts;
      90             :   long conn_deadline;
      91             : 
      92             :   ulong   waker_client_idx;
      93             :   ulong * waker_fseq;
      94             :   long    next_step_deadline;
      95             : 
      96             :   /* Keepalive via HTTP/2 PINGs (randomized) */
      97             :   long              keepalive_interval;
      98             :   fd_keepalive_t    keepalive[1];
      99             :   fd_rtt_estimate_t rtt[1];
     100             : 
     101             :   /* gRPC client */
     102             :   void *                   grpc_client_mem;
     103             :   ulong                    grpc_buf_max;
     104             :   fd_grpc_client_t *       grpc_client;
     105             :   fd_grpc_client_metrics_t grpc_metrics[1];
     106             :   ulong                    map_seed;
     107             : 
     108             :   /* Bundle authenticator */
     109             :   fd_bundle_auther_t auther;
     110             : 
     111             :   /* Bundle block builder info */
     112             :   uchar builder_pubkey[ 32 ];
     113             :   uchar builder_commission;  /* in [0,100] (percent) */
     114             :   uchar builder_info_avail : 1;  /* Block builder info available? (potentially stale) */
     115             :   uchar builder_info_wait  : 1;  /* Request already in-flight? */
     116             :   long  builder_info_valid_until;
     117             : 
     118             :   /* Bundle subscriptions */
     119             :   uchar packet_subscription_live : 1;  /* Want to subscribe to a stream? */
     120             :   uchar packet_subscription_wait : 1;  /* Request already in-flight? */
     121             :   uchar bundle_subscription_live : 1;
     122             :   uchar bundle_subscription_wait : 1;
     123             : 
     124             :   /* Bundle state */
     125             :   ulong bundle_seq;
     126             : 
     127             :   /* Error backoff */
     128             :   fd_rng_t rng[1];
     129             :   uint     backoff_iter;
     130             :   long     backoff_until;
     131             :   long     backoff_reset;
     132             : 
     133             :   /* Stem publish */
     134             :   fd_stem_context_t *       stem;
     135             :   fd_bundle_out_ctx_t       verify_out;
     136             :   fd_bundle_out_ctx_t       plugin_out;
     137             : 
     138             :   /* App metrics */
     139             :   fd_bundle_metrics_t metrics;
     140             : 
     141             :   /* Check engine light */
     142             :   uchar bundle_status_recent;  /* most recently observed 'check engine light' */
     143             :   uchar bundle_status_plugin;  /* last 'plugin' update written */
     144             :   uchar bundle_status_logged;
     145             :   long  last_bundle_status_log_nanos;
     146             : 
     147             :   ulong next_leader_slot; /* from replay became leader and next leader messages, or ULONG_MAX */
     148             :   ulong reset_slot;       /* from replay became leader and next leader messages, or ULONG_MAX */
     149             :   int   sleep_mode;       /* 1 means sleeping, 0 means connecting/connected */
     150             :   long  sleep_check_ns;   /* next wallclock time to re-evaluate sleeping */
     151             : 
     152             :   fd_clock_tile_t clock[1]; /* fast wallclock-ns source for fd_bundle_now */
     153             :   int   halt_signing;     /* 1 means signing is halted, 0 means signing is not halted */
     154             : 
     155             :   /* Staged values from during_frag, committed in after_frag */
     156             :   ulong next_leader_slot_staged;
     157             :   ulong reset_slot_staged;
     158             : 
     159             :   int   in_kind[ 64 ];
     160             :   struct {
     161             :     fd_wksp_t * mem;
     162             :     ulong       chunk0;
     163             :     ulong       wmark;
     164             :   } replay_in;
     165             : };
     166             : 
     167             : typedef struct fd_bundle_tile fd_bundle_tile_t;
     168             : 
     169             : /* Define 'request_ctx' IDs to identify different types of gRPC calls */
     170             : 
     171           0 : #define FD_BUNDLE_CLIENT_REQ_Bundle_SubscribePackets            4
     172           0 : #define FD_BUNDLE_CLIENT_REQ_Bundle_SubscribeBundles            5
     173           0 : #define FD_BUNDLE_CLIENT_REQ_Bundle_GetBlockBuilderFeeInfo      6
     174             : 
     175             : FD_PROTOTYPES_BEGIN
     176             : 
     177             : /* fd_bundle_now returns wallclock nanoseconds from the tile's
     178             :    fd_clock.  Backed by a weak symbol, allowing tests to override the
     179             :    clock source. */
     180             : 
     181             : long
     182             : fd_bundle_now( fd_bundle_tile_t const * ctx );
     183             : 
     184             : /* fd_bundle_client_grpc_callbacks provides callbacks for grpc_client. */
     185             : 
     186             : extern fd_grpc_client_callbacks_t fd_bundle_client_grpc_callbacks;
     187             : 
     188             : /* fd_bundle_client_step is an all-in-one routine to drive client logic.
     189             :    As long as the tile calls this periodically, the client will
     190             :    reconnect to the bundle server, authenticate, and subscribe to
     191             :    packets and bundles. */
     192             : 
     193             : void
     194             : fd_bundle_client_step( fd_bundle_tile_t * bundle,
     195             :                        int *              charge_busy );
     196             : 
     197             : /* fd_bundle_client_step_reconnect drives the 'reconnect' state machine.
     198             :    Once the HTTP/2 conn is established (SETTINGS exchanged), this
     199             :    function drives the auth logic, requests block builder info, sets up
     200             :    packet and bundle subscriptions, and PINGs. */
     201             : 
     202             : int
     203             : fd_bundle_client_step_reconnect( fd_bundle_tile_t * ctx,
     204             :                                  long               now );
     205             : 
     206             : /* fd_bundle_client_next_deadline returns when fd_bundle_client_step
     207             :    next needs to run absent any fd event.  While TCP is connecting,
     208             :    returns the connection setup deadline; while disconnected, returns
     209             :    the reconnect time (at least now).  Once connected, considers gRPC
     210             :    deadlines, keepalive, builder info expiry and connection setup.
     211             :    Returns now if decrypted bytes are buffered in the native TLS/gRPC
     212             :    client and TLS TX is not blocked; blocked TX waits for EPOLLOUT. */
     213             : 
     214             : long
     215             : fd_bundle_client_next_deadline( fd_bundle_tile_t const * ctx,
     216             :                                 long                     now );
     217             : 
     218             : /* fd_bundle_tile_backoff is called whenever an error occurs.  Stalls
     219             :    forward progress for a randomized amount of time to prevent error
     220             :    floods. */
     221             : 
     222             : void
     223             : fd_bundle_tile_backoff( fd_bundle_tile_t * ctx,
     224             :                         long               now );
     225             : 
     226             : /* fd_bundle_tile_should_stall returns 1 if forward progress should be
     227             :    temporarily prevented due to an error. */
     228             : 
     229             : FD_FN_PURE static inline int
     230             : fd_bundle_tile_should_stall( fd_bundle_tile_t const * ctx,
     231           0 :                              long                     now ) {
     232           0 :   return now < ctx->backoff_until;
     233           0 : }
     234             : 
     235             : /* fd_bundle_tile_housekeeping runs periodically at a low frequency. */
     236             : 
     237             : void
     238             : fd_bundle_tile_housekeeping( fd_bundle_tile_t * ctx );
     239             : 
     240             : /* fd_bundle_client_grpc_rx_start is the first RX callback of a stream. */
     241             : 
     242             : void
     243             : fd_bundle_client_grpc_rx_start(
     244             :     void * app_ctx,
     245             :     ulong  request_ctx
     246             : ) ;
     247             : 
     248             : /* fd_bundle_client_grpc_rx_msg is called by grpc_client when a gRPC
     249             :    message arrives (unary or server-streaming response). */
     250             : 
     251             : void
     252             : fd_bundle_client_grpc_rx_msg(
     253             :     void *       app_ctx,      /* (fd_bundle_tile_t *) */
     254             :     void const * protobuf,
     255             :     ulong        protobuf_sz,
     256             :     ulong        request_ctx   /* FD_BUNDLE_CLIENT_REQ_{...} */
     257             : );
     258             : 
     259             : /* fd_bundle_client_grpc_rx_end is called by grpc_client when a gRPC
     260             :    server-streaming response finishes. */
     261             : 
     262             : void
     263             : fd_bundle_client_grpc_rx_end(
     264             :     void *                app_ctx,
     265             :     ulong                 request_ctx,
     266             :     fd_grpc_resp_hdrs_t * resp
     267             : );
     268             : 
     269             : /* fd_bundle_client_grpc_rx_timeout is called by grpc_client when a
     270             :    gRPC request deadline gets exceeded. */
     271             : 
     272             : void
     273             : fd_bundle_client_grpc_rx_timeout(
     274             :     void * app_ctx,
     275             :     ulong  request_ctx, /* FD_BUNDLE_CLIENT_REQ_{...} */
     276             :     int    deadline_kind /* FD_GRPC_DEADLINE_{HEADER|RX_END} */
     277             : );
     278             : 
     279             : /* fd_bundle_client_status provides a "check engine light" using
     280             :    FD_BUNDLE_STATE_{DISCONNECTED,CONNECTING,CONNECTED}.
     281             : 
     282             :    DISCONNECTED means TCP or HTTP/2 is unavailable/dead, GOAWAY is being
     283             :    sent, or keepalive has timed out.  CONNECTING means HTTP/2 setup,
     284             :    authentication, builder info or either subscription is not ready.
     285             :    CONNECTED means these checks pass and the gRPC client is connected. */
     286             : 
     287             : int
     288             : fd_bundle_client_status( fd_bundle_tile_t const * ctx );
     289             : 
     290             : /* fd_bundle_request_ctx_cstr returns the gRPC method name for a
     291             :    FD_BUNDLE_CLIENT_REQ_* ID.  Returns "unknown" the ID is not
     292             :    recognized. */
     293             : 
     294             : FD_FN_CONST char const *
     295             : fd_bundle_request_ctx_cstr( ulong request_ctx );
     296             : 
     297             : /* fd_bundle_client_reset frees all connection-related resources. */
     298             : 
     299             : void
     300             : fd_bundle_client_reset( fd_bundle_tile_t * ctx );
     301             : 
     302             : /* fd_bundle_client_send_ping enqueues an HTTP/2 PING and starts the
     303             :    keepalive deadline.  Does nothing if the connection is unavailable,
     304             :    busy, or lacks TX ring space. */
     305             : 
     306             : void
     307             : fd_bundle_client_send_ping( fd_bundle_tile_t * ctx );
     308             : 
     309             : FD_PROTOTYPES_END
     310             : 
     311             : #endif /* HEADER_fd_src_disco_bundle_fd_bundle_tile_private_h */