Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_shred_fd_shredder_h
       2             : #define HEADER_fd_src_disco_shred_fd_shredder_h
       3             : 
       4             : #include "../keyguard/fd_keyguard_client.h"
       5             : #include "../../ballet/sha256/fd_sha256.h"
       6             : #include "../../disco/pack/fd_microblock.h"
       7             : #include "../../ballet/wsample/fd_wsample.h"
       8             : #include "../../ballet/ed25519/fd_ed25519.h"
       9             : #include "../../ballet/reedsol/fd_reedsol.h"
      10             : #include "../../ballet/bmtree/fd_bmtree.h"
      11             : #include "fd_fec_set.h"
      12             : #include "../../ballet/shred/fd_shred.h"
      13             : 
      14             : #define FD_SHREDDER_MAX_STAKE_WEIGHTS (1UL<<20)
      15             : 
      16             : 
      17             : #define FD_FEC_SET_MAX_BMTREE_DEPTH (7UL) /* 1+ceil(log2(DATA_SHREDS_MAX + PARITY_SHREDS_MAX)) */
      18             : 
      19           0 : #define FD_SHREDDER_ALIGN     (  128UL)
      20             : /* FD_SHREDDER_FOOTPRINT is not provided because it depends on the footprint
      21             :    of fd_sha256_batch_t, which is not invariant (the latter depends on the
      22             :    underlying implementation). Instead, a static inline function is provided. */
      23             : 
      24           0 : #define FD_SHREDDER_MAGIC (0xF17EDA2547EDDE70UL) /* FIREDAN SHREDDER V0 */
      25             : 
      26             : typedef void (fd_shredder_sign_fn)( void * ctx, uchar * sig, uchar const * merkle_root );
      27             : 
      28           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT      (3UL)
      29             : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_SZ       (8UL)
      30           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_DISABLED (ULONG_MAX)
      31             : 
      32             : union fd_shred_features_activation_private {
      33             :    /* slots for features of interest - update cnt as needed in the future. */
      34             :    ulong slots[ FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT ];
      35             :    struct {
      36             :       /* 0 */ ulong enforce_fixed_fec_set;
      37             :       /* 1 */ ulong switch_to_chacha8_turbine;
      38             :       /* 2 */ ulong discard_unexpected_data_complete_shreds;
      39             :    };
      40             : };
      41             : typedef union fd_shred_features_activation_private fd_shred_features_activation_t;
      42             : 
      43             : 
      44             : struct __attribute__((aligned(FD_SHREDDER_ALIGN))) fd_shredder_private {
      45             :   ulong  magic;
      46             :   ushort shred_version;
      47             : 
      48             :   fd_sha256_batch_t sha256 [ 1 ];
      49             :   fd_reedsol_t      reedsol[ 1 ];
      50             :   union __attribute__((aligned(FD_BMTREE_COMMIT_ALIGN))) {
      51             :     fd_bmtree_commit_t bmtree;
      52             :     uchar _bmtree_footprint[ FD_BMTREE_COMMIT_FOOTPRINT( FD_FEC_SET_MAX_BMTREE_DEPTH ) ];
      53             :   };
      54             :   fd_bmtree_node_t bmtree_leaves[ FD_REEDSOL_DATA_SHREDS_MAX + FD_REEDSOL_PARITY_SHREDS_MAX ];
      55             : 
      56             :   void const * entry_batch;
      57             :   ulong        sz;
      58             :   ulong        offset;
      59             : 
      60             :   void *                signer_ctx;
      61             :   fd_shredder_sign_fn * signer;
      62             : 
      63             :   fd_entry_batch_meta_t meta;
      64             :   ulong slot;
      65             :   ulong data_idx_offset;
      66             :   ulong parity_idx_offset;
      67             : };
      68             : 
      69             : typedef struct fd_shredder_private fd_shredder_t;
      70             : 
      71           0 : FD_FN_CONST static inline ulong fd_shredder_align    ( void ) { return FD_SHREDDER_ALIGN;     }
      72           0 : FD_FN_CONST static inline ulong fd_shredder_footprint( void ) { return sizeof(fd_shredder_t); }
      73             : 
      74             : /* fd_shredder_new formats a region of memory as a shredder object.
      75             :    pubkey must point to the first byte of 32 bytes containing the public
      76             :    key of the validator that will sign the shreds this shredder
      77             :    produces.  The value provided for shred_version will be stored in the
      78             :    shred_version field of each shred that this shredder produces. */
      79             : void          * fd_shredder_new(  void * mem, fd_shredder_sign_fn * signer, void * signer_ctx );
      80             : fd_shredder_t * fd_shredder_join( void * mem );
      81             : void *          fd_shredder_leave(  fd_shredder_t * shredder );
      82             : void *          fd_shredder_delete( void *          mem      );
      83             : 
      84           0 : static inline void fd_shredder_set_shred_version( fd_shredder_t * shredder, ushort shred_version ) { shredder->shred_version = shred_version; }
      85             : 
      86             : 
      87             : /* fd_shredder_count_{data_shreds, parity_shreds, fec_sets}: returns the
      88             :    number of data shreds, parity shreds, or FEC sets (respectively)
      89             :    required to send an entry batch of size `sz_bytes` bytes.  It uses
      90             :    chained unsigned Merkle shreds except for that when block_complete is
      91             :    non-zero, the last FEC set uses chained resigned Merkle shreds.
      92             :    DATA_CHAINED, DATA_CHAINED_RESIGNED}.  For data and parity shred
      93             :    counts, this is the total count across all FEC sets.
      94             : 
      95             :    We only produce FEC sets with 32 data and 32 parity shreds, so this
      96             :    form of counting is much simpler than before.  The only strangeness
      97             :    is with the last entry batch because resigned shreds hold less
      98             :    payload than chained shreds. Thus, we might be in a situation where
      99             :    an entry batch would fit in one chained FEC set but requires two
     100             :    resigned FEC sets.  In this case, Agave produces a chained FEC set
     101             :    with extra padding at the end followed by a full resigned FEC set.
     102             :    We'll follow the same approach.
     103             : 
     104             :    Let C=FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ and
     105             :    R=FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ.  Then when
     106             : 
     107             :    sz_bytes <= R:     a signle resigned FEC set, possibly with padding
     108             : 
     109             :    sz_bytes >  R:     ceiling( (sz_bytes-R)/C ) chained FEC sets, with
     110             :                       the last one possibly having padding, followed by
     111             :                       one full resigned FEC set
     112             : 
     113             :    The nice part is that the normal C way of computing ceiling division,
     114             :    floor( (sz_bytes-R+C-1)/C ), gives 0 when sz_bytes<=R, which means we
     115             :    can combine these two cases. */
     116             : 
     117             : #define FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ   (31840UL)
     118           0 : #define FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ  (30816UL) /* -32 bytes * 32 shreds */
     119           0 : #define FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ (28768UL) /* -64 bytes * 32 shreds */
     120             : 
     121             : #define FD_SHREDDER_NORMAL_FEC_SET_RAW_BUF_SZ   (63679UL) /* 2 * ...PAYLOAD_SZ - 1 */
     122             : #define FD_SHREDDER_CHAINED_FEC_SET_RAW_BUF_SZ  (61631UL) /* 2 * ...PAYLOAD_SZ - 1 */
     123             : #define FD_SHREDDER_RESIGNED_FEC_SET_RAW_BUF_SZ (57535UL) /* 2 * ...PAYLOAD_SZ - 1 */
     124             : 
     125             : FD_FN_CONST static inline ulong
     126           0 : fd_shredder_count_fec_sets(      ulong sz_bytes, int block_complete ) {
     127           0 :   return fd_ulong_if( block_complete,
     128           0 :       1UL + (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ - 1UL)/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ,
     129           0 :       (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - 1UL )/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ );
     130           0 : }
     131             : FD_FN_CONST static inline ulong
     132           0 : fd_shredder_count_data_shreds(   ulong sz_bytes, int block_complete ) {
     133           0 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     134           0 : }
     135             : FD_FN_CONST static inline ulong
     136           0 : fd_shredder_count_parity_shreds( ulong sz_bytes, int block_complete ) {
     137           0 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     138           0 : }
     139             : 
     140             : /* fd_shredder_init_batch begins the computation of shreds for an entry
     141             :    batch.  shredder must be a valid local join.  entry_batch points to
     142             :    the first byte of a region of memory entry_batch_sz bytes long.
     143             :    entry_batch_sz must be strictly positive.  The shredder object
     144             :    retains a read interest in the region of memory [entry_batch,
     145             :    entry_batch+entry_batch_sz) that lasts until fd_shredder_fini_batch
     146             :    is called.  This region of memory should not be modified while in use
     147             :    by the shredder.  meta contains the metadata for the batch that is
     148             :    necessary for shred production.  The shredder object does not retain
     149             :    a read interest in the memory pointed to by meta.
     150             : 
     151             :    Returns shredder, which will be in a new batch when the function
     152             :    returns. */
     153             : fd_shredder_t * fd_shredder_init_batch( fd_shredder_t               * shredder,
     154             :                                         void const                  * entry_batch,
     155             :                                         ulong                         entry_batch_sz,
     156             :                                         ulong                         slot,
     157             :                                         fd_entry_batch_meta_t const * meta );
     158             : 
     159             : /* fd_shredder_skip_batch updates the shredder state as necessary
     160             :    to skip processing this current batch.  shredder must be a valid
     161             :    local join.  entry_batch_sz must be strictly positive.
     162             : 
     163             :    Returns shredder, which will have data and parity shred indices
     164             :    updated as if the caller had called fd_shredder_init_batch with
     165             :    a batch of the specified size and meta.block_complete set to
     166             :    block_complete, followed by fd_shredder_next_fec_set exactly
     167             :    fd_shredder_count_fec_sets( entry_batch_sz ) times. */
     168             : fd_shredder_t * fd_shredder_skip_batch( fd_shredder_t * shredder,
     169             :                                         ulong           entry_batch_sz,
     170             :                                         ulong           slot,
     171             :                                         int             block_complete );
     172             : 
     173             : /* fd_shredder_next_fec_set extracts the next FEC set from the in
     174             :    progress batch.  Computes the entirety of both data and parity
     175             :    shreds, including the parity information, Merkle proofs, and
     176             :    signatures.  Stores the generated FEC set in result, which is
     177             :    clobbered.  Populates all fields of result except for
     178             :    {data,parity}_shred_present (which is only used for reconstruction).
     179             : 
     180             :    shredder must be a valid local join.  chained_merkle_root is a
     181             :    pointer to a 32-byte buffer containing the chained merkle root (the
     182             :    merkle root of the previous FEC set).  Upon return,
     183             :    chained_merkle_root is updated with the new root.
     184             : 
     185             :    Returns result on success and NULL if all of the entry batch's data
     186             :    has been consumed already by previous calls to this function.  On
     187             :    success, advances the position of the shredder within the batch
     188             :    without finishing the batch. */
     189             : fd_fec_set_t *
     190             : fd_shredder_next_fec_set( fd_shredder_t * shredder,
     191             :                           fd_fec_set_t *  result,
     192             :                           uchar *         chained_merkle_root );
     193             : 
     194             : /* fd_shredder_fini_batch finishes the in process batch.  shredder must
     195             :    be a valid local join that is currently in a batch.  Upon return,
     196             :    shredder will no longer be in a batch and will be ready to begin a
     197             :    new batch with init_batch.  Returns shredder. */
     198             : fd_shredder_t * fd_shredder_fini_batch( fd_shredder_t * shredder );
     199             : 
     200             : #endif /* HEADER_fd_src_disco_shred_fd_shredder_h */