Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_shred_fd_shredder_h
       2             : #define HEADER_fd_src_disco_shred_fd_shredder_h
       3             : 
       4             : #include "../../ballet/sha256/fd_sha256.h"
       5             : #include "../../disco/pack/fd_microblock.h"
       6             : #include "../../ballet/reedsol/fd_reedsol.h"
       7             : #include "../../ballet/bmtree/fd_bmtree.h"
       8             : #include "fd_fec_set.h"
       9             : 
      10             : #define FD_FEC_SET_MAX_BMTREE_DEPTH (7UL) /* 1+ceil(log2(DATA_SHREDS_MAX + PARITY_SHREDS_MAX)) */
      11             : 
      12           0 : #define FD_SHREDDER_ALIGN     (  128UL)
      13             : /* FD_SHREDDER_FOOTPRINT is not provided because it depends on the footprint
      14             :    of fd_sha256_batch_t, which is not invariant (the latter depends on the
      15             :    underlying implementation). Instead, a static inline function is provided. */
      16             : 
      17           0 : #define FD_SHREDDER_MAGIC (0xF17EDA2547EDDE70UL) /* FIREDAN SHREDDER V0 */
      18             : 
      19           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT      (3UL)
      20           0 : #define FD_SHRED_FEATURES_ACTIVATION_SLOT_DISABLED (ULONG_MAX)
      21             : 
      22             : union fd_shred_features_activation_private {
      23             :    /* slots for features of interest - update cnt as needed in the future. */
      24             :    ulong slots[ FD_SHRED_FEATURES_ACTIVATION_SLOT_CNT ];
      25             :    struct {
      26             :       /* 0 */ ulong enforce_fixed_fec_set;
      27             :       /* 1 */ ulong switch_to_chacha8_turbine;
      28             :       /* 2 */ ulong discard_unexpected_data_complete_shreds;
      29             :    };
      30             : };
      31             : typedef union fd_shred_features_activation_private fd_shred_features_activation_t;
      32             : 
      33             : 
      34             : struct __attribute__((aligned(FD_SHREDDER_ALIGN))) fd_shredder_private {
      35             :   ulong  magic;
      36             :   ushort shred_version;
      37             : 
      38             :   fd_sha256_batch_t sha256 [ 1 ];
      39             :   fd_reedsol_t      reedsol[ 1 ];
      40             :   union __attribute__((aligned(FD_BMTREE_COMMIT_ALIGN))) {
      41             :     fd_bmtree_commit_t bmtree;
      42             :     uchar _bmtree_footprint[ FD_BMTREE_COMMIT_FOOTPRINT( FD_FEC_SET_MAX_BMTREE_DEPTH ) ];
      43             :   };
      44             :   fd_bmtree_node_t bmtree_leaves[ FD_REEDSOL_DATA_SHREDS_MAX + FD_REEDSOL_PARITY_SHREDS_MAX ];
      45             : 
      46             :   void const * entry_batch;
      47             :   ulong        sz;
      48             :   ulong        offset;
      49             : 
      50             :   fd_entry_batch_meta_t meta;
      51             :   ulong slot;
      52             :   ulong data_idx_offset;
      53             :   ulong parity_idx_offset;
      54             : };
      55             : 
      56             : typedef struct fd_shredder_private fd_shredder_t;
      57             : 
      58           0 : FD_FN_CONST static inline ulong fd_shredder_align    ( void ) { return FD_SHREDDER_ALIGN;     }
      59           0 : FD_FN_CONST static inline ulong fd_shredder_footprint( void ) { return sizeof(fd_shredder_t); }
      60             : 
      61             : /* fd_shredder_new formats a region of memory as a shredder object.
      62             :    The shredder does not sign: the caller writes the signature field
      63             :    of every shred it produces. */
      64             : void          * fd_shredder_new(  void * mem );
      65             : fd_shredder_t * fd_shredder_join( void * mem );
      66             : void *          fd_shredder_leave(  fd_shredder_t * shredder );
      67             : void *          fd_shredder_delete( void *          mem      );
      68             : 
      69           0 : static inline void fd_shredder_set_shred_version( fd_shredder_t * shredder, ushort shred_version ) { shredder->shred_version = shred_version; }
      70             : 
      71             : 
      72             : /* fd_shredder_count_{data_shreds, parity_shreds, fec_sets}: returns the
      73             :    number of data shreds, parity shreds, or FEC sets (respectively)
      74             :    required to send an entry batch of size `sz_bytes` bytes.  It uses
      75             :    chained unsigned Merkle shreds except for that when block_complete is
      76             :    non-zero, the last FEC set uses chained resigned Merkle shreds.
      77             :    DATA_CHAINED, DATA_CHAINED_RESIGNED}.  For data and parity shred
      78             :    counts, this is the total count across all FEC sets.
      79             : 
      80             :    We only produce FEC sets with 32 data and 32 parity shreds, so this
      81             :    form of counting is much simpler than before.  The only strangeness
      82             :    is with the last entry batch because resigned shreds hold less
      83             :    payload than chained shreds. Thus, we might be in a situation where
      84             :    an entry batch would fit in one chained FEC set but requires two
      85             :    resigned FEC sets.  In this case, Agave produces a chained FEC set
      86             :    with extra padding at the end followed by a full resigned FEC set.
      87             :    We'll follow the same approach.
      88             : 
      89             :    Let C=FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ and
      90             :    R=FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ.  Then when
      91             : 
      92             :    sz_bytes <= R:     a single resigned FEC set, possibly with padding
      93             : 
      94             :    sz_bytes >  R:     ceiling( (sz_bytes-R)/C ) chained FEC sets, with
      95             :                       the last one possibly having padding, followed by
      96             :                       one full resigned FEC set
      97             : 
      98             :    The nice part is that the normal C way of computing ceiling division,
      99             :    floor( (sz_bytes-R+C-1)/C ), gives 0 when sz_bytes<=R, which means we
     100             :    can combine these two cases. */
     101             : 
     102             : #define FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ   (31840UL)
     103           0 : #define FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ  (30816UL) /* -32 bytes * 32 shreds */
     104           0 : #define FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ (28768UL) /* -64 bytes * 32 shreds */
     105             : 
     106             : FD_FN_CONST static inline ulong
     107           0 : fd_shredder_count_fec_sets(      ulong sz_bytes, int block_complete ) {
     108           0 :   return fd_ulong_if( block_complete,
     109           0 :       1UL + (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - FD_SHREDDER_RESIGNED_FEC_SET_PAYLOAD_SZ - 1UL)/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ,
     110           0 :       (sz_bytes + FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - 1UL )/FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ );
     111           0 : }
     112             : FD_FN_CONST static inline ulong
     113           0 : fd_shredder_count_data_shreds(   ulong sz_bytes, int block_complete ) {
     114           0 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     115           0 : }
     116             : FD_FN_CONST static inline ulong
     117           0 : fd_shredder_count_parity_shreds( ulong sz_bytes, int block_complete ) {
     118           0 :   return 32UL*fd_shredder_count_fec_sets( sz_bytes, block_complete );
     119           0 : }
     120             : 
     121             : /* fd_shredder_init_batch begins the computation of shreds for an entry
     122             :    batch.  shredder must be a valid local join.  entry_batch points to
     123             :    the first byte of a region of memory entry_batch_sz bytes long.
     124             :    entry_batch_sz must be strictly positive.  The shredder object
     125             :    retains a read interest in the region of memory [entry_batch,
     126             :    entry_batch+entry_batch_sz) that lasts until fd_shredder_fini_batch
     127             :    is called.  This region of memory should not be modified while in use
     128             :    by the shredder.  meta contains the metadata for the batch that is
     129             :    necessary for shred production.  The shredder object does not retain
     130             :    a read interest in the memory pointed to by meta.
     131             : 
     132             :    Returns shredder, which will be in a new batch when the function
     133             :    returns. */
     134             : fd_shredder_t * fd_shredder_init_batch( fd_shredder_t               * shredder,
     135             :                                         void const                  * entry_batch,
     136             :                                         ulong                         entry_batch_sz,
     137             :                                         ulong                         slot,
     138             :                                         fd_entry_batch_meta_t const * meta );
     139             : 
     140             : /* fd_shredder_skip_batch updates the shredder state as necessary
     141             :    to skip processing this current batch.  shredder must be a valid
     142             :    local join.  entry_batch_sz must be strictly positive.
     143             : 
     144             :    Returns shredder, which will have data and parity shred indices
     145             :    updated as if the caller had called fd_shredder_init_batch with
     146             :    a batch of the specified size and meta.block_complete set to
     147             :    block_complete, followed by fd_shredder_next_fec_set exactly
     148             :    fd_shredder_count_fec_sets( entry_batch_sz ) times. */
     149             : fd_shredder_t * fd_shredder_skip_batch( fd_shredder_t * shredder,
     150             :                                         ulong           entry_batch_sz,
     151             :                                         ulong           slot,
     152             :                                         int             block_complete );
     153             : 
     154             : /* fd_shredder_next_fec_set extracts the next FEC set from the in
     155             :    progress batch.  Computes the entirety of both data and parity
     156             :    shreds, including the parity information and Merkle proofs; the
     157             :    signature field of each shred is clobbered (its tail holds the
     158             :    Merkle leaf prefix) and must be overwritten by the caller.  Stores
     159             :    the generated FEC set in result, which is clobbered.  Populates all
     160             :    fields of result except for {data,parity}_shred_present (which is
     161             :    only used for reconstruction).
     162             : 
     163             :    shredder must be a valid local join.  chained_merkle_root is a
     164             :    pointer to a 32-byte buffer containing the chained merkle root (the
     165             :    merkle root of the previous FEC set).  Upon return,
     166             :    chained_merkle_root is updated with the new root.
     167             : 
     168             :    Returns result on success and NULL if all of the entry batch's data
     169             :    has been consumed already by previous calls to this function.  On
     170             :    success, advances the position of the shredder within the batch
     171             :    without finishing the batch. */
     172             : fd_fec_set_t *
     173             : fd_shredder_next_fec_set( fd_shredder_t * shredder,
     174             :                           fd_fec_set_t *  result,
     175             :                           uchar *         chained_merkle_root );
     176             : 
     177             : /* fd_shredder_fini_batch finishes the in process batch.  shredder must
     178             :    be a valid local join that is currently in a batch.  Upon return,
     179             :    shredder will no longer be in a batch and will be ready to begin a
     180             :    new batch with init_batch.  Returns shredder. */
     181             : fd_shredder_t * fd_shredder_fini_batch( fd_shredder_t * shredder );
     182             : 
     183             : #endif /* HEADER_fd_src_disco_shred_fd_shredder_h */