Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_shred_fd_shred_batch_h
       2             : #define HEADER_fd_src_disco_shred_fd_shred_batch_h
       3             : 
       4             : #include "../../util/fd_util_base.h"
       5             : #include "../shred/fd_shredder.h"
       6             : 
       7             : /* The shred tile partitions a block into batches of microblocks, and
       8             :    it partitions each batch into fixed-size FEC sets, with 32 data
       9             :    shreds and 32 parity shreds each.  Each batch is made up of an
      10             :    integer number of microblocks, which means that no microblock can
      11             :    cross the batch boundary, and that the last FEC set may need to be
      12             :    padded with 0s.  Since padding completes the payload of the last
      13             :    FEC set in the batch, and does not impact the batch itself, no FEC
      14             :    set can contain only padding.
      15             :    The number of FEC sets per batch is limited only by block-level
      16             :    limits, so the larger the value, the less relative padding overhead
      17             :    in every batch.  However, the larger the value, the longer it takes
      18             :    before the tile can start generating and sending FEC sets, unless
      19             :    you use some strange speculative tricks, which we do not use.
      20             :    Assuming n batches in a given block, the (upper-limit) quantity of
      21             :    FEC sets is determined by FD_SHRED_BATCH_FEC_SETS_WMARK for the
      22             :    first n-1 batches, and by FD_SHRED_BATCH_FEC_SETS_MAX for the last
      23             :    batch.  This is needed because when the microblock completing the
      24             :    block is received, we include it in the current batch irrespective
      25             :    of how full the batch might be.  The diagrams below (not at scale)
      26             :    show 2 FEC sets before the watermark (W), which indicates where to
      27             :    close the current batch, as well as 2 extra FEC sets available for
      28             :    the last batch in the block.
      29             : 
      30             :      Normal: (to be deprecated)
      31             :        +------------+------------+~~~~~~~~~~~+~~~~~~~~~~~~+
      32             :        |   FEC_set  |   FEC_set (W)    extra |      extra |
      33             :        +------------+------------+~~~~~~~~~~~+~~~~~~~~~~~~+
      34             : 
      35             :      Chained: (first n-1 batches)
      36             :        +-----------+-----------+~~~~~~~~~~+~~~~~~~~~~~+
      37             :        |   FEC_set |  FEC_set (W)   extra |     extra |
      38             :        +-----------+-----------+~~~~~~~~~~+~~~~~~~~~~~+
      39             : 
      40             : 
      41             :    In the current version, all n batches are chained.  In order to size
      42             :    the region labeled extra, we need to consider the worst case
      43             :    scenario: we have buffered W-1 bytes using the chained watermark,
      44             :    when a max-size microblock arrives indicating it is the last in the
      45             :    block.  This situation can only happen when running in low-power
      46             :    mode, because otherwise the last microblock would be a tick, which is
      47             :    small, but it does not seem to be worth plumbing in that sort of
      48             :    logic just to save a little memory that would not pollute cache
      49             :    anyway.
      50             : 
      51             :    Note that this payload reduction does not affect normal FEC sets,
      52             :    which are scheduled to be deprecated at some point (TODO). */
      53           0 : #define FD_SHRED_BATCH_FEC_SETS_WMARK ( 2UL )
      54           0 : #define FD_SHRED_BATCH_FEC_SETS_EXTRA ( 2UL )
      55             : /* FD_SHRED_BATCH_FEC_SETS_MAX affects the value of STEM_BURST inside
      56             :    the shred tile (refer to its calculation for details). */
      57           0 : #define FD_SHRED_BATCH_FEC_SETS_MAX   ( FD_SHRED_BATCH_FEC_SETS_WMARK + FD_SHRED_BATCH_FEC_SETS_EXTRA )
      58             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_FEC_SETS_EXTRA * FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ   ) >= ( FD_POH_SHRED_MTU ), FD_SHRED_BATCH_FEC_SETS_EXTRA );
      59             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_FEC_SETS_EXTRA * FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ  ) >= ( FD_POH_SHRED_MTU ), FD_SHRED_BATCH_FEC_SETS_EXTRA );
      60             : 
      61             : /* FD_SHRED_BATCH_WMARK: Following along the lines of dcache, batch
      62             :    microblocks until either the slot ends or the batch would exceed
      63             :    the corresponding watermark.  Only when the received microblock
      64             :    completes the block is the batch allowed to go beyond the given
      65             :    watermark, since all remaining FEC sets need to be generated and
      66             :    forwarded at once.  The watermark is relative to the beginning
      67             :    of the payload in the pending batch buffer (i.e. excluding the
      68             :    8 bytes needed for microblock_cnt in the batch header). */
      69             : #define FD_SHRED_BATCH_WMARK_NORMAL   ( FD_SHRED_BATCH_FEC_SETS_WMARK * FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ   - 8UL )
      70           0 : #define FD_SHRED_BATCH_WMARK_CHAINED  ( FD_SHRED_BATCH_FEC_SETS_WMARK * FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ  - 8UL )
      71             : /* Validate watermark payload capacity.  It must support at least
      72             :    FD_POH_SHRED_MTU bytes.  Prefer > to >= as a safety margin, to
      73             :    avoid any dependency on how we determine later on if the batch
      74             :    would exceed the watermark. */
      75             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_WMARK_NORMAL   ) > ( FD_POH_SHRED_MTU ), FD_SHRED_BATCH_WMARK_NORMAL   );
      76             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_WMARK_CHAINED  ) > ( FD_POH_SHRED_MTU ), FD_SHRED_BATCH_WMARK_CHAINED  );
      77             : 
      78             : /* There are two different raw buffer sizes depending on the FEC
      79             :    set type: normal and chained (see fd_shredder.h for further details).
      80             :    In order to support all three, we allocate the largest of them (i.e.
      81             :    normal).  TODO once normal FEC sets have been deprecated, replace
      82             :    NORMAL with CHAINED payload size in the calculation of
      83             :    FD_SHRED_BATCH_RAW_BUF_SZ below. */
      84             : #define FD_SHRED_BATCH_RAW_BUF_SZ     ( FD_SHRED_BATCH_FEC_SETS_MAX * FD_SHREDDER_NORMAL_FEC_SET_PAYLOAD_SZ )
      85             : /* Validate batch raw buffer size. */
      86             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_RAW_BUF_SZ ) >= ( FD_SHRED_BATCH_WMARK_NORMAL   + FD_POH_SHRED_MTU ), FD_SHRED_BATCH_RAW_BUF_SZ );
      87             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_RAW_BUF_SZ ) >= ( FD_SHRED_BATCH_WMARK_CHAINED  + FD_POH_SHRED_MTU ), FD_SHRED_BATCH_RAW_BUF_SZ );
      88             : 
      89             : /* FD_SHRED_BATCH_BLOCK_DATA_SZ_MAX is a conservative compile-time
      90             :    bound on entry bytes in a FD_SHRED_BLK_MAX-shred block
      91             :    (FD_FEC_BLK_MAX chained FEC sets).  It subtracts worst-case padding
      92             :    of about half the payload, 8-byte batch headers, and two FEC sets for
      93             :    Alpenglow block components.  Pack does not use this as its per-slot
      94             :    cap; that is fd_shred_batch_pack_data_max at become-leader.  This
      95             :    macro is the pre-leader default (FD_PACK_MAX_DATA_PER_BLOCK) and
      96             :    sizes skipped-tick buffers. */
      97             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_FEC_SETS_MAX ) == ( 4UL ), FD_SHRED_BATCH_FEC_SETS_MAX );
      98           0 : #define FD_SHRED_BATCH_BLOCK_DATA_OHEAD  ( (FD_FEC_BLK_MAX/2UL) * FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ + 8192UL + 8192UL )
      99             : FD_STATIC_ASSERT( ( FD_SHRED_BATCH_BLOCK_DATA_OHEAD ) < ( FD_FEC_BLK_MAX * FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ ), FD_SHRED_BATCH_BLOCK_DATA_OHEAD );
     100           0 : #define FD_SHRED_BATCH_BLOCK_DATA_SZ_MAX ( (FD_FEC_BLK_MAX-2UL) * FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ - FD_SHRED_BATCH_BLOCK_DATA_OHEAD )
     101             : 
     102             : /* fd_shred_batch_pack_data_max: how many entry bytes (microblock
     103             :    headers + txn payloads, excluding empty ticks) pack can emit in a
     104             :    slot without producing more than slot_max_data_shreds.
     105             : 
     106             :    max_microblock_sz is the largest pack-produced entry: the 48-byte
     107             :    entry header plus (max txns per microblock)*FD_TPU_MTU.
     108             : 
     109             :    We shred by filling a batch until the next microblock would not
     110             :    fit in two chained FEC sets (WMARK_CHAINED = 2*C-8 payload bytes),
     111             :    then padding out that batch:
     112             :      - The first microblock of a slot is flushed immediately, using
     113             :        ceil( (8+max_microblock_sz)/C ) FEC sets.
     114             :      - Each later closed batch holds at least
     115             :        WMARK_CHAINED-max_microblock_sz entry bytes and uses 2 FEC sets.
     116             :      - The last batch is the leftover (at most wmark) plus the
     117             :        completing microblock:
     118             :        ceil( (wmark + 8 + max_microblock_sz) / C ).  Production
     119             :        finishes on an empty tick; low-power can finish on a
     120             :        max-size microblock.
     121             : 
     122             :    Tick entries are not included.  An Alpenglow block's only tick is
     123             :    its closing alpentick, which the last batch's FEC sets cover.
     124             :    Returns 0 if the shred budget cannot cover the reserved FEC sets. */
     125             : FD_FN_CONST static inline ulong
     126             : fd_shred_batch_pack_data_max( ulong slot_max_data_shreds,
     127           0 :                               ulong max_microblock_sz ) {
     128           0 :   ulong const C     = FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ;
     129           0 :   ulong const wmark = FD_SHRED_BATCH_WMARK_CHAINED; /* 2*C - 8 */
     130             : 
     131           0 :   if( FD_UNLIKELY( max_microblock_sz>=wmark ) ) return 0UL;
     132             : 
     133           0 :   ulong fec_set_cnt          = slot_max_data_shreds / 32UL;
     134           0 :   ulong first_batch_fec_cnt  = ( 8UL + max_microblock_sz + C - 1UL ) / C;
     135           0 :   ulong last_batch_fec_cnt   = ( wmark + 8UL + max_microblock_sz + C - 1UL ) / C;
     136           0 :   if( FD_UNLIKELY( fec_set_cnt<first_batch_fec_cnt+last_batch_fec_cnt+2UL ) ) return 0UL;
     137             : 
     138           0 :   ulong middle_batch_fec_cnt = ( fec_set_cnt - first_batch_fec_cnt - last_batch_fec_cnt - 2UL ) & ~1UL; /* even, 2 FEC per batch */
     139           0 :   ulong min_batch            = wmark - max_microblock_sz;
     140           0 :   return ( middle_batch_fec_cnt / 2UL ) * min_batch;
     141           0 : }
     142             : 
     143             : #endif /* HEADER_fd_src_disco_shred_fd_shred_batch_h */