Line data    Source code

       1             : #ifndef HEADER_fd_src_discof_backup_fd_txncache_writer_h
       2             : #define HEADER_fd_src_discof_backup_fd_txncache_writer_h
       3             : 
       4             : /* fd_txncache_writer serializes the rooted part of a txncache into the
       5             :    snapshots/status_cache file of a Solana snapshot (bincode
       6             :    Vec<SlotDelta>), shaped the way Agave writes it: one slot delta per
       7             :    recent rooted slot with a block, holding the transactions that
       8             :    executed in that slot, grouped by the blockhash they referenced.
       9             :    Takes the txncache read lock only during initial setup.  The full
      10             :    scale walks over the txncache are lock free and retry when they race
      11             :    a mutation. */
      12             : 
      13             : #include "../../flamenco/runtime/fd_txncache.h"
      14             : #include "../../flamenco/runtime/fd_txncache_shmem.h"
      15             : 
      16             : /* FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS is the number of slot deltas a
      17             :    status cache holds, mirroring Agave's StatusCache::max_root_entries
      18             :    (default MAX_ROOT_ENTRIES = MAX_RECENT_BLOCKHASHES = 300; the
      19             :    constant status_cache::MAX_CACHE_ENTRIES up to v3.x).  Snapshot load
      20             :    walks the SlotHistory sysvar back from the snapshot slot and requires
      21             :    a slot delta for each of the newest max_root_entries slots that has
      22             :    a block.  No shipped Agave changes it from 300; if it ever becomes
      23             :    configurable, this and FD_SLOT_DELTA_MAX_ENTRIES must follow.
      24             :    https://github.com/anza-xyz/agave/blob/v3.1.8/runtime/src/snapshot_bank_utils.rs#L609 */
      25             : 
      26           0 : #define FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS (300UL)
      27             : 
      28           0 : #define FD_TXNCACHE_WRITER_MAX_BLOCKHASHES (FD_TXNCACHE_MAX_SLOT_DELTAS+1UL)
      29             : 
      30             : #define FD_TXNCACHE_WRITER_MAX_GROUPS (FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS*FD_TXNCACHE_WRITER_MAX_BLOCKHASHES)
      31             : 
      32             : /* FD_TXNCACHE_WRITER_BUF_MIN is the smallest output buffer
      33             :    fd_txncache_writer_serialize accepts (fits any single record).
      34             :    Serialization emits whole records only and stops when the next one
      35             :    does not fit; the caller comes back with more room. */
      36             : 
      37           0 : #define FD_TXNCACHE_WRITER_BUF_MIN (4096UL)
      38             : 
      39             : /* How many chain entries a walk visits between re-reads of the txncache
      40             :    mutation generation.  The walks hold no lock.  A concurrent
      41             :    compaction or root advancement is detected through the generation
      42             :    number. */
      43             : 
      44             : #define FD_TXNCACHE_WRITER_CHECK_INTERVAL (1024UL)
      45             : 
      46             : struct fd_txncache_writer_blockhash_desc {
      47             :   ulong  blockcache_idx; /* blockcache pool index */
      48             :   uint   generation;     /* blockcache generation at init, detects reuse */
      49             :   ushort slot_i;         /* execution slot index, or USHORT_MAX for genesis */
      50             :   uchar  blockhash[ 32UL ];
      51             :   ulong  txnhash_offset;
      52             : };
      53             : 
      54             : typedef struct fd_txncache_writer_blockhash_desc fd_txncache_writer_blockhash_desc_t;
      55             : 
      56             : /* A group is one non-empty (execution slot, referenced blockhash) pair
      57             :    and serializes as one wire group in one slot delta.  The dense group
      58             :    key is a position in slots[] paired with a position in
      59             :    blockhash_descs[]. */
      60             : struct fd_txncache_writer_group {
      61             :   ushort slot_i;
      62             :   ushort blockhash_i;
      63             :   uint   entry_cnt;        /* transactions in the group */
      64             :   uint   arena_entry_off;
      65             :   uint   filled_entry_cnt; /* entries copied so far in the current fill */
      66             : };
      67             : 
      68             : typedef struct fd_txncache_writer_group fd_txncache_writer_group_t;
      69             : 
      70             : typedef uchar fd_txnhash_t[ 20UL ];
      71             : 
      72             : struct fd_txncache_writer {
      73             :   uint              state;
      74             :   fd_txncache_t *   tc;
      75             :   ulong             snapshot_slot;
      76             : 
      77             :   /* SlotDeltas (ascending). */
      78             :   ulong             slots[ FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS ];
      79             :   ulong             slot_cnt;
      80             :   ulong             group_cnt_by_slot_i[ FD_TXNCACHE_WRITER_MAX_SLOT_DELTAS ];
      81             : 
      82             :   /* Referenced blockhash descriptors, oldest rooted blockcache first. */
      83             :   fd_txncache_writer_blockhash_desc_t blockhash_descs[ FD_TXNCACHE_WRITER_MAX_BLOCKHASHES ];
      84             :   ulong                               blockhash_cnt;
      85             :   ushort                              fork_id_to_blockhash_i[ USHORT_MAX+1UL ];
      86             : 
      87             :   /* The txncache's root generation captured at init.  Must not move
      88             :      during snapshot production. */
      89             :   ulong             root_gen;
      90             : 
      91             :   /* Groups in wire order (slot ascending, then blockhash descriptor ascending). */
      92             :   uint                       entry_cnt_by_key[ FD_TXNCACHE_WRITER_MAX_GROUPS ];
      93             :   uint                       group_i_by_key[ FD_TXNCACHE_WRITER_MAX_GROUPS ];
      94             :   fd_txncache_writer_group_t groups[ FD_TXNCACHE_WRITER_MAX_GROUPS ];
      95             :   ulong                      group_cnt;
      96             :   ulong                      entry_cnt; /* Transactions across all groups. */
      97             : 
      98             :   /* Scratchpad for collecting transaction hashes in serialization
      99             :      order. */
     100             :   fd_txnhash_t * arena;
     101             :   ulong          arena_entry_cnt_max;
     102             :   ulong          batch_hi;
     103             : 
     104             :   /* Serialization cursor. */
     105             :   ulong             slot_i;  /* Next slot delta. */
     106             :   ulong             group_i; /* Next group. */
     107             :   ulong             entry_i; /* Next entry of the current group. */
     108             : };
     109             : 
     110             : typedef struct fd_txncache_writer fd_txncache_writer_t;
     111             : 
     112             : FD_PROTOTYPES_BEGIN
     113             : 
     114             : /* Returns the arena size the writer recommends for a txncache whose
     115             :    blocks hold at most max_txn_per_slot transactions.
     116             : 
     117             :    The writer stages transaction hashes in an arena the caller provides.
     118             :    The txncache stores transactions by referenced blockhash while the
     119             :    wire format wants them by execution slot, so the writer counts every
     120             :    group in one walk of the txncache, then fills the arena with as many
     121             :    consecutive groups as fit per further walk.  A single group that does
     122             :    not fit the arena is fatal.  Caller should use this function to size
     123             :    the arena.
     124             : 
     125             :    The arena must be aligned to fd_txncache_writer_arena_align(). */
     126             : 
     127             : FD_FN_CONST ulong
     128             : fd_txncache_writer_arena_sz( ulong max_txn_per_slot );
     129             : 
     130             : FD_FN_CONST ulong
     131             : fd_txncache_writer_arena_align( void );
     132             : 
     133             : /* fd_txncache_writer_init prepares writer to serialize the status
     134             :    cache of tc as of the rooted fork fork_id, which must be the most
     135             :    recent root.  snapshot_slot is the fork's slot.  slot_history points
     136             :    to the slot_history_sz byte bincode encoding of the SlotHistory
     137             :    sysvar in the bank's sysvar cache, which decides which slots get a
     138             :    slot delta and names each execution slot.  arena is the arena_sz byte
     139             :    staging area.
     140             : 
     141             :    Walks the whole txncache once to count the transactions of every
     142             :    (slot, blockhash) group, so the serialized size is known on return.
     143             : 
     144             :    Returns writer on success.  Returns NULL if fork_id is not the most
     145             :    recent root or if more than one rooted blockhash descriptor lacks a
     146             :    named execution slot.  Replay should not be advancing the root while
     147             :    a snapshot is in progress, so snapmk treats NULL as fatal. */
     148             : 
     149             : fd_txncache_writer_t *
     150             : fd_txncache_writer_init( fd_txncache_writer_t * writer,
     151             :                          fd_txncache_t *        tc,
     152             :                          fd_txncache_fork_id_t  fork_id,
     153             :                          ulong                  snapshot_slot,
     154             :                          uchar const *          slot_history,
     155             :                          ulong                  slot_history_sz,
     156             :                          void *                 arena,
     157             :                          ulong                  arena_sz );
     158             : 
     159             : /* fd_txncache_writer_serialized_sz returns the exact size in bytes of
     160             :    the status cache an initialized writer will produce. */
     161             : 
     162             : FD_FN_PURE ulong
     163             : fd_txncache_writer_serialized_sz( fd_txncache_writer_t const * writer );
     164             : 
     165             : /* fd_txncache_writer_serialize writes the next chunk of the status
     166             :    cache to out_buf (buf_sz>=FD_TXNCACHE_WRITER_BUF_MIN).  Returns the
     167             :    chunk size, or zero once the whole status cache was written.  May
     168             :    walk the txncache to fill the arena with the next batch of groups. */
     169             : 
     170             : ulong
     171             : fd_txncache_writer_serialize( fd_txncache_writer_t * writer,
     172             :                               uchar *                out_buf,
     173             :                               ulong                  buf_sz );
     174             : 
     175             : FD_PROTOTYPES_END
     176             : 
     177             : #endif /* HEADER_fd_src_discof_backup_fd_txncache_writer_h */