Line data    Source code

       1             : #ifndef HEADER_fd_src_discof_backup_fd_backup_h
       2             : #define HEADER_fd_src_discof_backup_fd_backup_h
       3             : 
       4             : /* fd_backup.h provides the internal ABI for snapshot production IPC. */
       5             : 
       6             : #include "../restore/utils/fd_ssarchive.h"
       7             : #include "../../util/archive/fd_tar.h"
       8             : 
       9             : /* tango frag type in fd_frag_meta_t::ctl::orig
      10             :    Used by links snapmk_zp, snaprd_mk. */
      11           0 : #define FD_BACKUP_ORIG_START          1  /* mk->zp: start compressing */
      12           0 : #define FD_BACKUP_ORIG_ACC_CACHE      2  /* mk->zp: pointers to cached account */
      13           0 : #define FD_BACKUP_ORIG_ACC_DISK       3  /* mk->zp: disk offset to cold account */
      14           0 : #define FD_BACKUP_ORIG_FLUSH          4  /* mk->zp: flush compress buffer */
      15           0 : #define FD_BACKUP_ORIG_DONE           5  /* mk->zp: stop compressing; mk->replay: free bank_idx */
      16           0 : #define FD_BACKUP_ORIG_ACC_DISK_BATCH 6  /* mk->zp: batch of cold accounts within one rd frag */
      17           0 : #define FD_BACKUP_ORIG_DISK_FRAG      7  /* rd->mk: accdb file frag */
      18           0 : #define FD_BACKUP_ORIG_ACC_DELTA      8  /* mk->zp: address of account */
      19             : 
      20             : /* FD_BACKUP_CACHE_PARA controls the batch size of ultra-sparse random
      21             :    index lookups from acc_map.  Tunes memory-level parallelism settings
      22             :    when doing DRAM gather. */
      23             : 
      24           0 : #define FD_BACKUP_CACHE_PARA 128
      25           0 : #define FD_BACKUP_DISK_PARA  128
      26             : 
      27             : /* FD_BACKUP_RD_MTU is the max frag size on a snaprd_out link. */
      28             : 
      29           0 : #define FD_BACKUP_RD_MTU 262144UL
      30             : 
      31             : /* FD_BACKUP_ZSTD_LEVEL is the Zstandard compression level of snapshots
      32             :    produced by Firedancer. */
      33             : 
      34           0 : #define FD_BACKUP_ZSTD_LEVEL 1
      35             : /* levels 4-15 are compiled out of libfd_zstd (third_party/zstd/Local.mk) */
      36             : FD_STATIC_ASSERT( FD_BACKUP_ZSTD_LEVEL<=3 || FD_BACKUP_ZSTD_LEVEL>=16, zstd_level );
      37             : 
      38             : /* FD_SNAP_MAX bounds the number of managed snapshot files
      39             :    (max_full_snapshots_to_keep+max_incremental_snapshots_to_keep, plus
      40             :    up to two scratch slots for a snapshot class that is downloaded but
      41             :    not retained). */
      42             : 
      43           0 : #define FD_SNAP_MAX (2U*(uint)FD_SSARCHIVE_MAX_ENTRIES+2U)
      44             : 
      45             : /* Well-known snapshot file descriptors
      46             : 
      47             :    Firedancer's strict sandbox bans opening/creating new files via
      48             :    open(2).  Instead, Firedancer opens a fixed number of existing
      49             :    snapshots and creates placeholder files.  Each file is eventually
      50             :    recycled by truncating and renaming it. */
      51             : 
      52           0 : #define FD_SNAP_FD_BASE (200000)
      53           0 : #define FD_SNAP_FD(     i ) (FD_SNAP_FD_BASE                     +(int)(i))
      54           0 : #define FD_SNAP_DIO_FD( i ) (FD_SNAP_FD_BASE+   (int)FD_SNAP_MAX +(int)(i))
      55           0 : #define FD_SNAP_RO_FD(  i ) (FD_SNAP_FD_BASE+(2*(int)FD_SNAP_MAX)+(int)(i))
      56             : 
      57             : /* fd_backup_inode_t annotates a snapshot file descriptor. */
      58             : 
      59             : struct fd_backup_inode {
      60             :   char  name[ FD_SNAP_NAME_MAX ];
      61             :   ulong full_slot; /* ULONG_MAX if placeholder */
      62             :   ulong incr_slot; /* ULONG_MAX if placeholder or full snapshot */
      63             : };
      64             : 
      65             : typedef struct fd_backup_inode fd_backup_inode_t;
      66             : 
      67             : /* fd_backup_start_msg_t (FD_BACKUP_ORIG_START) is the snapshot pipeline
      68             :    start signal. */
      69             : 
      70             : struct fd_backup_start_msg {
      71             :   ulong  slot;      /* slot number */
      72             :   ulong  slot_lo;   /* lowest appendvec slot this archive may use */
      73             :   uint   snap_idx;  /* identifies file descriptor */
      74             :   ushort fork_id;   /* accdb fork ID */
      75             : };
      76             : typedef struct fd_backup_start_msg fd_backup_start_msg_t;
      77             : 
      78             : /* fd_backup_cache_msg_t (FD_BACKUP_ORIG_ACC_CACHE) is a batch of cached
      79             :    account compression jobs. */
      80             : 
      81             : struct fd_backup_cache_msg {
      82             :   uint        acc_idx[ FD_BACKUP_CACHE_PARA ]; /* UINT_MAX is sentinel */
      83             :   fd_pubkey_t pubkey [ FD_BACKUP_CACHE_PARA ];
      84             : };
      85             : typedef struct fd_backup_cache_msg fd_backup_cache_msg_t;
      86             : 
      87             : /* fd_backup_disk_msg_t (FD_BACKUP_ORIG_ACC_DISK) is a compression job
      88             :    for an account that was read from disk (possibly fragmented). */
      89             : 
      90             : struct fd_backup_disk_msg {
      91             :   fd_pubkey_t pubkey;
      92             :   fd_pubkey_t owner;
      93             :   uint        size;
      94             :   uint        acc_idx;
      95             :   uint        snap_sz;
      96             :   uint        data_sz;
      97             : };
      98             : typedef struct fd_backup_disk_msg fd_backup_disk_msg_t;
      99             : 
     100             : /* fd_backup_disk_batch_msg_t (FD_BACKUP_ORIG_ACC_DISK_BATCH) is a batch
     101             :    compression jobs for multiple disk accounts that were read into a
     102             :    contiguous buffer (fast path). */
     103             : 
     104             : struct fd_backup_disk_batch_msg {
     105             :   uint        acc_idx [ FD_BACKUP_DISK_PARA ]; /* UINT_MAX is sentinel */
     106             :   uint        frag_off[ FD_BACKUP_DISK_PARA ];
     107             : };
     108             : typedef struct fd_backup_disk_batch_msg fd_backup_disk_batch_msg_t;
     109             : 
     110             : /* fd_backup_delta_msg_t is a batch of incremental snapshot accounts by
     111             :    address. */
     112             : 
     113             : struct fd_backup_delta_msg {
     114             :   uint        cnt;
     115             :   fd_pubkey_t pubkey[ FD_BACKUP_CACHE_PARA ];
     116             : };
     117             : 
     118             : typedef struct fd_backup_delta_msg fd_backup_delta_msg_t;
     119             : 
     120             : /* fd_backup_frag_t is only used to determine MTU of link */
     121             : 
     122             : union fd_backup_frag {
     123             :   fd_backup_start_msg_t      start;
     124             :   fd_backup_cache_msg_t      cache;
     125             :   fd_backup_disk_msg_t       disk;
     126             :   fd_backup_disk_batch_msg_t disk_batch;
     127             :   fd_backup_delta_msg_t      delta;
     128             : };
     129             : typedef union fd_backup_frag fd_backup_frag_t;
     130             : 
     131             : /* snap_acc_hdr_t is a snapshot-format account header. */
     132             : 
     133             : union __attribute__((packed)) snap_acc_hdr {
     134             :   struct __attribute__((packed)) {
     135             :     /* 0x00 */ ulong       slot;
     136             :     /* 0x08 */ ulong       data_len;
     137             :     /* 0x10 */ fd_pubkey_t pubkey;
     138             :     /* 0x30 */ ulong       lamports;
     139             :     /* 0x38 */ ulong       rent_epoch;
     140             :     /* 0x40 */ fd_pubkey_t owner;
     141             :     /* 0x60 */ uchar       executable;
     142             :     /* 0x61 */ uchar       padding[7];
     143             :     /* 0x68 */ fd_hash_t   hash;
     144             :     /* 0x88 */
     145             :   };
     146             :   uchar raw[ 0x88 ];
     147             : };
     148             : 
     149             : typedef union snap_acc_hdr snap_acc_hdr_t;
     150             : 
     151             : FD_PROTOTYPES_BEGIN
     152             : 
     153             : /* Utils */
     154             : 
     155             : FD_FN_UNUSED static fd_tar_meta_t *
     156             : fd_backup_tar_file_hdr( fd_tar_meta_t * tar_meta,
     157           0 :                         ulong           sz ) {
     158           0 :   *tar_meta = (fd_tar_meta_t){
     159           0 :     .magic    = { 'u','s','t','a','r',' ' },
     160           0 :     .version  = { ' ','\0' },
     161           0 :     .mode     = "644",
     162           0 :     .uid      = "0",
     163           0 :     .gid      = "0",
     164           0 :     .typeflag = FD_TAR_TYPE_REGULAR,
     165           0 :     .chksum   = { ' ',' ',' ',' ',' ',' ',' ',' ' }
     166           0 :   };
     167           0 :   (void)fd_tar_meta_set_size( tar_meta, sz );
     168           0 :   return tar_meta;
     169           0 : }
     170             : 
     171             : FD_FN_UNUSED static fd_tar_meta_t *
     172           0 : fd_backup_tar_dir_hdr( fd_tar_meta_t * tar_meta ) {
     173           0 :   *tar_meta = (fd_tar_meta_t){
     174           0 :     .magic    = { 'u','s','t','a','r',' ' },
     175           0 :     .version  = { ' ','\0' },
     176           0 :     .mode     = "755",
     177           0 :     .uid      = "0",
     178           0 :     .gid      = "0",
     179           0 :     .typeflag = FD_TAR_TYPE_DIR,
     180           0 :     .chksum   = { ' ',' ',' ',' ',' ',' ',' ',' ' }
     181           0 :   };
     182           0 :   (void)fd_tar_meta_set_size( tar_meta, 0UL );
     183           0 :   return tar_meta;
     184           0 : }
     185             : 
     186             : FD_PROTOTYPES_END
     187             : 
     188             : #endif /* HEADER_fd_src_discof_backup_fd_backup_h */