Line data Source code
1 : #include "fd_backtest_rocksdb.h"
2 : #include "fd_shredcap.h"
3 : #include "../../flamenco/gossip/fd_gossip_message.h"
4 : #include "../../ballet/shred/fd_shred.h"
5 : #include "../../util/fd_util.h"
6 : #include "../../util/net/fd_pcapng.h"
7 : #include "../../util/net/fd_ip4.h"
8 : #include "../../util/net/fd_udp.h"
9 : #include "fd_libc_zstd.h"
10 :
11 : #include <errno.h>
12 : #include <stdio.h>
13 : #include <stdlib.h>
14 : #include <fcntl.h>
15 :
16 : /* Hardcoded constants */
17 :
18 0 : #define IF_IDX_NET (0)
19 0 : #define IF_IDX_SHREDCAP (1)
20 0 : #define SHRED_PORT ((ushort)8003)
21 :
22 : static int
23 0 : usage( int rc ) {
24 0 : fputs(
25 0 : "\n"
26 0 : "Usage: fd_blockstore2shredcap --rocksdb <path> --out <path>\n"
27 0 : "\n"
28 0 : "Extract rooted blocks from Agave RocksDB.\n"
29 0 : "Produces shredcap 0.1 (pcapng) file containing shreds and bank hashes.\n"
30 0 : "\n"
31 0 : " --rocksdb <path> Agave RocksDB directory\n"
32 0 : " --out <path> File path to new shredcap file (fails if file already exists)\n"
33 0 : " --start-slot <n> Start slot (inclusive)\n"
34 0 : " --end-slot <n> End slot (inclusive)\n"
35 0 : # if FD_HAS_ZSTD
36 0 : " --zstd Output compressed .pcapng.zst stream instead of raw pcapng\n"
37 0 : " --zstd-level Zstandard compression level\n"
38 0 : # endif
39 0 : "\n",
40 0 : stderr
41 0 : );
42 0 : return rc;
43 0 : }
44 :
45 : static void
46 : write_bank_hash( FILE * pcap,
47 : ulong slot,
48 : ulong shred_cnt,
49 0 : uchar const bank_hash[32] ) {
50 0 : struct __attribute__((packed)) {
51 0 : uint type;
52 0 : fd_shredcap_bank_hash_v0_t bank_hash_rec;
53 0 : } packet;
54 0 : memset( &packet, 0, sizeof(packet) );
55 :
56 0 : packet.type = FD_SHREDCAP_TYPE_BANK_HASH_V0;
57 0 : fd_shredcap_bank_hash_v0_t * bank_hash_rec = &packet.bank_hash_rec;
58 0 : bank_hash_rec->slot = slot;
59 0 : bank_hash_rec->data_shred_cnt = shred_cnt;
60 0 : memcpy( bank_hash_rec->bank_hash, bank_hash, 32UL );
61 :
62 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, sizeof(packet), NULL, 0UL, IF_IDX_SHREDCAP, 0L );
63 0 : }
64 :
65 : static void
66 : write_shred( FILE * pcap,
67 0 : void const * shred ) {
68 0 : ulong shred_sz = fd_shred_sz( shred );
69 0 : FD_TEST( shred_sz<=FD_SHRED_MAX_SZ );
70 :
71 0 : struct __attribute__((packed)) {
72 0 : fd_ip4_hdr_t ip4;
73 0 : fd_udp_hdr_t udp;
74 0 : uchar shred[ FD_SHRED_MAX_SZ ];
75 0 : } packet;
76 :
77 0 : packet.ip4 = (fd_ip4_hdr_t) {
78 0 : .verihl = FD_IP4_VERIHL( 4, 5 ),
79 0 : .tos = 0,
80 0 : .net_tot_len = fd_ushort_bswap( (ushort)( 28+shred_sz ) ),
81 0 : .net_id = 0,
82 0 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
83 0 : .ttl = 64,
84 0 : .protocol = FD_IP4_HDR_PROTOCOL_UDP,
85 0 : .check = 0,
86 0 : .saddr = FD_IP4_ADDR( 127,0,0,1 ),
87 0 : .daddr = FD_IP4_ADDR( 127,0,0,1 ),
88 0 : };
89 0 : packet.ip4.check = fd_ip4_hdr_check_fast( &packet.ip4 );
90 0 : packet.udp = (fd_udp_hdr_t) {
91 0 : .net_sport = fd_ushort_bswap( 42424 ),
92 0 : .net_dport = fd_ushort_bswap( SHRED_PORT ),
93 0 : .net_len = fd_ushort_bswap( (ushort)( 8+shred_sz ) ),
94 0 : .check = 0,
95 0 : };
96 0 : fd_memcpy( packet.shred, shred, shred_sz );
97 :
98 0 : struct __attribute__((packed)) {
99 0 : ushort option_type;
100 0 : ushort option_sz;
101 0 : uint pen;
102 0 : ushort magic;
103 0 : ushort gossip_tag;
104 0 : } option = {
105 0 : .option_type = 2989, /* Custom Option containing binary octects, copyable */
106 0 : .option_sz = 8,
107 0 : .pen = 31592, /* Jump Trading, LLC */
108 0 : .magic = 0x4071, /* SOL! */
109 0 : .gossip_tag = FD_GOSSIP_CONTACT_INFO_SOCKET_TVU
110 0 : };
111 :
112 0 : fd_pcapng_fwrite_pkt1( pcap, &packet, 28UL+shred_sz, &option, sizeof(option), IF_IDX_NET, 0L );
113 0 : }
114 :
115 : int
116 : main( int argc,
117 0 : char ** argv ) {
118 0 : if( fd_env_strip_cmdline_contains( &argc, &argv, "--help" ) ) return usage( 0 );
119 :
120 0 : char const * rocksdb_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--rocksdb", NULL, NULL );
121 0 : char const * out_path = fd_env_strip_cmdline_cstr( &argc, &argv, "--out", NULL, NULL );
122 0 : char const * out_short = fd_env_strip_cmdline_cstr( &argc, &argv, "--o", NULL, NULL );
123 0 : if( !out_path ) out_path = out_short;
124 :
125 0 : int use_zstd = fd_env_strip_cmdline_contains( &argc, &argv, "--zstd" );
126 0 : int zstd_level = fd_env_strip_cmdline_int ( &argc, &argv, "--zstd-level", NULL, 3 );
127 0 : ulong zstd_bufsz = fd_env_strip_cmdline_ulong ( &argc, &argv, "--zstd-bufsz", NULL, 4UL<<20 ); /* 4MB default */
128 : # if !FD_HAS_ZSTD
129 : if( use_zstd ) FD_LOG_ERR(( "This build does not support ZSTD compression" ));
130 : (void)zstd_level;
131 : # endif
132 :
133 0 : ulong start_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--start-slot", NULL, 0UL );
134 0 : ulong end_slot = fd_env_strip_cmdline_ulong( &argc, &argv, "--end-slot", NULL, ULONG_MAX );
135 :
136 0 : if( FD_UNLIKELY( !rocksdb_path ) ) {
137 0 : fputs( "Error: --rocksdb not specified\n", stderr );
138 0 : return usage( 1 );
139 0 : }
140 0 : if( FD_UNLIKELY( !out_path ) ) {
141 0 : fputs( "Error: --out not specified\n", stderr );
142 0 : return usage( 1 );
143 0 : }
144 :
145 0 : fd_boot( &argc, &argv );
146 :
147 0 : void * rocks_mem = aligned_alloc( fd_backtest_rocksdb_align(), fd_backtest_rocksdb_footprint() );
148 0 : if( FD_UNLIKELY( !rocks_mem ) ) FD_LOG_ERR(( "out of memory" ));
149 0 : fd_backtest_rocksdb_t * rocksdb = fd_backtest_rocksdb_join( fd_backtest_rocksdb_new( rocks_mem, rocksdb_path ) );
150 0 : if( FD_UNLIKELY( !rocksdb ) ) FD_LOG_ERR(( "failed to open RocksDB at %s", rocksdb_path ));
151 0 : fd_backtest_rocksdb_init( rocksdb, start_slot );
152 :
153 0 : int out_fd = open( out_path, O_WRONLY|O_CREAT|O_EXCL, 0644 );
154 0 : if( FD_UNLIKELY( out_fd<0 ) ) FD_LOG_ERR(( "failed to create file %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
155 0 : FILE * out = fdopen( out_fd, "wb" );
156 0 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "fdopen failed on %s (%i-%s)", out_path, errno, fd_io_strerror( errno ) ));
157 :
158 0 : # if FD_HAS_ZSTD
159 0 : if( use_zstd ) {
160 0 : out = fd_zstd_wstream_open( out, zstd_level, zstd_bufsz );
161 0 : if( FD_UNLIKELY( !out ) ) FD_LOG_ERR(( "failed to initialize ZSTD compression" ));
162 0 : }
163 0 : # endif
164 :
165 : /* Write pcapng header */
166 0 : {
167 0 : fd_pcapng_shb_opts_t shb_opts;
168 0 : fd_pcapng_shb_defaults( &shb_opts );
169 0 : if( FD_UNLIKELY( !fd_pcapng_fwrite_shb( &shb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
170 0 : }
171 0 : uint idb_cnt = 0U;
172 0 : {
173 0 : fd_pcapng_idb_opts_t idb_opts = {
174 0 : .name = "lo",
175 0 : .ip4_addr = { 127,0,0,1 }
176 0 : };
177 0 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_IPV4, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
178 0 : FD_TEST( idb_cnt++==IF_IDX_NET );
179 0 : }
180 0 : {
181 0 : fd_pcapng_idb_opts_t idb_opts = {
182 0 : .name = "shredcap0",
183 0 : };
184 0 : if( FD_UNLIKELY( !fd_pcapng_fwrite_idb( FD_PCAPNG_LINKTYPE_USER0, &idb_opts, out ) ) ) FD_LOG_ERR(( "pcap write error" ));
185 0 : FD_TEST( idb_cnt++==IF_IDX_SHREDCAP );
186 0 : }
187 :
188 0 : ulong slot_cnt = 0UL;
189 0 : for( ;; slot_cnt++ ) {
190 0 : ulong root_slot;
191 0 : ulong shred_cnt;
192 0 : int root_ok = fd_backtest_rocksdb_next_root_slot( rocksdb, &root_slot, &shred_cnt );
193 0 : if( !root_ok ) break;
194 0 : uchar const * bank_hash = fd_backtest_rocksdb_bank_hash( rocksdb, root_slot );
195 0 : if( FD_UNLIKELY( !bank_hash ) ) FD_LOG_ERR(( "failed to extract bank hash for root slot %lu", root_slot ));
196 0 : if( root_slot>end_slot ) break;
197 :
198 0 : write_bank_hash( out, root_slot, shred_cnt, bank_hash );
199 :
200 0 : for( ulong i=0UL; i<shred_cnt; i++ ) {
201 0 : void const * shred = fd_backtest_rocksdb_shred( rocksdb, root_slot, i );
202 0 : if( FD_UNLIKELY( !shred ) ) {
203 0 : FD_LOG_WARNING(( "missing shred %lu for slot %lu", i, root_slot ));
204 0 : break;
205 0 : }
206 0 : write_shred( out, shred );
207 0 : }
208 :
209 0 : }
210 :
211 0 : long off = ftell( out );
212 0 : FD_LOG_NOTICE(( "%s: wrote %lu slots, %ld bytes", out_path, slot_cnt, off ));
213 :
214 : /* FIXME missing destructor for backtest_rocksdb */
215 0 : if( FD_UNLIKELY( 0!=fclose( out ) ) ) {
216 0 : FD_LOG_ERR(( "fclose failed on %s (%i-%s), output file may be corrupt", out_path, errno, fd_io_strerror( errno ) ));
217 0 : }
218 0 : free( rocks_mem );
219 :
220 0 : fd_halt();
221 0 : return 0;
222 0 : }
|
