Line data Source code
1 : #include "fd_ssparse.h"
2 :
3 : #include "../../../util/log/fd_log.h"
4 : #include "../../../util/archive/fd_tar.h"
5 : #include "../../../flamenco/runtime/fd_runtime_const.h"
6 : #include "../../../flamenco/runtime/fd_system_ids.h"
7 :
8 : #include <stdlib.h>
9 : #include <errno.h>
10 :
11 0 : #define FD_SSPARSE_STATE_TAR_HEADER (0)
12 0 : #define FD_SSPARSE_STATE_SCROLL_TAR_HEADER (1)
13 0 : #define FD_SSPARSE_STATE_VERSION (2)
14 0 : #define FD_SSPARSE_STATE_MANIFEST (3)
15 0 : #define FD_SSPARSE_STATE_ACCOUNT_HEADER (4)
16 0 : #define FD_SSPARSE_STATE_ACCOUNT_DATA (5)
17 0 : #define FD_SSPARSE_STATE_ACCOUNT_PADDING (6)
18 0 : #define FD_SSPARSE_STATE_STATUS_CACHE (7)
19 0 : #define FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE (8)
20 : #define FD_SSPARSE_STATE_ACCOUNT_BATCH (9)
21 :
22 : fd_ssparse_t *
23 0 : fd_ssparse_init( fd_ssparse_t * ssparse ) {
24 0 : memset( ssparse, 0, sizeof(fd_ssparse_t) );
25 0 : ssparse->state = FD_SSPARSE_STATE_TAR_HEADER;
26 0 : return ssparse;
27 0 : }
28 :
29 : static int
30 : parse_tar_header_name( char const * name,
31 : ulong * id,
32 0 : ulong * slot ) {
33 0 : char name_buf[ FD_TAR_NAME_SZ ];
34 0 : fd_memcpy( name_buf, name, FD_TAR_NAME_SZ );
35 0 : name_buf[ FD_TAR_NAME_SZ-1 ] = '\0';
36 :
37 0 : char const * ptr = name_buf;
38 :
39 0 : if( FD_UNLIKELY( strncmp( ptr, "accounts/", 9UL ) ) ) {
40 0 : *id = ULONG_MAX;
41 0 : *slot = ULONG_MAX;
42 0 : return -1;
43 0 : }
44 :
45 0 : ptr += 9UL;
46 0 : char const * next = strchr( ptr, '.' );
47 0 : if( FD_UNLIKELY( !next ) ) {
48 0 : *id = ULONG_MAX;
49 0 : *slot = ULONG_MAX;
50 0 : return -1;
51 0 : }
52 0 : errno = 0;
53 0 : char * endptr;
54 0 : *slot = strtoul( ptr, &endptr, 10 );
55 0 : if( FD_UNLIKELY( errno==ERANGE || *endptr!='.' || endptr==ptr ) ) {
56 0 : *id = ULONG_MAX;
57 0 : *slot = ULONG_MAX;
58 0 : return -1;
59 0 : }
60 :
61 0 : errno = 0;
62 0 : ptr = next + 1;
63 0 : *id = strtoul( ptr, &endptr, 10 );
64 0 : if( FD_UNLIKELY( errno==ERANGE || *endptr!='\0' || endptr==ptr ) ) {
65 0 : *id = ULONG_MAX;
66 0 : *slot = ULONG_MAX;
67 0 : return -1;
68 0 : }
69 :
70 0 : return 0;
71 0 : }
72 :
73 : static int
74 : advance_tar( fd_ssparse_t * ssparse,
75 : uchar const * data,
76 : ulong data_sz,
77 0 : fd_ssparse_advance_result_t * result ) {
78 0 : ulong consume = fd_ulong_min( data_sz, 512UL - ssparse->tar.header_bytes_consumed );
79 0 : if( FD_UNLIKELY( !consume ) ) {
80 0 : FD_LOG_WARNING(( "unexpected end of data in tar header, data_sz=%lu, header_bytes_consumed=%lu", data_sz, ssparse->tar.header_bytes_consumed ));
81 0 : return FD_SSPARSE_ADVANCE_ERROR;
82 0 : }
83 :
84 0 : fd_memcpy( ssparse->tar.header+ssparse->tar.header_bytes_consumed, data, consume );
85 0 : ssparse->bytes_consumed += consume;
86 0 : result->bytes_consumed = consume;
87 0 : ssparse->tar.header_bytes_consumed += consume;
88 :
89 0 : if( FD_UNLIKELY( ssparse->tar.header_bytes_consumed<512UL ) ) return FD_SSPARSE_ADVANCE_AGAIN;
90 :
91 0 : fd_tar_meta_t const * hdr = (fd_tar_meta_t const *)ssparse->tar.header;
92 0 : ssparse->tar.header_bytes_consumed = 0UL;
93 :
94 : /* "ustar\x00" and "ustar \x00" (overlaps with version) are both
95 : valid values for magic. These are POSIX ustar and OLDGNU versions
96 : respectively. */
97 0 : if( FD_UNLIKELY( memcmp( hdr->magic, FD_TAR_MAGIC, FD_TAR_MAGIC_SZ ) ) ) {
98 0 : int not_zero = 0;
99 0 : for( ulong i=0UL; i<512UL; i++ ) not_zero |= ssparse->tar.header[ i ];
100 0 : if( FD_UNLIKELY( not_zero ) ) {
101 0 : FD_LOG_WARNING(( "invalid tar header magic `%." FD_EXPAND_THEN_STRINGIFY(FD_TAR_MAGIC_SZ) "s`", hdr->magic ));
102 0 : return FD_SSPARSE_ADVANCE_ERROR;
103 0 : }
104 :
105 0 : if( FD_LIKELY( ssparse->flags.seen_zero_tar_frame ) ) {
106 0 : if( FD_UNLIKELY( !ssparse->flags.seen_version || !ssparse->flags.seen_manifest || !ssparse->flags.seen_status_cache ) ) {
107 0 : FD_LOG_WARNING(( "unexpected end of file before version or manifest or status cache" ));
108 0 : return FD_SSPARSE_ADVANCE_ERROR;
109 0 : }
110 :
111 0 : return FD_SSPARSE_ADVANCE_DONE;
112 0 : }
113 :
114 0 : ssparse->flags.seen_zero_tar_frame = 1;
115 0 : return FD_SSPARSE_ADVANCE_AGAIN;
116 0 : }
117 :
118 0 : if( FD_UNLIKELY( ssparse->flags.seen_zero_tar_frame ) ) {
119 0 : FD_LOG_WARNING(( "unexpected valid tar header after zero frame" ));
120 0 : return FD_SSPARSE_ADVANCE_ERROR;
121 0 : }
122 :
123 0 : ssparse->tar.file_bytes = fd_tar_meta_get_size( hdr );
124 0 : if( FD_UNLIKELY( ssparse->tar.file_bytes==ULONG_MAX ) ) {
125 0 : FD_LOG_WARNING(( "invalid tar header size %." FD_EXPAND_THEN_STRINGIFY(FD_TAR_SIZE_SZ) "s "
126 0 : "for tar header name %." FD_EXPAND_THEN_STRINGIFY(FD_TAR_NAME_SZ) "s", hdr->size, hdr->name ));
127 0 : return FD_SSPARSE_ADVANCE_ERROR;
128 0 : }
129 :
130 0 : if( FD_UNLIKELY( hdr->typeflag==FD_TAR_TYPE_DIR ) ) {
131 0 : if( FD_UNLIKELY( ssparse->tar.file_bytes ) ) {
132 0 : FD_LOG_WARNING(( "invalid tar directory entry with non-zero size %lu", ssparse->tar.file_bytes ));
133 0 : return FD_SSPARSE_ADVANCE_ERROR;
134 0 : }
135 0 : return FD_SSPARSE_ADVANCE_AGAIN;
136 0 : }
137 :
138 0 : if( FD_UNLIKELY( !fd_tar_meta_is_reg( hdr ) ) ) {
139 0 : FD_LOG_WARNING(( "invalid tar header type %d", hdr->typeflag ));
140 0 : return FD_SSPARSE_ADVANCE_ERROR;
141 0 : }
142 0 : if( FD_UNLIKELY( !ssparse->tar.file_bytes ) ) {
143 0 : FD_LOG_WARNING(( "invalid tar header size %lu", ssparse->tar.file_bytes ));
144 0 : return FD_SSPARSE_ADVANCE_ERROR;
145 0 : }
146 :
147 : /* TODO: Check every header field here for validity? */
148 :
149 0 : int desired_state;
150 0 : if( FD_LIKELY( !strncmp( hdr->name, "version", 7UL ) ) ) {
151 0 : desired_state = FD_SSPARSE_STATE_VERSION;
152 0 : if( FD_UNLIKELY( ssparse->tar.file_bytes!=5UL ) ) {
153 0 : FD_LOG_WARNING(( "invalid version file size %lu", ssparse->tar.file_bytes ));
154 0 : return FD_SSPARSE_ADVANCE_ERROR;
155 0 : }
156 0 : } else if( FD_LIKELY( !strncmp( hdr->name, "accounts/", 9UL ) ) ) {
157 0 : ssparse->account.header_bytes_consumed = 0UL;
158 0 : desired_state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
159 0 : ulong id, slot;
160 0 : if( FD_UNLIKELY( -1==parse_tar_header_name( hdr->name, &id, &slot ) ) ) {
161 0 : FD_LOG_WARNING(( "invalid account append vec name %." FD_EXPAND_THEN_STRINGIFY(FD_TAR_NAME_SZ) "s", hdr->name ));
162 0 : return FD_SSPARSE_ADVANCE_ERROR;
163 0 : }
164 0 : ssparse->slot = slot;
165 0 : ssparse->acc_vec_bytes = ssparse->tar.file_bytes;
166 0 : } else if( FD_LIKELY( !strncmp( hdr->name, "snapshots/status_cache", 22UL ) ) ) desired_state = FD_SSPARSE_STATE_STATUS_CACHE;
167 0 : else if( FD_LIKELY( !strncmp( hdr->name, "snapshots/", 10UL ) ) ) {
168 0 : desired_state = FD_SSPARSE_STATE_MANIFEST;
169 0 : } else {
170 0 : FD_LOG_WARNING(( "unexpected tar header name `%." FD_EXPAND_THEN_STRINGIFY(FD_TAR_NAME_SZ) "s`", hdr->name ));
171 0 : return FD_SSPARSE_ADVANCE_ERROR;
172 0 : }
173 :
174 0 : ssparse->tar.file_bytes_consumed = 0UL;
175 :
176 0 : switch( desired_state ) {
177 0 : case FD_SSPARSE_STATE_VERSION:
178 0 : if( FD_UNLIKELY( ssparse->flags.seen_version ) ) {
179 0 : FD_LOG_WARNING(( "unexpected duplicate version file" ));
180 0 : return FD_SSPARSE_ADVANCE_ERROR;
181 0 : }
182 :
183 0 : ssparse->flags.seen_version = 1;
184 0 : ssparse->state = FD_SSPARSE_STATE_VERSION;
185 0 : break;
186 0 : case FD_SSPARSE_STATE_MANIFEST:
187 0 : if( FD_UNLIKELY( ssparse->flags.seen_manifest ) ) {
188 0 : FD_LOG_WARNING(( "unexpected duplicate manifest file" ));
189 0 : return FD_SSPARSE_ADVANCE_ERROR;
190 0 : }
191 :
192 0 : ssparse->flags.seen_manifest = 1;
193 0 : ssparse->state = FD_SSPARSE_STATE_MANIFEST;
194 0 : break;
195 0 : case FD_SSPARSE_STATE_ACCOUNT_HEADER:
196 0 : if( FD_UNLIKELY( !ssparse->flags.seen_manifest ) ) {
197 0 : FD_LOG_WARNING(( "unexpected account append vec file before manifest" ));
198 0 : return FD_SSPARSE_ADVANCE_ERROR;
199 0 : }
200 :
201 0 : ssparse->account.header_bytes_consumed = 0UL;
202 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE;
203 0 : result->appendvec.slot = ssparse->slot;
204 0 : result->appendvec.data_sz = ssparse->tar.file_bytes;
205 0 : return FD_SSPARSE_ADVANCE_APPENDVEC;
206 0 : case FD_SSPARSE_STATE_STATUS_CACHE:
207 0 : if( FD_UNLIKELY( ssparse->flags.seen_status_cache ) ) {
208 0 : FD_LOG_WARNING(( "unexpected status cache file" ));
209 0 : return FD_SSPARSE_ADVANCE_ERROR;
210 0 : }
211 :
212 0 : ssparse->flags.seen_status_cache = 1;
213 0 : ssparse->state = FD_SSPARSE_STATE_STATUS_CACHE;
214 0 : break;
215 0 : default:
216 0 : FD_LOG_ERR(( "unexpected tar header desired state %d", desired_state ));
217 0 : break;
218 0 : }
219 :
220 0 : return FD_SSPARSE_ADVANCE_AGAIN;
221 0 : }
222 :
223 : static int
224 : advance_version( fd_ssparse_t * ssparse,
225 : uchar const * data,
226 : ulong data_sz,
227 0 : fd_ssparse_advance_result_t * result ) {
228 0 : ulong consume = fd_ulong_min( data_sz, ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed );
229 0 : if( FD_UNLIKELY( !consume ) ) {
230 0 : FD_LOG_WARNING(( "unexpected end of data while parsing version file, data_sz=%lu, file_bytes_consumed=%lu, file_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->tar.file_bytes ));
231 0 : return FD_SSPARSE_ADVANCE_ERROR;
232 0 : }
233 :
234 0 : fd_memcpy( ssparse->version+ssparse->tar.file_bytes_consumed, data, consume );
235 :
236 0 : ssparse->tar.file_bytes_consumed += consume;
237 0 : ssparse->bytes_consumed += consume;
238 0 : result->bytes_consumed = consume;
239 :
240 0 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) return FD_SSPARSE_ADVANCE_AGAIN;
241 :
242 0 : FD_TEST( ssparse->tar.file_bytes_consumed==ssparse->tar.file_bytes );
243 0 : FD_TEST( ssparse->tar.file_bytes_consumed==5UL );
244 :
245 0 : if( FD_UNLIKELY( memcmp( ssparse->version, "1.2.0", 5UL ) ) ) {
246 0 : FD_LOG_WARNING(( "invalid version file %.*s", 5, ssparse->version ));
247 0 : return FD_SSPARSE_ADVANCE_ERROR;
248 0 : }
249 :
250 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
251 0 : return FD_SSPARSE_ADVANCE_AGAIN;
252 0 : }
253 :
254 : static int
255 : advance_status_cache( fd_ssparse_t * ssparse,
256 : uchar const * data,
257 : ulong data_sz,
258 0 : fd_ssparse_advance_result_t * result ) {
259 0 : ulong consume = fd_ulong_min( data_sz, ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed );
260 0 : if( FD_UNLIKELY( !consume ) ) {
261 0 : FD_LOG_WARNING(( "unexpected end of data while parsing status cache, data_sz=%lu, file_bytes_consumed=%lu, file_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->tar.file_bytes ));
262 0 : return FD_SSPARSE_ADVANCE_ERROR;
263 0 : }
264 :
265 0 : ssparse->tar.file_bytes_consumed += consume;
266 0 : ssparse->bytes_consumed += consume;
267 :
268 0 : result->bytes_consumed = consume;
269 0 : result->status_cache.data = data;
270 0 : result->status_cache.data_sz = consume;
271 0 : result->status_cache.done = ssparse->tar.file_bytes_consumed==ssparse->tar.file_bytes;
272 :
273 0 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) {
274 0 : return FD_SSPARSE_ADVANCE_STATUS_CACHE;
275 0 : }
276 0 : else { /* ssparse->tar.file_bytes_consumed==ssparse->tar.file_bytes */
277 : /* finished parsing status cache */
278 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
279 0 : return FD_SSPARSE_ADVANCE_STATUS_CACHE;
280 0 : }
281 0 : }
282 :
283 : static int
284 : advance_manifest( fd_ssparse_t * ssparse,
285 : uchar const * data,
286 : ulong data_sz,
287 0 : fd_ssparse_advance_result_t * result ) {
288 0 : ulong consume = fd_ulong_min( data_sz, ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed );
289 0 : if( FD_UNLIKELY( !consume ) ) {
290 0 : FD_LOG_WARNING(( "unexpected end of data while parsing manifest, data_sz=%lu, file_bytes_consumed=%lu, file_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->tar.file_bytes ));
291 0 : return FD_SSPARSE_ADVANCE_ERROR;
292 0 : }
293 :
294 0 : ssparse->tar.file_bytes_consumed += consume;
295 0 : ssparse->bytes_consumed += consume;
296 :
297 0 : result->bytes_consumed = consume;
298 0 : result->manifest.data = data;
299 0 : result->manifest.data_sz = consume;
300 :
301 0 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) return FD_SSPARSE_ADVANCE_MANIFEST;
302 :
303 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
304 0 : return FD_SSPARSE_ADVANCE_MANIFEST_DONE;
305 0 : }
306 :
307 : static int
308 : advance_next_tar( fd_ssparse_t * ssparse,
309 : uchar const * data,
310 : ulong data_sz,
311 0 : fd_ssparse_advance_result_t * result ) {
312 0 : (void)data;
313 : /* skip padding */
314 0 : ulong bytes_remaining = fd_ulong_align_up( ssparse->bytes_consumed, 512UL ) - ssparse->bytes_consumed;
315 0 : ulong pad_sz = bytes_remaining;
316 0 : pad_sz = fd_ulong_min( pad_sz, data_sz );
317 0 : if( FD_UNLIKELY( !pad_sz && bytes_remaining ) ) {
318 0 : FD_LOG_WARNING(( "unexpected end of data while parsing tar header padding, data_sz=%lu, bytes_consumed=%lu, bytes_remaining=%lu", data_sz, ssparse->bytes_consumed, bytes_remaining ));
319 0 : return FD_SSPARSE_ADVANCE_ERROR;
320 0 : }
321 :
322 0 : ssparse->bytes_consumed += pad_sz;
323 0 : result->bytes_consumed = pad_sz;
324 0 : bytes_remaining -= pad_sz;
325 :
326 0 : if( FD_LIKELY( !bytes_remaining ) ) ssparse->state = FD_SSPARSE_STATE_TAR_HEADER;
327 0 : return FD_SSPARSE_ADVANCE_AGAIN;
328 0 : }
329 :
330 : static int
331 : advance_account_batch( fd_ssparse_t * ssparse,
332 : uchar const * data,
333 : ulong data_sz,
334 0 : fd_ssparse_advance_result_t * result ) {
335 : /* Cannot create a batch unless the parser is aligned to an account. */
336 0 : if( FD_UNLIKELY( ssparse->account.header_bytes_consumed ) ) return FD_SSPARSE_ADVANCE_AGAIN;
337 :
338 : /* Each account is at least 136 bytes large. Don't attempt to create
339 : a batch unless at least 4 accounts fit. */
340 0 : ulong avail = fd_ulong_min( data_sz, ssparse->acc_vec_bytes - ssparse->tar.file_bytes_consumed );
341 0 : if( FD_UNLIKELY( avail<(4*136UL) ) ) return FD_SSPARSE_ADVANCE_AGAIN;
342 :
343 : /* Prefetch data to reduce cache misses in hot path. */
344 0 : for( ulong i=0UL; i<fd_ulong_min( avail, 4096UL ); i+=64UL ) {
345 0 : __builtin_prefetch( (uchar *)data+i, 0, 0 );
346 0 : }
347 :
348 : /* Skip over accounts until we reached EOF or batch is full */
349 0 : result->account_batch.batch_cnt = 0;
350 0 : ulong off = 0UL;
351 0 : for( ulong idx=0UL; idx<FD_SSPARSE_ACC_BATCH_MAX && off+136UL<=avail; idx++ ) {
352 0 : uchar const * acc_hdr = (uchar *)data+off;
353 :
354 : /* We want ConfigProgram accounts to go through the slow path,
355 : since they are published from there to consumers for monitoring. */
356 0 : if( FD_UNLIKELY( !memcmp( acc_hdr+64UL, fd_solana_config_program_id.key, sizeof(fd_hash_t) ) ) ) {
357 0 : if( FD_UNLIKELY( idx==0UL ) ) return FD_SSPARSE_ADVANCE_AGAIN; /* At the front of the batch, abort */
358 0 : else break; /* otherwise, break early. */
359 0 : }
360 :
361 0 : ulong acc_data_sz = fd_ulong_load_8_fast( acc_hdr+8 );
362 0 : if( FD_UNLIKELY( acc_data_sz>FD_RUNTIME_ACC_SZ_MAX ) ) {
363 0 : FD_LOG_WARNING(( "invalid account data size %lu", acc_data_sz ));
364 0 : return FD_SSPARSE_ADVANCE_ERROR;
365 0 : }
366 : /* acc_hdr[96] is the executable flag (uchar), must be 0 or 1 */
367 0 : if( FD_UNLIKELY( acc_hdr[ 96UL ]>1 ) ) {
368 0 : FD_LOG_WARNING(( "invalid account header executable %u", acc_hdr[ 96UL ] ));
369 0 : return FD_SSPARSE_ADVANCE_ERROR;
370 0 : }
371 0 : ulong next_off = off+136UL+acc_data_sz;
372 0 : ulong pad_sz = fd_ulong_align_up( ssparse->tar.file_bytes_consumed+next_off, 8UL ) -
373 0 : ( ssparse->tar.file_bytes_consumed+next_off );
374 0 : next_off += pad_sz;
375 0 : if( FD_UNLIKELY( next_off>avail ) ) break; /* account is fragmented */
376 0 : result->account_batch.batch_cnt = idx+1UL;
377 0 : result->account_batch.batch[ idx ] = acc_hdr;
378 0 : ssparse->account.header_bytes_consumed = 136UL;
379 0 : ssparse->account.data_bytes_consumed = acc_data_sz;
380 0 : ssparse->account.data_len = acc_data_sz;
381 0 : off = next_off;
382 0 : }
383 :
384 : /* Skip if no batches to process. */
385 0 : if( FD_UNLIKELY( !result->account_batch.batch_cnt ) ) {
386 0 : return FD_SSPARSE_ADVANCE_AGAIN;
387 0 : }
388 :
389 0 : ssparse->tar.file_bytes_consumed += off;
390 0 : ssparse->bytes_consumed += off;
391 0 : result->bytes_consumed = off;
392 :
393 : /* reset state */
394 :
395 0 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_PADDING;
396 :
397 0 : result->account_batch.slot = ssparse->slot;
398 :
399 0 : return FD_SSPARSE_ADVANCE_ACCOUNT_BATCH;
400 0 : }
401 :
402 : static int
403 : advance_account_header( fd_ssparse_t * ssparse,
404 : uchar const * data,
405 : ulong data_sz,
406 0 : fd_ssparse_advance_result_t * result ) {
407 0 : ulong consume = fd_ulong_min( 136UL-ssparse->account.header_bytes_consumed, fd_ulong_min( data_sz, ssparse->acc_vec_bytes-ssparse->tar.file_bytes_consumed ) );
408 :
409 0 : if( FD_UNLIKELY( !consume ) ) {
410 0 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed==ssparse->acc_vec_bytes ) ) {
411 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE;
412 0 : return FD_SSPARSE_ADVANCE_AGAIN;
413 0 : } else {
414 0 : FD_LOG_WARNING(( "unexpected end of data while advancing account header, data_sz=%lu, file_bytes_consumed=%lu, acc_vec_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->acc_vec_bytes ));
415 0 : return FD_SSPARSE_ADVANCE_ERROR;
416 0 : }
417 0 : }
418 :
419 0 : if( FD_UNLIKELY( consume<136UL ) ) {
420 0 : fd_memcpy( ssparse->account.header+ssparse->account.header_bytes_consumed, data, consume );
421 0 : } else if( ssparse->batch_enabled ) {
422 : /* fast path */
423 0 : int res = advance_account_batch( ssparse, data, data_sz, result );
424 0 : if( res==FD_SSPARSE_ADVANCE_ACCOUNT_BATCH ) return res;
425 : /* fall through and continue processing account header */
426 0 : }
427 :
428 0 : ssparse->account.header_bytes_consumed += consume;
429 0 : ssparse->tar.file_bytes_consumed += consume;
430 0 : ssparse->bytes_consumed += consume;
431 0 : result->bytes_consumed = consume;
432 :
433 0 : if( FD_UNLIKELY( ssparse->account.header_bytes_consumed<136UL ) ) return FD_SSPARSE_ADVANCE_AGAIN;
434 :
435 0 : uchar const * hdr = ssparse->account.header;
436 0 : if( FD_LIKELY( consume==136UL ) ) hdr = data;
437 :
438 0 : result->account_header.data_len = fd_ulong_load_8_fast( hdr+8UL );
439 0 : if( FD_UNLIKELY( result->account_header.data_len>FD_RUNTIME_ACC_SZ_MAX ) ) {
440 0 : FD_LOG_WARNING(( "invalid account header data length %lu", result->account_header.data_len ));
441 0 : return FD_SSPARSE_ADVANCE_ERROR;
442 0 : }
443 :
444 0 : result->account_header.pubkey = hdr+16UL;
445 0 : result->account_header.lamports = fd_ulong_load_8_fast( hdr+48UL );
446 0 : result->account_header.rent_epoch = fd_ulong_load_8_fast( hdr+56UL );
447 0 : result->account_header.owner = hdr+64UL;
448 0 : result->account_header.executable = hdr[ 96UL ];
449 0 : if( FD_UNLIKELY( result->account_header.executable>1 ) ) {
450 0 : char pubkey_str[ FD_BASE58_ENCODED_32_SZ ];
451 0 : fd_base58_encode_32( result->account_header.pubkey, NULL, pubkey_str );
452 0 : FD_LOG_WARNING(( "invalid account header executable %d for account %s", result->account_header.executable, pubkey_str ));
453 0 : return FD_SSPARSE_ADVANCE_ERROR;
454 0 : }
455 0 : result->account_header.hash = hdr+104UL;
456 0 : result->account_header.slot = ssparse->slot;
457 :
458 0 : ssparse->account.data_len = result->account_header.data_len;
459 0 : ssparse->account.data_bytes_consumed = 0UL;
460 0 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_DATA;
461 :
462 0 : return FD_SSPARSE_ADVANCE_ACCOUNT_HEADER;
463 0 : }
464 :
465 : static int
466 : advance_account_data( fd_ssparse_t * ssparse,
467 : uchar const * data,
468 : ulong data_sz,
469 0 : fd_ssparse_advance_result_t * result ) {
470 0 : if( FD_UNLIKELY( ssparse->account.data_bytes_consumed==ssparse->account.data_len ) ) {
471 0 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_PADDING;
472 0 : return FD_SSPARSE_ADVANCE_AGAIN;
473 0 : }
474 :
475 0 : ulong consume = fd_ulong_min( data_sz, ssparse->acc_vec_bytes-ssparse->tar.file_bytes_consumed );
476 0 : if( FD_UNLIKELY( !consume ) ) {
477 0 : FD_LOG_WARNING(( "account data extends beyond append vec size" ));
478 0 : return FD_SSPARSE_ADVANCE_ERROR;
479 0 : }
480 :
481 0 : consume = fd_ulong_min( consume, ssparse->account.data_len-ssparse->account.data_bytes_consumed );
482 0 : if( FD_UNLIKELY( !consume ) ) {
483 0 : FD_LOG_WARNING(( "unexpected end of data while parsing account data, data_sz=%lu, data_bytes_consumed=%lu, data_len=%lu", data_sz, ssparse->account.data_bytes_consumed, ssparse->account.data_len ));
484 0 : return FD_SSPARSE_ADVANCE_ERROR;
485 0 : }
486 :
487 0 : ssparse->tar.file_bytes_consumed += consume;
488 0 : ssparse->bytes_consumed += consume;
489 0 : ssparse->account.data_bytes_consumed += consume;
490 0 : result->bytes_consumed = consume;
491 :
492 0 : result->account_data.data_sz = consume;
493 0 : result->account_data.data = data;
494 :
495 0 : FD_TEST( ssparse->account.data_bytes_consumed<=ssparse->account.data_len );
496 0 : if( FD_LIKELY( ssparse->account.data_bytes_consumed==ssparse->account.data_len ) ) {
497 0 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_PADDING;
498 0 : }
499 :
500 0 : return FD_SSPARSE_ADVANCE_ACCOUNT_DATA;
501 0 : }
502 :
503 : static int
504 : advance_account_padding( fd_ssparse_t * ssparse,
505 : uchar const * data,
506 : ulong data_sz,
507 0 : fd_ssparse_advance_result_t * result ) {
508 0 : (void)data;
509 :
510 0 : ulong pad_sz = fd_ulong_align_up( ssparse->tar.file_bytes_consumed, 8UL ) - ssparse->tar.file_bytes_consumed;
511 0 : pad_sz = fd_ulong_min( pad_sz, ssparse->acc_vec_bytes - ssparse->tar.file_bytes_consumed );
512 0 : if( FD_UNLIKELY( !pad_sz ) ) {
513 0 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed==ssparse->acc_vec_bytes ) ) ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
514 0 : else ssparse->state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
515 :
516 0 : ssparse->account.header_bytes_consumed = 0UL;
517 0 : return FD_SSPARSE_ADVANCE_AGAIN;
518 0 : }
519 :
520 0 : ulong consume = fd_ulong_min( data_sz, pad_sz );
521 0 : if( FD_UNLIKELY( !consume ) ) {
522 0 : FD_LOG_WARNING(( "unexpected end of data while parsing account padding, data_sz=%lu, file_bytes_consumed=%lu, acc_vec_bytes=%lu", data_sz, ssparse->tar.file_bytes_consumed, ssparse->acc_vec_bytes ));
523 0 : return FD_SSPARSE_ADVANCE_ERROR;
524 0 : }
525 :
526 0 : ssparse->tar.file_bytes_consumed += consume;
527 0 : ssparse->bytes_consumed += consume;
528 0 : result->bytes_consumed = consume;
529 :
530 0 : ulong remaining = fd_ulong_align_up( ssparse->tar.file_bytes_consumed, 8UL ) - ssparse->tar.file_bytes_consumed;
531 0 : if( FD_LIKELY( !remaining ) ) {
532 0 : ssparse->account.header_bytes_consumed = 0UL;
533 0 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
534 0 : }
535 0 : return FD_SSPARSE_ADVANCE_AGAIN;
536 0 : }
537 :
538 : static int
539 : advance_account_garbage( fd_ssparse_t * ssparse,
540 : uchar const * data,
541 : ulong data_sz,
542 0 : fd_ssparse_advance_result_t * result ) {
543 0 : (void)data;
544 0 : ulong rem = ssparse->tar.file_bytes-ssparse->tar.file_bytes_consumed;
545 0 : if( FD_UNLIKELY( !rem ) ) {
546 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
547 0 : return FD_SSPARSE_ADVANCE_AGAIN;
548 0 : }
549 :
550 0 : if( FD_UNLIKELY( !data_sz ) ) {
551 0 : FD_LOG_WARNING(( "unexpected end of data while parsing append vec garbage, data_sz=%lu, remaining_bytes=%lu", data_sz, rem ));
552 0 : return FD_SSPARSE_ADVANCE_ERROR;
553 0 : }
554 :
555 0 : ulong consume = fd_ulong_min( data_sz, rem );
556 0 : ssparse->tar.file_bytes_consumed += consume;
557 0 : ssparse->bytes_consumed += consume;
558 0 : result->bytes_consumed = consume;
559 :
560 0 : if( FD_LIKELY( ssparse->tar.file_bytes_consumed<ssparse->tar.file_bytes ) ) return FD_SSPARSE_ADVANCE_AGAIN;
561 :
562 0 : ssparse->state = FD_SSPARSE_STATE_SCROLL_TAR_HEADER;
563 0 : return FD_SSPARSE_ADVANCE_AGAIN;
564 0 : }
565 :
566 : int
567 : fd_ssparse_advance( fd_ssparse_t * ssparse,
568 : uchar const * data,
569 : ulong data_sz,
570 0 : fd_ssparse_advance_result_t * result ) {
571 0 : result->bytes_consumed = 0UL;
572 :
573 0 : switch( ssparse->state ) {
574 0 : case FD_SSPARSE_STATE_TAR_HEADER: return advance_tar( ssparse, data, data_sz, result );
575 0 : case FD_SSPARSE_STATE_SCROLL_TAR_HEADER: return advance_next_tar( ssparse, data, data_sz, result );
576 0 : case FD_SSPARSE_STATE_VERSION: return advance_version( ssparse, data, data_sz, result );
577 0 : case FD_SSPARSE_STATE_MANIFEST: return advance_manifest( ssparse, data, data_sz, result );
578 0 : case FD_SSPARSE_STATE_ACCOUNT_HEADER: return advance_account_header( ssparse, data, data_sz, result );
579 0 : case FD_SSPARSE_STATE_ACCOUNT_DATA: return advance_account_data( ssparse, data, data_sz, result );
580 0 : case FD_SSPARSE_STATE_ACCOUNT_PADDING: return advance_account_padding( ssparse, data, data_sz, result );
581 0 : case FD_SSPARSE_STATE_STATUS_CACHE: return advance_status_cache( ssparse, data, data_sz, result );
582 0 : case FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE: return advance_account_garbage( ssparse, data, data_sz, result );
583 0 : default: FD_LOG_ERR(( "invalid state %d", ssparse->state ));
584 0 : }
585 0 : }
586 :
587 : void
588 : fd_ssparse_batch_enable( fd_ssparse_t * ssparse,
589 0 : int enabled ) {
590 0 : ssparse->batch_enabled = !!enabled;
591 0 : }
592 :
593 : void
594 0 : fd_ssparse_appendvec_parse( fd_ssparse_t * ssparse ) {
595 0 : FD_TEST( ssparse->state==FD_SSPARSE_STATE_SCROLL_ACCOUNT_GARBAGE );
596 0 : FD_TEST( !ssparse->tar.file_bytes_consumed );
597 0 : ssparse->state = FD_SSPARSE_STATE_ACCOUNT_HEADER;
598 0 : }
|
