Line data Source code
1 : #ifndef HEADER_fd_discof_restore_fd_snapin_tile_private_h
2 : #define HEADER_fd_discof_restore_fd_snapin_tile_private_h
3 :
4 : /* fd_snapin_tile_private.h contains private APIs for the "snapin" tile,
5 : which is the tile responsible for parsing a snapshot, and directing
6 : database writes. */
7 :
8 : #include "utils/fd_ssparse.h"
9 : #include "utils/fd_ssmanifest_parser.h"
10 : #include "utils/fd_slot_delta_parser.h"
11 : #include "utils/fd_ssctrl.h"
12 : #include "../../flamenco/accdb/fd_accdb_admin.h"
13 : #include "../../flamenco/accdb/fd_accdb_user.h"
14 : #include "../../flamenco/runtime/fd_txncache.h"
15 : #include "../../disco/stem/fd_stem.h"
16 : #include "../../vinyl/meta/fd_vinyl_meta.h"
17 :
18 : /* 300 here is from status_cache.rs::MAX_CACHE_ENTRIES which is the most
19 : root slots Agave could possibly serve in a snapshot. */
20 0 : #define FD_SNAPIN_TXNCACHE_MAX_ENTRIES (300UL*FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT)
21 :
22 : struct blockhash_group {
23 : uchar blockhash[ 32UL ];
24 : ulong txnhash_offset;
25 : };
26 :
27 : typedef struct blockhash_group blockhash_group_t;
28 :
29 : struct fd_snapin_out_link {
30 : ulong idx;
31 : fd_wksp_t * mem;
32 : ulong chunk0;
33 : ulong wmark;
34 : ulong chunk;
35 : ulong mtu;
36 : };
37 : typedef struct fd_snapin_out_link fd_snapin_out_link_t;
38 :
39 : struct buffered_account_batch {
40 : uchar const * batch[ FD_SSPARSE_ACC_BATCH_MAX ];
41 : ulong batch_cnt;
42 : ulong slot;
43 : /* index at which to start processing a buffered account batch */
44 : ulong remaining_idx;
45 : };
46 :
47 : typedef struct buffered_account_batch buffered_account_batch_t;
48 :
49 : struct fd_snapin_tile {
50 : int state;
51 : uint full : 1; /* loading a full snapshot? */
52 : uint use_vinyl : 1; /* using vinyl-backed accdb? */
53 : uint lthash_disabled : 1; /* disable lthash checking? */
54 :
55 : ulong seed;
56 : long boot_timestamp;
57 :
58 : fd_accdb_admin_t accdb_admin[1];
59 : fd_accdb_user_t accdb[1];
60 : fd_funk_t * funk;
61 :
62 : fd_txncache_t * txncache;
63 : uchar * acc_data;
64 :
65 : fd_funk_txn_xid_t xid[1]; /* txn XID */
66 :
67 : fd_stem_context_t * stem;
68 :
69 : fd_ssparse_t * ssparse;
70 : fd_ssmanifest_parser_t * manifest_parser;
71 : fd_slot_delta_parser_t * slot_delta_parser;
72 :
73 : buffered_account_batch_t buffered_batch;
74 :
75 : struct {
76 : int manifest_done;
77 : int status_cache_done;
78 : int manifest_processed;
79 : } flags;
80 :
81 : ulong advertised_slot;
82 : ulong bank_slot;
83 : ulong epoch;
84 :
85 : ulong full_genesis_creation_time_seconds;
86 : uchar advertised_hash[ FD_HASH_FOOTPRINT ];
87 :
88 : /* TODO: remove capitalization tracking when snapshot loading into
89 : funk goes away */
90 : ulong capitalization; /* tracks capitalization of all loaded accounts in the current snapshot */
91 : ulong dup_capitalization; /* tracks capitalization of duplicate accounts encountered during incremental snapshot loading */
92 : ulong manifest_capitalization; /* capitalization according to the current snapshot manifest */
93 :
94 : struct {
95 : ulong capitalization;
96 : } recovery; /* stores the capitalization value from the last full snapshot */
97 :
98 : ulong blockhash_offsets_len;
99 : blockhash_group_t * blockhash_offsets;
100 :
101 : ulong txncache_entries_len;
102 : ulong * txncache_entries_len_vinyl_ptr;
103 : fd_sstxncache_entry_t * txncache_entries;
104 :
105 : fd_txncache_fork_id_t txncache_root_fork_id;
106 :
107 : struct {
108 : ulong full_bytes_read;
109 : ulong incremental_bytes_read;
110 :
111 : /* Account counters (full + incremental) */
112 : ulong accounts_loaded;
113 : ulong accounts_replaced;
114 : ulong accounts_ignored;
115 :
116 : /* Account counters (snapshot taken for full snapshot only) */
117 : ulong full_accounts_loaded;
118 : ulong full_accounts_replaced;
119 : ulong full_accounts_ignored;
120 : } metrics;
121 :
122 : struct {
123 : fd_wksp_t * wksp;
124 : ulong chunk0;
125 : ulong wmark;
126 : ulong mtu;
127 : ulong pos;
128 : } in;
129 :
130 : ulong out_ct_idx;
131 : fd_snapin_out_link_t manifest_out;
132 : fd_snapin_out_link_t gui_out;
133 : fd_snapin_out_link_t hash_out;
134 :
135 : struct {
136 : uchar * pair;
137 : ulong pair_sz;
138 :
139 : uchar * dst;
140 : ulong dst_rem;
141 : ulong data_rem;
142 :
143 : fd_vinyl_meta_ele_t * meta_ele;
144 : } vinyl_op;
145 : };
146 :
147 : typedef struct fd_snapin_tile fd_snapin_tile_t;
148 :
149 : /* Funk APIs **********************************************************/
150 :
151 : FD_PROTOTYPES_BEGIN
152 :
153 : int fd_snapin_process_account_header_funk( fd_snapin_tile_t * ctx, fd_ssparse_advance_result_t * result );
154 : int fd_snapin_process_account_data_funk ( fd_snapin_tile_t * ctx, fd_ssparse_advance_result_t * result );
155 : int fd_snapin_process_account_batch_funk ( fd_snapin_tile_t * ctx, fd_ssparse_advance_result_t * result, buffered_account_batch_t * buffered_batch );
156 :
157 : void
158 : fd_snapin_read_account_funk( fd_snapin_tile_t * ctx,
159 : void const * acct_addr,
160 : fd_account_meta_t * meta,
161 : uchar * data,
162 : ulong data_max );
163 :
164 : FD_PROTOTYPES_END
165 :
166 : /* Vinyl APIs *********************************************************/
167 :
168 : FD_PROTOTYPES_BEGIN
169 :
170 : /* Internal APIs for inserting accounts */
171 :
172 : int fd_snapin_process_account_header_vinyl( fd_snapin_tile_t * ctx, fd_ssparse_advance_result_t * result );
173 : int fd_snapin_process_account_data_vinyl ( fd_snapin_tile_t * ctx, fd_ssparse_advance_result_t * result );
174 : int fd_snapin_process_account_batch_vinyl ( fd_snapin_tile_t * ctx, fd_ssparse_advance_result_t * result );
175 :
176 : FD_PROTOTYPES_END
177 :
178 : /* Generic APIs *******************************************************/
179 :
180 : FD_PROTOTYPES_BEGIN
181 :
182 : /* int return value for fd_snapin_process_account_header,
183 : fd_snapin_process_account_data, and fd_snapin_process_account_batch
184 : indicates whether to yield to stem for credit return */
185 :
186 : static inline int
187 : fd_snapin_process_account_header( fd_snapin_tile_t * ctx,
188 0 : fd_ssparse_advance_result_t * result ) {
189 0 : if( ctx->use_vinyl ) {
190 0 : return fd_snapin_process_account_header_vinyl( ctx, result );
191 0 : } else {
192 0 : return fd_snapin_process_account_header_funk( ctx, result );
193 0 : }
194 0 : return 0;
195 0 : }
196 :
197 : static inline int
198 : fd_snapin_process_account_data( fd_snapin_tile_t * ctx,
199 0 : fd_ssparse_advance_result_t * result ) {
200 0 : if( ctx->use_vinyl ) {
201 0 : return fd_snapin_process_account_data_vinyl( ctx, result );
202 0 : } else {
203 0 : return fd_snapin_process_account_data_funk( ctx, result );
204 0 : }
205 0 : return 0;
206 0 : }
207 :
208 : static inline int
209 : fd_snapin_process_account_batch( fd_snapin_tile_t * ctx,
210 : fd_ssparse_advance_result_t * result,
211 0 : buffered_account_batch_t * buffered_batch ) {
212 0 : if( ctx->use_vinyl ) {
213 0 : return fd_snapin_process_account_batch_vinyl( ctx, result );
214 0 : } else {
215 0 : return fd_snapin_process_account_batch_funk( ctx, result, buffered_batch );
216 0 : }
217 0 : return 0;
218 0 : }
219 :
220 : static inline void
221 : fd_snapin_read_account( fd_snapin_tile_t * ctx,
222 : void const * acct_addr,
223 : fd_account_meta_t * meta,
224 : uchar * data,
225 0 : ulong data_max ) {
226 : /* fd_snapin_read_account will no longer be required in the snapin
227 : tile once funk is deprecated from the snapshot load pipeline.
228 : Under vinyl, this is implemented in the snapwm tile. */
229 0 : if( ctx->use_vinyl ) {
230 0 : FD_LOG_ERR(( "read account is not supported under vinyl" ));
231 0 : } else {
232 0 : fd_snapin_read_account_funk( ctx, acct_addr, meta, data, data_max );
233 0 : }
234 0 : }
235 :
236 : /* fd_snapin_send_duplicate_account sends a duplicate account message
237 : with the signature FD_SNAPSHOT_HASH_MSG_SUB or
238 : FD_SNAPSHOT_HASH_MSG_SUB_HDR, depending on if this duplicate account
239 : contains valid account data. The message is only
240 : sent if lthash verification is enabled in the snapshot loader.
241 :
242 : lamports is account's lamports value. data is the account's data,
243 : which can be optionally null. data_len is the length of the account
244 : data. executable is the account's executable flag. owner points to
245 : the account's owner (32 bytes). pubkey points to the account's
246 : pubkey (32 bytes). early_exit is an optional pointer to an int flag
247 : that is set to 1 if the caller should yield to stem following this
248 : call. */
249 : static inline void
250 : fd_snapin_send_duplicate_account( fd_snapin_tile_t * ctx,
251 : ulong lamports,
252 : uchar const * data,
253 : ulong data_len,
254 : uchar executable,
255 : uchar const * owner,
256 : uchar const * pubkey,
257 : int has_data,
258 0 : int * early_exit ) {
259 0 : if( FD_UNLIKELY( ctx->lthash_disabled ) ) return;
260 :
261 0 : if( FD_LIKELY( has_data ) ) {
262 0 : fd_snapshot_full_account_t * existing_account = fd_chunk_to_laddr( ctx->hash_out.mem, ctx->hash_out.chunk );
263 0 : fd_snapshot_account_hdr_init( &existing_account->hdr, pubkey, owner, lamports, executable, data_len );
264 0 : fd_memcpy( existing_account->data, data, data_len );
265 0 : fd_stem_publish( ctx->stem, ctx->out_ct_idx, FD_SNAPSHOT_HASH_MSG_SUB, ctx->hash_out.chunk, sizeof(fd_snapshot_account_hdr_t)+data_len, 0UL, 0UL, 0UL );
266 0 : ctx->hash_out.chunk = fd_dcache_compact_next( ctx->hash_out.chunk, sizeof(fd_snapshot_account_hdr_t)+data_len, ctx->hash_out.chunk0, ctx->hash_out.wmark );
267 0 : } else {
268 0 : fd_snapshot_account_hdr_t * acc_hdr = fd_chunk_to_laddr( ctx->hash_out.mem, ctx->hash_out.chunk );
269 0 : fd_snapshot_account_hdr_init( acc_hdr, pubkey, owner, lamports, executable, data_len );
270 0 : fd_stem_publish( ctx->stem, ctx->out_ct_idx, FD_SNAPSHOT_HASH_MSG_SUB_HDR, ctx->hash_out.chunk, sizeof(fd_snapshot_account_hdr_t), 0UL, 0UL, 0UL );
271 0 : ctx->hash_out.chunk = fd_dcache_compact_next( ctx->hash_out.chunk, sizeof(fd_snapshot_account_hdr_t), ctx->hash_out.chunk0, ctx->hash_out.wmark );
272 0 : }
273 0 : if( FD_LIKELY( early_exit ) ) *early_exit = 1;
274 0 : }
275 :
276 : /* fd_snapin_send_duplicate_account_data sends a duplicate account
277 : message with the signature FD_SNAPSHOT_HASH_MSG_SUB_DATA. The
278 : message is only sent if lthash verification is enabled in the
279 : snapshot loader.
280 :
281 : data is the account's data, which cannot be null. data_len is the
282 : length of the account data. early_exit is an optional pointer to an
283 : int flag that is set to 1 if the caller should yield to stem
284 : following this call. */
285 : static inline void
286 : fd_snapin_send_duplicate_account_data( fd_snapin_tile_t * ctx,
287 : uchar const * data,
288 : ulong data_sz,
289 0 : int * early_exit ) {
290 0 : if( FD_UNLIKELY( ctx->lthash_disabled ) ) return;
291 :
292 0 : uchar * drop_account_data = fd_chunk_to_laddr( ctx->hash_out.mem, ctx->hash_out.chunk );
293 0 : fd_memcpy( drop_account_data, data, data_sz );
294 0 : fd_stem_publish( ctx->stem, ctx->out_ct_idx, FD_SNAPSHOT_HASH_MSG_SUB_DATA, ctx->hash_out.chunk, data_sz, 0UL, 0UL, 0UL );
295 0 : ctx->hash_out.chunk = fd_dcache_compact_next( ctx->hash_out.chunk, data_sz, ctx->hash_out.chunk0, ctx->hash_out.wmark );
296 0 : if( FD_LIKELY( early_exit ) ) *early_exit = 1;
297 0 : }
298 :
299 : FD_PROTOTYPES_END
300 :
301 : #endif /* HEADER_fd_discof_restore_fd_snapin_tile_private_h */
|
