Line data Source code
1 : #include <stdio.h> /* for vsnprintf */
2 : #include <stdarg.h> /* for va_list */
3 :
4 : #include "fd_sched.h"
5 : #include "../../flamenco/alpenglow/fd_block_marker_serde.h"
6 : #include "fd_execrp.h" /* for poh hash value */
7 : #include "../../ballet/sha256/fd_sha256.h"
8 : #include "../../disco/fd_disco_base.h" /* for FD_MAX_TXN_PER_SLOT_SHRED */
9 : #include "../../disco/fd_txn_p.h"
10 : #include "../../disco/metrics/fd_metrics.h" /* for fd_metrics_convert_ticks_to_nanoseconds and etc. */
11 : #include "../../disco/pack/fd_chkdup.h"
12 : #include "../../disco/shred/fd_shredder.h" /* FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ */
13 : #include "../../flamenco/runtime/fd_runtime.h" /* for fd_runtime_load_txn_address_lookup_tables */
14 : #include "../../flamenco/runtime/fd_system_ids.h"
15 : #include "../../flamenco/runtime/sysvar/fd_sysvar_slot_hashes.h" /* for ALUTs */
16 :
17 0 : #define FD_SCHED_MAX_PRINT_BUF_SZ (2UL<<20)
18 : #define FD_SCHED_POISON_MAX_ACCT_PER_SLOT (64UL)
19 :
20 0 : #define FD_SCHED_MAX_POH_HASHES_PER_TASK (4096UL) /* This seems to be the sweet spot. */
21 :
22 : /* 64 ticks per slot, and a single gigantic microblock containing min
23 : size transactions. */
24 : FD_STATIC_ASSERT( FD_MAX_TXN_PER_SLOT_SHRED==((FD_SHRED_DATA_PAYLOAD_MAX_PER_SLOT-65UL*sizeof(fd_microblock_hdr_t))/FD_TXN_MIN_SERIALIZED_SZ), max_txn_per_slot_shred );
25 :
26 : /* We size the buffer to be able to hold residual data from the previous
27 : FEC set that only becomes parseable after the next FEC set is
28 : ingested, as well as the incoming FEC set. The largest minimally
29 : parseable unit of data is a transaction. So that much data may
30 : straddle FEC set boundaries. Other minimally parseable units of data
31 : include the microblock header and the microblock count within a
32 : batch. */
33 0 : #define FD_SCHED_MAX_PAYLOAD_PER_FEC (63985UL)
34 : FD_STATIC_ASSERT( FD_SCHED_MAX_PAYLOAD_PER_FEC>=FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ, bump sched fec size bound ); /* FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ is the bound we want, but older ledgers have larger FEC sets. */
35 : #define FD_SCHED_MAX_RESIDUAL_SZ (FD_TXN_MTU>FD_BLOCK_MARKER_SER_MAX ? FD_TXN_MTU : FD_BLOCK_MARKER_SER_MAX)
36 : #define FD_SCHED_MAX_FEC_BUF_SZ (FD_SCHED_MAX_PAYLOAD_PER_FEC+FD_SCHED_MAX_RESIDUAL_SZ)
37 : FD_STATIC_ASSERT( FD_SCHED_MAX_RESIDUAL_SZ>=sizeof(fd_microblock_hdr_t), resize buffer for residual data );
38 : FD_STATIC_ASSERT( FD_SCHED_MAX_RESIDUAL_SZ>=sizeof(ulong), resize buffer for residual data );
39 :
40 0 : #define FD_SCHED_MAX_TXN_PER_FEC ((FD_SCHED_MAX_PAYLOAD_PER_FEC-1UL)/FD_TXN_MIN_SERIALIZED_SZ+1UL) /* 478 */
41 0 : #define FD_SCHED_MAX_MBLK_PER_FEC ((FD_SCHED_MAX_PAYLOAD_PER_FEC-1UL)/sizeof(fd_microblock_hdr_t)+1UL) /* 1334 */
42 :
43 : FD_STATIC_ASSERT( FD_SCHED_MIN_DEPTH>=FD_SCHED_MAX_TXN_PER_FEC, limits );
44 : FD_STATIC_ASSERT( FD_SCHED_MAX_DEPTH<=FD_RDISP_MAX_DEPTH, limits );
45 : FD_STATIC_ASSERT( FD_SCHED_MAX_DEPTH<=UINT_MAX, txn_idx_width );
46 :
47 0 : #define FD_SCHED_MAGIC (0xace8a79c181f89b6UL) /* echo -n "fd_sched_v0" | sha512sum | head -c 16 */
48 :
49 : struct fd_sched_mblk {
50 : ulong start_txn_idx; /* inclusive parse idx */
51 : ulong end_txn_idx; /* non-inclusive parse idx */
52 : ulong curr_txn_idx; /* next txn to mixin, parse idx in block */
53 : ulong hashcnt; /* number of pure hashes, excluding final mixin */
54 : ulong curr_hashcnt;
55 : fd_hash_t end_hash[ 1 ];
56 : fd_hash_t curr_hash[ 1 ];
57 : uint curr_sig_cnt;
58 : uint next;
59 : uint mixin_txn_idx; /* rdisp pool idx of the next txn to mixin */
60 : int is_tick;
61 : };
62 : typedef struct fd_sched_mblk fd_sched_mblk_t;
63 :
64 : #define SLIST_NAME mblk_slist
65 : #define SLIST_ELE_T fd_sched_mblk_t
66 0 : #define SLIST_IDX_T uint
67 0 : #define SLIST_NEXT next
68 : #include "../../util/tmpl/fd_slist.c"
69 :
70 : #define SET_NAME txn_bitset
71 : #define SET_MAX FD_SCHED_MAX_DEPTH
72 : #include "../../util/tmpl/fd_set.c"
73 :
74 : struct fd_sched_block {
75 : ulong slot;
76 : ulong parent_slot;
77 : ulong parent_idx; /* Index of the parent in the pool. */
78 : ulong child_idx; /* Index of the left-child in the pool. */
79 : ulong sibling_idx; /* Index of the right-sibling in the pool. */
80 :
81 : /* Counters. */
82 : uint txn_parsed_cnt;
83 : /* txn_queued_cnt = txn_parsed_cnt-txn_in_flight_cnt-txn_done_cnt */
84 : uint txn_exec_in_flight_cnt;
85 : uint txn_exec_done_cnt;
86 : uint txn_sigverify_in_flight_cnt;
87 : uint txn_sigverify_done_cnt;
88 : uint poh_hashing_in_flight_cnt; /* number of in-flight PoH batch tasks */
89 : uint poh_mblk_in_flight_cnt; /* number of individual in-flight PoH jobs across batches */
90 : uint poh_hashing_done_cnt;
91 : uint poh_hash_cmp_done_cnt; /* poh_hashing_done_cnt==poh_hash_cmp_done_cnt+len(mixin_in_progress) */
92 : uint cert_in_flight_cnt; /* in-flight footer cert verify tasks, see fd_sched_cert_dispatch */
93 : uint txn_done_cnt; /* A transaction is considered done when all types of tasks associated with it are done. */
94 : uint shred_cnt;
95 : uint shred_scan_idx; /* First shred boundary not before shred_scan_off. */
96 : uint shred_scan_off; /* Block byte offset at the start of shred_scan_idx. */
97 : uint mblk_cnt; /* Total number of microblocks, including ticks and non ticks.
98 : mblk_cnt==len(unhashed)+len(hashing_in_progress)+poh_mblk_in_flight_cnt+len(mixin_in_progress)+hash_cmp_done_cnt */
99 : uint mblk_tick_cnt; /* Total number of tick microblocks. */
100 : uint mblk_freed_cnt; /* This is ==hash_cmp_done_cnt in most cases, except for aborted
101 : blocks, where the freed cnt will catch up to mblk_cnt and surpass
102 : hash_cmp_done_cnt when the block is reaped. */
103 : uint mblk_unhashed_cnt; /* ==len(unhashed) */
104 : ulong hashcnt; /* How many hashes this block wants replay to do. A mixin/record counts as one hash. */
105 : ulong txn_pool_max_popcnt; /* Peak transaction pool occupancy during the time this block was replaying. */
106 : ulong mblk_pool_max_popcnt; /* Peak mblk pool occupancy. */
107 : ulong block_pool_max_popcnt; /* Peak block pool occupancy. */
108 : uint txn_idx_tail; /* Most recently parsed rdisp pool index. */
109 : uint txn_sigverify_next_idx; /* Next rdisp pool index to dispatch for sigverify, or 0 if caught up. */
110 : uint parse_mblk_idx; /* mblk pool index currently receiving parsed transactions. */
111 :
112 : /* PoH verify. */
113 : fd_hash_t poh_hash[ 1 ]; /* running end_hash of last parsed mblk */
114 : int last_mblk_is_tick;
115 : mblk_slist_t mblks_unhashed[ 1 ]; /* A microblock, once parsed out, is in one of these queues. It
116 : generally progresses from unhashed to hashing to mixin. When a
117 : microblock is being hashed/in-flight, it'll be transiently out of
118 : any of the queues. Once a microblock progresses through all stages
119 : of work, it'll be immediately freed. */
120 : mblk_slist_t mblks_hashing_in_progress[ 1 ];
121 : mblk_slist_t mblks_mixin_in_progress[ 1 ];
122 : uchar bmtree_mem[ FD_BMTREE_COMMIT_FOOTPRINT(0) ] __attribute__((aligned(FD_BMTREE_COMMIT_ALIGN)));
123 : fd_bmtree_commit_t * bmtree;
124 : ulong tick_hashcnt_wmk; /* All ticks in a valid block must accumulate the same number of
125 : hashes since the previous tick, or since block start, and this hash
126 : count must match hashes_per_tick for the block. */
127 : ulong curr_tick_hashcnt; /* Starts at 0, accumulates hashcnt, resets to 0 on the next tick. */
128 : ulong tick_height; /* Block is built off of a parent block with this many ticks. */
129 : ulong max_tick_height; /* Block should end with precisely this many ticks. */
130 : ulong hashes_per_tick; /* Fixed per block, feature gated, known after bank clone. */
131 : int inconsistent_hashes_per_tick;
132 : int zero_hash_tick;
133 :
134 : /* Parser state. */
135 : uchar txn[ FD_TXN_MAX_SZ ] __attribute__((aligned(alignof(fd_txn_t))));
136 : ulong mblks_rem; /* Number of microblocks remaining in the current batch. */
137 : ulong txns_rem; /* Number of transactions remaining in the current microblock. */
138 : uint fec_buf_sz; /* Size of the fec_buf in bytes. */
139 : uint fec_buf_soff; /* Starting offset into fec_buf for unparsed transactions. */
140 : uint fec_buf_boff; /* Byte offset into raw block data of the first byte currently in fec_buf */
141 : uint poison_cnt; /* [0, FD_SCHED_POISON_MAX_ACCT_PER_SLOT) */
142 : fd_acct_addr_t poison[ FD_SCHED_POISON_MAX_ACCT_PER_SLOT ]; /* Poison set to handle loader v3 implied programdata accounts.
143 :
144 : A transaction that lists a loader v3 program account P is charged
145 : for, and gated on, P's implicit programdata account PD, which the
146 : transaction does not have to declare. The dispatcher therefore
147 : establishes no dependency for said transaction against a
148 : transaction that writes PD without also touching P, for example a
149 : simple lamports transfer into PD, or a Close on an Unintialized PD.
150 : Every loader v3 instruction that changes a live PD's size or
151 : liveness (from alive to dead) does write lock P (except for the
152 : aforementioned Close on Unintialized PD). Simple transfers cannot
153 : debit PD since it's a PDA.
154 :
155 : So, when a loader v3 instruction that could modify PD is inserted,
156 : its writable accounts are added to a poison set, and any later
157 : transaction that writes a set member is inserted serializing. That
158 : sequences the unordered writer of PD against any implied use of PD
159 : for the rest of the block. This allows the runtime to treat the
160 : current fork's view of PD as stable.
161 :
162 : Note that this does not cover the case where the poisoning
163 : transaction (with the PD-modifying loader v3 instruction) and the
164 : PD-only transactions do not land in the same block. Non-determinism
165 : between the PD-only transaction and an implied PD transaction in
166 : that case is a protocol bug. */
167 : long fec_completed_ns; /* Network arrival (wallclock ns) of the FEC set being ingested; txns
168 : parse out only once the FEC completing their bytes arrives, so this
169 : is stamped on every txn parsed during the current ingest. */
170 : uint poison_serialize:1; /* Serialize everything in the rest of the block. Set when the poison
171 : set overflows or we couldn't maintain the poison property for any
172 : other reason. */
173 : uint fec_eob:1; /* FEC end-of-batch: set if the last FEC set in the batch is being
174 : ingested. */
175 : uint fec_sob:1; /* FEC start-of-batch: set if the parser expects to be parsing out a
176 : batch header. */
177 :
178 : /* Block state. */
179 : uint fec_eos:1; /* FEC end-of-stream: set if the last FEC set in the block has been
180 : ingested. */
181 : uint rooted:1; /* Set if the block is rooted. */
182 : uint dying:1; /* Set if the block has been abandoned and no transactions should be
183 : scheduled from it. */
184 : uint discarded:1; /* Set if the block went down with a discarded (not invalid) lineage:
185 : its own discard or an ancestor's. This field, along with
186 : dead_reason, can be in one of three possible combinations when a
187 : block is dead:
188 :
189 : dead_reason!=DEAD_ANCESTOR, discarded initialized to 0: invalid block due to dead_reason
190 : dead_reason==DEAD_ANCESTOR, discarded==0: invalid lineage
191 : dead_reason==DEAD_ANCESTOR||NONE, discarded==1: discarded lineage */
192 : uint refcnt:1; /* Starts at 1 when the block is added, set to 0 if caller has been
193 : informed to decrement refcnt for sched. */
194 : uint in_sched:1; /* Set if the block is being tracked by the scheduler. */
195 : uint in_rdisp:1; /* Set if the block is being tracked by the dispatcher, either as staged
196 : or unstaged. */
197 : uint block_start_signaled:1; /* Set if the start-of-block sentinel has been dispatched. */
198 : uint block_end_signaled:1; /* Set if the end-of-block sentinel has been dispatched. */
199 : uint block_start_done:1; /* Set if the start-of-block processing has been completed. */
200 : uint block_end_done:1; /* Set if the end-of-block processing has been completed. */
201 : uint staged:1; /* Set if the block is in a dispatcher staging lane; a staged block is
202 : tracked by the dispatcher. */
203 : ulong staging_lane; /* Ignored if staged==0. */
204 : ulong luf_depth; /* Depth of longest unstaged fork starting from this node; only
205 : stageable unstaged descendants are counted. */
206 : int dead_reason; /* One of FD_SCHED_DEAD_REASON_*; records the first reason the block
207 : was ruled invalid (set-once via record_dead_reason), NONE otherwise. */
208 : uchar fec_buf[ FD_SCHED_MAX_FEC_BUF_SZ ]; /* The previous FEC set could have some residual data that only becomes
209 : parseable after the next FEC set is ingested. */
210 :
211 : /* Alpenglow block footer, deserialized out of the marker batch at
212 : parse time. footer_seen is set if the block carries a footer
213 : marker and it is seen by sched; footer is only meaningful in that
214 : case. */
215 : fd_block_footer_t footer;
216 :
217 : /* Alpenglow block structure, mirroring agave's BlockComponentStage
218 : as a set of "seen" flags. Only maintained when sched->is_alpenglow.
219 : Together with mblk_cnt they determine what may come next:
220 :
221 : agave stage header footer alpentick mblk_cnt
222 : PreParentMarker 0 0 0 0
223 : AcceptingGenesisOrEntries 1 0 0 0 (and no genesis cert yet)
224 : AcceptingEntriesOrFooter 1 0 0 *
225 : AcceptingAlpentick 1 1 0 *
226 : Done 1 1 1 * */
227 : int header_seen;
228 : int genesis_cert_seen;
229 : int update_parent_seen;
230 : int footer_seen;
231 : int alpentick_seen;
232 : };
233 : typedef struct fd_sched_block fd_sched_block_t;
234 :
235 : FD_STATIC_ASSERT( sizeof(fd_sched_mblk_t)==120UL, fd_sched_mblk );
236 : FD_STATIC_ASSERT( sizeof(fd_sched_txn_info_t)==192UL, fd_sched_txn_info );
237 : FD_STATIC_ASSERT( sizeof(fd_sched_block_t)==75840UL, fd_sched_block );
238 : FD_STATIC_ASSERT( sizeof(fd_hash_t)==sizeof(((fd_microblock_hdr_t *)0)->hash), unexpected poh hash size );
239 :
240 :
241 : struct fd_sched_metrics {
242 : uint block_added_cnt;
243 : uint block_added_staged_cnt;
244 : uint block_added_unstaged_cnt;
245 : uint block_added_dead_ood_cnt;
246 : uint block_removed_cnt;
247 : uint block_abandoned_cnt;
248 : uint block_bad_cnt;
249 : uint block_promoted_cnt;
250 : uint block_demoted_cnt;
251 : uint deactivate_no_child_cnt;
252 : uint deactivate_no_txn_cnt;
253 : uint deactivate_pruned_cnt;
254 : uint deactivate_abandoned_cnt;
255 : uint lane_switch_cnt;
256 : uint lane_promoted_cnt;
257 : uint lane_demoted_cnt;
258 : uint fork_observed_cnt;
259 : uint alut_success_cnt;
260 : uint alut_serializing_cnt;
261 : uint poison_latch_overflow_cnt;
262 : uint poison_latch_alt_cnt;
263 : uint poison_cnt_wmk;
264 : uint txn_poison_serializing_cnt;
265 : uint txn_poisoned_cnt;
266 : uint txn_abandoned_parsed_cnt;
267 : uint txn_abandoned_exec_done_cnt;
268 : uint txn_abandoned_done_cnt;
269 : uint txn_max_in_flight_cnt;
270 : ulong txn_weighted_in_flight_cnt;
271 : ulong txn_weighted_in_flight_tickcount;
272 : ulong txn_none_in_flight_tickcount;
273 : ulong txn_parsed_cnt;
274 : ulong txn_exec_done_cnt;
275 : ulong txn_sigverify_done_cnt;
276 : ulong txn_mixin_done_cnt;
277 : ulong txn_done_cnt;
278 : ulong mblk_parsed_cnt;
279 : ulong mblk_poh_hashed_cnt;
280 : ulong mblk_poh_done_cnt;
281 : ulong bytes_ingested_cnt;
282 : ulong bytes_ingested_unparsed_cnt;
283 : ulong bytes_dropped_cnt;
284 : ulong fec_cnt;
285 : };
286 : typedef struct fd_sched_metrics fd_sched_metrics_t;
287 :
288 : #define DEQUE_NAME ref_q
289 0 : #define DEQUE_T ulong
290 : #include "../../util/tmpl/fd_deque_dynamic.c"
291 :
292 : struct fd_sched {
293 : fd_acct_addr_t aluts[ 256 ]; /* Resolve ALUT accounts into this buffer for more parallelism. */
294 : char print_buf[ FD_SCHED_MAX_PRINT_BUF_SZ ];
295 : ulong print_buf_sz;
296 : ushort * shred_sz; /* Payload size of each ingested data shred, max_shreds_per_block per block pool idx. */
297 : ulong max_shreds_per_block; /* Immutable. */
298 : ulong max_txn_per_slot; /* Immutable. */
299 : ulong max_mblk_per_slot; /* Immutable, FD_SCHED_MAX_MBLK_PER_SLOT scaled with max_shreds_per_block. */
300 : fd_chkdup_t chkdup[ 1 ];
301 : fd_sched_metrics_t metrics[ 1 ];
302 : int is_alpenglow; /* set if alpenglow is enabled. */
303 : ulong canary; /* == FD_SCHED_MAGIC */
304 : ulong depth; /* Immutable. */
305 : ulong block_cnt_max; /* Immutable. */
306 : ulong exec_cnt; /* Immutable. */
307 : ulong poh_simd_min; /* Immutable. */
308 : ulong poh_simd_max; /* Immutable. */
309 : int bypass_poh_verify; /* Test/fuzz: skip the PoH end_hash compare in maybe_mixin. */
310 : int bypass_alut_resolution; /* Test/fuzz: skip ALUT resolution (no accdb). */
311 : long txn_in_flight_last_tick;
312 : long next_ready_last_tick;
313 : ulong next_ready_last_bank_idx;
314 : ulong root_idx;
315 : fd_rdisp_t * rdisp;
316 : ulong txn_exec_ready_bitset[ 1 ];
317 : ulong sigverify_ready_bitset[ 1 ];
318 : ulong poh_ready_bitset[ 1 ];
319 : ulong cert_ready_bitset[ 1 ];
320 : ulong active_bank_idx; /* Index of the actively replayed block, or ULONG_MAX if no block is
321 : actively replayed; has to have a transaction to dispatch; staged
322 : blocks that have no transactions to dispatch are not eligible for
323 : being active. */
324 : ulong last_active_bank_idx;
325 : ulong staged_bitset; /* Bit i set if staging lane i is occupied. */
326 : ulong staged_head_bank_idx[ FD_SCHED_MAX_STAGING_LANES ]; /* Head of the linear chain in each staging lane, ignored if bit i is
327 : not set in the bitset. */
328 : ulong staged_popcnt_wmk;
329 : ulong txn_pool_free_cnt;
330 : fd_txn_p_t * txn_pool; /* Just a flat array. */
331 : fd_sched_txn_info_t * txn_info_pool; /* Just a flat array. */
332 : fd_sched_mblk_t * mblk_pool; /* Just a flat array. */
333 : ulong mblk_pool_free_cnt;
334 : uint mblk_pool_free_head;
335 : ulong tile_to_bank_idx[ FD_SCHED_MAX_EXEC_TILE_CNT ]; /* Index of the bank that the exec tile is executing against. */
336 : fd_sched_poh_hash_t poh_inflight[ FD_SCHED_MAX_EXEC_TILE_CNT ]; /* PoH dispatch metadata, indexed by exec tile. */
337 : txn_bitset_t exec_done_set[ txn_bitset_word_cnt ]; /* Indexed by txn_idx. */
338 : txn_bitset_t sigverify_done_set[ txn_bitset_word_cnt ]; /* Indexed by txn_idx. */
339 : txn_bitset_t poh_mixin_done_set[ txn_bitset_word_cnt ]; /* Indexed by txn_idx. */
340 : fd_sched_block_t * block_pool; /* Just a flat array. */
341 : ulong block_pool_popcnt;
342 : ulong * ref_q;
343 : };
344 : typedef struct fd_sched fd_sched_t;
345 :
346 :
347 : /* Internal helpers. */
348 :
349 : static int
350 : verify_ticks_eager( fd_sched_block_t * block );
351 :
352 : static int
353 : verify_ticks_final( fd_sched_block_t * block );
354 :
355 : static int
356 : ag_on_final( fd_sched_block_t * block );
357 :
358 : static void
359 : add_block( fd_sched_t * sched,
360 : ulong bank_idx,
361 : ulong parent_bank_idx );
362 :
363 : FD_WARN_UNUSED static int
364 : fd_sched_parse( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx );
365 :
366 : FD_WARN_UNUSED static int
367 : fd_sched_parse_txn( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx );
368 :
369 : static void
370 : dispatch_sigverify( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, fd_sched_task_t * out );
371 :
372 : static uint
373 : shred_split( fd_sched_t * sched,
374 : fd_sched_block_t * block,
375 : uint query );
376 :
377 : static void
378 : dispatch_poh( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, ulong max_cnt, fd_sched_task_t * out );
379 :
380 : static int
381 : poh_retire_mblk( fd_sched_t * sched, fd_sched_block_t * block, uint mblk_idx );
382 :
383 : static int
384 : poh_retire_ready_mblks( fd_sched_t * sched, fd_sched_block_t * block );
385 :
386 : static inline ulong
387 0 : poh_batch_max_cnt( fd_sched_t const * sched, ulong queued_cnt, ulong free_cnt, ulong threshold ) {
388 0 : ulong tile_cnt = free_cnt-threshold;
389 0 : ulong per_tile = (queued_cnt+tile_cnt-1UL)/tile_cnt;
390 0 : ulong max_cnt = fd_ulong_min( fd_ulong_max( per_tile, 1UL ), sched->poh_simd_max );
391 0 : return fd_ulong_if( max_cnt<sched->poh_simd_min, 1UL, max_cnt );
392 0 : }
393 :
394 : FD_WARN_UNUSED static int
395 : maybe_mixin( fd_sched_t * sched, fd_sched_block_t * block );
396 :
397 : static void
398 0 : free_mblk( fd_sched_t * sched, fd_sched_block_t * block, uint mblk_idx ) {
399 0 : sched->mblk_pool[ mblk_idx ].next = sched->mblk_pool_free_head;
400 0 : sched->mblk_pool_free_head = mblk_idx;
401 0 : sched->mblk_pool_free_cnt++;
402 0 : block->mblk_freed_cnt++;
403 0 : }
404 :
405 : static void
406 0 : free_mblk_slist( fd_sched_t * sched, fd_sched_block_t * block, mblk_slist_t * list ) {
407 0 : while( !mblk_slist_is_empty( list, sched->mblk_pool ) ) {
408 0 : uint idx = (uint)mblk_slist_idx_pop_head( list, sched->mblk_pool );
409 0 : free_mblk( sched, block, idx );
410 0 : }
411 0 : }
412 :
413 : static void
414 : try_activate_block( fd_sched_t * sched );
415 :
416 : static void
417 : check_or_set_active_block( fd_sched_t * sched );
418 :
419 : static void
420 : subtree_abandon( fd_sched_t * sched, fd_sched_block_t * block );
421 :
422 : static void
423 : subtree_release_refcnt( fd_sched_t * sched, fd_sched_block_t * block );
424 :
425 : static void
426 : subtree_prune( fd_sched_t * sched, ulong bank_idx, ulong except_idx );
427 :
428 : static void
429 : maybe_switch_block( fd_sched_t * sched, ulong bank_idx );
430 :
431 : FD_FN_UNUSED static ulong
432 : find_and_stage_longest_unstaged_fork( fd_sched_t * sched, int lane_idx );
433 :
434 : static ulong
435 : compute_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx );
436 :
437 : static ulong
438 : stage_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx, int lane_idx );
439 :
440 : static int
441 : lane_is_demotable( fd_sched_t * sched, int lane_idx );
442 :
443 : static ulong
444 : demote_lane( fd_sched_t * sched, int lane_idx );
445 :
446 : static inline fd_sched_block_t *
447 0 : block_pool_ele( fd_sched_t * sched, ulong idx ) {
448 0 : FD_TEST( idx<sched->block_cnt_max || idx==ULONG_MAX );
449 0 : return idx==ULONG_MAX ? NULL : sched->block_pool+idx;
450 0 : }
451 :
452 : FD_FN_UNUSED static inline int
453 0 : block_is_void( fd_sched_block_t * block ) {
454 0 : /* We've seen everything in the block and no transaction got parsed
455 0 : out. */
456 0 : return block->fec_eos && block->txn_parsed_cnt==0;
457 0 : }
458 :
459 : static inline int
460 0 : block_should_signal_end( fd_sched_block_t * block ) {
461 : /* Under the current policy of eager synchronous PoH mixin, hashing
462 : done plus fec_eos imply that all mixins have been done. */
463 0 : if( FD_UNLIKELY( !( !block->fec_eos || ((block->mblk_cnt==block->poh_hashing_done_cnt&&block->mblk_cnt==block->poh_hash_cmp_done_cnt)||block->mblk_cnt!=block->poh_hashing_done_cnt) ) ) ) FD_LOG_CRIT(( "invariant violation: slot %lu fec_eos %d mblk_cnt %u poh_hashing_done_cnt %u poh_hash_cmp_done_cnt %u", block->slot, block->fec_eos, block->mblk_cnt, block->poh_hashing_done_cnt, block->poh_hash_cmp_done_cnt ));
464 0 : return block->fec_eos && block->txn_parsed_cnt==block->txn_done_cnt && block->mblk_cnt==block->poh_hashing_done_cnt && block->block_start_done && !block->block_end_signaled;
465 0 : }
466 :
467 : static inline int
468 0 : block_will_signal_end( fd_sched_block_t * block ) {
469 0 : return block->fec_eos && !block->block_end_signaled;
470 0 : }
471 :
472 : /* Is there something known to be dispatchable in the block? This is an
473 : important liveness property. A block that doesn't contain any known
474 : dispatchable tasks will be deactivated or demoted. */
475 : static inline int
476 0 : block_is_dispatchable( fd_sched_block_t * block ) {
477 0 : ulong exec_queued_cnt = block->txn_parsed_cnt-block->txn_exec_in_flight_cnt-block->txn_exec_done_cnt;
478 0 : ulong sigverify_queued_cnt = block->txn_parsed_cnt-block->txn_sigverify_in_flight_cnt-block->txn_sigverify_done_cnt;
479 0 : ulong poh_queued_cnt = block->mblk_cnt-block->poh_mblk_in_flight_cnt-block->poh_hashing_done_cnt;
480 0 : return exec_queued_cnt>0UL ||
481 0 : sigverify_queued_cnt>0UL ||
482 0 : poh_queued_cnt>0UL ||
483 0 : !block->block_start_signaled ||
484 0 : block_will_signal_end( block );
485 0 : }
486 :
487 : static inline int
488 0 : block_is_in_flight( fd_sched_block_t * block ) {
489 0 : return block->txn_exec_in_flight_cnt || block->txn_sigverify_in_flight_cnt || block->poh_hashing_in_flight_cnt || block->cert_in_flight_cnt || (block->block_end_signaled && !block->block_end_done);
490 0 : }
491 :
492 : static inline int
493 0 : block_is_done( fd_sched_block_t * block ) {
494 0 : return block->fec_eos && block->txn_parsed_cnt==block->txn_done_cnt && block->mblk_cnt==block->poh_hash_cmp_done_cnt && block->block_start_done && block->block_end_done && !block->cert_in_flight_cnt;
495 0 : }
496 :
497 : static inline int
498 0 : block_is_stageable( fd_sched_block_t * block ) {
499 0 : int rv = !block_is_done( block ) && !block->dying;
500 0 : if( FD_UNLIKELY( rv && !block->in_rdisp ) ) {
501 : /* Invariant: stageable blocks may be currently staged or unstaged,
502 : but must be in the dispatcher either way. When a block
503 : transitions to DONE, it will be immediately removed from the
504 : dispatcher. When a block transitions to DYING, it will be
505 : eventually abandoned from the dispatcher. */
506 0 : FD_LOG_CRIT(( "invariant violation: stageable block->in_rdisp==0, txn_parsed_cnt %u, txn_done_cnt %u, fec_eos %u,, slot %lu, parent slot %lu",
507 0 : block->txn_parsed_cnt, block->txn_done_cnt, (uint)block->fec_eos, block->slot, block->parent_slot ));
508 0 : }
509 0 : return rv;
510 0 : }
511 :
512 : static inline int
513 0 : block_is_promotable( fd_sched_block_t * block ) {
514 0 : return block_is_stageable( block ) && block_is_dispatchable( block ) && !block->staged;
515 0 : }
516 :
517 : static inline int
518 0 : block_is_demotable( fd_sched_block_t * block ) {
519 : /* A block can only be demoted from rdisp if it is empty, meaning no
520 : PENDING, READY, or DISPATCHED transactions. This is equivalent to
521 : having no in-flight transactions (DISPATCHED) and no queued
522 : transactions (PENDING or READY). This function actually implements
523 : a stronger requirement. We consider a block demotable only if
524 : there are no in-flight or queued tasks of any kind. */
525 0 : return !block_is_in_flight( block ) && !block_is_dispatchable( block ) && block->staged;
526 0 : }
527 :
528 : static inline int
529 0 : block_is_activatable( fd_sched_block_t * block ) {
530 0 : return block_is_stageable( block ) && block_is_dispatchable( block ) && block->staged;
531 0 : }
532 :
533 : static inline int
534 0 : block_should_deactivate( fd_sched_block_t * block ) {
535 : /* We allow a grace period, during which a block has nothing to
536 : dispatch, but has something in-flight. The block is allowed to
537 : stay activated and ingest FEC sets during this time. The block
538 : will be deactivated if there's still nothing to dispatch by the
539 : time all in-flight tasks are completed. */
540 0 : return !block_is_activatable( block ) && !block_is_in_flight( block );
541 0 : }
542 :
543 : static inline int
544 0 : block_is_prunable( fd_sched_block_t * block ) {
545 0 : return !block->in_rdisp && !block_is_in_flight( block );
546 0 : }
547 :
548 : static inline ulong
549 0 : block_to_idx( fd_sched_t * sched, fd_sched_block_t * block ) { return (ulong)(block-sched->block_pool); }
550 :
551 : __attribute__((format(printf,2,3)))
552 : static void
553 : fd_sched_printf( fd_sched_t * sched,
554 : char const * fmt,
555 0 : ... ) {
556 0 : va_list ap;
557 0 : ulong len;
558 0 : va_start( ap, fmt );
559 0 : int ret = vsnprintf( sched->print_buf+sched->print_buf_sz,
560 0 : FD_SCHED_MAX_PRINT_BUF_SZ-sched->print_buf_sz,
561 0 : fmt, ap );
562 0 : va_end( ap );
563 0 : len = fd_ulong_if( ret<0, 0UL, fd_ulong_min( (ulong)ret, FD_SCHED_MAX_PRINT_BUF_SZ-sched->print_buf_sz-1UL ) );
564 0 : sched->print_buf[ sched->print_buf_sz+len ] = '\0';
565 0 : sched->print_buf_sz += len;
566 0 : }
567 :
568 : FD_FN_UNUSED static void
569 0 : print_histogram( fd_sched_t * sched, fd_histf_t * hist, ulong converter, char * title ) {
570 0 : fd_sched_printf( sched, " +---------------------+----------------------+--------------+\n" );
571 0 : fd_sched_printf( sched, " | %-19s | | Count |\n", title );
572 0 : fd_sched_printf( sched, " +---------------------+----------------------+--------------+\n" );
573 0 :
574 0 : ulong total_count = 0;
575 0 : for( ulong i=0UL; i<fd_histf_bucket_cnt( hist ); i++ ) {
576 0 : total_count += fd_histf_cnt( hist, i );
577 0 : }
578 0 :
579 0 : for( ulong i=0UL; i< fd_histf_bucket_cnt( hist ); i++ ) {
580 0 : ulong bucket_count = fd_histf_cnt( hist, i );
581 0 :
582 0 : char * lt_str;
583 0 : char lt_buf[ 64 ];
584 0 : if( FD_UNLIKELY( i==fd_histf_bucket_cnt( hist )-1UL ) ) {
585 0 : lt_str = "+Inf";
586 0 : } else {
587 0 : ulong edge = fd_histf_right( hist, i );
588 0 : if( converter==FD_METRICS_CONVERTER_NANOSECONDS ) {
589 0 : edge = fd_metrics_convert_ticks_to_nanoseconds( edge-1UL );
590 0 : FD_TEST( fd_cstr_printf_check( lt_buf, sizeof( lt_buf ), NULL, "<= %lu nanos", edge ) );
591 0 : } else if( converter==FD_METRICS_CONVERTER_NONE ) {
592 0 : FD_TEST( fd_cstr_printf_check( lt_buf, sizeof( lt_buf ), NULL, "<= %lu", edge-1UL ) );
593 0 : }
594 0 : lt_str = lt_buf;
595 0 : }
596 0 :
597 0 : /* Create visual bar - scale to max 20 characters. */
598 0 : char bar_buf[ 22 ];
599 0 : if( bucket_count>0UL && total_count>0UL ) {
600 0 : ulong bar_length = (bucket_count*20UL)/total_count;
601 0 : if( !bar_length ) bar_length = 1;
602 0 : for( ulong j=0UL; j<bar_length; j++ ) { bar_buf[ j ] = '*'; }
603 0 : bar_buf[ bar_length ] = '\0';
604 0 : } else {
605 0 : bar_buf[ 0 ] = '\0';
606 0 : }
607 0 :
608 0 : fd_sched_printf( sched, " | %19s | %-20s | %12lu |\n", lt_str, bar_buf, bucket_count );
609 0 : }
610 0 : }
611 :
612 : FD_FN_UNUSED static void
613 0 : print_block_metrics( fd_sched_t * sched, fd_sched_block_t * block ) {
614 0 : fd_sched_printf( sched, "block idx %lu, block slot %lu, parent_slot %lu, fec_eos %d, rooted %d, txn_parsed_cnt %u, txn_exec_done_cnt %u, txn_sigverify_done_cnt %u, poh_hashing_done_cnt %u, poh_hash_cmp_done_cnt %u, txn_done_cnt %u, shred_cnt %u, mblk_cnt %u, mblk_freed_cnt %u, mblk_tick_cnt %u, mblk_unhashed_cnt %u, hashcnt %lu, txn_pool_max_popcnt %lu/%lu, mblk_pool_max_popcnt %lu/%lu, block_pool_max_popcnt %lu/%lu, mblks_rem %lu, txns_rem %lu, fec_buf_sz %u, fec_buf_boff %u, fec_buf_soff %u, fec_eob %d, fec_sob %d\n",
615 0 : block_to_idx( sched, block ), block->slot, block->parent_slot, block->fec_eos, block->rooted, block->txn_parsed_cnt, block->txn_exec_done_cnt, block->txn_sigverify_done_cnt, block->poh_hashing_done_cnt, block->poh_hash_cmp_done_cnt, block->txn_done_cnt, block->shred_cnt, block->mblk_cnt, block->mblk_freed_cnt, block->mblk_tick_cnt, block->mblk_unhashed_cnt, block->hashcnt, block->txn_pool_max_popcnt, sched->depth, block->mblk_pool_max_popcnt, sched->depth, block->block_pool_max_popcnt, sched->block_cnt_max, block->mblks_rem, block->txns_rem, block->fec_buf_sz, block->fec_buf_boff, block->fec_buf_soff, block->fec_eob, block->fec_sob );
616 0 : }
617 :
618 : FD_FN_UNUSED static void
619 0 : print_block_debug( fd_sched_t * sched, fd_sched_block_t * block ) {
620 0 : fd_sched_printf( sched, "block idx %lu, block slot %lu, parent_slot %lu, staged %d (lane %lu), dying %d, in_rdisp %d, fec_eos %d, rooted %d, block_start_signaled %d, block_end_signaled %d, block_start_done %d, block_end_done %d, txn_parsed_cnt %u, txn_exec_in_flight_cnt %u, txn_exec_done_cnt %u, txn_sigverify_in_flight_cnt %u, txn_sigverify_done_cnt %u, poh_hashing_in_flight_cnt %u, poh_mblk_in_flight_cnt %u, cert_in_flight_cnt %u, poh_hashing_done_cnt %u, poh_hash_cmp_done_cnt %u, txn_done_cnt %u, shred_cnt %u, mblk_cnt %u, mblk_freed_cnt %u, mblk_tick_cnt %u, mblk_unhashed_cnt %u, hashcnt %lu, txn_pool_max_popcnt %lu/%lu, mblk_pool_max_popcnt %lu/%lu, block_pool_max_popcnt %lu/%lu, tick_hashcnt_wmk %lu, curr_tick_hashcnt %lu, hashes_per_tick %lu, mblks_rem %lu, txns_rem %lu, fec_buf_sz %u, fec_buf_boff %u, fec_buf_soff %u, fec_eob %d, fec_sob %d\n",
621 0 : block_to_idx( sched, block ), block->slot, block->parent_slot, block->staged, block->staging_lane, block->dying, block->in_rdisp, block->fec_eos, block->rooted, block->block_start_signaled, block->block_end_signaled, block->block_start_done, block->block_end_done, block->txn_parsed_cnt, block->txn_exec_in_flight_cnt, block->txn_exec_done_cnt, block->txn_sigverify_in_flight_cnt, block->txn_sigverify_done_cnt, block->poh_hashing_in_flight_cnt, block->poh_mblk_in_flight_cnt, block->cert_in_flight_cnt, block->poh_hashing_done_cnt, block->poh_hash_cmp_done_cnt, block->txn_done_cnt, block->shred_cnt, block->mblk_cnt, block->mblk_freed_cnt, block->mblk_tick_cnt, block->mblk_unhashed_cnt, block->hashcnt, block->txn_pool_max_popcnt, sched->depth, block->mblk_pool_max_popcnt, sched->depth, block->block_pool_max_popcnt, sched->block_cnt_max, block->tick_hashcnt_wmk, block->curr_tick_hashcnt, block->hashes_per_tick, block->mblks_rem, block->txns_rem, block->fec_buf_sz, block->fec_buf_boff, block->fec_buf_soff, block->fec_eob, block->fec_sob );
622 0 : }
623 :
624 : FD_FN_UNUSED static void
625 0 : print_block_and_parent( fd_sched_t * sched, fd_sched_block_t * block ) {
626 0 : print_block_debug( sched, block );
627 0 : fd_sched_block_t * parent = block_pool_ele( sched, block->parent_idx );
628 0 : if( FD_LIKELY( parent ) ) print_block_debug( sched, parent );
629 0 : }
630 :
631 : FD_FN_UNUSED static void
632 0 : print_metrics( fd_sched_t * sched ) {
633 0 : fd_sched_printf( sched, "metrics: block_added_cnt %u, block_added_staged_cnt %u, block_added_unstaged_cnt %u, block_added_dead_ood_cnt %u, block_removed_cnt %u, block_abandoned_cnt %u, block_bad_cnt %u, block_promoted_cnt %u, block_demoted_cnt %u, deactivate_no_child_cnt %u, deactivate_no_txn_cnt %u, deactivate_pruned_cnt %u, deactivate_abandoned_cnt %u, lane_switch_cnt %u, lane_promoted_cnt %u, lane_demoted_cnt %u, fork_observed_cnt %u, alut_success_cnt %u, alut_serializing_cnt %u, poison_latch_overflow_cnt %u, poison_latch_alt_cnt %u, poison_cnt_wmk %u, txn_poison_serializing_cnt %u, txn_poisoned_cnt %u, txn_abandoned_parsed_cnt %u, txn_abandoned_exec_done_cnt %u, txn_abandoned_done_cnt %u, txn_max_in_flight_cnt %u, txn_weighted_in_flight_cnt %lu, txn_weighted_in_flight_tickcount %lu, txn_none_in_flight_tickcount %lu, txn_parsed_cnt %lu, txn_exec_done_cnt %lu, txn_sigverify_done_cnt %lu, txn_mixin_done_cnt %lu, txn_done_cnt %lu, mblk_parsed_cnt %lu, mblk_poh_hashed_cnt %lu, mblk_poh_done_cnt %lu, bytes_ingested_cnt %lu, bytes_ingested_unparsed_cnt %lu, bytes_dropped_cnt %lu, fec_cnt %lu\n",
634 0 : sched->metrics->block_added_cnt, sched->metrics->block_added_staged_cnt, sched->metrics->block_added_unstaged_cnt, sched->metrics->block_added_dead_ood_cnt, sched->metrics->block_removed_cnt, sched->metrics->block_abandoned_cnt, sched->metrics->block_bad_cnt, sched->metrics->block_promoted_cnt, sched->metrics->block_demoted_cnt, sched->metrics->deactivate_no_child_cnt, sched->metrics->deactivate_no_txn_cnt, sched->metrics->deactivate_pruned_cnt, sched->metrics->deactivate_abandoned_cnt, sched->metrics->lane_switch_cnt, sched->metrics->lane_promoted_cnt, sched->metrics->lane_demoted_cnt, sched->metrics->fork_observed_cnt, sched->metrics->alut_success_cnt, sched->metrics->alut_serializing_cnt, sched->metrics->poison_latch_overflow_cnt, sched->metrics->poison_latch_alt_cnt, sched->metrics->poison_cnt_wmk, sched->metrics->txn_poison_serializing_cnt, sched->metrics->txn_poisoned_cnt, sched->metrics->txn_abandoned_parsed_cnt, sched->metrics->txn_abandoned_exec_done_cnt, sched->metrics->txn_abandoned_done_cnt, sched->metrics->txn_max_in_flight_cnt, sched->metrics->txn_weighted_in_flight_cnt, sched->metrics->txn_weighted_in_flight_tickcount, sched->metrics->txn_none_in_flight_tickcount, sched->metrics->txn_parsed_cnt, sched->metrics->txn_exec_done_cnt, sched->metrics->txn_sigverify_done_cnt, sched->metrics->txn_mixin_done_cnt, sched->metrics->txn_done_cnt, sched->metrics->mblk_parsed_cnt, sched->metrics->mblk_poh_hashed_cnt, sched->metrics->mblk_poh_done_cnt, sched->metrics->bytes_ingested_cnt, sched->metrics->bytes_ingested_unparsed_cnt, sched->metrics->bytes_dropped_cnt, sched->metrics->fec_cnt );
635 0 : }
636 :
637 : FD_FN_UNUSED static void
638 0 : print_sched( fd_sched_t * sched ) {
639 0 : fd_sched_printf( sched, "sched canary 0x%lx, exec_cnt %lu, root_idx %lu, txn_exec_ready_bitset[ 0 ] 0x%lx, sigverify_ready_bitset[ 0 ] 0x%lx, poh_ready_bitset[ 0 ] 0x%lx, cert_ready_bitset[ 0 ] 0x%lx, active_idx %lu, staged_bitset %lu, staged_head_idx[0] %lu, staged_head_idx[1] %lu, staged_head_idx[2] %lu, staged_head_idx[3] %lu, staged_popcnt_wmk %lu, txn_pool_free_cnt %lu/%lu, block_pool_popcnt %lu/%lu\n",
640 0 : sched->canary, sched->exec_cnt, sched->root_idx, sched->txn_exec_ready_bitset[ 0 ], sched->sigverify_ready_bitset[ 0 ], sched->poh_ready_bitset[ 0 ], sched->cert_ready_bitset[ 0 ], sched->active_bank_idx, sched->staged_bitset, sched->staged_head_bank_idx[ 0 ], sched->staged_head_bank_idx[ 1 ], sched->staged_head_bank_idx[ 2 ], sched->staged_head_bank_idx[ 3 ], sched->staged_popcnt_wmk, sched->txn_pool_free_cnt, sched->depth, sched->block_pool_popcnt, sched->block_cnt_max );
641 0 : fd_sched_block_t * active_block = block_pool_ele( sched, sched->active_bank_idx );
642 0 : if( active_block ) print_block_debug( sched, active_block );
643 0 : for( int l=0; l<(int)FD_SCHED_MAX_STAGING_LANES; l++ ) {
644 0 : if( fd_ulong_extract_bit( sched->staged_bitset, l ) ) {
645 0 : fd_sched_block_t * block = block_pool_ele( sched, sched->staged_head_bank_idx[ l ] );
646 0 : print_block_debug( sched, block );
647 0 : }
648 0 : }
649 0 : }
650 :
651 : FD_FN_UNUSED static void
652 0 : print_all( fd_sched_t * sched, fd_sched_block_t * block ) {
653 0 : print_metrics( sched );
654 0 : print_sched( sched );
655 0 : print_block_and_parent( sched, block );
656 0 : }
657 :
658 : static inline void
659 : record_dead_reason( fd_sched_block_t * block,
660 0 : int dead_reason ) {
661 0 : if( FD_LIKELY( block->dead_reason==FD_SCHED_DEAD_REASON_NONE ) ) {
662 0 : block->dead_reason = dead_reason;
663 0 : block->discarded = 0;
664 0 : }
665 0 : }
666 :
667 : /* Record that the block goes down with its lineage rather than for a
668 : defect of its own, aka DEAD_ANCESTOR. Also record the parent's
669 : deadness flavor so we can tell a discarded (but not invalid) lineage
670 : from an invalid one.
671 :
672 : Gated on the block having no dead reason of its own, so a verdict
673 : sched already reached is never overwritten. That gate neither bounds
674 : how often the inherit runs nor sees a ruling the replay tile made,
675 : which leaves dead_reason NONE. Callers are responsible for invoking
676 : this only on a block that is newly going down. */
677 : static inline void
678 0 : record_lineage_death( fd_sched_block_t * block, fd_sched_block_t const * parent ) {
679 0 : if( FD_LIKELY( block->dead_reason==FD_SCHED_DEAD_REASON_NONE ) ) {
680 0 : record_dead_reason( block, FD_SCHED_DEAD_REASON_DEAD_ANCESTOR );
681 0 : block->discarded = parent->discarded;
682 0 : }
683 0 : }
684 :
685 : static void
686 0 : handle_bad_block( fd_sched_t * sched, fd_sched_block_t * block, int dead_reason ) {
687 0 : record_dead_reason( block, dead_reason );
688 0 : sched->print_buf_sz = 0UL;
689 0 : print_all( sched, block );
690 0 : FD_LOG_DEBUG(( "%s", sched->print_buf ));
691 0 : subtree_abandon( sched, block );
692 0 : sched->metrics->block_bad_cnt++;
693 0 : check_or_set_active_block( sched );
694 0 : }
695 :
696 : /* Returns the number of shred boundaries strictly before query.
697 : Transaction offsets are visited in nondecreasing order, so each
698 : shred length is scanned at most once per block. */
699 : static uint
700 : shred_split( fd_sched_t * sched,
701 : fd_sched_block_t * block,
702 0 : uint query ) {
703 0 : FD_TEST( block->shred_scan_idx<=block->shred_cnt );
704 0 : FD_TEST( block->shred_scan_off<=query );
705 0 : ushort const * shred_sz = sched->shred_sz + block_to_idx( sched, block )*sched->max_shreds_per_block;
706 0 : while( block->shred_scan_idx<block->shred_cnt ) {
707 0 : uint next_off = block->shred_scan_off + (uint)shred_sz[ block->shred_scan_idx ];
708 0 : if( next_off>=query ) break;
709 0 : block->shred_scan_off = next_off;
710 0 : block->shred_scan_idx++;
711 0 : }
712 0 : return block->shred_scan_idx;
713 0 : }
714 :
715 :
716 : /* Public functions. */
717 :
718 : ulong
719 0 : fd_sched_align( void ) {
720 0 : return fd_ulong_max( alignof(fd_sched_t),
721 0 : fd_ulong_max( fd_rdisp_align(),
722 0 : fd_ulong_max( alignof(fd_sched_block_t), 64UL ))); /* Minimally cache line aligned. */
723 0 : }
724 :
725 : ulong
726 : fd_sched_footprint( ulong depth,
727 : ulong block_cnt_max,
728 : ulong max_shreds_per_block,
729 0 : ulong max_txn_per_slot ) {
730 0 : if( FD_UNLIKELY( depth<FD_SCHED_MIN_DEPTH || depth>FD_SCHED_MAX_DEPTH ) ) return 0UL; /* bad depth */
731 0 : if( FD_UNLIKELY( !block_cnt_max ) ) return 0UL; /* bad block_cnt_max */
732 0 : if( FD_UNLIKELY( depth>UINT_MAX-1UL ) ) return 0UL; /* mblk_pool use uint as pointers */
733 0 : if( FD_UNLIKELY( !max_shreds_per_block || max_shreds_per_block>UINT_MAX ) ) return 0UL; /* shred_cnt is uint */
734 0 : if( FD_UNLIKELY( !max_txn_per_slot || max_txn_per_slot >UINT_MAX ) ) return 0UL; /* txn_parsed_cnt is uint */
735 :
736 0 : ulong l = FD_LAYOUT_INIT;
737 0 : l = FD_LAYOUT_APPEND( l, fd_sched_align(), sizeof(fd_sched_t) );
738 0 : l = FD_LAYOUT_APPEND( l, fd_rdisp_align(), fd_rdisp_footprint( depth, block_cnt_max ) ); /* dispatcher */
739 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_block_t), block_cnt_max*sizeof(fd_sched_block_t) ); /* block pool */
740 0 : l = FD_LAYOUT_APPEND( l, alignof(ushort), block_cnt_max*max_shreds_per_block*sizeof(ushort) ); /* shred_sz */
741 0 : l = FD_LAYOUT_APPEND( l, ref_q_align(), ref_q_footprint( block_cnt_max ) );
742 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_txn_p_t), depth*sizeof(fd_txn_p_t) ); /* txn_pool */
743 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_txn_info_t), depth*sizeof(fd_sched_txn_info_t) ); /* txn_info_pool */
744 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_mblk_t), depth*sizeof(fd_sched_mblk_t) ); /* mblk_pool */
745 0 : return FD_LAYOUT_FINI( l, fd_sched_align() );
746 0 : }
747 :
748 : void *
749 : fd_sched_new( void * mem,
750 : fd_rng_t * rng,
751 : ulong depth,
752 : ulong block_cnt_max,
753 : ulong max_shreds_per_block,
754 : ulong max_txn_per_slot,
755 : ulong exec_cnt,
756 0 : int is_alpenglow ) {
757 :
758 0 : if( FD_UNLIKELY( !mem ) ) {
759 0 : FD_LOG_WARNING(( "NULL mem" ));
760 0 : return NULL;
761 0 : }
762 :
763 0 : if( FD_UNLIKELY( !rng ) ) {
764 0 : FD_LOG_WARNING(( "NULL rng" ));
765 0 : return NULL;
766 0 : }
767 :
768 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_sched_align() ) ) ) {
769 0 : FD_LOG_WARNING(( "misaligned mem (%p)", mem ));
770 0 : return NULL;
771 0 : }
772 :
773 0 : if( FD_UNLIKELY( depth<FD_SCHED_MIN_DEPTH || depth>FD_SCHED_MAX_DEPTH ) ) {
774 0 : FD_LOG_WARNING(( "bad depth (%lu)", depth ));
775 0 : return NULL;
776 0 : }
777 :
778 0 : if( FD_UNLIKELY( !block_cnt_max ) ) {
779 0 : FD_LOG_WARNING(( "bad block_cnt_max (%lu)", block_cnt_max ));
780 0 : return NULL;
781 0 : }
782 :
783 0 : if( FD_UNLIKELY( depth>UINT_MAX-1UL ) ) {
784 0 : FD_LOG_WARNING(( "bad depth (%lu)", depth ));
785 0 : return NULL;
786 0 : }
787 :
788 0 : if( FD_UNLIKELY( !max_shreds_per_block || max_shreds_per_block>UINT_MAX ) ) {
789 0 : FD_LOG_WARNING(( "bad max_shreds_per_block (%lu)", max_shreds_per_block ));
790 0 : return NULL;
791 0 : }
792 :
793 0 : if( FD_UNLIKELY( !max_txn_per_slot || max_txn_per_slot>UINT_MAX ) ) {
794 0 : FD_LOG_WARNING(( "bad max_txn_per_slot (%lu)", max_txn_per_slot ));
795 0 : return NULL;
796 0 : }
797 :
798 0 : if( FD_UNLIKELY( !exec_cnt || exec_cnt>FD_SCHED_MAX_EXEC_TILE_CNT ) ) {
799 0 : FD_LOG_WARNING(( "bad exec_cnt (%lu)", exec_cnt ));
800 0 : return NULL;
801 0 : }
802 :
803 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
804 0 : fd_sched_t * sched = FD_SCRATCH_ALLOC_APPEND( l, fd_sched_align(), sizeof(fd_sched_t) );
805 0 : void * _rdisp = FD_SCRATCH_ALLOC_APPEND( l, fd_rdisp_align(), fd_rdisp_footprint( depth, block_cnt_max ) );
806 0 : void * _bpool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_block_t), block_cnt_max*sizeof(fd_sched_block_t) );
807 0 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(ushort), block_cnt_max*max_shreds_per_block*sizeof(ushort) );
808 0 : void * _ref_q = FD_SCRATCH_ALLOC_APPEND( l, ref_q_align(), ref_q_footprint( block_cnt_max ) );
809 0 : fd_txn_p_t * _txn_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txn_p_t), depth*sizeof(fd_txn_p_t) );
810 0 : fd_sched_txn_info_t * _txn_info_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_txn_info_t), depth*sizeof(fd_sched_txn_info_t) );
811 0 : fd_sched_mblk_t * _mblk_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_mblk_t), depth*sizeof(fd_sched_mblk_t) );
812 0 : FD_SCRATCH_ALLOC_FINI( l, fd_sched_align() );
813 :
814 0 : sched->txn_pool = _txn_pool;
815 0 : sched->txn_info_pool = _txn_info_pool;
816 0 : sched->mblk_pool = _mblk_pool;
817 :
818 0 : fd_rdisp_new( _rdisp, depth, block_cnt_max, fd_rng_ulong( rng ) );
819 :
820 0 : fd_sched_block_t * bpool = (fd_sched_block_t *)_bpool;
821 0 : for( ulong i=0; i<block_cnt_max; i++ ) {
822 0 : bpool[ i ].in_sched = 0;
823 0 : mblk_slist_new( bpool[ i ].mblks_unhashed );
824 0 : mblk_slist_new( bpool[ i ].mblks_hashing_in_progress );
825 0 : mblk_slist_new( bpool[ i ].mblks_mixin_in_progress );
826 0 : }
827 :
828 0 : FD_TEST( fd_chkdup_new( sched->chkdup, rng ) );
829 :
830 0 : fd_memset( sched->metrics, 0, sizeof(fd_sched_metrics_t) );
831 0 : sched->txn_in_flight_last_tick = fd_tickcount(); /* boot counts as idle */
832 0 : sched->next_ready_last_tick = LONG_MAX;
833 0 : sched->next_ready_last_bank_idx = ULONG_MAX;
834 :
835 0 : sched->is_alpenglow = is_alpenglow;
836 0 : sched->canary = FD_SCHED_MAGIC;
837 0 : sched->depth = depth;
838 0 : sched->block_cnt_max = block_cnt_max;
839 0 : sched->max_shreds_per_block = max_shreds_per_block;
840 0 : sched->max_txn_per_slot = max_txn_per_slot;
841 0 : sched->max_mblk_per_slot = fd_ulong_min( FD_SCHED_MAX_MBLK_PER_SLOT*max_shreds_per_block/FD_SHRED_BLK_MAX, UINT_MAX ); /* mblk_cnt is uint */
842 0 : sched->exec_cnt = exec_cnt;
843 0 : sched->poh_simd_max = fd_sha256_simd_lane_max(); FD_CHECK_ERR( sched->poh_simd_max<=FD_SCHED_POH_PARA, "overly wide PoH SHA batch" );
844 0 : sched->poh_simd_min = fd_sha256_simd_lane_min();
845 0 : sched->bypass_poh_verify = 0;
846 0 : sched->bypass_alut_resolution = 0;
847 0 : sched->root_idx = ULONG_MAX;
848 0 : sched->active_bank_idx = ULONG_MAX;
849 0 : sched->last_active_bank_idx = ULONG_MAX;
850 0 : sched->staged_bitset = 0UL;
851 0 : sched->staged_popcnt_wmk = 0UL;
852 :
853 0 : sched->txn_exec_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
854 0 : sched->sigverify_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
855 0 : sched->poh_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
856 0 : sched->cert_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
857 :
858 0 : sched->txn_pool_free_cnt = depth-1UL; /* -1 because index 0 is unusable as a sentinel reserved by the dispatcher */
859 :
860 0 : for( ulong i=0UL; i<depth-1UL; i++ ) sched->mblk_pool[ i ].next = (uint)(i+1UL);
861 0 : sched->mblk_pool[ depth-1UL ].next = UINT_MAX;
862 0 : sched->mblk_pool_free_head = 0U;
863 0 : sched->mblk_pool_free_cnt = depth;
864 :
865 0 : txn_bitset_new( sched->exec_done_set );
866 0 : txn_bitset_new( sched->sigverify_done_set );
867 0 : txn_bitset_new( sched->poh_mixin_done_set );
868 :
869 0 : sched->block_pool_popcnt = 0UL;
870 :
871 0 : ref_q_new( _ref_q, block_cnt_max );
872 :
873 0 : return sched;
874 0 : }
875 :
876 : fd_sched_t *
877 0 : fd_sched_join( void * mem ) {
878 :
879 0 : if( FD_UNLIKELY( !mem ) ) {
880 0 : FD_LOG_WARNING(( "NULL mem" ));
881 0 : return NULL;
882 0 : }
883 :
884 0 : fd_sched_t * sched = (fd_sched_t *)mem;
885 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
886 0 : ulong depth = sched->depth;
887 0 : ulong block_cnt_max = sched->block_cnt_max;
888 0 : ulong max_shreds_per_block = sched->max_shreds_per_block;
889 :
890 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
891 0 : /* */ FD_SCRATCH_ALLOC_APPEND( l, fd_sched_align(), sizeof(fd_sched_t) );
892 0 : void * _rdisp = FD_SCRATCH_ALLOC_APPEND( l, fd_rdisp_align(), fd_rdisp_footprint( depth, block_cnt_max ) );
893 0 : void * _bpool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_block_t), block_cnt_max*sizeof(fd_sched_block_t) );
894 0 : void * _shred_sz = FD_SCRATCH_ALLOC_APPEND( l, alignof(ushort), block_cnt_max*max_shreds_per_block*sizeof(ushort) );
895 0 : void * _ref_q = FD_SCRATCH_ALLOC_APPEND( l, ref_q_align(), ref_q_footprint( block_cnt_max ) );
896 0 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txn_p_t), depth*sizeof(fd_txn_p_t) );
897 0 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_txn_info_t), depth*sizeof(fd_sched_txn_info_t) );
898 0 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_mblk_t), depth*sizeof(fd_sched_mblk_t) );
899 0 : FD_SCRATCH_ALLOC_FINI( l, fd_sched_align() );
900 :
901 0 : sched->rdisp = fd_rdisp_join( _rdisp );
902 0 : sched->ref_q = ref_q_join( _ref_q );
903 0 : sched->block_pool = _bpool;
904 0 : sched->shred_sz = _shred_sz;
905 :
906 0 : for( ulong i=0; i<block_cnt_max; i++ ) {
907 0 : mblk_slist_join( sched->block_pool[ i ].mblks_unhashed );
908 0 : mblk_slist_join( sched->block_pool[ i ].mblks_hashing_in_progress );
909 0 : mblk_slist_join( sched->block_pool[ i ].mblks_mixin_in_progress );
910 0 : }
911 :
912 0 : txn_bitset_join( sched->exec_done_set );
913 0 : txn_bitset_join( sched->sigverify_done_set );
914 0 : txn_bitset_join( sched->poh_mixin_done_set );
915 :
916 0 : return sched;
917 0 : }
918 :
919 : int
920 0 : fd_sched_fec_can_ingest( fd_sched_t * sched, fd_sched_fec_t * fec ) {
921 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
922 0 : FD_TEST( fec->bank_idx<sched->block_cnt_max );
923 0 : FD_TEST( fec->parent_bank_idx<sched->block_cnt_max );
924 :
925 0 : if( FD_UNLIKELY( fec->fec->data_sz>FD_SCHED_MAX_PAYLOAD_PER_FEC ) ) {
926 0 : sched->print_buf_sz = 0UL;
927 0 : print_metrics( sched );
928 0 : print_sched( sched );
929 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
930 0 : FD_LOG_CRIT(( "invalid FEC set: fec->data_sz %u, slot %lu, parent slot %lu", fec->fec->data_sz, fec->slot, fec->parent_slot ));
931 0 : }
932 :
933 0 : ulong fec_buf_sz = 0UL;
934 0 : fd_sched_block_t * block = block_pool_ele( sched, fec->bank_idx );
935 0 : if( FD_LIKELY( !fec->is_first_in_block ) ) {
936 0 : fec_buf_sz += block->fec_buf_sz-block->fec_buf_soff;
937 0 : } else {
938 : /* No residual data as this is a fresh new block. */
939 0 : }
940 : /* Addition is safe and won't overflow because we checked the FEC set
941 : size above. */
942 0 : fec_buf_sz += fec->fec->data_sz;
943 : /* Assuming every transaction is min size, do we have enough free
944 : entries in the txn pool? For a more precise txn count, we would
945 : have to do some parsing. */
946 0 : return sched->txn_pool_free_cnt>=fec_buf_sz/FD_TXN_MIN_SERIALIZED_SZ && sched->mblk_pool_free_cnt>=fec_buf_sz/sizeof(fd_microblock_hdr_t);
947 0 : }
948 :
949 : ulong
950 0 : fd_sched_can_ingest_cnt( fd_sched_t * sched ) {
951 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
952 : /* Worst case, we need one byte from the incoming data to extract a
953 : transaction out of the residual data, and the rest of the incoming
954 : data contributes toward min sized transactions. */
955 0 : return fd_ulong_min( sched->txn_pool_free_cnt/FD_SCHED_MAX_TXN_PER_FEC, sched->mblk_pool_free_cnt/FD_SCHED_MAX_MBLK_PER_FEC );
956 0 : }
957 :
958 : int
959 0 : fd_sched_is_drained( fd_sched_t * sched ) {
960 0 : int nothing_inflight = sched->exec_cnt==(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ]&sched->sigverify_ready_bitset[ 0 ]&sched->poh_ready_bitset[ 0 ]&sched->cert_ready_bitset[ 0 ] );
961 0 : int nothing_queued = sched->active_bank_idx==ULONG_MAX;
962 0 : return nothing_inflight && nothing_queued;
963 0 : }
964 :
965 : ulong
966 : fd_sched_cert_dispatch( fd_sched_t * sched,
967 : ulong bank_idx,
968 0 : ulong cnt ) {
969 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
970 0 : FD_TEST( bank_idx==sched->active_bank_idx );
971 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
972 0 : FD_TEST( block->block_end_signaled && !block->block_end_done );
973 0 : ulong idle = sched->txn_exec_ready_bitset[ 0 ]&sched->sigverify_ready_bitset[ 0 ]&sched->poh_ready_bitset[ 0 ]&sched->cert_ready_bitset[ 0 ];
974 0 : ulong claimed = 0UL;
975 0 : for( ; idle && cnt; cnt--, idle=fd_ulong_pop_lsb( idle ) ) {
976 0 : int t = fd_ulong_find_lsb( idle );
977 0 : claimed = fd_ulong_set_bit( claimed, t );
978 0 : sched->tile_to_bank_idx[ t ] = bank_idx;
979 0 : block->cert_in_flight_cnt++;
980 0 : }
981 0 : sched->cert_ready_bitset[ 0 ] &= ~claimed;
982 0 : return claimed;
983 0 : }
984 :
985 : int
986 0 : fd_sched_block_start_pending( fd_sched_t * sched ) {
987 0 : return sched->active_bank_idx!=ULONG_MAX && !block_pool_ele( sched, sched->active_bank_idx )->block_start_signaled;
988 0 : }
989 :
990 : FD_WARN_UNUSED int
991 : fd_sched_fec_ingest( fd_sched_t * sched,
992 0 : fd_sched_fec_t * fec ) {
993 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
994 0 : FD_TEST( fec->bank_idx<sched->block_cnt_max );
995 0 : FD_TEST( fec->parent_bank_idx<sched->block_cnt_max );
996 0 : FD_TEST( ref_q_empty( sched->ref_q ) );
997 :
998 0 : fd_sched_block_t * block = block_pool_ele( sched, fec->bank_idx );
999 :
1000 0 : if( FD_UNLIKELY( fec->fec->data_sz>FD_SCHED_MAX_PAYLOAD_PER_FEC ) ) {
1001 0 : sched->print_buf_sz = 0UL;
1002 0 : print_all( sched, block );
1003 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1004 0 : FD_LOG_CRIT(( "invalid FEC set: fec->data_sz %u, slot %lu, parent slot %lu", fec->fec->data_sz, fec->slot, fec->parent_slot ));
1005 0 : }
1006 :
1007 0 : sched->metrics->fec_cnt++;
1008 :
1009 0 : if( FD_UNLIKELY( fec->is_first_in_block ) ) {
1010 : /* This is a new block. */
1011 0 : add_block( sched, fec->bank_idx, fec->parent_bank_idx );
1012 0 : block->slot = fec->slot;
1013 0 : block->parent_slot = fec->parent_slot;
1014 :
1015 0 : if( FD_UNLIKELY( block->dying ) ) {
1016 : /* The child of a dead block is also dead. We added it to our
1017 : fork tree just so we could track an entire lineage of dead
1018 : children and propagate the dead property to the entire lineage,
1019 : in case there were frags for more than one dead children
1020 : in-flight at the time the parent was abandoned. That being
1021 : said, we shouldn't need to add the dead child to the
1022 : dispatcher. */
1023 0 : sched->metrics->block_added_dead_ood_cnt++;
1024 :
1025 : /* Release the refcnt right away since we don't intend to replay
1026 : it at all. We do this so its bank does not linger and stall
1027 : root advancement until the next root notify. */
1028 0 : subtree_release_refcnt( sched, block );
1029 :
1030 : /* Ignore the FEC set for a dead block. */
1031 0 : sched->metrics->bytes_dropped_cnt += fec->fec->data_sz;
1032 0 : return 0;
1033 0 : }
1034 :
1035 : /* Try to find a staging lane for this block. */
1036 0 : int alloc_lane = 0;
1037 0 : fd_sched_block_t * parent_block = block_pool_ele( sched, fec->parent_bank_idx );
1038 0 : if( FD_LIKELY( parent_block->staged ) ) {
1039 : /* Parent is staged. So see if we can continue down the same
1040 : staging lane. */
1041 0 : ulong staging_lane = parent_block->staging_lane;
1042 0 : ulong child_idx = parent_block->child_idx;
1043 0 : while( child_idx!=ULONG_MAX ) {
1044 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
1045 0 : if( child->staged && child->staging_lane==staging_lane ) {
1046 : /* Found a child on the same lane. So we're done. */
1047 0 : staging_lane = FD_RDISP_UNSTAGED;
1048 0 : break;
1049 0 : }
1050 0 : child_idx = child->sibling_idx;
1051 0 : }
1052 : /* No child is staged on the same lane as the parent. So stage
1053 : this block. This is the common case. */
1054 0 : if( FD_LIKELY( staging_lane!=FD_RDISP_UNSTAGED ) ) {
1055 0 : block->in_rdisp = 1;
1056 0 : block->staged = 1;
1057 0 : block->staging_lane = staging_lane;
1058 0 : fd_rdisp_add_block( sched->rdisp, fec->bank_idx, staging_lane );
1059 0 : sched->metrics->block_added_cnt++;
1060 0 : sched->metrics->block_added_staged_cnt++;
1061 0 : FD_LOG_DEBUG(( "block %lu:%lu entered lane %lu: add", block->slot, fec->bank_idx, staging_lane ));
1062 0 : } else {
1063 0 : alloc_lane = 1;
1064 0 : }
1065 0 : } else {
1066 0 : if( block_is_stageable( parent_block ) ) {
1067 : /* Parent is unstaged but stageable. So let's be unstaged too.
1068 : This is not only a policy decision to be lazy and not promote
1069 : the parent at the moment, but also an important invariant
1070 : that we maintain for deadlock freeness in the face of staging
1071 : lane shortage. See the comments in lane eviction for how
1072 : this invariant is relevant. */
1073 0 : block->in_rdisp = 1;
1074 0 : block->staged = 0;
1075 0 : fd_rdisp_add_block( sched->rdisp, fec->bank_idx, FD_RDISP_UNSTAGED );
1076 0 : sched->metrics->block_added_cnt++;
1077 0 : sched->metrics->block_added_unstaged_cnt++;
1078 0 : FD_LOG_DEBUG(( "block %lu:%lu entered lane unstaged: add", block->slot, fec->bank_idx ));
1079 0 : } else {
1080 0 : alloc_lane = 1;
1081 0 : }
1082 0 : }
1083 0 : if( FD_UNLIKELY( alloc_lane ) ) {
1084 : /* We weren't able to inherit the parent's staging lane. So try
1085 : to find a new staging lane. */
1086 0 : if( FD_LIKELY( sched->staged_bitset!=fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) { /* Optimize for lane available. */
1087 0 : int lane_idx = fd_ulong_find_lsb( ~sched->staged_bitset );
1088 0 : if( FD_UNLIKELY( lane_idx>=(int)FD_SCHED_MAX_STAGING_LANES ) ) {
1089 0 : FD_LOG_CRIT(( "invariant violation: lane_idx %d, sched->staged_bitset %lx",
1090 0 : lane_idx, sched->staged_bitset ));
1091 0 : }
1092 0 : sched->staged_bitset = fd_ulong_set_bit( sched->staged_bitset, lane_idx );
1093 0 : sched->staged_head_bank_idx[ lane_idx ] = fec->bank_idx;
1094 0 : sched->staged_popcnt_wmk = fd_ulong_max( sched->staged_popcnt_wmk, (ulong)fd_ulong_popcnt( sched->staged_bitset ) );
1095 0 : block->in_rdisp = 1;
1096 0 : block->staged = 1;
1097 0 : block->staging_lane = (ulong)lane_idx;
1098 0 : fd_rdisp_add_block( sched->rdisp, fec->bank_idx, block->staging_lane );
1099 0 : sched->metrics->block_added_cnt++;
1100 0 : sched->metrics->block_added_staged_cnt++;
1101 0 : FD_LOG_DEBUG(( "block %lu:%lu entered lane %lu: add", block->slot, fec->bank_idx, block->staging_lane ));
1102 0 : } else {
1103 : /* No lanes available. */
1104 0 : block->in_rdisp = 1;
1105 0 : block->staged = 0;
1106 0 : fd_rdisp_add_block( sched->rdisp, fec->bank_idx, FD_RDISP_UNSTAGED );
1107 0 : sched->metrics->block_added_cnt++;
1108 0 : sched->metrics->block_added_unstaged_cnt++;
1109 0 : FD_LOG_DEBUG(( "block %lu:%lu entered lane unstaged: add", block->slot, fec->bank_idx ));
1110 0 : }
1111 0 : }
1112 0 : }
1113 :
1114 0 : block->txn_pool_max_popcnt = fd_ulong_max( block->txn_pool_max_popcnt, sched->depth - sched->txn_pool_free_cnt - 1UL );
1115 0 : block->mblk_pool_max_popcnt = fd_ulong_max( block->mblk_pool_max_popcnt, sched->depth - sched->mblk_pool_free_cnt );
1116 0 : block->block_pool_max_popcnt = fd_ulong_max( block->block_pool_max_popcnt, sched->block_pool_popcnt );
1117 :
1118 0 : if( FD_UNLIKELY( block->dying ) ) {
1119 : /* Ignore the FEC set for a dead block. */
1120 0 : sched->metrics->bytes_dropped_cnt += fec->fec->data_sz;
1121 0 : return 1;
1122 0 : }
1123 :
1124 0 : if( FD_UNLIKELY( !block->in_rdisp ) ) {
1125 : /* Invariant: block must be in the dispatcher at this point. */
1126 0 : sched->print_buf_sz = 0UL;
1127 0 : print_all( sched, block );
1128 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1129 0 : FD_LOG_CRIT(( "invariant violation: block->in_rdisp==0, slot %lu, parent slot %lu",
1130 0 : block->slot, block->parent_slot ));
1131 0 : }
1132 :
1133 0 : if( FD_UNLIKELY( block->fec_eos ) ) {
1134 : /* This means something is wrong upstream. We're getting more FEC
1135 : sets for a block that has already ended, or so we were told. */
1136 0 : sched->print_buf_sz = 0UL;
1137 0 : print_all( sched, block );
1138 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1139 0 : FD_LOG_CRIT(( "invariant violation: block->fec_eos set but getting more FEC sets, slot %lu, parent slot %lu", fec->slot, fec->parent_slot ));
1140 0 : }
1141 0 : if( FD_UNLIKELY( block->fec_eob ) ) {
1142 : /* fec_eob is only ever set from is_last_in_batch, and the parser is
1143 : expected to finish a batch in full upon getting the FEC set that
1144 : ends the batch. An incomplete parse is caught eagerly below,
1145 : which marks the block dying and is caught above. Getting here
1146 : means the eager check below regressed. */
1147 0 : sched->print_buf_sz = 0UL;
1148 0 : print_all( sched, block );
1149 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1150 0 : FD_LOG_CRIT(( "invariant violation: block->fec_eob set but getting more FEC sets, slot %lu, parent slot %lu", fec->slot, fec->parent_slot ));
1151 0 : }
1152 0 : if( FD_UNLIKELY( block->child_idx!=ULONG_MAX ) ) {
1153 : /* This means something is wrong upstream. FEC sets are not being
1154 : delivered in replay order. We got a child block FEC set before
1155 : this block was completely delivered. */
1156 0 : sched->print_buf_sz = 0UL;
1157 0 : print_all( sched, block );
1158 0 : fd_sched_block_t * child_block = block_pool_ele( sched, block->child_idx );
1159 0 : print_block_debug( sched, child_block );
1160 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1161 0 : FD_LOG_CRIT(( "invariant violation: block->child_idx %lu, slot %lu, parent slot %lu", block->child_idx, fec->slot, fec->parent_slot ));
1162 0 : }
1163 :
1164 0 : FD_TEST( block->fec_buf_sz>=block->fec_buf_soff );
1165 0 : if( FD_LIKELY( block->fec_buf_sz>block->fec_buf_soff ) ) {
1166 : /* If there is residual data from the previous FEC set within the
1167 : same batch, we move it to the beginning of the buffer and append
1168 : the new FEC set. */
1169 0 : memmove( block->fec_buf, block->fec_buf+block->fec_buf_soff, block->fec_buf_sz-block->fec_buf_soff );
1170 0 : }
1171 0 : block->fec_buf_boff += block->fec_buf_soff;
1172 0 : block->fec_buf_sz -= block->fec_buf_soff;
1173 0 : block->fec_buf_soff = 0;
1174 : /* Addition is safe and won't overflow because we checked the FEC
1175 : set size above. */
1176 0 : if( FD_UNLIKELY( block->fec_buf_sz+fec->fec->data_sz>FD_SCHED_MAX_FEC_BUF_SZ ) ) {
1177 : /* The residual carried over from the previous parse is bounded: it
1178 : can only be a partial transaction or a microblock/batch header
1179 : that straddles a FEC set boundary. Any trailing bytes past the
1180 : last microblock within the same batch are discarded by the
1181 : parser, so they never contribute to the residual. So residual <=
1182 : max(sizeof(ulong), sizeof(fd_microblock_hdr_t), FD_TXN_MTU), and
1183 : the buffer is sized to always fit the residual plus a single FEC
1184 : set. Otherwise, it's a bad block. Instead of crashing, we
1185 : should refuse to replay down the fork. */
1186 0 : FD_LOG_INFO(( "bad block: UNPARSEABLE_CONTENT, fec_buf_sz %u, fec->data_sz %u, slot %lu, parent slot %lu", block->fec_buf_sz, fec->fec->data_sz, fec->slot, fec->parent_slot ));
1187 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_UNPARSEABLE_CONTENT );
1188 0 : sched->metrics->bytes_dropped_cnt += fec->fec->data_sz;
1189 0 : return 0;
1190 0 : }
1191 :
1192 : /* Append the new FEC set to the end of the buffer. */
1193 0 : fd_memcpy( block->fec_buf+block->fec_buf_sz, fec->data, fec->fec->data_sz );
1194 0 : block->fec_buf_sz += (uint)fec->fec->data_sz;
1195 0 : sched->metrics->bytes_ingested_cnt += fec->fec->data_sz;
1196 :
1197 0 : block->fec_eob = fec->is_last_in_batch;
1198 0 : block->fec_eos = fec->is_last_in_block;
1199 :
1200 0 : ushort * shred_sz = sched->shred_sz + fec->bank_idx*sched->max_shreds_per_block;
1201 0 : ulong tracked_shred_cnt = fd_ulong_min( fec->shred_cnt, FD_FEC_SHRED_CNT );
1202 0 : uint tracked_data_sz = 0U;
1203 0 : for( ulong i=0UL; i<tracked_shred_cnt; i++ ) tracked_data_sz += fec->fec->shred_sz[ i ];
1204 0 : if( FD_LIKELY( fec->shred_cnt<=FD_FEC_SHRED_CNT ) ) FD_TEST( tracked_data_sz==fec->fec->data_sz );
1205 0 : else FD_TEST( tracked_data_sz<=fec->fec->data_sz );
1206 :
1207 0 : uint prev_shred_off = 0U;
1208 0 : for( ulong i=0; i<fec->shred_cnt; i++ ) {
1209 0 : FD_TEST( block->shred_cnt<sched->max_shreds_per_block );
1210 0 : uint shred_off;
1211 0 : if( FD_LIKELY( i<FD_FEC_SHRED_CNT ) ) {
1212 0 : shred_off = prev_shred_off + fec->fec->shred_sz[ i ];
1213 0 : } else if( FD_UNLIKELY( i!=fec->shred_cnt-1UL ) ) {
1214 : /* We don't track shred boundaries after 32 shreds, assume they're
1215 : sized uniformly */
1216 0 : ulong num_overflow_shreds = fec->shred_cnt-FD_FEC_SHRED_CNT;
1217 0 : ulong overflow_idx = i-FD_FEC_SHRED_CNT;
1218 0 : ulong overflow_data_sz = (ulong)fec->fec->data_sz-tracked_data_sz;
1219 0 : shred_off = tracked_data_sz + (uint)(overflow_data_sz / num_overflow_shreds * (overflow_idx + 1UL));
1220 0 : } else {
1221 0 : shred_off = (uint)fec->fec->data_sz;
1222 0 : }
1223 0 : FD_TEST( shred_off>=prev_shred_off && shred_off-prev_shred_off<=USHORT_MAX );
1224 0 : shred_sz[ block->shred_cnt++ ] = (ushort)(shred_off-prev_shred_off);
1225 0 : prev_shred_off = shred_off;
1226 0 : }
1227 :
1228 0 : block->fec_completed_ns = fec->completed_ns;
1229 :
1230 0 : int err = fd_sched_parse( sched, block, fec->alut_ctx );
1231 :
1232 0 : if( FD_UNLIKELY( err!=FD_SCHED_DEAD_REASON_NONE ) ) {
1233 0 : handle_bad_block( sched, block, err );
1234 0 : sched->metrics->bytes_dropped_cnt += block->fec_buf_sz-block->fec_buf_soff;
1235 0 : return 0;
1236 0 : }
1237 :
1238 0 : if( FD_UNLIKELY( (fec->is_last_in_batch||fec->is_last_in_block) && (block->txns_rem||block->mblks_rem||block->fec_eob) ) ) {
1239 : /* A malformed block that fails to parse out exactly as many
1240 : transactions and microblocks as it should.
1241 :
1242 : Upon getting a last-in-batch FEC, everything in the ongoing batch
1243 : should completely parse, and the eob flag should be reset. There
1244 : should be no go around for a last-in-batch FEC. */
1245 0 : FD_LOG_INFO(( "bad block: SHORT_BLOCK, bytes_rem %u, txns_rem %lu, mblks_rem %lu, fec_eob %d, slot %lu, parent slot %lu", block->fec_buf_sz-block->fec_buf_soff, block->txns_rem, block->mblks_rem, block->fec_eob, block->slot, block->parent_slot ));
1246 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_SHORT_BLOCK );
1247 0 : return 0;
1248 0 : }
1249 :
1250 0 : if( FD_UNLIKELY( block->fec_eos && !block->last_mblk_is_tick ) ) {
1251 : /* The last microblock should be a tick.
1252 :
1253 : Note that this early parse-time detection could cause us to throw
1254 : a slightly different error from Agave, in the case that there are
1255 : too few ticks, since the tick count check precedes the trailing
1256 : entry check in Agave. That being said, ultimately a
1257 : TRAILING_ENTRY renders a block invalid, regardless of anything
1258 : else. */
1259 0 : FD_LOG_INFO(( "bad block: TRAILING_ENTRY, slot %lu, parent slot %lu, mblk_cnt %u", block->slot, block->parent_slot, block->mblk_cnt ));
1260 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_TRAILING_ENTRY );
1261 0 : return 0;
1262 0 : }
1263 :
1264 0 : if( FD_UNLIKELY( block->fec_eos && sched->is_alpenglow ) ) {
1265 0 : int dead_reason = ag_on_final( block );
1266 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
1267 0 : handle_bad_block( sched, block, dead_reason );
1268 0 : return 0;
1269 0 : }
1270 0 : }
1271 :
1272 : /* We just received a FEC set, which may have made all transactions in
1273 : a partially parsed microblock available. If this were a malformed
1274 : block that ends in a non-tick microblock, there's not going to be a
1275 : hashing task from the missing ending tick to drain the mixin queue.
1276 : So we try to drain the mixin queue right here. Another option is
1277 : to drain it at dispatch time, when we are about to dispatch the end
1278 : of block signal, right before the check for whether block should
1279 : end. */
1280 0 : int mixin_res;
1281 0 : while( (mixin_res=maybe_mixin( sched, block )) ) {
1282 0 : if( FD_UNLIKELY( mixin_res==-1 ) ) {
1283 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_ENTRY_HASH_MISMATCH_INGEST );
1284 0 : return 0;
1285 0 : }
1286 0 : FD_TEST( mixin_res==1||mixin_res==2 );
1287 0 : }
1288 :
1289 : /* Check if we need to set the active block. */
1290 0 : check_or_set_active_block( sched );
1291 :
1292 0 : return 1;
1293 0 : }
1294 :
1295 : ulong
1296 0 : fd_sched_task_next_ready( fd_sched_t * sched, fd_sched_task_t * out ) {
1297 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1298 0 : FD_TEST( ref_q_empty( sched->ref_q ) );
1299 :
1300 0 : ulong exec_ready_bitset0 = sched->txn_exec_ready_bitset[ 0 ];
1301 0 : ulong exec_fully_ready_bitset = sched->sigverify_ready_bitset[ 0 ] & sched->poh_ready_bitset[ 0 ] & sched->cert_ready_bitset[ 0 ] & exec_ready_bitset0;
1302 0 : if( FD_UNLIKELY( !exec_fully_ready_bitset ) ) {
1303 : /* Early exit if no exec tiles available. */
1304 0 : return 0UL;
1305 0 : }
1306 :
1307 0 : if( FD_UNLIKELY( sched->active_bank_idx==ULONG_MAX ) ) {
1308 : /* No need to try activating a block. If we're in this state,
1309 : there's truly nothing to execute. We will activate something
1310 : when we ingest a FEC set with transactions. */
1311 0 : return 0UL;
1312 0 : }
1313 :
1314 0 : out->task_type = FD_SCHED_TT_NULL;
1315 :
1316 : /* We could in theory reevaluate staging lane allocation here and do
1317 : promotion/demotion as needed. It's a policy decision to minimize
1318 : fork churn for now and just execute down the same active fork. */
1319 :
1320 0 : ulong bank_idx = sched->active_bank_idx;
1321 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1322 0 : if( FD_UNLIKELY( block_should_deactivate( block ) ) ) {
1323 0 : sched->print_buf_sz = 0UL;
1324 0 : print_all( sched, block );
1325 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1326 0 : FD_LOG_CRIT(( "invariant violation: active block %lu:%lu is not activatable nor has anything in-flight", block->slot, sched->active_bank_idx ));
1327 0 : }
1328 :
1329 0 : block->txn_pool_max_popcnt = fd_ulong_max( block->txn_pool_max_popcnt, sched->depth - sched->txn_pool_free_cnt - 1UL );
1330 0 : block->mblk_pool_max_popcnt = fd_ulong_max( block->mblk_pool_max_popcnt, sched->depth - sched->mblk_pool_free_cnt );
1331 0 : block->block_pool_max_popcnt = fd_ulong_max( block->block_pool_max_popcnt, sched->block_pool_popcnt );
1332 :
1333 0 : if( FD_UNLIKELY( !block->block_start_signaled ) ) {
1334 0 : out->task_type = FD_SCHED_TT_BLOCK_START;
1335 0 : out->block_start->bank_idx = bank_idx;
1336 0 : out->block_start->parent_bank_idx = block->parent_idx;
1337 0 : out->block_start->slot = block->slot;
1338 0 : block->block_start_signaled = 1;
1339 0 : sched->next_ready_last_tick = fd_tickcount();
1340 0 : sched->next_ready_last_bank_idx = bank_idx;
1341 0 : return 1UL;
1342 0 : }
1343 :
1344 0 : ulong exec_tile_idx0 = fd_ulong_if( !!exec_fully_ready_bitset, (ulong)fd_ulong_find_lsb( exec_fully_ready_bitset ), ULONG_MAX );
1345 0 : ulong exec_queued_cnt = block->txn_parsed_cnt-block->txn_exec_in_flight_cnt-block->txn_exec_done_cnt;
1346 0 : if( FD_LIKELY( exec_queued_cnt>0UL && fd_ulong_popcnt( exec_fully_ready_bitset ) ) ) { /* Optimize for no fork switching. */
1347 : /* Transaction execution has the highest priority. Current mainnet
1348 : block times are very much dominated by critical path transaction
1349 : execution. To achieve the fastest block replay speed, we can't
1350 : afford to make any mistake in critical path dispatching. Any
1351 : deviation from perfect critical path dispatching is basically
1352 : irrecoverable. As such, we try to keep all the exec tiles busy
1353 : with transaction execution, but we allow at most one transaction
1354 : to be in-flight per exec tile. This is to ensure that whenever a
1355 : critical path transaction completes, we have at least one exec
1356 : tile, e.g. the one that just completed said transaction, readily
1357 : available to continue executing down the critical path. */
1358 0 : out->txn_exec->txn_idx = fd_rdisp_get_next_ready( sched->rdisp, bank_idx );
1359 0 : if( FD_UNLIKELY( out->txn_exec->txn_idx==0UL ) ) {
1360 : /* There are transactions queued but none ready for execution.
1361 : This implies that there must be in-flight transactions on whose
1362 : completion the queued transactions depend. So we return and
1363 : wait for those in-flight transactions to retire. This is a
1364 : policy decision to execute as much as we can down the current
1365 : fork. */
1366 0 : if( FD_UNLIKELY( !block->txn_exec_in_flight_cnt ) ) {
1367 0 : sched->print_buf_sz = 0UL;
1368 0 : print_all( sched, block );
1369 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1370 0 : FD_LOG_CRIT(( "invariant violation: no ready transaction found but block->txn_exec_in_flight_cnt==0" ));
1371 0 : }
1372 :
1373 : /* Next up are PoH tasks. Same dispatching policy as sigverify
1374 : tasks. */
1375 0 : ulong poh_ready_bitset = exec_fully_ready_bitset;
1376 0 : ulong poh_hashing_queued_cnt = block->mblk_cnt-block->poh_mblk_in_flight_cnt-block->poh_hashing_done_cnt;
1377 0 : int poh_threshold = fd_int_if( block->txn_exec_in_flight_cnt>0U, 0, 1 );
1378 0 : if( FD_LIKELY( poh_hashing_queued_cnt>0UL && fd_ulong_popcnt( poh_ready_bitset )>poh_threshold ) ) {
1379 0 : ulong max_cnt = poh_batch_max_cnt( sched, poh_hashing_queued_cnt, (ulong)fd_ulong_popcnt( poh_ready_bitset ), (ulong)poh_threshold );
1380 0 : dispatch_poh( sched, block, bank_idx, fd_ulong_find_lsb( poh_ready_bitset ), max_cnt, out );
1381 0 : sched->next_ready_last_tick = fd_tickcount();
1382 0 : sched->next_ready_last_bank_idx = bank_idx;
1383 0 : return 1UL;
1384 0 : }
1385 :
1386 : /* Dispatch more sigverify tasks only if at least one exec tile is
1387 : executing transactions or completely idle. Allow at most one
1388 : sigverify task in-flight per tile, and only dispatch to
1389 : completely idle tiles. */
1390 0 : ulong sigverify_ready_bitset = exec_fully_ready_bitset;
1391 0 : ulong sigverify_queued_cnt = block->txn_parsed_cnt-block->txn_sigverify_in_flight_cnt-block->txn_sigverify_done_cnt;
1392 0 : if( FD_LIKELY( sigverify_queued_cnt>0UL && fd_ulong_popcnt( sigverify_ready_bitset )>fd_int_if( block->txn_exec_in_flight_cnt>0U, 0, 1 ) ) ) {
1393 0 : dispatch_sigverify( sched, block, bank_idx, fd_ulong_find_lsb( sigverify_ready_bitset ), out );
1394 0 : sched->next_ready_last_tick = sched->txn_info_pool[ out->txn_sigverify->txn_idx ].tick_sigverify_disp = fd_tickcount();
1395 0 : sched->next_ready_last_bank_idx = bank_idx;
1396 0 : return 1UL;
1397 0 : }
1398 0 : return 0UL;
1399 0 : }
1400 0 : out->task_type = FD_SCHED_TT_TXN_EXEC;
1401 0 : out->txn_exec->bank_idx = bank_idx;
1402 0 : out->txn_exec->slot = block->slot;
1403 0 : out->txn_exec->exec_idx = exec_tile_idx0;
1404 0 : FD_TEST( out->txn_exec->exec_idx!=ULONG_MAX );
1405 :
1406 0 : long now = fd_tickcount();
1407 0 : ulong delta = (ulong)(now-sched->txn_in_flight_last_tick);
1408 0 : ulong txn_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( exec_ready_bitset0 );
1409 0 : sched->metrics->txn_none_in_flight_tickcount += fd_ulong_if( txn_exec_busy_cnt==0UL, delta, 0UL );
1410 0 : sched->metrics->txn_weighted_in_flight_tickcount += fd_ulong_if( txn_exec_busy_cnt!=0UL, delta, 0UL );
1411 0 : sched->metrics->txn_weighted_in_flight_cnt += delta*txn_exec_busy_cnt;
1412 0 : sched->txn_in_flight_last_tick = now;
1413 :
1414 0 : sched->txn_info_pool[ out->txn_exec->txn_idx ].tick_exec_disp = now;
1415 0 : sched->txn_info_pool[ out->txn_exec->txn_idx ].exec_tile_idx = exec_tile_idx0;
1416 :
1417 0 : sched->txn_exec_ready_bitset[ 0 ] = fd_ulong_clear_bit( exec_ready_bitset0, (int)exec_tile_idx0);
1418 0 : sched->tile_to_bank_idx[ exec_tile_idx0 ] = bank_idx;
1419 :
1420 0 : block->txn_exec_in_flight_cnt++;
1421 0 : sched->metrics->txn_max_in_flight_cnt = fd_uint_max( sched->metrics->txn_max_in_flight_cnt, block->txn_exec_in_flight_cnt );
1422 :
1423 0 : if( FD_UNLIKELY( (~sched->txn_exec_ready_bitset[ 0 ])&(~sched->sigverify_ready_bitset[ 0 ])&(~sched->poh_ready_bitset[ 0 ])&fd_ulong_mask_lsb( (int)sched->exec_cnt ) ) ) FD_LOG_CRIT(( "invariant violation: txn_exec_ready_bitset 0x%lx sigverify_ready_bitset 0x%lx poh_ready_bitset 0x%lx", sched->txn_exec_ready_bitset[ 0 ], sched->sigverify_ready_bitset[ 0 ], sched->poh_ready_bitset[ 0 ] ));
1424 0 : ulong total_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ]&sched->sigverify_ready_bitset[ 0 ]&sched->poh_ready_bitset[ 0 ] );
1425 0 : if( FD_UNLIKELY( block->txn_exec_in_flight_cnt+block->txn_sigverify_in_flight_cnt+block->poh_hashing_in_flight_cnt!=total_exec_busy_cnt ) ) {
1426 : /* Ideally we'd simply assert that the two sides of the equation
1427 : are equal. But abandoned blocks throw a wrench into this. We
1428 : allow abandoned blocks to have in-flight transactions that are
1429 : naturally drained while we try to dispatch from another block.
1430 : In such cases, the total number of in-flight transactions
1431 : should include the abandoned blocks too. The contract is that
1432 : blocks with in-flight transactions cannot be abandoned or
1433 : demoted from rdisp. So a dying block has to be the head of one
1434 : of the staging lanes. */
1435 : // FIXME This contract no longer true if we implement immediate
1436 : // demotion of abandoned blocks.
1437 0 : ulong total_in_flight = 0UL;
1438 0 : for( int l=0; l<(int)FD_SCHED_MAX_STAGING_LANES; l++ ) {
1439 0 : if( fd_ulong_extract_bit( sched->staged_bitset, l ) ) {
1440 0 : fd_sched_block_t * staged_block = block_pool_ele( sched, sched->staged_head_bank_idx[ l ] );
1441 0 : if( FD_UNLIKELY( block_is_in_flight( staged_block )&&!(staged_block==block||staged_block->dying) ) ) {
1442 0 : sched->print_buf_sz = 0UL;
1443 0 : print_all( sched, staged_block );
1444 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1445 0 : FD_LOG_CRIT(( "invariant violation: in-flight block is neither active nor dying" ));
1446 0 : }
1447 0 : total_in_flight += staged_block->txn_exec_in_flight_cnt;
1448 0 : total_in_flight += staged_block->txn_sigverify_in_flight_cnt;
1449 0 : total_in_flight += staged_block->poh_hashing_in_flight_cnt;
1450 0 : }
1451 0 : }
1452 0 : if( FD_UNLIKELY( total_in_flight!=total_exec_busy_cnt ) ) {
1453 0 : sched->print_buf_sz = 0UL;
1454 0 : print_all( sched, block );
1455 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1456 0 : FD_LOG_CRIT(( "invariant violation: total_in_flight %lu != total_exec_busy_cnt %lu", total_in_flight, total_exec_busy_cnt ));
1457 0 : }
1458 0 : FD_LOG_DEBUG(( "exec_busy_cnt %lu checks out", total_exec_busy_cnt ));
1459 0 : }
1460 0 : sched->next_ready_last_tick = now;
1461 0 : sched->next_ready_last_bank_idx = bank_idx;
1462 0 : return 1UL;
1463 0 : }
1464 :
1465 : /* At this point txn_queued_cnt==0 */
1466 :
1467 : /* Next up are PoH tasks. Same dispatching policy as sigverify. */
1468 0 : ulong poh_ready_bitset = exec_fully_ready_bitset;
1469 0 : ulong poh_hashing_queued_cnt = block->mblk_cnt-block->poh_mblk_in_flight_cnt-block->poh_hashing_done_cnt;
1470 0 : int poh_threshold = fd_int_if( block->fec_eos||block->txn_exec_in_flight_cnt>0U||sched->exec_cnt==1UL, 0, 1 );
1471 0 : if( FD_LIKELY( poh_hashing_queued_cnt>0UL && fd_ulong_popcnt( poh_ready_bitset )>poh_threshold ) ) {
1472 0 : ulong max_cnt = poh_batch_max_cnt( sched, poh_hashing_queued_cnt, (ulong)fd_ulong_popcnt( poh_ready_bitset ), (ulong)poh_threshold );
1473 0 : dispatch_poh( sched, block, bank_idx, fd_ulong_find_lsb( poh_ready_bitset ), max_cnt, out );
1474 0 : sched->next_ready_last_tick = fd_tickcount();
1475 0 : sched->next_ready_last_bank_idx = bank_idx;
1476 0 : return 1UL;
1477 0 : }
1478 :
1479 : /* Try to dispatch a sigverify task, but leave one exec tile idle for
1480 : critical path execution, unless there's not going to be any more
1481 : real transactions for the critical path. In the degenerate case of
1482 : only one exec tile, keep it busy. */
1483 0 : ulong sigverify_ready_bitset = exec_fully_ready_bitset;
1484 0 : ulong sigverify_queued_cnt = block->txn_parsed_cnt-block->txn_sigverify_in_flight_cnt-block->txn_sigverify_done_cnt;
1485 0 : if( FD_LIKELY( sigverify_queued_cnt>0UL && fd_ulong_popcnt( sigverify_ready_bitset )>fd_int_if( block->fec_eos||block->txn_exec_in_flight_cnt>0U||sched->exec_cnt==1UL, 0, 1 ) ) ) {
1486 0 : dispatch_sigverify( sched, block, bank_idx, fd_ulong_find_lsb( sigverify_ready_bitset ), out );
1487 0 : sched->next_ready_last_tick = sched->txn_info_pool[ out->txn_sigverify->txn_idx ].tick_sigverify_disp = fd_tickcount();
1488 0 : sched->next_ready_last_bank_idx = bank_idx;
1489 0 : return 1UL;
1490 0 : }
1491 :
1492 0 : if( FD_UNLIKELY( block_should_signal_end( block ) ) ) {
1493 0 : FD_TEST( block->block_start_signaled );
1494 0 : int tick_reason = verify_ticks_final( block );
1495 0 : if( FD_UNLIKELY( tick_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
1496 : /* Tick verification can't be done at parse time (except for
1497 : TRAILING_ENTRY), because we may not know the expected number of
1498 : hashes yet. It can't be driven by transaction dispatch or
1499 : completion, because the block may be empty. Similarly, it can't
1500 : be driven by PoH hashing, because a bad block may simply not
1501 : have any microblocks. */
1502 0 : handle_bad_block( sched, block, tick_reason );
1503 0 : out->task_type = FD_SCHED_TT_MARK_DEAD;
1504 0 : out->mark_dead->bank_idx = bank_idx;
1505 0 : sched->next_ready_last_tick = fd_tickcount();
1506 0 : sched->next_ready_last_bank_idx = bank_idx;
1507 0 : return 1UL;
1508 0 : }
1509 0 : out->task_type = FD_SCHED_TT_BLOCK_END;
1510 0 : out->block_end->bank_idx = bank_idx;
1511 0 : block->block_end_signaled = 1;
1512 0 : FD_TEST( block->refcnt );
1513 0 : block->refcnt = 0;
1514 0 : FD_TEST( ref_q_avail( sched->ref_q ) );
1515 0 : ref_q_push_tail( sched->ref_q, bank_idx );
1516 0 : sched->next_ready_last_tick = fd_tickcount();
1517 0 : sched->next_ready_last_bank_idx = bank_idx;
1518 0 : return 1UL;
1519 0 : }
1520 :
1521 : /* Nothing queued for the active block. If we haven't received all
1522 : the FEC sets for it, then return and wait for more FEC sets, while
1523 : there are in-flight transactions. This is a policy decision to
1524 : minimize fork churn and allow for executing down the current fork
1525 : as much as we can. If we have received all the FEC sets for it,
1526 : then we'd still like to return and wait for the in-flight
1527 : transactions to retire, before switching to a different block.
1528 :
1529 : Either way, there should be in-flight transactions. We deactivate
1530 : the active block the moment we exhausted transactions from it.
1531 :
1532 : We don't assert block_is_in_flight() here because there might be
1533 : in-flight tasks from dying blocks that are preventing us from
1534 : dispatching anything momentarily. So we assert the global exec
1535 : tile busy count. This doesn't lose too much assertion coverage
1536 : since dying blocks are few and far between. */
1537 0 : ulong total_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( exec_fully_ready_bitset );
1538 0 : if( FD_UNLIKELY( !total_exec_busy_cnt ) ) {
1539 0 : sched->print_buf_sz = 0UL;
1540 0 : print_all( sched, block );
1541 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1542 0 : FD_LOG_CRIT(( "invariant violation: expected in-flight tasks but none found" ));
1543 0 : }
1544 :
1545 0 : return 0UL;
1546 0 : }
1547 :
1548 : int
1549 0 : fd_sched_task_done( fd_sched_t * sched, ulong task_type, ulong txn_idx, ulong exec_idx, void * data ) {
1550 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1551 :
1552 0 : ulong bank_idx = ULONG_MAX;
1553 0 : switch( task_type ) {
1554 0 : case FD_SCHED_TT_BLOCK_START:
1555 0 : case FD_SCHED_TT_BLOCK_END: {
1556 0 : (void)txn_idx;
1557 0 : (void)data;
1558 0 : bank_idx = sched->active_bank_idx;
1559 0 : break;
1560 0 : }
1561 0 : case FD_SCHED_TT_TXN_EXEC:
1562 0 : case FD_SCHED_TT_TXN_SIGVERIFY: {
1563 0 : (void)data;
1564 0 : FD_TEST( txn_idx < sched->depth );
1565 0 : bank_idx = sched->tile_to_bank_idx[ exec_idx ];
1566 0 : break;
1567 0 : }
1568 0 : case FD_SCHED_TT_POH_HASH:
1569 0 : case FD_SCHED_TT_CERT_VERIFY: {
1570 0 : (void)txn_idx;
1571 0 : bank_idx = sched->tile_to_bank_idx[ exec_idx ];
1572 0 : break;
1573 0 : }
1574 0 : default: FD_LOG_CRIT(( "unsupported task_type %lu", task_type ));
1575 0 : }
1576 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1577 :
1578 0 : if( FD_UNLIKELY( !block->in_sched ) ) {
1579 0 : FD_LOG_CRIT(( "invariant violation: block->in_sched==0, block %lu:%lu, parent slot %lu",
1580 0 : block->slot, bank_idx, block->parent_slot ));
1581 0 : }
1582 0 : if( FD_UNLIKELY( !block->staged ) ) {
1583 : /* Invariant: only staged blocks can have in-flight transactions. */
1584 0 : FD_LOG_CRIT(( "invariant violation: block->staged==0, block %lu:%lu, parent slot %lu",
1585 0 : block->slot, bank_idx, block->parent_slot ));
1586 0 : }
1587 0 : if( FD_UNLIKELY( !block->in_rdisp ) ) {
1588 : /* Invariant: staged blocks must be in the dispatcher. */
1589 0 : FD_LOG_CRIT(( "invariant violation: block->in_rdisp==0, block %lu:%lu, parent slot %lu",
1590 0 : block->slot, bank_idx, block->parent_slot ));
1591 0 : }
1592 :
1593 0 : block->txn_pool_max_popcnt = fd_ulong_max( block->txn_pool_max_popcnt, sched->depth - sched->txn_pool_free_cnt - 1UL );
1594 0 : block->mblk_pool_max_popcnt = fd_ulong_max( block->mblk_pool_max_popcnt, sched->depth - sched->mblk_pool_free_cnt );
1595 0 : block->block_pool_max_popcnt = fd_ulong_max( block->block_pool_max_popcnt, sched->block_pool_popcnt );
1596 :
1597 0 : int exec_tile_idx = (int)exec_idx;
1598 :
1599 0 : switch( task_type ) {
1600 0 : case FD_SCHED_TT_BLOCK_START: {
1601 0 : FD_TEST( !block->block_start_done );
1602 0 : block->block_start_done = 1;
1603 0 : break;
1604 0 : }
1605 0 : case FD_SCHED_TT_BLOCK_END: {
1606 : /* It may seem redundant to be invoking task_done() on these
1607 : somewhat fake tasks. But these are necessary to drive state
1608 : transition for empty blocks or slow blocks. */
1609 0 : FD_TEST( !block->block_end_done );
1610 0 : block->block_end_done = 1;
1611 0 : sched->print_buf_sz = 0UL;
1612 0 : print_block_metrics( sched, block );
1613 0 : FD_LOG_DEBUG(( "block %lu:%lu replayed fully: %s", block->slot, bank_idx, sched->print_buf ));
1614 0 : break;
1615 0 : }
1616 0 : case FD_SCHED_TT_TXN_EXEC: {
1617 0 : long now = fd_tickcount();
1618 0 : ulong delta = (ulong)(now-sched->txn_in_flight_last_tick);
1619 0 : ulong txn_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ] );
1620 0 : sched->metrics->txn_weighted_in_flight_tickcount += delta;
1621 0 : sched->metrics->txn_weighted_in_flight_cnt += delta*txn_exec_busy_cnt;
1622 0 : sched->txn_in_flight_last_tick = now;
1623 :
1624 0 : sched->txn_info_pool[ txn_idx ].tick_exec_done = now;
1625 :
1626 0 : block->txn_exec_done_cnt++;
1627 0 : block->txn_exec_in_flight_cnt--;
1628 0 : FD_TEST( !fd_ulong_extract_bit( sched->txn_exec_ready_bitset[ 0 ], exec_tile_idx ) );
1629 0 : sched->txn_exec_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->txn_exec_ready_bitset[ 0 ], exec_tile_idx );
1630 0 : sched->metrics->txn_exec_done_cnt++;
1631 0 : txn_bitset_insert( sched->exec_done_set, txn_idx );
1632 0 : sched->txn_info_pool[ txn_idx ].flags |= FD_SCHED_TXN_EXEC_DONE;
1633 0 : if( txn_bitset_test( sched->sigverify_done_set, txn_idx ) && txn_bitset_test( sched->poh_mixin_done_set, txn_idx ) ) {
1634 : /* Release the txn_idx if all tasks on it are done. This is
1635 : guaranteed to only happen once per transaction because
1636 : whichever one completed first would not release. */
1637 0 : if( FD_UNLIKELY( block->txn_idx_tail==(uint)txn_idx ) ) block->txn_idx_tail = 0U;
1638 0 : fd_rdisp_complete_txn( sched->rdisp, txn_idx, 1 );
1639 0 : sched->txn_pool_free_cnt++;
1640 0 : block->txn_done_cnt++;
1641 0 : sched->metrics->txn_done_cnt++;
1642 0 : } else {
1643 0 : fd_rdisp_complete_txn( sched->rdisp, txn_idx, 0 );
1644 0 : }
1645 0 : break;
1646 0 : }
1647 0 : case FD_SCHED_TT_TXN_SIGVERIFY: {
1648 0 : sched->txn_info_pool[ txn_idx ].tick_sigverify_done = fd_tickcount();
1649 0 : block->txn_sigverify_done_cnt++;
1650 0 : block->txn_sigverify_in_flight_cnt--;
1651 0 : FD_TEST( !fd_ulong_extract_bit( sched->sigverify_ready_bitset[ 0 ], exec_tile_idx ) );
1652 0 : sched->sigverify_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->sigverify_ready_bitset[ 0 ], exec_tile_idx );
1653 0 : sched->metrics->txn_sigverify_done_cnt++;
1654 0 : txn_bitset_insert( sched->sigverify_done_set, txn_idx );
1655 0 : sched->txn_info_pool[ txn_idx ].flags |= FD_SCHED_TXN_SIGVERIFY_DONE;
1656 0 : if( txn_bitset_test( sched->exec_done_set, txn_idx ) && txn_bitset_test( sched->poh_mixin_done_set, txn_idx ) ) {
1657 : /* Release the txn_idx if all tasks on it are done. This is
1658 : guaranteed to only happen once per transaction because
1659 : whichever one completed first would not release. */
1660 0 : if( FD_UNLIKELY( block->txn_idx_tail==(uint)txn_idx ) ) block->txn_idx_tail = 0U;
1661 0 : fd_rdisp_complete_txn( sched->rdisp, txn_idx, 1 );
1662 0 : sched->txn_pool_free_cnt++;
1663 0 : block->txn_done_cnt++;
1664 0 : sched->metrics->txn_done_cnt++;
1665 0 : }
1666 0 : break;
1667 0 : }
1668 0 : case FD_SCHED_TT_POH_HASH: {
1669 0 : block->poh_hashing_in_flight_cnt--;
1670 0 : FD_TEST( !fd_ulong_extract_bit( sched->poh_ready_bitset[ 0 ], exec_tile_idx ) );
1671 0 : sched->poh_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->poh_ready_bitset[ 0 ], exec_tile_idx );
1672 0 : fd_execrp_poh_hash_done_msg_t * msg = fd_type_pun( data );
1673 0 : FD_TEST( msg->cnt && msg->cnt<=FD_EXECRP_POH_PARA );
1674 0 : fd_sched_poh_hash_t const * task = sched->poh_inflight+exec_tile_idx;
1675 0 : FD_TEST( msg->cnt==task->cnt );
1676 :
1677 : /* Once the block is found to be bad, the remaining elements are
1678 : put back on a list so that they get freed along with the
1679 : block. */
1680 0 : int dead_reason = FD_SCHED_DEAD_REASON_NONE;
1681 0 : for( ulong i=0UL; i<msg->cnt; i++ ) {
1682 0 : block->poh_mblk_in_flight_cnt--;
1683 0 : uint mblk_idx = (uint)task->mblk_idx[ i ];
1684 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
1685 0 : mblk->curr_hashcnt += task->hashcnt;
1686 0 : memcpy( mblk->curr_hash, msg->hash+i, sizeof(fd_hash_t) );
1687 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) || mblk->curr_hashcnt<mblk->hashcnt ) {
1688 0 : mblk_slist_idx_push_tail( block->mblks_hashing_in_progress, mblk_idx, sched->mblk_pool );
1689 0 : continue;
1690 0 : }
1691 0 : dead_reason = poh_retire_mblk( sched, block, mblk_idx );
1692 0 : }
1693 0 : if( FD_LIKELY( dead_reason==FD_SCHED_DEAD_REASON_NONE ) ) dead_reason = poh_retire_ready_mblks( sched, block );
1694 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
1695 0 : handle_bad_block( sched, block, dead_reason );
1696 0 : return dead_reason;
1697 0 : }
1698 0 : break;
1699 0 : }
1700 0 : case FD_SCHED_TT_CERT_VERIFY: {
1701 0 : FD_TEST( block->cert_in_flight_cnt );
1702 0 : block->cert_in_flight_cnt--;
1703 0 : FD_TEST( !fd_ulong_extract_bit( sched->cert_ready_bitset[ 0 ], exec_tile_idx ) );
1704 0 : sched->cert_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->cert_ready_bitset[ 0 ], exec_tile_idx );
1705 0 : break;
1706 0 : }
1707 0 : }
1708 :
1709 0 : if( FD_UNLIKELY( block->dying && !block_is_in_flight( block ) ) ) {
1710 0 : if( FD_UNLIKELY( sched->active_bank_idx==bank_idx ) ) {
1711 0 : FD_LOG_CRIT(( "invariant violation: active block %lu:%lu shouldn't be dying, parent slot %lu",
1712 0 : block->slot, bank_idx, block->parent_slot ));
1713 0 : }
1714 0 : FD_LOG_DEBUG(( "dying block %lu:%lu drained", block->slot, bank_idx ));
1715 0 : subtree_abandon( sched, block );
1716 0 : try_activate_block( sched );
1717 0 : return 0;
1718 0 : }
1719 :
1720 0 : if( FD_UNLIKELY( !block->dying && sched->active_bank_idx!=bank_idx ) ) {
1721 : /* Block is not dead. So we should be actively replaying it. */
1722 0 : fd_sched_block_t * active_block = block_pool_ele( sched, sched->active_bank_idx );
1723 0 : FD_LOG_CRIT(( "invariant violation: sched->active_bank_idx %lu, slot %lu, parent slot %lu, bank_idx %lu, slot %lu, parent slot %lu",
1724 0 : sched->active_bank_idx, active_block->slot, active_block->parent_slot,
1725 0 : bank_idx, block->slot, block->parent_slot ));
1726 0 : }
1727 :
1728 0 : maybe_switch_block( sched, bank_idx );
1729 :
1730 0 : return 0;
1731 0 : }
1732 :
1733 : void
1734 0 : fd_sched_block_abandon( fd_sched_t * sched, ulong bank_idx, int cause ) {
1735 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1736 0 : FD_TEST( bank_idx<sched->block_cnt_max );
1737 :
1738 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1739 0 : if( FD_UNLIKELY( !block->in_sched ) ) {
1740 0 : FD_LOG_CRIT(( "invariant violation: block->in_sched==0, block %lu:%lu, parent slot %lu",
1741 0 : block->slot, bank_idx, block->parent_slot ));
1742 0 : }
1743 :
1744 0 : if( FD_LIKELY( cause==FD_SCHED_ABANDON_DISCARDED && !block->dying ) ) block->discarded = 1;
1745 :
1746 0 : FD_LOG_INFO(( "abandoning block %lu:%lu", block->slot, bank_idx ));
1747 0 : sched->print_buf_sz = 0UL;
1748 0 : print_all( sched, block );
1749 0 : FD_LOG_DEBUG(( "%s", sched->print_buf ));
1750 :
1751 0 : subtree_abandon( sched, block );
1752 0 : try_activate_block( sched );
1753 0 : }
1754 :
1755 : int
1756 0 : fd_sched_get_dead_reason( fd_sched_t * sched, ulong bank_idx ) {
1757 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1758 0 : FD_TEST( bank_idx<sched->block_cnt_max );
1759 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1760 0 : if( FD_UNLIKELY( !block->in_sched ) ) return FD_SCHED_DEAD_REASON_NONE;
1761 0 : return block->dead_reason;
1762 0 : }
1763 :
1764 : int
1765 0 : fd_sched_block_is_discarded( fd_sched_t * sched, ulong bank_idx ) {
1766 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1767 0 : FD_TEST( bank_idx<sched->block_cnt_max );
1768 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1769 0 : if( FD_UNLIKELY( !block->in_sched ) ) return 0;
1770 0 : return (int)block->discarded;
1771 0 : }
1772 :
1773 : void
1774 0 : fd_sched_cancel( fd_sched_t * sched, ulong bank_idx ) {
1775 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1776 0 : FD_TEST( bank_idx<sched->block_cnt_max );
1777 :
1778 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1779 0 : if( FD_UNLIKELY( !block->in_sched ) ) return;
1780 0 : FD_LOG_INFO(( "canceling subtree at block %lu:%lu", block->slot, bank_idx ));
1781 0 : fd_sched_block_t * parent = block_pool_ele( sched, block->parent_idx );
1782 0 : if( FD_LIKELY( parent ) ) {
1783 : /* Splice the block out of its parent's children list. */
1784 0 : ulong block_idx = block_to_idx( sched, block );
1785 0 : ulong * idx_p = &parent->child_idx;
1786 0 : while( *idx_p!=block_idx ) {
1787 0 : idx_p = &(block_pool_ele( sched, *idx_p )->sibling_idx);
1788 0 : }
1789 0 : *idx_p = block->sibling_idx;
1790 0 : }
1791 0 : subtree_prune( sched, bank_idx, ULONG_MAX );
1792 0 : }
1793 :
1794 : void
1795 0 : fd_sched_block_add_done( fd_sched_t * sched, ulong bank_idx, ulong parent_bank_idx, ulong slot ) {
1796 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1797 0 : FD_TEST( bank_idx<sched->block_cnt_max );
1798 :
1799 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1800 0 : add_block( sched, bank_idx, parent_bank_idx );
1801 0 : block->slot = slot;
1802 0 : block->fec_eos = 1;
1803 0 : block->block_start_signaled = 1;
1804 0 : block->block_end_signaled = 1;
1805 0 : block->block_start_done = 1;
1806 0 : block->block_end_done = 1;
1807 0 : block->refcnt = 0;
1808 0 : if( FD_LIKELY( parent_bank_idx!=ULONG_MAX ) ) {
1809 0 : fd_sched_block_t * parent_block = block_pool_ele( sched, parent_bank_idx );
1810 0 : block->parent_slot = parent_block->slot;
1811 0 : }
1812 0 : if( FD_UNLIKELY( parent_bank_idx==ULONG_MAX ) ) {
1813 : /* Assumes that a NULL parent implies the snapshot slot. */
1814 0 : block->parent_slot = ULONG_MAX;
1815 0 : block->rooted = 1;
1816 0 : sched->root_idx = bank_idx;
1817 0 : }
1818 0 : }
1819 :
1820 : void
1821 0 : fd_sched_advance_root( fd_sched_t * sched, ulong root_idx ) {
1822 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1823 0 : FD_TEST( root_idx<sched->block_cnt_max );
1824 0 : FD_TEST( sched->root_idx<sched->block_cnt_max );
1825 0 : FD_TEST( ref_q_empty( sched->ref_q ) );
1826 :
1827 0 : fd_sched_block_t * new_root = block_pool_ele( sched, root_idx );
1828 0 : fd_sched_block_t * old_root = block_pool_ele( sched, sched->root_idx );
1829 0 : if( FD_UNLIKELY( !old_root->rooted ) ) {
1830 0 : FD_LOG_CRIT(( "invariant violation: old_root is not rooted, slot %lu, parent slot %lu",
1831 0 : old_root->slot, old_root->parent_slot ));
1832 0 : }
1833 :
1834 : /* Early exit if the new root is the same as the old root. */
1835 0 : if( FD_UNLIKELY( root_idx==sched->root_idx ) ) {
1836 0 : FD_LOG_INFO(( "new root is the same as the old root, slot %lu, parent slot %lu",
1837 0 : new_root->slot, new_root->parent_slot ));
1838 0 : return;
1839 0 : }
1840 :
1841 0 : subtree_prune( sched, sched->root_idx, root_idx );
1842 :
1843 0 : new_root->parent_idx = ULONG_MAX;
1844 0 : sched->root_idx = root_idx;
1845 0 : }
1846 :
1847 : void
1848 0 : fd_sched_root_notify( fd_sched_t * sched, ulong root_idx ) {
1849 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1850 0 : FD_TEST( root_idx<sched->block_cnt_max );
1851 0 : FD_TEST( sched->root_idx<sched->block_cnt_max );
1852 0 : FD_TEST( ref_q_empty( sched->ref_q ) );
1853 :
1854 0 : fd_sched_block_t * block = block_pool_ele( sched, root_idx );
1855 0 : fd_sched_block_t * old_root = block_pool_ele( sched, sched->root_idx );
1856 0 : if( FD_UNLIKELY( !old_root->rooted ) ) {
1857 0 : FD_LOG_CRIT(( "invariant violation: old_root is not rooted, slot %lu, parent slot %lu",
1858 0 : old_root->slot, old_root->parent_slot ));
1859 0 : }
1860 :
1861 : /* Early exit if the new root is the same as the old root. */
1862 0 : if( FD_UNLIKELY( root_idx==sched->root_idx ) ) {
1863 0 : FD_LOG_INFO(( "new root is the same as the old root, slot %lu, parent slot %lu",
1864 0 : block->slot, block->parent_slot ));
1865 0 : return;
1866 0 : }
1867 :
1868 : /* Mark every node from the new root up through its parents to the
1869 : old root as being rooted. */
1870 0 : fd_sched_block_t * curr = block;
1871 0 : fd_sched_block_t * prev = NULL;
1872 0 : while( curr ) {
1873 0 : if( FD_UNLIKELY( !block_is_done( curr ) ) ) {
1874 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is not done, slot %lu, parent slot %lu",
1875 0 : curr->slot, curr->parent_slot ));
1876 0 : }
1877 0 : if( FD_UNLIKELY( curr->dying ) ) {
1878 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is dying, slot %lu, parent slot %lu",
1879 0 : curr->slot, curr->parent_slot ));
1880 0 : }
1881 0 : if( FD_UNLIKELY( curr->staged ) ) {
1882 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is staged, slot %lu, parent slot %lu",
1883 0 : curr->slot, curr->parent_slot ));
1884 0 : }
1885 0 : if( FD_UNLIKELY( curr->in_rdisp ) ) {
1886 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is in the dispatcher, slot %lu, parent slot %lu",
1887 0 : curr->slot, curr->parent_slot ));
1888 0 : }
1889 0 : curr->rooted = 1;
1890 0 : prev = curr;
1891 0 : curr = block_pool_ele( sched, curr->parent_idx );
1892 0 : }
1893 :
1894 : /* If we didn't reach the old root, the new root is not a descendant. */
1895 0 : if( FD_UNLIKELY( prev!=old_root ) ) {
1896 0 : FD_LOG_CRIT(( "invariant violation: new root is not a descendant of old root, new root slot %lu, parent slot %lu, old root slot %lu, parent slot %lu",
1897 0 : block->slot, block->parent_slot, old_root->slot, old_root->parent_slot ));
1898 0 : }
1899 :
1900 0 : ulong old_active_bank_idx = sched->active_bank_idx;
1901 0 : uint old_deactivate_abandon = sched->metrics->deactivate_abandoned_cnt;
1902 :
1903 : /* Now traverse from old root towards new root, and abandon all
1904 : minority forks. */
1905 0 : curr = old_root;
1906 0 : while( curr && curr->rooted && curr!=block ) { /* curr!=block to avoid abandoning good forks. */
1907 0 : fd_sched_block_t * rooted_child_block = NULL;
1908 0 : ulong child_idx = curr->child_idx;
1909 0 : while( child_idx!=ULONG_MAX ) {
1910 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
1911 0 : child_idx = child->sibling_idx;
1912 0 : if( child->rooted ) {
1913 0 : rooted_child_block = child;
1914 0 : } else {
1915 : /* This is a minority fork. Consensus converged elsewhere, so
1916 : the fork is discarded rather than invalid. Mark the subtree
1917 : root as discarded so subtree_abandon() propagates the flag
1918 : down.
1919 :
1920 : Gated on the block not already going down. A block that is
1921 : already dying was abandoned for a reason of its own, either
1922 : sched's own verdict or runtime's verdict on its validity, and
1923 : losing the fork choice race afterwards must not re-label it
1924 : discarded. */
1925 0 : if( FD_LIKELY( !child->dying ) ) child->discarded = 1;
1926 :
1927 0 : ulong abandoned_cnt = sched->metrics->block_abandoned_cnt;
1928 0 : subtree_abandon( sched, child );
1929 0 : abandoned_cnt = sched->metrics->block_abandoned_cnt-abandoned_cnt;
1930 0 : if( FD_UNLIKELY( abandoned_cnt ) ) FD_LOG_DEBUG(( "abandoned %lu blocks on minority fork starting at block %lu:%lu", abandoned_cnt, child->slot, block_to_idx( sched, child ) ));
1931 0 : }
1932 0 : }
1933 0 : curr = rooted_child_block;
1934 0 : }
1935 :
1936 : /* If the active block got abandoned, we need to reset it. */
1937 0 : if( sched->active_bank_idx==ULONG_MAX ) {
1938 : /* Count a minority-fork deactivation once, as MinorityFork only */
1939 0 : sched->metrics->deactivate_abandoned_cnt = old_deactivate_abandon;
1940 0 : sched->metrics->deactivate_pruned_cnt += fd_uint_if( old_active_bank_idx!=ULONG_MAX, 1U, 0U );
1941 0 : try_activate_block( sched );
1942 0 : }
1943 0 : }
1944 :
1945 : ulong
1946 0 : fd_sched_pruned_block_next( fd_sched_t * sched ) {
1947 0 : if( !ref_q_empty( sched->ref_q ) ) {
1948 0 : ulong bank_idx = ref_q_pop_head( sched->ref_q );
1949 0 : return bank_idx;
1950 0 : }
1951 0 : return ULONG_MAX;
1952 0 : }
1953 :
1954 : void
1955 0 : fd_sched_set_poh_params( fd_sched_t * sched, ulong bank_idx, ulong tick_height, ulong max_tick_height, ulong hashes_per_tick, fd_hash_t const * start_poh ) {
1956 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1957 0 : FD_TEST( bank_idx<sched->block_cnt_max );
1958 0 : FD_TEST( max_tick_height>tick_height );
1959 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1960 0 : block->tick_height = tick_height;
1961 0 : block->max_tick_height = max_tick_height;
1962 0 : block->hashes_per_tick = hashes_per_tick;
1963 : #if FD_SCHED_SKIP_POH
1964 : /* No-op. */
1965 : (void)start_poh;
1966 : #else
1967 0 : if( FD_LIKELY( block->mblk_cnt ) ) {
1968 : /* Fix up the first mblk's curr_hash. */
1969 0 : FD_TEST( block->mblk_unhashed_cnt );
1970 0 : FD_TEST( !mblk_slist_is_empty( block->mblks_unhashed, sched->mblk_pool ) );
1971 0 : FD_TEST( !block->mblk_freed_cnt );
1972 0 : fd_sched_mblk_t * first_mblk = sched->mblk_pool + mblk_slist_idx_peek_head( block->mblks_unhashed, sched->mblk_pool );
1973 0 : memcpy( first_mblk->curr_hash, start_poh, sizeof(fd_hash_t) );
1974 0 : } else {
1975 0 : memcpy( block->poh_hash, start_poh, sizeof(fd_hash_t) );
1976 0 : }
1977 0 : #endif
1978 0 : }
1979 :
1980 : void
1981 0 : fd_sched_set_bypass_poh_verify( fd_sched_t * sched, int bypass_poh_verify ) {
1982 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1983 0 : sched->bypass_poh_verify = !!bypass_poh_verify;
1984 0 : }
1985 :
1986 : void
1987 0 : fd_sched_set_bypass_alut_resolution( fd_sched_t * sched, int bypass_alut_resolution ) {
1988 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1989 0 : sched->bypass_alut_resolution = !!bypass_alut_resolution;
1990 0 : }
1991 :
1992 : fd_txn_p_t *
1993 0 : fd_sched_get_txn( fd_sched_t * sched, ulong txn_idx ) {
1994 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
1995 0 : if( FD_UNLIKELY( txn_idx>=sched->depth ) ) {
1996 0 : return NULL;
1997 0 : }
1998 0 : return sched->txn_pool+txn_idx;
1999 0 : }
2000 :
2001 : fd_sched_txn_info_t *
2002 0 : fd_sched_get_txn_info( fd_sched_t * sched, ulong txn_idx ) {
2003 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2004 0 : if( FD_UNLIKELY( txn_idx>=sched->depth ) ) {
2005 0 : return NULL;
2006 0 : }
2007 0 : return sched->txn_info_pool+txn_idx;
2008 0 : }
2009 :
2010 : fd_hash_t *
2011 0 : fd_sched_get_poh( fd_sched_t * sched, ulong bank_idx ) {
2012 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2013 0 : FD_TEST( bank_idx<sched->block_cnt_max );
2014 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2015 0 : FD_TEST( block->fec_eos );
2016 0 : FD_TEST( block->mblk_cnt );
2017 0 : return block->poh_hash;
2018 0 : }
2019 :
2020 : uint
2021 0 : fd_sched_get_shred_cnt( fd_sched_t * sched, ulong bank_idx ) {
2022 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2023 0 : FD_TEST( bank_idx<sched->block_cnt_max );
2024 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2025 0 : return block->shred_cnt;
2026 0 : }
2027 :
2028 : fd_block_footer_t const *
2029 0 : fd_sched_get_footer( fd_sched_t * sched, ulong bank_idx ) {
2030 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2031 0 : FD_TEST( bank_idx<sched->block_cnt_max );
2032 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2033 0 : return block->footer_seen ? &block->footer : NULL;
2034 0 : }
2035 :
2036 : void
2037 : fd_sched_metrics_write( fd_sched_t * sched,
2038 0 : fd_clock_tile_t const * clock ) {
2039 0 : FD_MGAUGE_SET( REPLAY, SCHED_ACTIVE_BANK_INDEX, sched->active_bank_idx );
2040 0 : FD_MGAUGE_SET( REPLAY, SCHED_LAST_DISPATCH_BANK_INDEX, sched->next_ready_last_bank_idx );
2041 0 : FD_MGAUGE_SET( REPLAY, SCHED_LAST_DISPATCH_TIME, sched->next_ready_last_tick==LONG_MAX ? 0UL : (ulong)fd_clock_tile_tickcount_to_wallclock( clock, sched->next_ready_last_tick ) );
2042 0 : FD_MGAUGE_SET( REPLAY, SCHED_STAGING_LANE_USED, (ulong)fd_ulong_popcnt( sched->staged_bitset ) );
2043 0 : FD_MGAUGE_SET( REPLAY, SCHED_STAGING_LANE_PEAK, sched->staged_popcnt_wmk );
2044 0 : ulong staging_lane_head_bank_idx[ FD_METRICS_ENUM_STAGING_LANE_CNT ];
2045 0 : for( ulong lane=0UL; lane<FD_METRICS_ENUM_STAGING_LANE_CNT; lane++ ) {
2046 0 : staging_lane_head_bank_idx[ lane ] = fd_ulong_if( fd_ulong_extract_bit( sched->staged_bitset, (int)lane ), sched->staged_head_bank_idx[ lane ], ULONG_MAX );
2047 0 : }
2048 0 : FD_MGAUGE_ENUM_COPY( REPLAY, SCHED_STAGING_LANE_HEAD_BANK_INDEX, staging_lane_head_bank_idx );
2049 0 : FD_MGAUGE_SET( REPLAY, SCHED_TXN_POOL_USED, sched->depth-sched->txn_pool_free_cnt-1UL );
2050 0 : FD_MGAUGE_SET( REPLAY, SCHED_TXN_POOL_CAPACITY, sched->depth-1UL );
2051 0 : FD_MGAUGE_SET( REPLAY, SCHED_MICROBLOCK_POOL_USED, sched->depth-sched->mblk_pool_free_cnt );
2052 0 : FD_MGAUGE_SET( REPLAY, SCHED_MICROBLOCK_POOL_CAPACITY, sched->depth );
2053 0 : FD_MGAUGE_SET( REPLAY, SCHED_BLOCK_POOL_USED, sched->block_pool_popcnt );
2054 0 : FD_MGAUGE_SET( REPLAY, SCHED_BLOCK_POOL_CAPACITY, sched->block_cnt_max );
2055 :
2056 0 : ulong sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_CNT ];
2057 0 : sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_V_STAGED_IDX ] = sched->metrics->block_added_staged_cnt;
2058 0 : sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_V_UNSTAGED_IDX ] = sched->metrics->block_added_unstaged_cnt;
2059 0 : sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_V_DEAD_IDX ] = sched->metrics->block_added_dead_ood_cnt;
2060 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_BLOCK_ADDED, sched_block_added );
2061 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_REPLAYED, sched->metrics->block_removed_cnt );
2062 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_ABANDONED, sched->metrics->block_abandoned_cnt );
2063 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_REJECTED, sched->metrics->block_bad_cnt );
2064 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_PROMOTED, sched->metrics->block_promoted_cnt );
2065 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_DEMOTED, sched->metrics->block_demoted_cnt );
2066 0 : ulong sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_CNT ];
2067 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_NO_CHILD_IDX ] = sched->metrics->deactivate_no_child_cnt;
2068 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_NO_WORK_IDX ] = sched->metrics->deactivate_no_txn_cnt;
2069 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_ABANDONED_IDX ] = sched->metrics->deactivate_abandoned_cnt;
2070 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_MINORITY_FORK_IDX ] = sched->metrics->deactivate_pruned_cnt;
2071 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_BLOCK_DEACTIVATED, sched_deactivate );
2072 0 : FD_MCNT_SET( REPLAY, SCHED_LANE_SWITCHED, sched->metrics->lane_switch_cnt );
2073 0 : FD_MCNT_SET( REPLAY, SCHED_LANE_PROMOTED, sched->metrics->lane_promoted_cnt );
2074 0 : FD_MCNT_SET( REPLAY, SCHED_LANE_DEMOTED, sched->metrics->lane_demoted_cnt );
2075 0 : FD_MCNT_SET( REPLAY, SCHED_FORK_OBSERVED, sched->metrics->fork_observed_cnt );
2076 0 : ulong sched_alut[ FD_METRICS_ENUM_SCHED_LUT_RESOLVE_RESULT_CNT ];
2077 0 : sched_alut[ FD_METRICS_ENUM_SCHED_LUT_RESOLVE_RESULT_V_SUCCESS_IDX ] = sched->metrics->alut_success_cnt;
2078 0 : sched_alut[ FD_METRICS_ENUM_SCHED_LUT_RESOLVE_RESULT_V_FAILED_IDX ] = sched->metrics->alut_serializing_cnt;
2079 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_TXN_LUT_RESOLVE_ATTEMPTED, sched_alut );
2080 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_PARSED_ABANDONED, sched->metrics->txn_abandoned_parsed_cnt );
2081 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_EXECUTED_ABANDONED, sched->metrics->txn_abandoned_exec_done_cnt );
2082 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_DONE_ABANDONED, sched->metrics->txn_abandoned_done_cnt );
2083 : /* Fold in the open interval, which the next dispatch or completion credits */
2084 0 : ulong open_ticks = (ulong)( fd_tickcount()-sched->txn_in_flight_last_tick );
2085 0 : ulong busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ] );
2086 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_IN_FLIGHT_INTEGRAL_NANOS, sched->metrics->txn_weighted_in_flight_cnt+open_ticks*busy_cnt );
2087 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_BUSY_DURATION_NANOS, sched->metrics->txn_weighted_in_flight_tickcount+fd_ulong_if( !!busy_cnt, open_ticks, 0UL ) );
2088 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_IDLE_DURATION_NANOS, sched->metrics->txn_none_in_flight_tickcount+fd_ulong_if( !busy_cnt, open_ticks, 0UL ) );
2089 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_PARSED, sched->metrics->txn_parsed_cnt );
2090 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_EXECUTED, sched->metrics->txn_exec_done_cnt );
2091 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_SIGNATURE_VERIFIED, sched->metrics->txn_sigverify_done_cnt );
2092 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_POH_MIXED, sched->metrics->txn_mixin_done_cnt );
2093 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_DONE, sched->metrics->txn_done_cnt );
2094 0 : FD_MCNT_SET( REPLAY, SCHED_MICROBLOCK_PARSED, sched->metrics->mblk_parsed_cnt );
2095 0 : FD_MCNT_SET( REPLAY, SCHED_MICROBLOCK_HASHED, sched->metrics->mblk_poh_hashed_cnt );
2096 0 : FD_MCNT_SET( REPLAY, SCHED_MICROBLOCK_DONE, sched->metrics->mblk_poh_done_cnt );
2097 0 : FD_MCNT_SET( REPLAY, SCHED_INGESTED_BYTES, sched->metrics->bytes_ingested_cnt );
2098 0 : FD_MCNT_SET( REPLAY, SCHED_TRAILING_SKIPPED_BYTES, sched->metrics->bytes_ingested_unparsed_cnt );
2099 0 : FD_MCNT_SET( REPLAY, SCHED_DROPPED_BYTES, sched->metrics->bytes_dropped_cnt );
2100 0 : FD_MCNT_SET( REPLAY, SCHED_FEC_INGESTED, sched->metrics->fec_cnt );
2101 0 : }
2102 :
2103 : char *
2104 0 : fd_sched_get_state_cstr( fd_sched_t * sched ) {
2105 0 : sched->print_buf_sz = 0UL;
2106 0 : print_metrics( sched );
2107 0 : print_sched( sched );
2108 0 : return sched->print_buf;
2109 0 : }
2110 :
2111 0 : void * fd_sched_leave ( fd_sched_t * sched ) { return sched; }
2112 0 : void * fd_sched_delete( void * mem ) { return mem; }
2113 :
2114 :
2115 : /* Internal helpers. */
2116 :
2117 : static void
2118 : add_block( fd_sched_t * sched,
2119 : ulong bank_idx,
2120 0 : ulong parent_bank_idx ) {
2121 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2122 0 : FD_TEST( !block->in_sched );
2123 0 : sched->block_pool_popcnt++;
2124 :
2125 0 : block->txn_parsed_cnt = 0U;
2126 0 : block->txn_exec_in_flight_cnt = 0U;
2127 0 : block->txn_exec_done_cnt = 0U;
2128 0 : block->txn_sigverify_in_flight_cnt = 0U;
2129 0 : block->txn_sigverify_done_cnt = 0U;
2130 0 : block->poh_hashing_in_flight_cnt = 0U;
2131 0 : block->poh_mblk_in_flight_cnt = 0U;
2132 0 : block->poh_hashing_done_cnt = 0U;
2133 0 : block->poh_hash_cmp_done_cnt = 0U;
2134 0 : block->cert_in_flight_cnt = 0U;
2135 0 : block->txn_done_cnt = 0U;
2136 0 : block->shred_cnt = 0U;
2137 0 : block->shred_scan_idx = 0U;
2138 0 : block->shred_scan_off = 0U;
2139 0 : block->mblk_cnt = 0U;
2140 0 : block->mblk_freed_cnt = 0U;
2141 0 : block->mblk_tick_cnt = 0U;
2142 0 : block->mblk_unhashed_cnt = 0U;
2143 0 : block->hashcnt = 0UL;
2144 0 : block->dead_reason = FD_SCHED_DEAD_REASON_NONE;
2145 0 : block->txn_pool_max_popcnt = sched->depth - sched->txn_pool_free_cnt - 1UL;
2146 0 : block->mblk_pool_max_popcnt = sched->depth - sched->mblk_pool_free_cnt;
2147 0 : block->block_pool_max_popcnt = sched->block_pool_popcnt;
2148 0 : block->txn_idx_tail = 0U;
2149 0 : block->txn_sigverify_next_idx = 0U;
2150 0 : block->parse_mblk_idx = UINT_MAX;
2151 :
2152 0 : mblk_slist_remove_all( block->mblks_unhashed, sched->mblk_pool );
2153 0 : mblk_slist_remove_all( block->mblks_hashing_in_progress, sched->mblk_pool );
2154 0 : mblk_slist_remove_all( block->mblks_mixin_in_progress, sched->mblk_pool );
2155 0 : block->last_mblk_is_tick = 0;
2156 0 : block->tick_hashcnt_wmk = 0UL;
2157 0 : block->curr_tick_hashcnt = 0UL;
2158 0 : block->tick_height = ULONG_MAX;
2159 0 : block->max_tick_height = ULONG_MAX;
2160 0 : block->hashes_per_tick = ULONG_MAX;
2161 0 : block->inconsistent_hashes_per_tick = 0;
2162 0 : block->zero_hash_tick = 0;
2163 :
2164 0 : block->header_seen = 0;
2165 0 : block->genesis_cert_seen = 0;
2166 0 : block->update_parent_seen = 0;
2167 0 : block->footer_seen = 0;
2168 0 : block->alpentick_seen = 0;
2169 :
2170 0 : block->mblks_rem = 0UL;
2171 0 : block->txns_rem = 0UL;
2172 0 : block->fec_buf_sz = 0U;
2173 0 : block->fec_buf_boff = 0U;
2174 0 : block->fec_buf_soff = 0U;
2175 0 : block->poison_cnt = 0U;
2176 0 : block->poison_serialize = 0;
2177 0 : block->fec_eob = 0;
2178 0 : block->fec_sob = 1;
2179 :
2180 0 : block->fec_eos = 0;
2181 0 : block->rooted = 0;
2182 0 : block->dying = 0;
2183 0 : block->discarded = 0;
2184 0 : block->fec_completed_ns = 0L;
2185 0 : block->refcnt = 1;
2186 0 : block->in_sched = 1;
2187 0 : block->in_rdisp = 0;
2188 0 : block->block_start_signaled = 0;
2189 0 : block->block_end_signaled = 0;
2190 0 : block->block_start_done = 0;
2191 0 : block->block_end_done = 0;
2192 0 : block->staged = 0;
2193 :
2194 0 : block->luf_depth = 0UL;
2195 :
2196 : /* New leaf node, no child, no sibling. */
2197 0 : block->child_idx = ULONG_MAX;
2198 0 : block->sibling_idx = ULONG_MAX;
2199 0 : block->parent_idx = ULONG_MAX;
2200 :
2201 0 : if( FD_UNLIKELY( parent_bank_idx==ULONG_MAX ) ) {
2202 0 : return;
2203 0 : }
2204 :
2205 : /* node->parent link */
2206 0 : fd_sched_block_t * parent_block = block_pool_ele( sched, parent_bank_idx );
2207 0 : block->parent_idx = parent_bank_idx;
2208 :
2209 : /* parent->node and sibling->node links */
2210 0 : ulong child_idx = bank_idx;
2211 0 : if( FD_LIKELY( parent_block->child_idx==ULONG_MAX ) ) { /* Optimize for no forking. */
2212 0 : parent_block->child_idx = child_idx;
2213 0 : } else {
2214 0 : fd_sched_block_t * curr_block = block_pool_ele( sched, parent_block->child_idx );
2215 0 : while( curr_block->sibling_idx!=ULONG_MAX ) {
2216 0 : curr_block = block_pool_ele( sched, curr_block->sibling_idx );
2217 0 : }
2218 0 : curr_block->sibling_idx = child_idx;
2219 0 : sched->metrics->fork_observed_cnt++;
2220 0 : }
2221 :
2222 0 : if( FD_UNLIKELY( parent_block->dying ) ) {
2223 0 : record_lineage_death( block, parent_block );
2224 0 : block->dying = 1;
2225 0 : }
2226 0 : }
2227 :
2228 : /* Alpenglow block structure. agave's BlockComponentProcessor rules
2229 : an Alpenglow block invalid unless its components are laid out as
2230 :
2231 : header | [genesis cert] | entries* | [update parent] | entries* | footer | alpentick
2232 :
2233 : with exactly one header and one footer, and the alpentick as the
2234 : final component. The helpers below drive the same state machine off
2235 : the batches sched parses. */
2236 :
2237 : static int
2238 : ag_on_marker( fd_sched_t * sched,
2239 : fd_sched_block_t * block,
2240 0 : fd_block_marker_t const * marker ) {
2241 0 : (void)sched;
2242 0 : switch( marker->kind ) {
2243 :
2244 0 : case FD_BLOCK_MARKER_KIND_HEADER:
2245 0 : if( FD_UNLIKELY( block->header_seen ) ) {
2246 0 : FD_LOG_INFO(( "bad block: MULTIPLE_BLOCK_HEADERS, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
2247 0 : return FD_SCHED_DEAD_REASON_MULTIPLE_BLOCK_HEADERS;
2248 0 : }
2249 : /* Checking for shred header and parent_slot mismatch occurs at the
2250 : rotor level, so it does not need to be checked in sched. We do
2251 : this because currently replay tile never sees shred headers. */
2252 0 : block->header_seen = 1;
2253 0 : return FD_SCHED_DEAD_REASON_NONE;
2254 :
2255 0 : case FD_BLOCK_MARKER_KIND_GENESIS_CERT:
2256 0 : if( FD_UNLIKELY( !block->header_seen ) ) {
2257 0 : FD_LOG_INFO(( "bad block: MISSING_PARENT_MARKER, slot %lu, parent slot %lu, genesis cert before header", block->slot, block->parent_slot ));
2258 0 : return FD_SCHED_DEAD_REASON_MISSING_PARENT_MARKER;
2259 0 : }
2260 0 : if( FD_UNLIKELY( block->mblk_cnt || block->genesis_cert_seen || block->footer_seen ) ) {
2261 0 : FD_LOG_INFO(( "bad block: GENESIS_CERT_OUT_OF_ORDER, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
2262 0 : return FD_SCHED_DEAD_REASON_GENESIS_CERT_OUT_OF_ORDER;
2263 0 : }
2264 0 : block->genesis_cert_seen = 1;
2265 0 : return FD_SCHED_DEAD_REASON_NONE;
2266 :
2267 0 : case FD_BLOCK_MARKER_KIND_FOOTER:
2268 0 : if( FD_UNLIKELY( !block->header_seen ) ) {
2269 0 : FD_LOG_INFO(( "bad block: MISSING_PARENT_MARKER, slot %lu, parent slot %lu, footer before header", block->slot, block->parent_slot ));
2270 0 : return FD_SCHED_DEAD_REASON_MISSING_PARENT_MARKER;
2271 0 : }
2272 0 : if( FD_UNLIKELY( block->footer_seen ) ) {
2273 0 : FD_LOG_INFO(( "bad block: MULTIPLE_BLOCK_FOOTERS, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
2274 0 : return FD_SCHED_DEAD_REASON_MULTIPLE_BLOCK_FOOTERS;
2275 0 : }
2276 0 : block->footer = marker->footer;
2277 0 : block->footer_seen = 1;
2278 0 : return FD_SCHED_DEAD_REASON_NONE;
2279 :
2280 0 : case FD_BLOCK_MARKER_KIND_UPDATE_PARENT:
2281 0 : block->update_parent_seen = 1;
2282 0 : return FD_SCHED_DEAD_REASON_SPURIOUS_UPDATE_PARENT;
2283 :
2284 0 : default:
2285 0 : FD_LOG_INFO(( "bad block: slot %lu, parent slot %lu, unknown block marker kind %u", block->slot, block->parent_slot, marker->kind ));
2286 0 : return FD_SCHED_DEAD_REASON_BAD_BLOCK_MARKER;
2287 0 : }
2288 0 : }
2289 :
2290 : /* Called when a batch header declares one or more microblocks. */
2291 :
2292 : static int
2293 0 : ag_on_entry_batch( fd_sched_block_t * block ) {
2294 0 : if( FD_UNLIKELY( !block->header_seen ) ) {
2295 0 : FD_LOG_INFO(( "bad block: MISSING_PARENT_MARKER, slot %lu, parent slot %lu, entries before header", block->slot, block->parent_slot ));
2296 0 : return FD_SCHED_DEAD_REASON_MISSING_PARENT_MARKER;
2297 0 : }
2298 0 : if( FD_UNLIKELY( block->alpentick_seen ) ) {
2299 0 : FD_LOG_INFO(( "bad block: ENTRY_AFTER_BLOCK_FOOTER, slot %lu, parent slot %lu, batch after alpentick", block->slot, block->parent_slot ));
2300 0 : return FD_SCHED_DEAD_REASON_ENTRY_AFTER_BLOCK_FOOTER;
2301 0 : }
2302 : /* Only the alpentick may follow the footer: a batch of exactly one
2303 : microblock which must turn out to be a tick (checked when its
2304 : header parses). */
2305 0 : if( FD_UNLIKELY( block->footer_seen && block->mblks_rem!=1UL ) ) {
2306 0 : FD_LOG_INFO(( "bad block: ENTRY_AFTER_BLOCK_FOOTER, slot %lu, parent slot %lu, batch of %lu microblocks after footer", block->slot, block->parent_slot, block->mblks_rem ));
2307 0 : return FD_SCHED_DEAD_REASON_ENTRY_AFTER_BLOCK_FOOTER;
2308 0 : }
2309 0 : return FD_SCHED_DEAD_REASON_NONE;
2310 0 : }
2311 :
2312 : static int
2313 0 : ag_on_final( fd_sched_block_t * block ) {
2314 0 : if( FD_UNLIKELY( !block->footer_seen ) ) {
2315 0 : FD_LOG_INFO(( "bad block: MISSING_BLOCK_FOOTER, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
2316 0 : return FD_SCHED_DEAD_REASON_MISSING_BLOCK_FOOTER;
2317 0 : }
2318 0 : if( FD_UNLIKELY( !block->alpentick_seen ) ) {
2319 0 : FD_LOG_INFO(( "bad block: INVALID_ALPENTICK_POSITION, slot %lu, parent slot %lu, block ended after footer without alpentick", block->slot, block->parent_slot ));
2320 0 : return FD_SCHED_DEAD_REASON_INVALID_ALPENTICK_POSITION;
2321 0 : }
2322 0 : return FD_SCHED_DEAD_REASON_NONE;
2323 0 : }
2324 :
2325 : /* Agave invokes verify_ticks() anywhere between once per slot and once
2326 : per entry batch, before transactions are parsed or dispatched for
2327 : execution. We can't do quite the same thing due to out-of-order
2328 : scheduling and the fact that we allow parsing to run well ahead of
2329 : block boundaries. Out-of-order scheduling is good, so is overlapping
2330 : parsing with execution. The easiest thing for us would be to just
2331 : delay verify_ticks() wholesale till the end of a slot, except that
2332 : this opens us up to bogus tick and hash counts, potentially causing
2333 : runaway consumption of compute cycles. Of all the checks that are
2334 : performed in verify_ticks(), two types are relevant to mitigating
2335 : this risk. One is constraining the number of ticks, and the other is
2336 : constraining the number of hashes per tick. So we implement these
2337 : checks here, and perform them on the fly as eagerly as possible.
2338 :
2339 : Returns FD_SCHED_DEAD_REASON_NONE (0) on success, else the
2340 : FD_SCHED_DEAD_REASON_* the block should be ruled invalid for. Does
2341 : not modify the block. */
2342 : static int
2343 0 : verify_ticks_eager( fd_sched_block_t * block ) {
2344 0 : FD_TEST( block->hashes_per_tick!=ULONG_MAX ); /* PoH params initialized. */
2345 :
2346 0 : if( FD_UNLIKELY( block->mblk_tick_cnt+block->tick_height>block->max_tick_height ) ) {
2347 0 : FD_LOG_INFO(( "bad block: TOO_MANY_TICKS, slot %lu, parent slot %lu, tick_cnt %u, tick_height %lu, max_tick_height %lu", block->slot, block->parent_slot, block->mblk_tick_cnt, block->tick_height, block->max_tick_height ));
2348 0 : return FD_SCHED_DEAD_REASON_TOO_MANY_TICKS;
2349 0 : }
2350 0 : if( FD_UNLIKELY( block->hashes_per_tick>1UL && block->zero_hash_tick ) ) {
2351 0 : FD_LOG_INFO(( "bad block: ZERO_HASH_TICK, slot %lu, parent slot %lu, has at least one zero hash tick", block->slot, block->parent_slot ));
2352 0 : return FD_SCHED_DEAD_REASON_ZERO_HASH_TICK;
2353 0 : }
2354 0 : if( FD_UNLIKELY( block->hashes_per_tick>1UL && block->mblk_tick_cnt && (block->hashes_per_tick!=block->tick_hashcnt_wmk||block->inconsistent_hashes_per_tick) ) ) {
2355 0 : FD_LOG_INFO(( "bad block: WRONG_HASHES_PER_TICK, slot %lu, parent slot %lu, expected %lu, got %lu", block->slot, block->parent_slot, block->hashes_per_tick, block->tick_hashcnt_wmk ));
2356 0 : return FD_SCHED_DEAD_REASON_WRONG_HASHES_PER_TICK;
2357 0 : }
2358 0 : if( FD_UNLIKELY( block->hashes_per_tick>1UL && block->curr_tick_hashcnt>block->hashes_per_tick ) ) { /* >1 to ignore low power hashing or no hashing cases */
2359 : /* We couldn't really check this at parse time because we may not
2360 : have the expected hashes per tick value yet. We couldn't delay
2361 : this till after all PoH hashing is done, because this would be a
2362 : DoS vector. This can't be merged with the above check, because a
2363 : malformed block might not end with a tick. As in, a block might
2364 : end with a non-tick microblock with a high hashcnt. Note that
2365 : checking the hashcnt between ticks transitively places an upper
2366 : bound on the hashcnt of individual microblocks, thus mitigating
2367 : the DoS vector. */
2368 0 : FD_LOG_INFO(( "bad block: TICK_HASHES_OVERFLOW, slot %lu, parent slot %lu, observed cumulative tick_hashcnt %lu, expected %lu", block->slot, block->parent_slot, block->curr_tick_hashcnt, block->hashes_per_tick ));
2369 0 : return FD_SCHED_DEAD_REASON_TICK_HASHES_OVERFLOW;
2370 0 : }
2371 :
2372 0 : return FD_SCHED_DEAD_REASON_NONE;
2373 0 : }
2374 :
2375 : /* https://github.com/anza-xyz/agave/blob/v3.0.6/ledger/src/blockstore_processor.rs#L1057
2376 :
2377 : The only check we don't do here is TRAILING_ENTRY, which can be done
2378 : independently when we parse the final FEC set of a block.
2379 :
2380 : Returns FD_SCHED_DEAD_REASON_NONE (0) on success, else the
2381 : FD_SCHED_DEAD_REASON_* the block should be ruled invalid for. Does
2382 : not modify the block. */
2383 : static int
2384 0 : verify_ticks_final( fd_sched_block_t * block ) {
2385 0 : FD_TEST( block->fec_eos );
2386 :
2387 0 : if( FD_UNLIKELY( block->mblk_tick_cnt+block->tick_height<block->max_tick_height ) ) {
2388 0 : FD_LOG_INFO(( "bad block: TOO_FEW_TICKS, slot %lu, parent slot %lu, tick_cnt %u, tick_height %lu, max_tick_height %lu", block->slot, block->parent_slot, block->mblk_tick_cnt, block->tick_height, block->max_tick_height ));
2389 0 : return FD_SCHED_DEAD_REASON_TOO_FEW_TICKS;
2390 0 : }
2391 :
2392 0 : return verify_ticks_eager( block );
2393 0 : }
2394 :
2395 : int
2396 : fd_sched_block_verify_ticks( fd_sched_t * sched,
2397 : ulong bank_idx,
2398 : ulong tick_height,
2399 : ulong max_tick_height,
2400 0 : ulong hashes_per_tick ) {
2401 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2402 0 : FD_TEST( bank_idx<sched->block_cnt_max );
2403 0 : FD_TEST( max_tick_height>tick_height );
2404 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2405 : /* Set the tick window directly; skip fd_sched_set_poh_params
2406 : PoH fixup, unneeded here and unsafe to repeat per FEC set. */
2407 0 : block->tick_height = tick_height;
2408 0 : block->max_tick_height = max_tick_height;
2409 0 : block->hashes_per_tick = hashes_per_tick;
2410 : /* fec_eos => slot fully ingested, so the TOO_FEW check is meaningful
2411 : too. */
2412 0 : return block->fec_eos ? verify_ticks_final( block ) : verify_ticks_eager( block );
2413 0 : }
2414 :
2415 0 : #define CHECK( cond ) do { \
2416 0 : if( FD_UNLIKELY( !(cond) ) ) { \
2417 0 : return FD_SCHED_DEAD_REASON_NONE; \
2418 0 : } \
2419 0 : } while( 0 )
2420 :
2421 : /* CHECK that it is safe to read at least n more bytes. */
2422 0 : #define CHECK_LEFT( n ) CHECK( (n)<=(block->fec_buf_sz-block->fec_buf_soff) )
2423 :
2424 : /* Consume as much as possible from the buffer. By the end of this
2425 : function, there could be residual data left over in the buffer. That
2426 : residual has to be either a partially received microblock header or a
2427 : transaction that straddles a FEC set boundary. These bytes are kept
2428 : and will be concatenated with the next FEC set. This residual is
2429 : bounded.
2430 :
2431 : Trailing bytes within a batch are dropped immediately. These
2432 : trailing bytes can span arbitrarily many FEC sets. */
2433 : FD_WARN_UNUSED static int
2434 0 : fd_sched_parse( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx ) {
2435 0 : while( 1 ) {
2436 0 : while( block->txns_rem>0UL ) {
2437 0 : int err = fd_sched_parse_txn( sched, block, alut_ctx );
2438 0 : if( FD_UNLIKELY( -1==err ) ) return FD_SCHED_DEAD_REASON_NONE;
2439 0 : else if( FD_UNLIKELY( err ) ) return err;
2440 0 : }
2441 :
2442 0 : if( block->txns_rem==0UL && block->mblks_rem>0UL ) {
2443 0 : if( FD_UNLIKELY( block->mblk_cnt>=sched->max_mblk_per_slot ) ) {
2444 : /* Microblock count is enforced as invariant
2445 : mblk_cnt+mblks_rem<=max_mblk_per_slot
2446 : when we read the declared count in a batch header. So we
2447 : shouldn't get here. */
2448 0 : sched->print_buf_sz = 0UL;
2449 0 : print_all( sched, block );
2450 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2451 0 : FD_LOG_CRIT(( "invariant violation: slot %lu, parent slot %lu, mblks_rem %lu, mblk_cnt %u (%u ticks) >= %lu", block->slot, block->parent_slot, block->mblks_rem, block->mblk_cnt, block->mblk_tick_cnt, sched->max_mblk_per_slot ));
2452 0 : }
2453 :
2454 0 : CHECK_LEFT( sizeof(fd_microblock_hdr_t) );
2455 0 : fd_microblock_hdr_t * hdr = (fd_microblock_hdr_t *)fd_type_pun( block->fec_buf+block->fec_buf_soff );
2456 0 : block->fec_buf_soff += (uint)sizeof(fd_microblock_hdr_t);
2457 :
2458 0 : if( FD_UNLIKELY( hdr->txn_cnt>fd_ulong_sat_sub( sched->max_txn_per_slot, block->txn_parsed_cnt ) ) ) {
2459 0 : FD_LOG_INFO(( "bad block: TOO_MANY_TXNS, microblock header declared too many transactions, slot %lu, parent slot %lu, txn_parsed_cnt %u, hdr->txn_cnt %lu", block->slot, block->parent_slot, block->txn_parsed_cnt, hdr->txn_cnt ));
2460 0 : return FD_SCHED_DEAD_REASON_TOO_MANY_TXNS;
2461 0 : }
2462 0 : if( FD_UNLIKELY( !sched->is_alpenglow && hdr->hash_cnt>fd_ulong_sat_sub( FD_RUNTIME_MAX_HASHES_PER_TICK, block->curr_tick_hashcnt ) ) ) {
2463 0 : FD_LOG_INFO(( "bad block: TICK_HASHES_OVERFLOW_INGEST, slot %lu, parent slot %lu, curr_tick_hashcnt %lu, hdr->hash_cnt %lu", block->slot, block->parent_slot, block->curr_tick_hashcnt, hdr->hash_cnt ));
2464 0 : return FD_SCHED_DEAD_REASON_TICK_HASHES_OVERFLOW_INGEST;
2465 0 : }
2466 0 : if( FD_UNLIKELY( sched->is_alpenglow && hdr->hash_cnt!=1UL ) ) {
2467 0 : FD_LOG_INFO(( "bad block: ALPENGLOW_HASH_CNT, slot %lu, parent slot %lu, mblk idx %u declared hash_cnt %lu", block->slot, block->parent_slot, block->mblk_cnt, hdr->hash_cnt ));
2468 0 : return FD_SCHED_DEAD_REASON_ALPENGLOW_HASH_CNT;
2469 0 : }
2470 0 : if( FD_UNLIKELY( sched->is_alpenglow && block->footer_seen ) ) {
2471 0 : if( FD_UNLIKELY( hdr->txn_cnt ) ) {
2472 0 : FD_LOG_INFO(( "bad block: ENTRY_AFTER_BLOCK_FOOTER, slot %lu, parent slot %lu, transaction microblock after footer", block->slot, block->parent_slot ));
2473 0 : return FD_SCHED_DEAD_REASON_ENTRY_AFTER_BLOCK_FOOTER;
2474 0 : }
2475 0 : block->alpentick_seen = 1;
2476 0 : }
2477 :
2478 0 : block->mblks_rem--;
2479 0 : block->txns_rem = hdr->txn_cnt;
2480 :
2481 : /* One might think that every microblock needs to have at least
2482 : one hash, otherwise the block should be considered invalid. A
2483 : vanilla validator certainly produces microblocks that conform
2484 : to this. But a modded validator could in theory produce
2485 : zero-hash microblocks. Agave's replay stage will happily take
2486 : those microblocks. The Agave implementation-defined way of
2487 : doing PoH verify is as follows:
2488 :
2489 : For a tick microblock, do the same number of hashes as
2490 : specified by the microblock. Zero hashes are not allowed.
2491 :
2492 : For a transaction microblock, if the number of hashes specified
2493 : by the microblock is <= 1, then do zero pure hashes, and simply
2494 : do a mixin/record. Otherwise, do (number of hashes-1) amount
2495 : of pure hashing, and then do a mixin. However, note that for
2496 : the purposes of tick_verify, the number of hashes specified by
2497 : the microblock is taken verbatim.
2498 :
2499 : An additional constraint is that Agave expects non-tick
2500 : microblocks to leave the cumulative tick hash count at least
2501 : one away from hashes_per_tick at the end of a batch:
2502 : https://github.com/anza-xyz/agave/blob/v4.0.0-rc.0/entry/src/entry.rs#L672
2503 :
2504 : Observe that this is not a net new constraint. Since ticks are
2505 : not allowed to have zero hashes, and ticks have to end at
2506 : exactly hashes_per_tick, this check is redundant.
2507 :
2508 :
2509 : Some additional references to Agave:
2510 :
2511 : On the consumer side, non-tick microblocks can have a zero hash
2512 : count, and the mixin will happen anyways:
2513 : https://github.com/anza-xyz/agave/blob/v4.0.0-rc.0/entry/src/entry.rs#L326
2514 : https://github.com/anza-xyz/agave/blob/v4.0.0-rc.0/entry/src/entry.rs#L542
2515 :
2516 : Ticks cannot have a zero hash count:
2517 : https://github.com/anza-xyz/agave/blob/v4.1.1/entry/src/entry.rs#L684
2518 :
2519 : On the producer side, Agave reserves at least one hash for the
2520 : tick, so an Agave produced tick would satisfy the verifier:
2521 : https://github.com/anza-xyz/agave/blob/v4.0.0-rc.0/entry/src/poh.rs#L78
2522 : https://github.com/anza-xyz/agave/blob/v4.0.0-rc.0/entry/src/poh.rs#L101 */
2523 0 : block->curr_tick_hashcnt = fd_ulong_sat_add( hdr->hash_cnt, block->curr_tick_hashcnt ); /* For tick_verify, take the number of hashes verbatim. */
2524 0 : sched->metrics->mblk_parsed_cnt++;
2525 0 : if( FD_UNLIKELY( !hdr->txn_cnt ) ) {
2526 : /* This is a tick microblock. */
2527 0 : if( FD_UNLIKELY( block->mblk_tick_cnt && block->tick_hashcnt_wmk!=block->curr_tick_hashcnt ) ) {
2528 0 : block->inconsistent_hashes_per_tick = 1;
2529 0 : if( FD_LIKELY( block->hashes_per_tick!=ULONG_MAX && block->hashes_per_tick>1UL ) ) {
2530 : /* >1 to ignore low power hashing or hashing disabled */
2531 0 : FD_LOG_INFO(( "bad block: INCONSISTENT_TICK_HASHES, slot %lu, parent slot %lu, tick idx %u, tick_hashcnt_wmk %lu, curr hashcnt %lu, hashes_per_tick %lu", block->slot, block->parent_slot, block->mblk_tick_cnt, block->tick_hashcnt_wmk, block->curr_tick_hashcnt, block->hashes_per_tick ));
2532 0 : return FD_SCHED_DEAD_REASON_INCONSISTENT_TICK_HASHES;
2533 0 : }
2534 0 : }
2535 0 : if( FD_UNLIKELY( !hdr->hash_cnt ) ) {
2536 0 : block->zero_hash_tick = 1;
2537 0 : if( FD_LIKELY( block->hashes_per_tick!=ULONG_MAX && block->hashes_per_tick>1UL ) ) {
2538 0 : FD_LOG_INFO(( "bad block: ZERO_HASH_TICK_INGEST, slot %lu, parent slot %lu, tick idx %u has zero hashes", block->slot, block->parent_slot, block->mblk_tick_cnt ));
2539 0 : return FD_SCHED_DEAD_REASON_ZERO_HASH_TICK_INGEST;
2540 0 : }
2541 0 : }
2542 0 : block->tick_hashcnt_wmk = fd_ulong_max( block->curr_tick_hashcnt, block->tick_hashcnt_wmk );
2543 0 : block->curr_tick_hashcnt = 0UL;
2544 0 : block->mblk_tick_cnt++;
2545 0 : }
2546 :
2547 0 : FD_TEST( sched->mblk_pool_free_cnt ); /* can_ingest should have guaranteed sufficient free capacity. */
2548 0 : uint mblk_idx = sched->mblk_pool_free_head;
2549 0 : sched->mblk_pool_free_head = sched->mblk_pool[ mblk_idx ].next;
2550 0 : sched->mblk_pool_free_cnt--;
2551 :
2552 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
2553 0 : mblk->start_txn_idx = block->txn_parsed_cnt;
2554 0 : mblk->end_txn_idx = mblk->start_txn_idx+hdr->txn_cnt;
2555 0 : mblk->curr_txn_idx = mblk->start_txn_idx;
2556 0 : mblk->hashcnt = fd_ulong_sat_sub( hdr->hash_cnt, fd_ulong_if( !hdr->txn_cnt, 0UL, 1UL ) ); /* For pure hashing, implement saturating sub for non-tick microblocks. */
2557 0 : mblk->curr_hashcnt = 0UL;
2558 0 : mblk->curr_sig_cnt = 0U;
2559 0 : mblk->mixin_txn_idx = 0U;
2560 0 : mblk->is_tick = !hdr->txn_cnt;
2561 0 : memcpy( mblk->end_hash, hdr->hash, sizeof(fd_hash_t) );
2562 0 : memcpy( mblk->curr_hash, block->poh_hash, sizeof(fd_hash_t) );
2563 0 : block->parse_mblk_idx = mblk_idx;
2564 :
2565 : /* Update block tracking. */
2566 0 : block->hashcnt += mblk->hashcnt+fd_ulong_if( !hdr->txn_cnt, 0UL, 1UL );
2567 0 : memcpy( block->poh_hash, hdr->hash, sizeof(fd_hash_t) );
2568 0 : block->last_mblk_is_tick = mblk->is_tick;
2569 0 : block->mblk_cnt++;
2570 :
2571 : #if FD_SCHED_SKIP_POH
2572 : block->poh_hashing_done_cnt++;
2573 : block->poh_hash_cmp_done_cnt++;
2574 : free_mblk( sched, block, mblk_idx );
2575 : #else
2576 0 : mblk_slist_idx_push_tail( block->mblks_unhashed, mblk_idx, sched->mblk_pool );
2577 0 : block->mblk_unhashed_cnt++;
2578 0 : #endif
2579 0 : continue;
2580 0 : }
2581 0 : if( block->txns_rem==0UL && block->mblks_rem==0UL && block->fec_sob ) {
2582 0 : CHECK_LEFT( sizeof(ulong) );
2583 0 : FD_TEST( block->fec_buf_soff==0U );
2584 0 : if( FD_UNLIKELY( sched->is_alpenglow && !FD_LOAD( ulong, block->fec_buf ) && !block->fec_eob ) ) {
2585 : /* Block marker split across FEC sets */
2586 0 : fd_block_marker_t peek[1];
2587 0 : if( FD_UNLIKELY( FD_BLOCK_MARKER_DE_ERR_SZ==fd_block_marker_de( peek, block->fec_buf, (ulong)block->fec_buf_sz ) &&
2588 0 : block->fec_buf_sz<FD_BLOCK_MARKER_SER_MAX ) ) return FD_SCHED_DEAD_REASON_NONE;
2589 0 : }
2590 0 : block->mblks_rem = FD_LOAD( ulong, block->fec_buf );
2591 0 : block->fec_buf_soff += (uint)sizeof(ulong);
2592 :
2593 0 : if( FD_UNLIKELY( block->mblks_rem>fd_ulong_sat_sub( sched->max_mblk_per_slot, block->mblk_cnt ) ) ) {
2594 0 : FD_LOG_INFO(( "bad block: TOO_MANY_MICROBLOCKS, slot %lu, parent slot %lu, mblk_cnt %u (%u ticks) + hdr->mblk_cnt %lu >= %lu", block->slot, block->parent_slot, block->mblk_cnt, block->mblk_tick_cnt, block->mblks_rem, sched->max_mblk_per_slot ));
2595 0 : return FD_SCHED_DEAD_REASON_TOO_MANY_MICROBLOCKS;
2596 0 : }
2597 :
2598 0 : block->fec_sob = 0;
2599 :
2600 0 : if( FD_UNLIKELY( !block->mblks_rem && sched->is_alpenglow ) ) {
2601 : /* A zero-microblock batch is a block marker */
2602 0 : fd_block_marker_t marker[1];
2603 0 : int err = fd_block_marker_de( marker, block->fec_buf, (ulong)block->fec_buf_sz );
2604 0 : if( FD_UNLIKELY( err ) ) {
2605 0 : FD_LOG_INFO(( "bad block: slot %lu, unable to parse block marker (err %d)", block->slot, err ));
2606 0 : return FD_SCHED_DEAD_REASON_BAD_BLOCK_MARKER;
2607 0 : }
2608 0 : int dead_reason = ag_on_marker( sched, block, marker );
2609 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) return dead_reason;
2610 0 : block->fec_buf_soff = (uint)( FD_BLOCK_MARKER_PREAMBLE_SZ+FD_LOAD( ushort, block->fec_buf+FD_BLOCK_MARKER_PREAMBLE_SZ-sizeof(ushort) ) ); /* the marker is not trailing bytes */
2611 0 : } else if( FD_UNLIKELY( block->mblks_rem && sched->is_alpenglow ) ) {
2612 0 : int dead_reason = ag_on_entry_batch( block );
2613 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) return dead_reason;
2614 0 : } else if( FD_UNLIKELY( !block->mblks_rem && !sched->is_alpenglow ) ) {
2615 0 : FD_LOG_INFO(( "bad block: ZERO_MICROBLOCKS, slot %lu, parent slot %lu, mblk_cnt %u (%u ticks)", block->slot, block->parent_slot, block->mblk_cnt, block->mblk_tick_cnt ));
2616 0 : return FD_SCHED_DEAD_REASON_ZERO_MICROBLOCKS;
2617 0 : }
2618 0 : continue;
2619 0 : }
2620 0 : if( block->txns_rem==0UL && block->mblks_rem==0UL ) {
2621 0 : break;
2622 0 : }
2623 0 : }
2624 0 : if( !block->fec_sob && block->txns_rem==0UL && block->mblks_rem==0UL ) {
2625 : /* All microblocks announced by the current batch header have been
2626 : parsed out. Anything still in the buffer must be trailing bytes.
2627 : Drop them now because trailing bytes can span many FEC sets and
2628 : accumulating them would overflow the parse buffer. Any followup
2629 : FEC sets within the same batch will simply be discarded
2630 : wholesale. */
2631 0 : sched->metrics->bytes_ingested_unparsed_cnt += block->fec_buf_sz-block->fec_buf_soff;
2632 0 : block->fec_buf_boff += block->fec_buf_sz;
2633 0 : block->fec_buf_soff = 0U;
2634 0 : block->fec_buf_sz = 0U;
2635 0 : }
2636 0 : if( block->fec_eob ) {
2637 0 : block->fec_sob = 1;
2638 0 : block->fec_eob = 0;
2639 0 : }
2640 0 : return FD_SCHED_DEAD_REASON_NONE;
2641 0 : }
2642 :
2643 : static inline fd_acct_addr_t const *
2644 0 : get_acct( fd_txn_t const * txn, fd_acct_addr_t const * imms, fd_acct_addr_t const * alts, ushort idx ) {
2645 0 : if( FD_LIKELY( idx<txn->acct_addr_cnt ) ) return imms+idx;
2646 0 : if( FD_UNLIKELY( !alts ) ) return NULL;
2647 0 : return alts+(idx-txn->acct_addr_cnt);
2648 0 : }
2649 :
2650 : /* Returns 1 if the transaction write locks any account in the block's
2651 : poison set. */
2652 : static int
2653 0 : block_poison_hit( fd_sched_block_t const * block, fd_txn_t const * txn, fd_acct_addr_t const * imms, fd_acct_addr_t const * alts ) {
2654 0 : if( FD_LIKELY( !block->poison_cnt ) ) return 0;
2655 :
2656 0 : for( fd_txn_acct_iter_t iter = fd_txn_acct_iter_init( txn, FD_TXN_ACCT_CAT_WRITABLE ); iter!=fd_txn_acct_iter_end(); iter=fd_txn_acct_iter_next( iter ) ) {
2657 0 : fd_acct_addr_t const * acct = get_acct( txn, imms, alts, (ushort)fd_txn_acct_iter_idx( iter ) );
2658 : /* Shouldn't really happen, since ALT-serializing transactions don't
2659 : enter this function, so all account pubkeys are available. */
2660 0 : if( FD_UNLIKELY( !acct ) ) return 1;
2661 0 : for( uint j=0U; j<block->poison_cnt; j++ ) {
2662 0 : if( FD_UNLIKELY( !memcmp( block->poison[ j ].b, acct->b, 32UL ) ) ) return 1;
2663 0 : }
2664 0 : }
2665 :
2666 0 : return 0;
2667 0 : }
2668 :
2669 : /* Returns 0 on success, 1 if the poison set overflowed. */
2670 : static int
2671 0 : block_poison_insert( fd_sched_t * sched, fd_sched_block_t * block, fd_acct_addr_t const * acct ) {
2672 0 : for( uint j=0U; j<block->poison_cnt; j++ ) {
2673 0 : if( FD_UNLIKELY( !memcmp( block->poison[ j ].b, acct->b, 32UL ) ) ) return 0; /* Dedup. */
2674 0 : }
2675 :
2676 0 : if( FD_UNLIKELY( block->poison_cnt>=FD_SCHED_POISON_MAX_ACCT_PER_SLOT ) ) {
2677 0 : block->poison_serialize = 1;
2678 0 : sched->metrics->poison_latch_overflow_cnt++;
2679 0 : return 1;
2680 0 : }
2681 :
2682 0 : block->poison[ block->poison_cnt++ ] = *acct;
2683 0 : sched->metrics->poison_cnt_wmk = fd_uint_max( sched->metrics->poison_cnt_wmk, block->poison_cnt );
2684 0 : return 0;
2685 0 : }
2686 :
2687 : /* Adds relevant accounts to the poison set. This function is
2688 : conservative and stateless. Account data is not necessarily
2689 : available at insertion time, so we cannot tell which writable account
2690 : is the programdata PDA. So the rule is simply: if loader v3 appears
2691 : anywhere in the transaction account list, poison every writable
2692 : account of the transaction.
2693 :
2694 : That is sound because instruction level writability can never exceed
2695 : transaction level writability thanks to CPI rejecting writability
2696 : escalations. So any account a loader v3 instruction could modify, at
2697 : the top-level or in a CPI, must be transaction level writable.
2698 : Additionally, the loader being in the account keys is necessary for
2699 : any invocation of it. */
2700 : static void
2701 : block_poison_add( fd_sched_t * sched,
2702 : fd_sched_block_t * block,
2703 : fd_txn_t const * txn,
2704 : fd_acct_addr_t const * imms,
2705 : fd_acct_addr_t const * alts,
2706 0 : ulong alt_cnt ) {
2707 : /* The poison set is never consulted again once it enters a
2708 : serializing regime, so there is no point in adding anything to it.
2709 : Returning here also keeps the serializing counters at one increment
2710 : per block. */
2711 0 : if( FD_UNLIKELY( block->poison_serialize ) ) return;
2712 :
2713 : /* If we can't expand an ALT, we'd have to conservatively assume
2714 : there's loader v3 listed in it. And if there also happens to be a
2715 : writable account in an unresolvable ALT, we can't add it and we no
2716 : longer maintain poison list integrity, so we conservatively
2717 : serialize the rest of the block. */
2718 0 : int alt_unresolvable = alt_cnt && !alts;
2719 0 : int loader_present = 0;
2720 0 : for( ushort i=0; i<txn->acct_addr_cnt; i++ ) {
2721 0 : if( FD_UNLIKELY( !memcmp( imms[ i ].b, fd_solana_bpf_loader_upgradeable_program_id.key, 32UL ) ) ) {
2722 0 : loader_present = 1;
2723 0 : break;
2724 0 : }
2725 0 : }
2726 0 : if( !loader_present && !alt_unresolvable ) {
2727 0 : for( ulong i=0UL; i<alt_cnt; i++ ) {
2728 0 : if( FD_UNLIKELY( !memcmp( alts[ i ].b, fd_solana_bpf_loader_upgradeable_program_id.key, 32UL ) ) ) {
2729 0 : loader_present = 1;
2730 0 : break;
2731 0 : }
2732 0 : }
2733 0 : }
2734 0 : if( FD_LIKELY( !loader_present && !alt_unresolvable ) ) return;
2735 :
2736 0 : for( fd_txn_acct_iter_t iter = fd_txn_acct_iter_init( txn, FD_TXN_ACCT_CAT_WRITABLE ); iter!=fd_txn_acct_iter_end(); iter=fd_txn_acct_iter_next( iter ) ) {
2737 0 : fd_acct_addr_t const * acct = get_acct( txn, imms, alts, (ushort)fd_txn_acct_iter_idx( iter ) );
2738 0 : if( FD_UNLIKELY( !acct ) ) {
2739 0 : block->poison_serialize = 1;
2740 0 : sched->metrics->poison_latch_alt_cnt++;
2741 0 : break;
2742 0 : }
2743 0 : if( FD_UNLIKELY( block_poison_insert( sched, block, acct ) ) ) break;
2744 0 : }
2745 0 : }
2746 :
2747 : FD_WARN_UNUSED static int
2748 0 : fd_sched_parse_txn( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx ) {
2749 0 : fd_txn_t * txn = fd_type_pun( block->txn );
2750 :
2751 0 : uchar * payload = block->fec_buf+block->fec_buf_soff;
2752 0 : ulong remaining = block->fec_buf_sz-block->fec_buf_soff;
2753 0 : ulong pay_sz = 0UL;
2754 0 : ulong txn_sz = fd_txn_parse_core( payload,
2755 0 : remaining,
2756 0 : txn,
2757 0 : NULL,
2758 0 : &pay_sz );
2759 :
2760 : /* Can't parse out a full transaction yet, EAGAIN. */
2761 0 : if( FD_UNLIKELY( !pay_sz || !txn_sz ) ) return -1;
2762 :
2763 : /* The pool slots this transaction will land in are cold: warm the
2764 : lines written below (the start of the payload copy, the metadata,
2765 : the start of the parsed txn and the whole info record) while the
2766 : ALUT resolution and the dispatcher insert run. A hint only. */
2767 0 : ulong next_idx = fd_rdisp_peek_free_txn( sched->rdisp );
2768 0 : if( FD_LIKELY( next_idx && next_idx<sched->depth ) ) {
2769 0 : fd_txn_p_t * next_p = sched->txn_pool+next_idx;
2770 0 : __builtin_prefetch( next_p->payload, 1, 3 );
2771 0 : __builtin_prefetch( next_p->payload+64UL, 1, 3 );
2772 0 : __builtin_prefetch( &next_p->payload_sz, 1, 3 );
2773 0 : __builtin_prefetch( TXN( next_p ), 1, 3 );
2774 0 : ulong info_lo = fd_ulong_align_dn( (ulong)(sched->txn_info_pool+next_idx ), 64UL );
2775 0 : ulong info_hi = fd_ulong_align_up( (ulong)(sched->txn_info_pool+next_idx+1UL ), 64UL );
2776 0 : for( ulong line=info_lo; line<info_hi; line+=64UL ) __builtin_prefetch( (void const *)line, 1, 3 );
2777 0 : }
2778 :
2779 0 : if( FD_UNLIKELY( block->txn_parsed_cnt>=sched->max_txn_per_slot ) ) {
2780 : /* Transaction count is enforced as invariant
2781 : txn_parsed_cnt+txns_rem<=max_txn_per_slot
2782 : when we read the declared count in a microblock header. So we
2783 : shouldn't get here. */
2784 0 : sched->print_buf_sz = 0UL;
2785 0 : print_all( sched, block );
2786 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2787 0 : FD_LOG_CRIT(( "invariant violation: slot %lu, parent slot %lu, txn_parsed_cnt %u", block->slot, block->parent_slot, block->txn_parsed_cnt ));
2788 0 : }
2789 :
2790 0 : ulong imm_cnt = fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_IMM );
2791 0 : ulong alt_cnt = fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_ALT );
2792 :
2793 : /* Try to expand ALUTs. */
2794 0 : int serializing = 0;
2795 0 : if( alt_cnt>0UL ) {
2796 0 : if( FD_UNLIKELY( sched->bypass_alut_resolution ) ) {
2797 : /* test/fuzz: no accdb to query, so treat ALUT txns as serializing. */
2798 0 : serializing = 1;
2799 0 : } else {
2800 : /* Copy the slot hashes sysvar out and release the read BEFORE
2801 : resolving the ALTs. fd_runtime_load_txn_address_lookup_tables
2802 : issues its own fd_accdb_read_one per lookup table, and the accdb
2803 : acquire state is a single non-nestable flag — holding this read
2804 : open across that call would trip the IDLE assertion in
2805 : fd_accdb_acquire. The slot_hashes view aliases the record data,
2806 : so it must view the copy, not the released record. */
2807 0 : static uchar slot_hashes_buf[ FD_SYSVAR_SLOT_HASHES_BINCODE_SZ ];
2808 0 : ulong slot_hashes_sz = 0UL;
2809 0 : int have_slot_hashes = 0;
2810 0 : fd_acc_t ro = fd_accdb_read_one( alut_ctx->accdb, alut_ctx->fork_id, fd_sysvar_slot_hashes_id.uc );
2811 0 : if( FD_LIKELY( ro.lamports && ro.data_len<=sizeof(slot_hashes_buf) ) ) {
2812 0 : fd_memcpy( slot_hashes_buf, ro.data, ro.data_len );
2813 0 : slot_hashes_sz = ro.data_len;
2814 0 : have_slot_hashes = 1;
2815 0 : }
2816 0 : fd_accdb_unread_one( alut_ctx->accdb, &ro );
2817 :
2818 0 : fd_slot_hashes_t slot_hashes_view[1];
2819 0 : if( FD_LIKELY( have_slot_hashes &&
2820 0 : fd_sysvar_slot_hashes_view( slot_hashes_view, slot_hashes_buf, slot_hashes_sz ) ) ) {
2821 0 : serializing = !!fd_runtime_load_txn_address_lookup_tables( txn, payload, alut_ctx->accdb, alut_ctx->fork_id, alut_ctx->els, slot_hashes_view, sched->aluts );
2822 0 : sched->metrics->alut_success_cnt += (uint)!serializing;
2823 0 : } else {
2824 0 : serializing = 1;
2825 0 : }
2826 0 : }
2827 0 : }
2828 :
2829 : /* Capture alt_cnt before it's clamped below. Poisoning needs to
2830 : distinguish between "no ALT keys" from "ALT keys failed to
2831 : resolve". */
2832 0 : ulong poison_alt_cnt = alt_cnt;
2833 0 : fd_acct_addr_t const * poison_alts = (alt_cnt && !serializing) ? sched->aluts : NULL;
2834 :
2835 : /* Transactions should not have duplicate accounts.
2836 : https://github.com/anza-xyz/agave/blob/v3.1.11/ledger/src/blockstore_processor.rs#L778-L790 */
2837 0 : fd_acct_addr_t const * imms = fd_txn_get_acct_addrs( txn, payload );
2838 0 : fd_acct_addr_t * alts = (!alt_cnt||serializing) ? NULL : sched->aluts;
2839 0 : alt_cnt = alts ? alt_cnt : 0UL;
2840 0 : if( FD_UNLIKELY( fd_chkdup_check( sched->chkdup, imms, imm_cnt, alts, alt_cnt ) ) ) {
2841 0 : FD_LOG_INFO(( "bad block: DUPLICATE_ACCOUNT, slot %lu, parent slot %lu, txn_parsed_cnt %u", block->slot, block->parent_slot, block->txn_parsed_cnt ));
2842 0 : return FD_SCHED_DEAD_REASON_DUPLICATE_ACCOUNT;
2843 0 : }
2844 :
2845 : /* After any error return but before poisoning. */
2846 0 : sched->metrics->alut_serializing_cnt += (uint)serializing;
2847 :
2848 : /* At this point, we've decided whether the transaction is
2849 : ALT-serializing or not. If it didn't get serialized by ALT
2850 : expansion failure, check if it should be serialized by poisoning.
2851 : Add to the poison set after checking, so a poisoning transaction
2852 : doesn't get serialized against its own entries. Adding to the
2853 : poison set is unconditional. */
2854 0 : if( FD_UNLIKELY( !serializing && ( block->poison_serialize||block_poison_hit( block, txn, imms, poison_alts ) ) ) ) {
2855 0 : serializing = 1;
2856 0 : sched->metrics->txn_poisoned_cnt += block->poison_serialize ? 0U : 1U;
2857 0 : sched->metrics->txn_poison_serializing_cnt += block->poison_serialize ? 1U : 0U;
2858 0 : }
2859 0 : block_poison_add( sched, block, txn, imms, poison_alts, poison_alt_cnt );
2860 :
2861 0 : ulong bank_idx = (ulong)(block-sched->block_pool);
2862 0 : ulong txn_idx = fd_rdisp_add_txn( sched->rdisp, bank_idx, txn, payload, alts, serializing, NULL );
2863 0 : FD_TEST( txn_idx && txn_idx<sched->depth );
2864 :
2865 : /* This transaction either needs to be consumed by sigverify or PoH.
2866 : If either of the two are caught up, mark it for consumption.
2867 : Otherwise, link it to the tail of the chain. */
2868 :
2869 0 : ulong sigverify_dispatched_cnt = (ulong)block->txn_sigverify_done_cnt + (ulong)block->txn_sigverify_in_flight_cnt;
2870 0 : int sigverify_caught_up = sigverify_dispatched_cnt==block->txn_parsed_cnt;
2871 0 : #if !FD_SCHED_SKIP_POH
2872 0 : fd_sched_mblk_t * parse_mblk = sched->mblk_pool + block->parse_mblk_idx;
2873 0 : int poh_caught_up = parse_mblk->curr_txn_idx==block->txn_parsed_cnt;
2874 0 : #endif
2875 :
2876 0 : sched->txn_info_pool[ txn_idx ].next_idx = 0U;
2877 : /* Link to the previous transaction. */
2878 0 : if( FD_LIKELY( block->txn_idx_tail ) ) {
2879 0 : FD_TEST( block->txn_idx_tail<sched->depth );
2880 0 : sched->txn_info_pool[ block->txn_idx_tail ].next_idx = (uint)txn_idx;
2881 0 : }
2882 : /* If either consumer is caught up, update next pointer to the new
2883 : transaction. */
2884 0 : if( FD_LIKELY( sigverify_caught_up ) ) block->txn_sigverify_next_idx = (uint)txn_idx;
2885 0 : #if !FD_SCHED_SKIP_POH
2886 0 : if( FD_LIKELY( poh_caught_up ) ) parse_mblk->mixin_txn_idx = (uint)txn_idx;
2887 0 : #endif
2888 0 : block->txn_idx_tail = (uint)txn_idx;
2889 :
2890 0 : sched->metrics->txn_parsed_cnt++;
2891 0 : sched->txn_pool_free_cnt--;
2892 0 : fd_txn_p_t * txn_p = sched->txn_pool + txn_idx;
2893 0 : txn_p->payload_sz = (ushort)pay_sz;
2894 :
2895 0 : txn_p->start_shred_idx = (ushort)fd_uint_min( shred_split( sched, block, block->fec_buf_boff+block->fec_buf_soff ), USHORT_MAX ); /* monitoring-only ushorts; saturate under bench shred counts */
2896 0 : txn_p->start_shred_idx = fd_ushort_if( txn_p->start_shred_idx>0U, (ushort)(txn_p->start_shred_idx-1U), txn_p->start_shred_idx );
2897 0 : txn_p->end_shred_idx = (ushort)fd_uint_min( shred_split( sched, block, block->fec_buf_boff+block->fec_buf_soff+(uint)pay_sz ), USHORT_MAX );
2898 :
2899 0 : fd_memcpy( txn_p->payload, payload, pay_sz );
2900 0 : fd_memcpy( TXN(txn_p), txn, txn_sz );
2901 0 : txn_bitset_remove( sched->exec_done_set, txn_idx );
2902 0 : txn_bitset_remove( sched->sigverify_done_set, txn_idx );
2903 0 : txn_bitset_remove( sched->poh_mixin_done_set, txn_idx );
2904 0 : sched->txn_info_pool[ txn_idx ].flags = 0UL;
2905 0 : sched->txn_info_pool[ txn_idx ].received_ns = block->fec_completed_ns;
2906 0 : sched->txn_info_pool[ txn_idx ].txn_err = 0;
2907 0 : sched->txn_info_pool[ txn_idx ].is_simple_vote = 0;
2908 :
2909 0 : sched->txn_info_pool[ txn_idx ].tick_parsed = fd_tickcount();
2910 0 : sched->txn_info_pool[ txn_idx ].tick_sigverify_disp = LONG_MAX;
2911 0 : sched->txn_info_pool[ txn_idx ].tick_sigverify_done = LONG_MAX;
2912 0 : sched->txn_info_pool[ txn_idx ].tick_exec_disp = LONG_MAX;
2913 0 : sched->txn_info_pool[ txn_idx ].tick_exec_done = LONG_MAX;
2914 0 : sched->txn_info_pool[ txn_idx ].tick_load_start = LONG_MAX;
2915 0 : sched->txn_info_pool[ txn_idx ].tick_check_start = LONG_MAX;
2916 0 : sched->txn_info_pool[ txn_idx ].tick_exec_start = LONG_MAX;
2917 0 : sched->txn_info_pool[ txn_idx ].tick_commit_start = LONG_MAX;
2918 0 : sched->txn_info_pool[ txn_idx ].tick_commit_end = LONG_MAX;
2919 :
2920 0 : sched->txn_info_pool[ txn_idx ].slot = block->slot;
2921 0 : sched->txn_info_pool[ txn_idx ].bank_seq = ULONG_MAX;
2922 0 : sched->txn_info_pool[ txn_idx ].index_in_slot = block->txn_parsed_cnt;
2923 0 : sched->txn_info_pool[ txn_idx ].exec_tile_idx = ULONG_MAX;
2924 0 : sched->txn_info_pool[ txn_idx ].sigverify_exec_tile_idx = ULONG_MAX;
2925 0 : sched->txn_info_pool[ txn_idx ].compute_units_consumed = 0U;
2926 0 : sched->txn_info_pool[ txn_idx ].max_compute_units = ULONG_MAX;
2927 0 : sched->txn_info_pool[ txn_idx ].transaction_fee = 0UL;
2928 0 : sched->txn_info_pool[ txn_idx ].priority_fee = 0UL;
2929 0 : sched->txn_info_pool[ txn_idx ].tips = 0UL;
2930 0 : block->fec_buf_soff += (uint)pay_sz;
2931 0 : block->txn_parsed_cnt++;
2932 : #if FD_SCHED_SKIP_SIGVERIFY
2933 : txn_bitset_insert( sched->sigverify_done_set, txn_idx );
2934 : block->txn_sigverify_done_cnt++;
2935 : #endif
2936 : #if FD_SCHED_SKIP_POH
2937 : txn_bitset_insert( sched->poh_mixin_done_set, txn_idx );
2938 : #endif
2939 0 : block->txns_rem--;
2940 0 : return FD_SCHED_DEAD_REASON_NONE;
2941 0 : }
2942 :
2943 : #undef CHECK
2944 : #undef CHECK_LEFT
2945 :
2946 : static void
2947 0 : dispatch_sigverify( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, fd_sched_task_t * out ) {
2948 : /* Dispatch transactions for sigverify in parse order. */
2949 0 : uint txn_idx = block->txn_sigverify_next_idx;
2950 0 : FD_TEST( txn_idx && txn_idx<sched->depth );
2951 0 : out->task_type = FD_SCHED_TT_TXN_SIGVERIFY;
2952 0 : out->txn_sigverify->bank_idx = bank_idx;
2953 0 : out->txn_sigverify->txn_idx = txn_idx;
2954 0 : out->txn_sigverify->exec_idx = (ulong)exec_tile_idx;
2955 0 : sched->txn_info_pool[ out->txn_sigverify->txn_idx ].sigverify_exec_tile_idx = (ulong)exec_tile_idx;
2956 0 : sched->sigverify_ready_bitset[ 0 ] = fd_ulong_clear_bit( sched->sigverify_ready_bitset[ 0 ], exec_tile_idx );
2957 0 : sched->tile_to_bank_idx[ exec_tile_idx ] = bank_idx;
2958 0 : block->txn_sigverify_in_flight_cnt++;
2959 0 : block->txn_sigverify_next_idx = sched->txn_info_pool[ txn_idx ].next_idx;
2960 0 : if( FD_UNLIKELY( (~sched->txn_exec_ready_bitset[ 0 ])&(~sched->sigverify_ready_bitset[ 0 ])&(~sched->poh_ready_bitset[ 0 ])&fd_ulong_mask_lsb( (int)sched->exec_cnt ) ) ) FD_LOG_CRIT(( "invariant violation: txn_exec_ready_bitset 0x%lx sigverify_ready_bitset 0x%lx poh_ready_bitset 0x%lx", sched->txn_exec_ready_bitset[ 0 ], sched->sigverify_ready_bitset[ 0 ], sched->poh_ready_bitset[ 0 ] ));
2961 0 : }
2962 :
2963 : /* Retires a microblock whose PoH hashing has completed, and eagerly
2964 : does as much mixin as possible. Returns FD_SCHED_DEAD_REASON_NONE on
2965 : success, else the dead reason. */
2966 : static int
2967 0 : poh_retire_mblk( fd_sched_t * sched, fd_sched_block_t * block, uint mblk_idx ) {
2968 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
2969 0 : FD_TEST( mblk->curr_hashcnt==mblk->hashcnt );
2970 0 : block->poh_hashing_done_cnt++;
2971 0 : sched->metrics->mblk_poh_hashed_cnt++;
2972 0 : if( FD_LIKELY( !mblk->is_tick ) ) {
2973 0 : mblk_slist_idx_push_tail( block->mblks_mixin_in_progress, mblk_idx, sched->mblk_pool );
2974 0 : } else {
2975 0 : block->poh_hash_cmp_done_cnt++;
2976 0 : sched->metrics->mblk_poh_done_cnt++;
2977 0 : free_mblk( sched, block, mblk_idx );
2978 0 : if( FD_UNLIKELY( memcmp( mblk->curr_hash, mblk->end_hash, sizeof(fd_hash_t) ) ) ) {
2979 0 : FD_BASE58_ENCODE_32_BYTES( mblk->curr_hash->hash, our_str );
2980 0 : FD_BASE58_ENCODE_32_BYTES( mblk->end_hash->hash, ref_str );
2981 0 : FD_LOG_INFO(( "bad block: TICK_HASH_MISMATCH, mblk %u, ours %s, claimed %s, hashcnt %lu, slot %lu, parent slot %lu", mblk_idx, our_str, ref_str, mblk->hashcnt, block->slot, block->parent_slot ));
2982 0 : return FD_SCHED_DEAD_REASON_TICK_HASH_MISMATCH;
2983 0 : }
2984 0 : }
2985 0 : int mixin_res;
2986 0 : while( (mixin_res=maybe_mixin( sched, block )) ) {
2987 0 : if( FD_UNLIKELY( mixin_res==-1 ) ) return FD_SCHED_DEAD_REASON_ENTRY_HASH_MISMATCH;
2988 0 : FD_TEST( mixin_res==1||mixin_res==2 );
2989 0 : }
2990 0 : return FD_SCHED_DEAD_REASON_NONE;
2991 0 : }
2992 :
2993 : /* Retires microblocks at the head of the unhashed queue that have
2994 : nothing left for PoH to hash. Returns FD_SCHED_DEAD_REASON_NONE on
2995 : success, else the dead reason. */
2996 : static int
2997 0 : poh_retire_ready_mblks( fd_sched_t * sched, fd_sched_block_t * block ) {
2998 0 : while( block->mblk_unhashed_cnt ) {
2999 0 : uint mblk_idx = (uint)mblk_slist_idx_peek_head( block->mblks_unhashed, sched->mblk_pool );
3000 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
3001 0 : if( FD_LIKELY( mblk->curr_hashcnt!=mblk->hashcnt ) ) break;
3002 0 : mblk_slist_idx_pop_head( block->mblks_unhashed, sched->mblk_pool );
3003 0 : block->mblk_unhashed_cnt--;
3004 0 : int dead_reason = poh_retire_mblk( sched, block, mblk_idx );
3005 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) return dead_reason;
3006 0 : }
3007 0 : return verify_ticks_eager( block );
3008 0 : }
3009 :
3010 : /* Assembles a PoH task for the given exec tile. Assumes there is a
3011 : PoH task available for dispatching. A microblock with nothing left
3012 : to hash is dispatched as a degenerate zero-hashcnt task, so that it
3013 : retires through fd_sched_task_done like any other PoH task. */
3014 : static void
3015 0 : dispatch_poh( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, ulong max_cnt, fd_sched_task_t * out ) {
3016 0 : FD_TEST( max_cnt>=1UL && max_cnt<=FD_SCHED_POH_PARA );
3017 : /* Every element of a batch advances by the same hashcnt, so the
3018 : batch hashcnt is the smallest remaining hashcnt of its elements.
3019 : Elements with more work left are re-queued on completion. */
3020 0 : fd_sched_poh_hash_t * poh = out->poh_hash;
3021 0 : ulong cnt = 0UL;
3022 0 : ulong hashcnt = ULONG_MAX;
3023 0 : while( cnt<max_cnt ) {
3024 0 : mblk_slist_t * list;
3025 0 : if( FD_LIKELY( !mblk_slist_is_empty( block->mblks_hashing_in_progress, sched->mblk_pool ) ) ) {
3026 : /* There's a PoH task in progress, just continue working on that. */
3027 0 : list = block->mblks_hashing_in_progress;
3028 0 : } else if( FD_LIKELY( block->mblk_unhashed_cnt ) ) {
3029 0 : list = block->mblks_unhashed;
3030 0 : } else {
3031 0 : break; /* Nothing more to batch. */
3032 0 : }
3033 0 : uint mblk_idx = (uint)mblk_slist_idx_pop_head( list, sched->mblk_pool );
3034 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
3035 0 : ulong hashcnt_todo = mblk->hashcnt-mblk->curr_hashcnt;
3036 : /* A microblock with nothing left to hash can't share a batch with
3037 : one that has work left. The batch hashcnt would be zero, and the
3038 : latter would never make progress. */
3039 0 : if( FD_UNLIKELY( cnt && (!hashcnt_todo)!=(!hashcnt) ) ) {
3040 0 : mblk_slist_idx_push_head( list, mblk_idx, sched->mblk_pool );
3041 0 : break;
3042 0 : }
3043 0 : if( list==block->mblks_unhashed ) block->mblk_unhashed_cnt--;
3044 0 : hashcnt = fd_ulong_min( hashcnt, hashcnt_todo );
3045 0 : poh->mblk_idx[ cnt ] = mblk_idx;
3046 0 : memcpy( poh->hash+cnt, mblk->curr_hash, sizeof(fd_hash_t) );
3047 0 : cnt++;
3048 0 : block->poh_mblk_in_flight_cnt++;
3049 0 : }
3050 0 : FD_TEST( cnt ); /* poh_hashing_queued_cnt>0 implies at least one queued microblock. */
3051 :
3052 : /* See FD_SCHED_MAX_POH_HASHES_PER_TASK. A batch's per iteration
3053 : cost depends on its width. */
3054 0 : ulong iters_max = FD_SCHED_MAX_POH_HASHES_PER_TASK;
3055 0 : if( cnt>=sched->poh_simd_min ) iters_max = fd_ulong_max( (FD_SCHED_MAX_POH_HASHES_PER_TASK<<8)/fd_sha256_simd_iter_cost_q8( cnt ), 1UL );
3056 0 : hashcnt = fd_ulong_min( hashcnt, iters_max );
3057 0 : out->task_type = FD_SCHED_TT_POH_HASH;
3058 0 : poh->bank_idx = bank_idx;
3059 0 : poh->exec_idx = (ulong)exec_tile_idx;
3060 0 : poh->cnt = cnt;
3061 0 : poh->hashcnt = hashcnt;
3062 0 : sched->poh_inflight[ exec_tile_idx ] = *poh;
3063 :
3064 0 : sched->poh_ready_bitset[ 0 ] = fd_ulong_clear_bit( sched->poh_ready_bitset[ 0 ], exec_tile_idx );
3065 0 : sched->tile_to_bank_idx[ exec_tile_idx ] = bank_idx;
3066 0 : block->poh_hashing_in_flight_cnt++;
3067 0 : if( FD_UNLIKELY( (~sched->txn_exec_ready_bitset[ 0 ])&(~sched->sigverify_ready_bitset[ 0 ])&(~sched->poh_ready_bitset[ 0 ])&fd_ulong_mask_lsb( (int)sched->exec_cnt ) ) ) FD_LOG_CRIT(( "invariant violation: txn_exec_ready_bitset 0x%lx sigverify_ready_bitset 0x%lx poh_ready_bitset 0x%lx", sched->txn_exec_ready_bitset[ 0 ], sched->sigverify_ready_bitset[ 0 ], sched->poh_ready_bitset[ 0 ] ));
3068 0 : }
3069 :
3070 : /* Does up to one transaction mixin. Returns 1 if one mixin was done, 2
3071 : if that mixin also completed a microblock, 0 if no transaction mixin
3072 : was available, -1 if there is a PoH verify error. */
3073 : FD_WARN_UNUSED static int
3074 0 : maybe_mixin( fd_sched_t * sched, fd_sched_block_t * block ) {
3075 0 : if( FD_UNLIKELY( mblk_slist_is_empty( block->mblks_mixin_in_progress, sched->mblk_pool ) ) ) return 0;
3076 0 : FD_TEST( block->poh_hashing_done_cnt-block->poh_hash_cmp_done_cnt>0 );
3077 :
3078 : /* The microblock we would like to do mixin on is at the head of the
3079 : queue. It may have had some mixin, it may have never had any
3080 : mixin. In the case of the former, we should continue to mixin the
3081 : same head microblock until it's done, lest the per-block bmtree
3082 : gets clobbered when we start a new one. */
3083 0 : ulong mblk_idx = mblk_slist_idx_pop_head( block->mblks_mixin_in_progress, sched->mblk_pool );
3084 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
3085 :
3086 0 : if( FD_UNLIKELY( mblk->end_txn_idx>block->txn_parsed_cnt ) ) {
3087 : /* A partially parsed microblock is by definition at the end of the
3088 : FEC stream. If such a microblock is in progress, there should be
3089 : no other microblock in this block so far that hasn't been
3090 : dispatched, because microblocks are dispatched in parse order. */
3091 0 : if( FD_UNLIKELY( block->mblk_unhashed_cnt ) ) {
3092 0 : sched->print_buf_sz = 0UL;
3093 0 : print_all( sched, block );
3094 0 : FD_LOG_CRIT(( "invariant violation end_txn_idx %lu: %s", mblk->end_txn_idx, sched->print_buf ));
3095 0 : }
3096 :
3097 : /* If we've decided to start mixin on a partially parsed microblock,
3098 : there better be nothing else in-progress. Otherwise, they might
3099 : clobber the per-block bmtree for mixin. */
3100 0 : if( FD_UNLIKELY( mblk->curr_txn_idx!=mblk->start_txn_idx && (block->poh_mblk_in_flight_cnt||!mblk_slist_is_empty( block->mblks_hashing_in_progress, sched->mblk_pool )||!mblk_slist_is_empty( block->mblks_mixin_in_progress, sched->mblk_pool )) ) ) {
3101 0 : sched->print_buf_sz = 0UL;
3102 0 : print_all( sched, block );
3103 0 : FD_LOG_CRIT(( "invariant violation end_txn_idx %lu start_txn_idx %lu curr_txn_idx %lu: %s", mblk->end_txn_idx, mblk->start_txn_idx, mblk->curr_txn_idx, sched->print_buf ));
3104 0 : }
3105 0 : }
3106 :
3107 : /* Very rarely, we've finished hashing, but not all transactions in
3108 : the microblock have been parsed out. This can happen if we haven't
3109 : received all the FEC sets for this microblock. We can't yet fully
3110 : mixin the microblock. So we'll stick it back into the end of the
3111 : queue, and try to see if there's a fully parsed microblock.
3112 : Unless, there's truly nothing else to mixin. Then we would start
3113 : mixin with the partially parsed microblock. We do this because the
3114 : txn pool is meant to be an OOO scheduling window not tied to
3115 : max_live_slots sizing requirements, so there shouldn't be a way for
3116 : external input to tie up txn pool entries for longer than
3117 : necessary. */
3118 0 : if( FD_UNLIKELY( mblk->curr_txn_idx>=block->txn_parsed_cnt || /* Nothing more to mixin for this microblock. */
3119 0 : (mblk->end_txn_idx>block->txn_parsed_cnt && /* There is something to mixin, but the microblock isn't fully parsed yet ... */
3120 0 : mblk->curr_txn_idx==mblk->start_txn_idx && /* ... and we haven't started mixin on it yet ... */
3121 0 : (block->poh_mblk_in_flight_cnt || /* ... and another microblock is in-progress and might preempt this microblock and clobber the bmtree, so we shouldn't start the partial microblock just yet. Counting microblocks rather than tasks matters: a batched PoH task is retired one microblock at a time, and the ones not yet retired are off every list. */
3122 0 : !mblk_slist_is_empty( block->mblks_hashing_in_progress, sched->mblk_pool ) ||
3123 0 : !mblk_slist_is_empty( block->mblks_mixin_in_progress, sched->mblk_pool ))) ) ) {
3124 0 : mblk_slist_idx_push_tail( block->mblks_mixin_in_progress, mblk_idx, sched->mblk_pool );
3125 :
3126 : /* No other microblock in the mixin queue. */
3127 0 : if( FD_UNLIKELY( block->poh_hashing_done_cnt-block->poh_hash_cmp_done_cnt==1 ) ) return 0;
3128 :
3129 : /* At this point, there's at least one more microblock in the mixin
3130 : queue we could try. It's a predecessor (in parse order) that
3131 : finished hashing later than the partially parsed microblock at
3132 : the head of the mixin queue. */
3133 :
3134 : /* It should never clobber the bmtree for a microblock that has had some mixin done on it. */
3135 0 : if( FD_UNLIKELY( mblk->curr_txn_idx!=mblk->start_txn_idx ) ) {
3136 0 : sched->print_buf_sz = 0UL;
3137 0 : print_all( sched, block );
3138 0 : FD_LOG_CRIT(( "invariant violation curr_txn_idx %lu start_txn_idx %lu: %s", mblk->curr_txn_idx, mblk->start_txn_idx, sched->print_buf ));
3139 0 : }
3140 :
3141 0 : mblk_idx = mblk_slist_idx_pop_head( block->mblks_mixin_in_progress, sched->mblk_pool );
3142 0 : mblk = sched->mblk_pool+mblk_idx;
3143 :
3144 : /* It should be a fresh microblock for mixin. */
3145 0 : FD_TEST( mblk->curr_txn_idx==mblk->start_txn_idx );
3146 : /* Invariant: at any given point in time, there can be at most one
3147 : microblock that hasn't been fully parsed yet, due to the nature
3148 : of sequential parsing. So this microblock has to be fully
3149 : parsed. */
3150 0 : FD_TEST( mblk->end_txn_idx<=block->txn_parsed_cnt );
3151 0 : }
3152 :
3153 0 : FD_TEST( mblk->curr_txn_idx<mblk->end_txn_idx );
3154 :
3155 : /* Now mixin. */
3156 0 : if( FD_LIKELY( mblk->curr_txn_idx==mblk->start_txn_idx ) ) block->bmtree = fd_bmtree_commit_init( block->bmtree_mem, 32UL, 1UL, 0UL ); /* Optimize for single-transaction microblocks, which are the majority. */
3157 :
3158 0 : ulong txn_gidx = mblk->mixin_txn_idx;
3159 0 : uint next_txn_pool_idx = sched->txn_info_pool[ txn_gidx ].next_idx;
3160 0 : fd_txn_p_t * _txn = sched->txn_pool+txn_gidx;
3161 0 : fd_txn_t * txn = TXN(_txn);
3162 0 : for( ulong j=0; j<txn->signature_cnt; j++ ) {
3163 0 : fd_bmtree_node_t node[ 1 ];
3164 0 : fd_bmtree_hash_leaf( node, _txn->payload+txn->signature_off+FD_TXN_SIGNATURE_SZ*j, 64UL, 1UL );
3165 0 : fd_bmtree_commit_append( block->bmtree, node, 1UL );
3166 0 : mblk->curr_sig_cnt++;
3167 0 : }
3168 :
3169 : /* Release the txn_idx. */
3170 0 : txn_bitset_insert( sched->poh_mixin_done_set, txn_gidx );
3171 0 : sched->metrics->txn_mixin_done_cnt++;
3172 0 : if( txn_bitset_test( sched->exec_done_set, txn_gidx ) && txn_bitset_test( sched->sigverify_done_set, txn_gidx ) ) {
3173 0 : if( FD_UNLIKELY( block->txn_idx_tail==(uint)txn_gidx ) ) block->txn_idx_tail = 0U;
3174 0 : fd_rdisp_complete_txn( sched->rdisp, txn_gidx, 1 );
3175 0 : sched->txn_pool_free_cnt++;
3176 0 : block->txn_done_cnt++;
3177 0 : sched->metrics->txn_done_cnt++;
3178 0 : }
3179 :
3180 0 : mblk->curr_txn_idx++;
3181 0 : mblk->mixin_txn_idx = next_txn_pool_idx;
3182 0 : int rv = 2;
3183 0 : if( FD_LIKELY( mblk->curr_txn_idx==mblk->end_txn_idx ) ) {
3184 : /* Ready to compute the final hash for this microblock. */
3185 0 : block->poh_hash_cmp_done_cnt++;
3186 0 : sched->metrics->mblk_poh_done_cnt++;
3187 0 : uchar * root = fd_bmtree_commit_fini( block->bmtree );
3188 0 : uchar mixin_buf[ 64 ];
3189 0 : fd_memcpy( mixin_buf, mblk->curr_hash, 32UL );
3190 0 : fd_memcpy( mixin_buf+32UL, root, 32UL );
3191 0 : fd_sha256_hash( mixin_buf, 64UL, mblk->curr_hash );
3192 0 : free_mblk( sched, block, (uint)mblk_idx );
3193 : /* Bypass PoH verification for fuzzing/testing throughput. */
3194 0 : if( FD_UNLIKELY( !sched->bypass_poh_verify && memcmp( mblk->curr_hash, mblk->end_hash, sizeof(fd_hash_t) ) ) ) {
3195 0 : FD_BASE58_ENCODE_32_BYTES( mblk->curr_hash->hash, our_str );
3196 0 : FD_BASE58_ENCODE_32_BYTES( mblk->end_hash->hash, ref_str );
3197 0 : FD_LOG_INFO(( "bad block: poh hash mismatch on mblk %lu, ours %s, claimed %s, hashcnt %lu, txns [%lu,%lu), %u sigs, slot %lu, parent slot %lu", mblk_idx, our_str, ref_str, mblk->hashcnt, mblk->start_txn_idx, mblk->end_txn_idx, mblk->curr_sig_cnt, block->slot, block->parent_slot ));
3198 0 : return -1;
3199 0 : }
3200 0 : } else {
3201 : /* There are more transactions to mixin in this microblock. */
3202 0 : mblk_slist_idx_push_head( block->mblks_mixin_in_progress, mblk_idx, sched->mblk_pool );
3203 0 : rv = 1;
3204 0 : }
3205 :
3206 0 : return rv;
3207 0 : }
3208 :
3209 : static void
3210 0 : try_activate_block( fd_sched_t * sched ) {
3211 : /* Early return if there's already an active block. */
3212 0 : if( FD_LIKELY( sched->active_bank_idx!=ULONG_MAX ) ) return;
3213 :
3214 : /* See if there are any allocated staging lanes that we can activate
3215 : for scheduling ... */
3216 0 : ulong staged_bitset = sched->staged_bitset;
3217 0 : while( staged_bitset ) {
3218 0 : int lane_idx = fd_ulong_find_lsb( staged_bitset );
3219 0 : staged_bitset = fd_ulong_pop_lsb( staged_bitset );
3220 :
3221 0 : ulong head_idx = sched->staged_head_bank_idx[ lane_idx ];
3222 0 : fd_sched_block_t * head_block = block_pool_ele( sched, head_idx );
3223 0 : fd_sched_block_t * parent_block = block_pool_ele( sched, head_block->parent_idx );
3224 : //FIXME: restore these invariant checks when we have immediate demotion of dying blocks
3225 : //Today, dying blocks can remain staged if they have in-flight transactions.
3226 : // if( FD_UNLIKELY( parent_block->dying ) ) {
3227 : // /* Invariant: no child of a dying block should be staged. */
3228 : // FD_LOG_CRIT(( "invariant violation: staged_head_bank_idx %lu, slot %lu, parent slot %lu on lane %d has parent_block->dying set, slot %lu, parent slot %lu",
3229 : // head_idx, head_block->slot, head_block->parent_slot, lane_idx, parent_block->slot, parent_block->parent_slot ));
3230 : // }
3231 : // if( FD_UNLIKELY( head_block->dying ) ) {
3232 : // /* Invariant: no dying block should be staged. */
3233 : // FD_LOG_CRIT(( "invariant violation: staged_head_bank_idx %lu, slot %lu, prime %lu on lane %u has head_block->dying set",
3234 : // head_idx, (ulong)head_block->block_id.slot, (ulong)head_block->block_id.prime, lane_idx ));
3235 : // }
3236 0 : if( block_is_done( parent_block ) && block_is_activatable( head_block ) ) {
3237 : /* ... Yes, on this staging lane the parent block is done. So we
3238 : can activate the staged child. */
3239 0 : if( FD_UNLIKELY( head_idx!=sched->last_active_bank_idx ) ) { /* Unlikely because only possible under forking or on slot boundary. */
3240 0 : if( FD_UNLIKELY( sched->last_active_bank_idx!=ULONG_MAX && sched->last_active_bank_idx!=head_block->parent_idx ) ) { /* Forking is rare, the first activation is not a switch. */
3241 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: lane switch to %d", head_block->slot, head_idx, lane_idx ));
3242 0 : sched->metrics->lane_switch_cnt++;
3243 0 : } else {
3244 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: lane %d waking up on slot boundary", head_block->slot, head_idx, lane_idx ));
3245 0 : }
3246 0 : }
3247 0 : sched->active_bank_idx = head_idx;
3248 0 : return;
3249 0 : }
3250 0 : }
3251 :
3252 : /* ... No, promote unstaged blocks. */
3253 0 : ulong root_idx = sched->root_idx;
3254 0 : if( FD_UNLIKELY( root_idx==ULONG_MAX ) ) {
3255 0 : FD_LOG_CRIT(( "invariant violation: root_idx==ULONG_MAX indicating fd_sched is uninitialized" ));
3256 0 : }
3257 : /* Find and stage the longest stageable unstaged fork. This is a
3258 : policy decision. */
3259 0 : ulong depth = compute_longest_unstaged_fork( sched, root_idx );
3260 0 : if( FD_LIKELY( depth>0UL ) ) {
3261 0 : if( FD_UNLIKELY( sched->staged_bitset==fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) {
3262 : /* No more staging lanes available. All of them are occupied by
3263 : slow squatters. Only empty blocks can be demoted, and so
3264 : blocks with in-flight transactions, including dying in-flight
3265 : blocks, shouldn't be demoted. We demote all demotable lanes.
3266 : Demotion isn't all that expensive, since demotable blocks have
3267 : no transactions in them. If a demoted block proves to be
3268 : active still, it'll naturally promote back into a staging lane.
3269 :
3270 : In fact, all lanes should be demotable at this point. None of
3271 : the lanes have anything dispatchable, otherwise we would have
3272 : simply activated one of the dispatchable lanes. None of the
3273 : lanes have anything in-flight either, as we allow for a grace
3274 : period while something is in-flight, before we deactivate any
3275 : block. In principle, we could get rid of the grace period and
3276 : deactivate right away. In that case, it's okay if nothing is
3277 : demotable at the moment, as that simply implies that all lanes
3278 : have in-flight tasks. We would get another chance to try to
3279 : demote when the last in-flight task on any lane completes.
3280 :
3281 : Another interesting side effect of the current dispatching and
3282 : lane switching policy is that each lane should have exactly one
3283 : block in it at this point. A parent block by definition can't
3284 : be partially ingested. Any parent block that is fully ingested
3285 : and dispatchable would have made the lane dispatchable, and we
3286 : wouldn't be here. Any parent that is fully ingested and fully
3287 : dispatched would be fully done after the grace period. So
3288 : there could only be one block per lane, and it is
3289 : simultaneously the head and the tail of the lane.
3290 :
3291 : A note on why this whole thing does not deadlock:
3292 :
3293 : One might reasonably wonder what happens if all the lanes are
3294 : non-empty, non-dead, but for some reason couldn't be activated
3295 : for dispatching. We would deadlock in this case, as no lane
3296 : dispatches to the point of being demotable, and no unstaged
3297 : block can be promoted. Such is not in fact possible. The only
3298 : way a dispatchable lane can be ineligible for activation is if
3299 : it has a parent block that isn't done yet. So a deadlock
3300 : happens when this parent block, or any of its dispatchable
3301 : ancestors, is unstaged. An important invariant we maintain is
3302 : that a staged block can't have an unstaged stageable parent.
3303 : This invariant, by induction, gives us the guarantee that at
3304 : least one of the lanes can be activated. */
3305 0 : for( int l=0; l<(int)FD_SCHED_MAX_STAGING_LANES; l++ ) {
3306 : /* We would be able to assert that all lanes are demotable,
3307 : except that abandoned blocks are given no grace period for
3308 : deactivation. So there could be lanes transiently occupied
3309 : by dying blocks that are neither demotable (due to in-flight
3310 : tasks) nor activatable (due to being dying). If
3311 : rdisp_demote() supported non-empty blocks, then we could
3312 : probably restore the assertion. */
3313 0 : if( FD_UNLIKELY( !lane_is_demotable( sched, l ) ) ) continue;
3314 0 : ulong demoted_cnt = demote_lane( sched, l );
3315 0 : if( FD_UNLIKELY( demoted_cnt!=1UL ) ) {
3316 0 : FD_LOG_CRIT(( "invariant violation: %lu blocks demoted from lane %d, expected 1 demotion", demoted_cnt, l ));
3317 0 : }
3318 0 : sched->metrics->lane_demoted_cnt++;
3319 0 : }
3320 : /* We weren't able to successfully demote anything. This is
3321 : likely because all lanes are occupied by dying blocks with
3322 : in-flight tasks. We would get another chance to try to demote
3323 : when the last in-flight task on any lane completes. */
3324 0 : if( FD_UNLIKELY( sched->staged_bitset==fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) return;
3325 0 : }
3326 0 : FD_TEST( sched->staged_bitset!=fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) );
3327 0 : int lane_idx = fd_ulong_find_lsb( ~sched->staged_bitset );
3328 0 : if( FD_UNLIKELY( lane_idx>=(int)FD_SCHED_MAX_STAGING_LANES ) ) {
3329 0 : FD_LOG_CRIT(( "invariant violation: lane_idx %d, sched->staged_bitset %lx",
3330 0 : lane_idx, sched->staged_bitset ));
3331 0 : }
3332 0 : ulong head_bank_idx = stage_longest_unstaged_fork( sched, root_idx, lane_idx );
3333 0 : if( FD_UNLIKELY( head_bank_idx==ULONG_MAX ) ) {
3334 : /* We found a promotable fork depth>0. This should not happen. */
3335 0 : FD_LOG_CRIT(( "invariant violation: head_bank_idx==ULONG_MAX" ));
3336 0 : }
3337 : /* We don't bother with promotion unless the block is immediately
3338 : dispatchable. So it's okay to set the active block here. This
3339 : doesn't cause out-of-order block replay because any parent block
3340 : must be fully done. If the parent block were dead, this fork
3341 : would be marked dead too and ineligible for promotion. If the
3342 : parent block were not dead and not done and staged, we wouldn't
3343 : be trying to promote an unstaged fork. If the parent block were
3344 : not dead and not done and unstaged, it would've been part of this
3345 : unstaged fork. */
3346 0 : fd_sched_block_t * head_block = block_pool_ele( sched, head_bank_idx );
3347 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: unstaged promotion to lane %d", head_block->slot, head_bank_idx, lane_idx ));
3348 0 : sched->active_bank_idx = head_bank_idx;
3349 0 : return;
3350 0 : }
3351 : /* No unstaged blocks to promote. So we're done. Yay. */
3352 0 : }
3353 :
3354 : static void
3355 0 : check_or_set_active_block( fd_sched_t * sched ) {
3356 0 : if( FD_UNLIKELY( sched->active_bank_idx==ULONG_MAX ) ) {
3357 0 : try_activate_block( sched );
3358 0 : } else {
3359 0 : fd_sched_block_t * active_block = block_pool_ele( sched, sched->active_bank_idx );
3360 0 : if( FD_UNLIKELY( block_should_deactivate( active_block ) ) ) {
3361 0 : sched->print_buf_sz = 0UL;
3362 0 : print_all( sched, active_block );
3363 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
3364 0 : FD_LOG_CRIT(( "invariant violation: should have been deactivated" ));
3365 0 : }
3366 0 : }
3367 0 : }
3368 :
3369 : /* This function has two main jobs:
3370 : - Mark everything on the fork tree dying.
3371 : - Take blocks out of rdisp if possible. */
3372 : static void
3373 0 : subtree_mark_and_maybe_prune_rdisp( fd_sched_t * sched, fd_sched_block_t * block ) {
3374 0 : if( FD_UNLIKELY( block->rooted ) ) {
3375 0 : FD_LOG_CRIT(( "invariant violation: rooted block should not be abandoned, slot %lu, parent slot %lu",
3376 0 : block->slot, block->parent_slot ));
3377 0 : }
3378 : /* All minority fork nodes pass through this function eventually. So
3379 : this is a good point to check per-node invariants for minority
3380 : forks. */
3381 0 : if( FD_UNLIKELY( block->staged && !block->in_rdisp ) ) {
3382 0 : FD_LOG_CRIT(( "invariant violation: staged block is not in the dispatcher, slot %lu, parent slot %lu",
3383 0 : block->slot, block->parent_slot ));
3384 0 : }
3385 :
3386 0 : fd_sched_block_t * parent = block_pool_ele( sched, block->parent_idx );
3387 0 : if( FD_UNLIKELY( !parent || !parent->in_sched ) ) {
3388 : /* Only the root has no parent. Abandon should never be called on
3389 : the root. So any block we are trying to abandon should have a
3390 : parent. */
3391 0 : FD_LOG_CRIT(( "invariant violation: parent not found slot %lu, parent slot %lu",
3392 0 : block->slot, block->parent_slot ));
3393 0 : }
3394 :
3395 : /* Gated on the parent going down and on this block not already being
3396 : on its way down.
3397 :
3398 : The parent test is because this function also runs on the block the
3399 : public abandon API was called on, whose parent is very much alive.
3400 : Inheriting there would stamp DEAD_ANCESTOR on a block that went
3401 : down on its own, and overwrite the discarded bit the caller just
3402 : set with the live parent's 0.
3403 :
3404 : The block test is because the walk also revisits blocks that
3405 : already went down, and an ancestor abandoned later must not
3406 : re-label them. This cannot be decided based on dead_reason alone
3407 : because a verdict the replay tile made leaves it NONE.
3408 :
3409 : Descendants work fine either way, because dying is set below before
3410 : we recurse, so every child rightfully sees a dying parent. */
3411 0 : if( FD_UNLIKELY( parent->dying && !block->dying ) ) record_lineage_death( block, parent );
3412 :
3413 : /* Setting the flag is non-optional and can happen more than once. */
3414 0 : block->dying = 1;
3415 :
3416 : /* Removal from dispatcher should only happen once. */
3417 0 : if( block->in_rdisp ) {
3418 :
3419 : /* The dispatcher expects blocks to be abandoned in the same order
3420 : that they were added on each lane. There are no requirements on
3421 : the order of abandoning if two blocks are not on the same lane,
3422 : or if a block is unstaged. This means that in general we
3423 : shouldn't abandon a child block if the parent hasn't been
3424 : abandoned yet, if and only if they are on the same lane. So wait
3425 : until we can abandon the parent, and then descend down the fork
3426 : tree to ensure orderly abandoning. */
3427 0 : int in_order = !parent->in_rdisp || /* parent is not in the dispatcher */
3428 0 : !parent->staged || /* parent is in the dispatcher but not staged */
3429 0 : !block->staged || /* parent is in the dispatcher and staged but this block is unstaged */
3430 0 : block->staging_lane!=parent->staging_lane; /* this block is on a different staging lane than its parent */
3431 :
3432 0 : if( FD_UNLIKELY( in_order && block->staged && sched->active_bank_idx==sched->staged_head_bank_idx[ block->staging_lane ] && sched->active_bank_idx!=ULONG_MAX ) ) {
3433 0 : FD_TEST( block_pool_ele( sched, sched->active_bank_idx )==block );
3434 0 : FD_LOG_DEBUG(( "reset active_bank_idx %lu: abandon", sched->active_bank_idx ));
3435 0 : sched->last_active_bank_idx = sched->active_bank_idx;
3436 0 : sched->active_bank_idx = ULONG_MAX;
3437 0 : sched->metrics->deactivate_abandoned_cnt++;
3438 0 : }
3439 :
3440 : /* We inform the dispatcher of an abandon only when there are no
3441 : more in-flight transactions. Otherwise, if the dispatcher
3442 : recycles the same txn_id that was just abandoned, and we receive
3443 : completion of an in-flight transaction whose txn_id was just
3444 : recycled. */
3445 : // FIXME The recycling might be fine now that we no longer use
3446 : // txn_id to index into anything. We might be able to just drop
3447 : // txn_id on abandoned blocks. Though would this leak transaction
3448 : // content if the txn_id is recycled?
3449 : // Note that subtree pruning from sched isn't dependent on the
3450 : // in-flight check being present here, as is_prunable already checks
3451 : // for in-flight==0.
3452 0 : int abandon = in_order && !block_is_in_flight( block );
3453 :
3454 0 : if( abandon ) {
3455 0 : block->in_rdisp = 0;
3456 0 : fd_rdisp_abandon_block( sched->rdisp, (ulong)(block-sched->block_pool) );
3457 0 : block->txn_idx_tail = 0U;
3458 0 : sched->txn_pool_free_cnt += block->txn_parsed_cnt-block->txn_done_cnt; /* in_flight_cnt==0 */
3459 :
3460 0 : sched->metrics->block_abandoned_cnt++;
3461 0 : sched->metrics->txn_abandoned_parsed_cnt += block->txn_parsed_cnt;
3462 0 : sched->metrics->txn_abandoned_exec_done_cnt += block->txn_exec_done_cnt;
3463 0 : sched->metrics->txn_abandoned_done_cnt += block->txn_done_cnt;
3464 :
3465 : //FIXME when demote supports non-empty blocks, we should demote
3466 : //the block from the lane unconditionally and immediately,
3467 : //regardless of whether it's safe to abandon or not. So a block
3468 : //would go immediately from staged to unstaged and eventually to
3469 : //abandoned.
3470 0 : if( FD_LIKELY( block->staged ) ) {
3471 0 : FD_LOG_DEBUG(( "block %lu:%lu exited lane %lu: abandon", block->slot, block_to_idx( sched, block ), block->staging_lane ));
3472 0 : block->staged = 0;
3473 : /* Now release the staging lane. This will release the lane as
3474 : soon as we abandon the head block on a lane. Technically a
3475 : release should only happen when we remove the tail block on a
3476 : lane. This is fine though. The way we abandon guarantees by
3477 : induction that an entire lane will be abandoned. Only the
3478 : head block on a lane can possibly have in-flight
3479 : transactions, and so once a head block becomes eligible for
3480 : abandoning, the entire lane all the way to the tail block,
3481 : will be eligible. */
3482 0 : sched->staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, (int)block->staging_lane );
3483 0 : sched->staged_head_bank_idx[ block->staging_lane ] = ULONG_MAX;
3484 0 : }
3485 0 : }
3486 0 : }
3487 :
3488 : /* Abandon the entire fork chaining off of this block. */
3489 0 : ulong child_idx = block->child_idx;
3490 0 : while( child_idx!=ULONG_MAX ) {
3491 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3492 0 : subtree_mark_and_maybe_prune_rdisp( sched, child );
3493 0 : child_idx = child->sibling_idx;
3494 0 : }
3495 0 : }
3496 :
3497 : /* This function tells sched that the subtree is no longer worth
3498 : replaying. It can happen as a result of one of the following.
3499 : - Bad block at head of lane.
3500 : - Bad block at tail of lane.
3501 : - Root notify.
3502 :
3503 : It's safe to call this function more than once on the same block. */
3504 : static void
3505 0 : subtree_abandon( fd_sched_t * sched, fd_sched_block_t * block ) {
3506 0 : subtree_mark_and_maybe_prune_rdisp( sched, block );
3507 : /* We do not gate refcnt releases on the entire subtree being
3508 : prunable. In the root notify case there can be in-flight tasks in
3509 : the middle of a subtree, and gating on whole-subtree prunability
3510 : would defer the release of an otherwise prunable block, e.g. an
3511 : inactive sibling of an in-flight block, to a later abandon such as
3512 : a subsequent root notify. That could stall bank reclamation.
3513 :
3514 : The sched fork tree still stays in sync with the banks tree. This
3515 : is to prevent attacks targeting lifetime discrepancy. So while we
3516 : know that the subtree can be pruned from sched at this point, we
3517 : only release refcnts here and stop short of actually pruning, which
3518 : remains solely initiated by banks. */
3519 0 : subtree_release_refcnt( sched, block );
3520 0 : }
3521 :
3522 : /* Release the bank refcnt for every block in the subtree that sched is
3523 : done with. This is evaluated per block. A block's refcnt is
3524 : released as soon as that block individually qualifies, independent of
3525 : whether its siblings or descendants still have in-flight tasks. This
3526 : is safe because the actual pruning is initiated separately by banks,
3527 : and gated on the whole subtree's refcnts reaching zero. Releasing
3528 : refcnts eagerly per block get us there sooner. Blocks that are still
3529 : in-flight will be released when their last task drains. This
3530 : function is idempotent. */
3531 : static void
3532 0 : subtree_release_refcnt( fd_sched_t * sched, fd_sched_block_t * block ) {
3533 0 : FD_TEST( block->in_sched );
3534 0 : if( block->refcnt && block_is_prunable( block ) ) {
3535 0 : FD_TEST( block->dying ); /* The happy path releases the refcnt on full replay. Only bad blocks end up here. */
3536 0 : FD_TEST( !block->block_end_done ); /* Implied by an outstanding refcnt. The happy path releases the refcnt on full replay. */
3537 0 : block->refcnt = 0;
3538 0 : FD_TEST( ref_q_avail( sched->ref_q ) );
3539 0 : ref_q_push_tail( sched->ref_q, block_to_idx( sched, block ) );
3540 0 : if( FD_LIKELY( block->block_start_done ) ) FD_LOG_DEBUG(( "block %lu:%lu replayed partially, releasing refcnt without full replay", block->slot, block_to_idx( sched, block ) ));
3541 0 : else FD_LOG_DEBUG(( "block %lu:%lu replayed nothing, releasing refcnt without any replay", block->slot, block_to_idx( sched, block ) ));
3542 0 : }
3543 :
3544 0 : ulong child_idx = block->child_idx;
3545 0 : while( child_idx!=ULONG_MAX ) {
3546 0 : fd_sched_block_t * child_block = block_pool_ele( sched, child_idx );
3547 0 : subtree_release_refcnt( sched, child_block );
3548 0 : child_idx = child_block->sibling_idx;
3549 0 : }
3550 0 : }
3551 :
3552 : static void
3553 0 : subtree_prune( fd_sched_t * sched, ulong bank_idx, ulong except_idx ) {
3554 0 : fd_sched_block_t * head = block_pool_ele( sched, bank_idx );
3555 0 : head->parent_idx = ULONG_MAX;
3556 0 : fd_sched_block_t * tail = head;
3557 :
3558 0 : while( head ) {
3559 0 : FD_TEST( !head->refcnt );
3560 0 : FD_TEST( head->in_sched );
3561 0 : head->in_sched = 0;
3562 :
3563 0 : ulong child_idx = head->child_idx;
3564 0 : while( child_idx!=ULONG_MAX ) {
3565 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3566 : /* Add children to be visited. We abuse the parent_idx field to
3567 : link up the next block to visit. */
3568 0 : if( child_idx!=except_idx ) {
3569 0 : tail->parent_idx = child_idx;
3570 0 : tail = child;
3571 0 : tail->parent_idx = ULONG_MAX;
3572 0 : }
3573 0 : child_idx = child->sibling_idx;
3574 0 : }
3575 :
3576 : /* Prune the current block. We will never publish halfway into a
3577 : staging lane, because anything that we are publishing away should
3578 : be out of the dispatcher at this point, much less staged.
3579 : - Anything on the rooted fork should have finished replaying
3580 : gracefully and be out of the dispatcher.
3581 : - Anything on minority forks should have been marked dying and
3582 : be taken out of the dispatcher.
3583 :
3584 : There should be no more in-flight tasks either. Prunes are
3585 : initiated by banks, and the refcnt on the bank won't drop to zero
3586 : unless all in-flight tasks have been drained. So the fact that
3587 : we're pruning implies that the bank thinks there's nothing more
3588 : in-flight. */
3589 0 : if( FD_UNLIKELY( block_is_in_flight( head ) ) ) {
3590 0 : FD_LOG_CRIT(( "invariant violation: block has tasks in flight (%u exec %u sigverify %u poh %u cert), slot %lu, parent slot %lu",
3591 0 : head->txn_exec_in_flight_cnt, head->txn_sigverify_in_flight_cnt, head->poh_hashing_in_flight_cnt, head->cert_in_flight_cnt, head->slot, head->parent_slot ));
3592 0 : }
3593 0 : if( FD_UNLIKELY( head->in_rdisp ) ) {
3594 : /* We should have removed it from the dispatcher when we were
3595 : notified of the new root, or when in-flight transactions were
3596 : drained. */
3597 0 : FD_LOG_CRIT(( "invariant violation: block is in the dispatcher, slot %lu, parent slot %lu", head->slot, head->parent_slot ));
3598 0 : }
3599 :
3600 : /* Return remaining mblk descriptors to the shared pool. */
3601 0 : free_mblk_slist( sched, head, head->mblks_unhashed );
3602 0 : free_mblk_slist( sched, head, head->mblks_hashing_in_progress );
3603 0 : free_mblk_slist( sched, head, head->mblks_mixin_in_progress );
3604 :
3605 0 : if( FD_UNLIKELY( !head->block_end_done ) ) {
3606 0 : sched->print_buf_sz = 0UL;
3607 0 : print_block_metrics( sched, head );
3608 0 : if( FD_LIKELY( head->block_start_done ) ) FD_LOG_DEBUG(( "block %lu:%lu replayed partially, pruning without full replay: %s", head->slot, block_to_idx( sched, head ), sched->print_buf ));
3609 0 : else FD_LOG_DEBUG(( "block %lu:%lu replayed nothing, pruning without any replay: %s", head->slot, block_to_idx( sched, head ), sched->print_buf ));
3610 0 : }
3611 :
3612 0 : sched->block_pool_popcnt--;
3613 :
3614 0 : fd_sched_block_t * next = block_pool_ele( sched, head->parent_idx );
3615 :
3616 : /* We don't have to clear the indices here since no one should be
3617 : accessing them. Defensive programming. */
3618 0 : head->parent_idx = ULONG_MAX;
3619 0 : head->child_idx = ULONG_MAX;
3620 0 : head->sibling_idx = ULONG_MAX;
3621 :
3622 0 : head = next;
3623 0 : }
3624 0 : }
3625 :
3626 : static void
3627 0 : maybe_switch_block( fd_sched_t * sched, ulong bank_idx ) {
3628 : /* This only happens rarely when there are dying in-flight blocks.
3629 : Early exit and don't let dying blocks affect replay. */
3630 0 : if( FD_UNLIKELY( bank_idx!=sched->active_bank_idx ) ) return;
3631 :
3632 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3633 0 : if( FD_UNLIKELY( block_is_done( block ) ) ) {
3634 0 : fd_rdisp_remove_block( sched->rdisp, bank_idx );
3635 0 : FD_LOG_DEBUG(( "block %lu:%lu exited lane %lu: remove", block->slot, bank_idx, block->staging_lane ));
3636 0 : block->in_rdisp = 0;
3637 0 : block->staged = 0;
3638 0 : sched->metrics->block_removed_cnt++;
3639 0 : FD_LOG_DEBUG(( "reset active_bank_idx %lu: remove", sched->active_bank_idx ));
3640 0 : sched->last_active_bank_idx = sched->active_bank_idx;
3641 0 : sched->active_bank_idx = ULONG_MAX;
3642 :
3643 : /* See if there is a child block down the same staging lane. This
3644 : is a policy decision to minimize fork churn. We could in theory
3645 : reevaluate staging lane allocation here and do promotion/demotion
3646 : as needed. */
3647 0 : ulong child_idx = block->child_idx;
3648 0 : while( child_idx!=ULONG_MAX ) {
3649 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3650 0 : if( FD_LIKELY( child->staged && child->staging_lane==block->staging_lane ) ) {
3651 : /* There is a child block down the same staging lane ... */
3652 0 : if( FD_LIKELY( !child->dying ) ) {
3653 : /* ... and the child isn't dead */
3654 0 : if( FD_UNLIKELY( !block_is_activatable( child ) ) ) {
3655 : /* ... but the child is not activatable, likely because
3656 : there are no transactions available yet. */
3657 0 : sched->metrics->deactivate_no_txn_cnt++;
3658 0 : try_activate_block( sched );
3659 0 : return;
3660 0 : }
3661 : /* ... and it's immediately dispatchable, so switch the active
3662 : block to it, and have the child inherit the head status of
3663 : the lane. This is the common case. */
3664 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: child inheritance on lane %lu", child->slot, child_idx, child->staging_lane ));
3665 0 : sched->active_bank_idx = child_idx;
3666 0 : sched->staged_head_bank_idx[ block->staging_lane ] = child_idx;
3667 0 : if( FD_UNLIKELY( !fd_ulong_extract_bit( sched->staged_bitset, (int)block->staging_lane ) ) ) {
3668 0 : FD_LOG_CRIT(( "invariant violation: staged_bitset 0x%lx bit %lu is not set, slot %lu, parent slot %lu, child slot %lu, parent slot %lu",
3669 0 : sched->staged_bitset, block->staging_lane, block->slot, block->parent_slot, child->slot, child->parent_slot ));
3670 0 : }
3671 0 : return;
3672 0 : } else {
3673 : /* ... but the child block is considered dead, likely because
3674 : the parser considers it invalid. */
3675 0 : FD_LOG_INFO(( "child block %lu is already dead", child->slot ));
3676 0 : subtree_abandon( sched, child );
3677 0 : break;
3678 0 : }
3679 0 : }
3680 0 : child_idx = child->sibling_idx;
3681 0 : }
3682 : /* There isn't a child block down the same staging lane. This is
3683 : the last block in the staging lane. Release the staging lane. */
3684 0 : sched->staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, (int)block->staging_lane );
3685 0 : sched->staged_head_bank_idx[ block->staging_lane ] = ULONG_MAX;
3686 0 : sched->metrics->deactivate_no_child_cnt++;
3687 0 : try_activate_block( sched );
3688 0 : } else if( block_should_deactivate( block ) ) {
3689 : /* We exhausted the active block, but it's not fully done yet. We
3690 : are just not getting FEC sets for it fast enough. This could
3691 : happen when the network path is congested, or when the leader
3692 : simply went down. Reset the active block. */
3693 0 : sched->last_active_bank_idx = sched->active_bank_idx;
3694 0 : sched->active_bank_idx = ULONG_MAX;
3695 0 : sched->metrics->deactivate_no_txn_cnt++;
3696 0 : try_activate_block( sched );
3697 0 : }
3698 0 : }
3699 :
3700 : FD_FN_UNUSED static ulong
3701 0 : find_and_stage_longest_unstaged_fork( fd_sched_t * sched, int lane_idx ) {
3702 0 : ulong root_idx = sched->root_idx;
3703 0 :
3704 0 : if( FD_UNLIKELY( root_idx==ULONG_MAX ) ) {
3705 0 : FD_LOG_CRIT(( "invariant violation: root_idx==ULONG_MAX indicating fd_sched is uninitialized" ));
3706 0 : }
3707 0 :
3708 0 : /* First pass: compute the longest unstaged fork depth for each node
3709 0 : in the fork tree. */
3710 0 : ulong depth = compute_longest_unstaged_fork( sched, root_idx );
3711 0 :
3712 0 : /* Second pass: stage blocks on the longest unstaged fork. */
3713 0 : ulong head_bank_idx = stage_longest_unstaged_fork( sched, root_idx, lane_idx );
3714 0 :
3715 0 : if( FD_UNLIKELY( (depth>0UL && head_bank_idx==ULONG_MAX) || (depth==0UL && head_bank_idx!=ULONG_MAX) ) ) {
3716 0 : FD_LOG_CRIT(( "invariant violation: depth %lu, head_bank_idx %lu",
3717 0 : depth, head_bank_idx ));
3718 0 : }
3719 0 :
3720 0 : return head_bank_idx;
3721 0 : }
3722 :
3723 : /* Returns length of the longest stageable unstaged fork, if there is
3724 : one, and 0 otherwise. */
3725 : static ulong
3726 0 : compute_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx ) {
3727 0 : if( FD_UNLIKELY( bank_idx==ULONG_MAX ) ) {
3728 0 : FD_LOG_CRIT(( "invariant violation: bank_idx==ULONG_MAX" ));
3729 0 : }
3730 :
3731 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3732 :
3733 0 : ulong max_child_depth = 0UL;
3734 0 : ulong child_idx = block->child_idx;
3735 0 : while( child_idx!=ULONG_MAX ) {
3736 0 : ulong child_depth = compute_longest_unstaged_fork( sched, child_idx );
3737 0 : if( child_depth > max_child_depth ) {
3738 0 : max_child_depth = child_depth;
3739 0 : }
3740 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3741 0 : child_idx = child->sibling_idx;
3742 0 : }
3743 :
3744 0 : block->luf_depth = max_child_depth + fd_ulong_if( block_is_promotable( block ), 1UL, 0UL );
3745 0 : return block->luf_depth;
3746 0 : }
3747 :
3748 : static ulong
3749 0 : stage_longest_unstaged_fork_helper( fd_sched_t * sched, ulong bank_idx, int lane_idx ) {
3750 0 : if( FD_UNLIKELY( bank_idx==ULONG_MAX ) ) {
3751 0 : FD_LOG_CRIT(( "invariant violation: bank_idx==ULONG_MAX" ));
3752 0 : }
3753 :
3754 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3755 :
3756 0 : int stage_it = fd_int_if( block_is_promotable( block ), 1, 0 );
3757 0 : ulong rv = fd_ulong_if( stage_it, bank_idx, ULONG_MAX );
3758 0 : if( FD_LIKELY( stage_it ) ) {
3759 0 : block->staged = 1;
3760 0 : block->staging_lane = (ulong)lane_idx;
3761 0 : fd_rdisp_promote_block( sched->rdisp, bank_idx, block->staging_lane );
3762 0 : sched->metrics->block_promoted_cnt++;
3763 0 : FD_LOG_DEBUG(( "block %lu:%lu entered lane %lu: promote", block->slot, bank_idx, block->staging_lane ));
3764 0 : }
3765 :
3766 : /* Base case: leaf node. */
3767 0 : if( block->child_idx==ULONG_MAX ) return rv;
3768 :
3769 0 : ulong max_depth = 0UL;
3770 0 : ulong best_child_idx = ULONG_MAX;
3771 0 : ulong child_idx = block->child_idx;
3772 0 : while( child_idx!=ULONG_MAX ) {
3773 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3774 0 : if( child->luf_depth>max_depth ) {
3775 0 : max_depth = child->luf_depth;
3776 0 : best_child_idx = child_idx;
3777 0 : }
3778 0 : child_idx = child->sibling_idx;
3779 0 : }
3780 :
3781 : /* Recursively stage descendants. */
3782 0 : if( best_child_idx!=ULONG_MAX ) {
3783 0 : ulong head_bank_idx = stage_longest_unstaged_fork_helper( sched, best_child_idx, lane_idx );
3784 0 : rv = fd_ulong_if( rv!=ULONG_MAX, rv, head_bank_idx );
3785 0 : }
3786 :
3787 0 : return rv;
3788 0 : }
3789 :
3790 : /* Returns idx of head block of staged lane on success, idx_null
3791 : otherwise. */
3792 : static ulong
3793 0 : stage_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx, int lane_idx ) {
3794 0 : ulong head_bank_idx = stage_longest_unstaged_fork_helper( sched, bank_idx, lane_idx );
3795 0 : if( FD_LIKELY( head_bank_idx!=ULONG_MAX ) ) {
3796 0 : sched->metrics->lane_promoted_cnt++;
3797 0 : sched->staged_bitset = fd_ulong_set_bit( sched->staged_bitset, lane_idx );
3798 : /* No need to update staged_popcnt_wmk because the fact that there
3799 : are unstaged blocks implies we already maxed out lanes at one
3800 : point. */
3801 0 : sched->staged_head_bank_idx[ lane_idx ] = head_bank_idx;
3802 0 : }
3803 0 : return head_bank_idx;
3804 0 : }
3805 :
3806 : /* Check if an entire staging lane can be demoted. Returns 1 if all
3807 : blocks in the lane are demotable, 0 otherwise. */
3808 : static int
3809 0 : lane_is_demotable( fd_sched_t * sched, int lane_idx ) {
3810 0 : ulong bank_idx = sched->staged_head_bank_idx[ lane_idx ];
3811 :
3812 0 : while( bank_idx!=ULONG_MAX ) {
3813 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3814 0 : FD_TEST( block->staged );
3815 0 : FD_TEST( block->staging_lane==(ulong)lane_idx );
3816 :
3817 0 : if( FD_UNLIKELY( !block_is_demotable( block ) ) ) {
3818 : /* Found a non-demotable block. Early exit. */
3819 0 : return 0;
3820 0 : }
3821 :
3822 : /* Find the child in the same staging lane. */
3823 0 : ulong child_idx = block->child_idx;
3824 0 : ulong next_bank_idx = ULONG_MAX;
3825 0 : while( child_idx!=ULONG_MAX ) {
3826 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3827 0 : if( child->staged && child->staging_lane==(ulong)lane_idx ) {
3828 0 : next_bank_idx = child_idx;
3829 0 : break;
3830 0 : }
3831 0 : child_idx = child->sibling_idx;
3832 0 : }
3833 0 : bank_idx = next_bank_idx;
3834 0 : }
3835 :
3836 0 : return 1;
3837 0 : }
3838 :
3839 : /* Demote all blocks in a staging lane. Assumes that all blocks in the
3840 : lane are demotable. Returns the number of blocks demoted. */
3841 : static ulong
3842 0 : demote_lane( fd_sched_t * sched, int lane_idx ) {
3843 0 : ulong bank_idx = sched->staged_head_bank_idx[ lane_idx ];
3844 0 : uint demoted_cnt = 0U;
3845 :
3846 0 : while( bank_idx!=ULONG_MAX ) {
3847 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3848 0 : FD_TEST( block->staged );
3849 0 : FD_TEST( block->staging_lane==(ulong)lane_idx );
3850 :
3851 0 : int ret = fd_rdisp_demote_block( sched->rdisp, bank_idx );
3852 0 : if( FD_UNLIKELY( ret!=0 ) ) {
3853 0 : FD_LOG_CRIT(( "fd_rdisp_demote_block failed for block %lu:%lu, lane %d", block->slot, bank_idx, lane_idx ));
3854 0 : }
3855 0 : FD_LOG_DEBUG(( "block %lu:%lu exited lane %lu: demote", block->slot, bank_idx, block->staging_lane ));
3856 0 : block->staged = 0;
3857 0 : demoted_cnt++;
3858 :
3859 : /* Find the child in the same staging lane. */
3860 0 : ulong child_idx = block->child_idx;
3861 0 : ulong next_bank_idx = ULONG_MAX;
3862 0 : while( child_idx!=ULONG_MAX ) {
3863 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3864 0 : if( child->staged && child->staging_lane==(ulong)lane_idx ) {
3865 0 : next_bank_idx = child_idx;
3866 0 : break;
3867 0 : }
3868 0 : child_idx = child->sibling_idx;
3869 0 : }
3870 0 : bank_idx = next_bank_idx;
3871 0 : }
3872 :
3873 : /* Clear the lane. */
3874 0 : sched->staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, lane_idx );
3875 0 : sched->staged_head_bank_idx[ lane_idx ] = ULONG_MAX;
3876 :
3877 0 : sched->metrics->block_demoted_cnt += demoted_cnt;
3878 0 : FD_LOG_DEBUG(( "demoted %u blocks in lane %d", demoted_cnt, lane_idx ));
3879 0 : return demoted_cnt;
3880 0 : }
|
