Line data Source code
1 : #include <stdio.h> /* for vsnprintf */
2 : #include <stdarg.h> /* for va_list */
3 :
4 : #include "fd_sched.h"
5 : #include "../../util/fd_hash32.h" /* for lthash lane maps */
6 : #include "../../flamenco/alpenglow/fd_block_marker_serde.h"
7 : #include "fd_execrp.h" /* for poh hash value */
8 : #include "../../ballet/sha256/fd_sha256.h"
9 : #include "../../disco/fd_disco_base.h" /* for FD_MAX_TXN_PER_SLOT_SHRED */
10 : #include "../../disco/fd_txn_p.h"
11 : #include "../../disco/metrics/fd_metrics.h" /* for fd_metrics_convert_ticks_to_nanoseconds and etc. */
12 : #include "../../disco/pack/fd_chkdup.h"
13 : #include "../../disco/shred/fd_shredder.h" /* FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ */
14 : #include "../../flamenco/runtime/fd_runtime.h" /* for fd_runtime_load_txn_address_lookup_tables */
15 : #include "../../flamenco/runtime/fd_system_ids.h"
16 : #include "../../flamenco/runtime/sysvar/fd_sysvar_slot_hashes.h" /* for ALUTs */
17 :
18 24 : #define FD_SCHED_MAX_PRINT_BUF_SZ (2UL<<20)
19 :
20 : /* 64 ticks per slot, and a single gigantic microblock containing min
21 : size transactions. */
22 : FD_STATIC_ASSERT( FD_MAX_TXN_PER_SLOT_SHRED==((FD_SHRED_DATA_PAYLOAD_MAX_PER_SLOT-65UL*sizeof(fd_microblock_hdr_t))/FD_TXN_MIN_SERIALIZED_SZ), max_txn_per_slot_shred );
23 :
24 : /* We size the buffer to be able to hold residual data from the previous
25 : FEC set that only becomes parseable after the next FEC set is
26 : ingested, as well as the incoming FEC set. The largest minimally
27 : parseable unit of data is a transaction. So that much data may
28 : straddle FEC set boundaries. Other minimally parseable units of data
29 : include the microblock header and the microblock count within a
30 : batch. */
31 0 : #define FD_SCHED_MAX_PAYLOAD_PER_FEC (63985UL)
32 : FD_STATIC_ASSERT( FD_SCHED_MAX_PAYLOAD_PER_FEC>=FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ, bump sched fec size bound ); /* FD_SHREDDER_CHAINED_FEC_SET_PAYLOAD_SZ is the bound we want, but older ledgers have larger FEC sets. */
33 : #define FD_SCHED_MAX_RESIDUAL_SZ (FD_TXN_MTU>FD_BLOCK_MARKER_SER_MAX ? FD_TXN_MTU : FD_BLOCK_MARKER_SER_MAX)
34 : #define FD_SCHED_MAX_FEC_BUF_SZ (FD_SCHED_MAX_PAYLOAD_PER_FEC+FD_SCHED_MAX_RESIDUAL_SZ)
35 : FD_STATIC_ASSERT( FD_SCHED_MAX_RESIDUAL_SZ>=sizeof(fd_microblock_hdr_t), resize buffer for residual data );
36 : FD_STATIC_ASSERT( FD_SCHED_MAX_RESIDUAL_SZ>=sizeof(ulong), resize buffer for residual data );
37 :
38 : /* Shred payload sizes are kept per block only for the shreds that the
39 : parser's shred scan has not passed yet. See shred_split(). The scan
40 : trails the parse position by at most the residual, which fits in
41 : FD_SCHED_MAX_RESIDUAL_SZ, and an ingest appends one FEC set. So this
42 : covers any FEC set made of sanely sized shreds. Only a block of
43 : absurdly small shreds has more of them outstanding at once. Such a
44 : block loses its shred attribution, which is monitoring only. */
45 267100 : #define FD_SCHED_SHRED_SZ_WINDOW (256UL)
46 : FD_STATIC_ASSERT( (FD_SCHED_SHRED_SZ_WINDOW&(FD_SCHED_SHRED_SZ_WINDOW-1UL))==0UL, shred_sz_window );
47 :
48 0 : #define FD_SCHED_MAX_TXN_PER_FEC ((FD_SCHED_MAX_PAYLOAD_PER_FEC-1UL)/FD_TXN_MIN_SERIALIZED_SZ+1UL) /* 478 */
49 0 : #define FD_SCHED_MAX_MBLK_PER_FEC ((FD_SCHED_MAX_PAYLOAD_PER_FEC-1UL)/sizeof(fd_microblock_hdr_t)+1UL) /* 1334 */
50 :
51 : FD_STATIC_ASSERT( FD_SCHED_MIN_DEPTH>=FD_SCHED_MAX_TXN_PER_FEC, limits );
52 : FD_STATIC_ASSERT( FD_SCHED_MAX_DEPTH<=FD_RDISP_MAX_DEPTH, limits );
53 : FD_STATIC_ASSERT( FD_SCHED_MAX_DEPTH<=UINT_MAX, txn_idx_width );
54 :
55 :
56 : /* Out-of-band LtHash.
57 :
58 : Recall that a bank's LtHash is the sum over every account of the hash
59 : of its current value. A block that writes an account has to subtract
60 : the hash of the value it found and add the hash of the value it
61 : leaves. In-band, the exec tile does both inside the commit phase of
62 : every writing transaction, so the hashing sits on the critical path
63 : of the block. Our goal is to take it off that path and hide it in
64 : idle exec tile cycles instead.
65 :
66 : The two hashes have different constraints. The subtraction depends
67 : only on the account's value in the parent fork, so it can run at any
68 : time after we have seen that the block writes the account, which we
69 : know at parse. The addition must hash the value the block's last
70 : writer leaves, so it has to run after that writer completes. We let
71 : the dispatcher sequence it for us. An addition is a pseudo
72 : transaction that writes the account, created through
73 : fd_rdisp_add_pseudo_txn(), and the dispatcher hands it back only once
74 : every earlier writer of the account is done. Once the block has all
75 : its FEC sets we know the last writers, and the fec_eos drain creates
76 : the pseudo transactions for every account still owed an addition, a
77 : chunk at a time. Before that we may speculate on an account whose
78 : writers so far have all completed, guessing that no more are coming,
79 : and keep the value in the speculation pool so a later write to the
80 : account can undo it. Subtractions and additions go out in batches of
81 : accounts whose hashes the exec tile sums. A speculation goes out
82 : alone. The scheduler folds the sums into the block's delta, which
83 : the caller adds to the bank's LtHash at block end. Hash tasks go
84 : only to tiles that would otherwise sit idle, and speculation is the
85 : lowest priority of all.
86 :
87 : Writes the scheduler cannot see at parse are registered later:
88 : start-of-block writes through fd_sched_block_start_writes(), and the
89 : writable accounts of lookup tables that failed to resolve, which the
90 : exec tile hands back after execution. */
91 :
92 : /* Every account a staged block owes an addition for stays in the
93 : dispatcher's writable set, and so in its account pool, potentially
94 : past the life of the transactions that reference it, up to
95 : FD_RDISP_MAX_WRITERS_PER_BLOCK per lane. The dispatcher is given
96 : that many account entries on top of what live transactions can
97 : reference. */
98 2005 : #define FD_SCHED_LTHASH_ACCT_EXTRA (FD_SCHED_MAX_STAGING_LANES*FD_RDISP_MAX_WRITERS_PER_BLOCK)
99 :
100 : /* A demoted block parks its writable set in the dispatcher's unstaged
101 : entries, which draw on the same extra capacity. Each block hashing
102 : out of band reserves one block's worth of it. See
103 : lthash_pick_mode().
104 :
105 : A block that already has all its FEC sets could reserve less, since
106 : its set can only grow by the start-of-block and lookup table
107 : registrations. We can estimate these and bound the reservation more
108 : tightly. We don't bother because it's not likely to squeeze more
109 : blocks out of it. */
110 1000 : #define FD_SCHED_LTHASH_OOB_BLOCK_MAX (FD_SCHED_LTHASH_ACCT_EXTRA/FD_RDISP_MAX_WRITERS_PER_BLOCK)
111 : FD_STATIC_ASSERT( FD_SCHED_LTHASH_OOB_BLOCK_MAX<=64UL, lthash_oob_reserved_bitmap );
112 :
113 : /* The speculation pool holds the values of speculative additions so a
114 : later intra-block rewrite can undo the addition. Speculation stops
115 : when this pool gets full. There is a threshold for free speculation
116 : entries per exec tile before a deactivated block is activated to
117 : engage in speculation. This is meant to reduce churn. */
118 16517072 : #define FD_SCHED_LTHASH_SPEC_MAX (8192UL)
119 0 : #define FD_SCHED_LTHASH_SPEC_THRESHOLD_PER_TILE (2UL)
120 :
121 : FD_STATIC_ASSERT( FD_SCHED_LTHASH_BATCH_MAX>=1UL && FD_SCHED_LTHASH_BATCH_MAX<=FD_EXECRP_LTHASH_BATCH_MAX, lthash_batch );
122 : FD_STATIC_ASSERT( FD_SCHED_LTHASH_BATCH_BUSY_MAX>=1UL && FD_SCHED_LTHASH_BATCH_BUSY_MAX<=FD_SCHED_LTHASH_BATCH_MAX, lthash_batch_busy_max );
123 :
124 : FD_STATIC_ASSERT( FD_SCHED_LTHASH_TXN_WRITERS_MAX+FD_SCHED_LTHASH_START_WRITES_MAX==FD_RDISP_MAX_WRITERS_PER_BLOCK, lthash_writers );
125 :
126 :
127 1000 : #define FD_SCHED_MAGIC (0xace8a79c181f89b6UL) /* echo -n "fd_sched_v0" | sha512sum | head -c 16 */
128 :
129 : struct fd_sched_mblk {
130 : ulong start_txn_idx; /* inclusive parse idx */
131 : ulong end_txn_idx; /* non-inclusive parse idx */
132 : ulong curr_txn_idx; /* next txn to mixin, parse idx in block */
133 : fd_hash_t end_hash[ 1 ];
134 : fd_hash_t curr_hash[ 1 ];
135 : uint curr_sig_cnt;
136 : uint next;
137 : uint mixin_txn_idx; /* rdisp pool idx of the next txn to mixin */
138 : int is_tick;
139 : };
140 : typedef struct fd_sched_mblk fd_sched_mblk_t;
141 :
142 : #define SLIST_NAME mblk_slist
143 : #define SLIST_ELE_T fd_sched_mblk_t
144 0 : #define SLIST_IDX_T uint
145 23702 : #define SLIST_NEXT next
146 : #include "../../util/tmpl/fd_slist.c"
147 :
148 : #define SET_NAME txn_bitset
149 : #define SET_MAX FD_SCHED_MAX_DEPTH
150 : #include "../../util/tmpl/fd_set.c"
151 :
152 : /* Out-of-band LtHash keeps per-account state for computing a block's
153 : LtHash delta. Each staging lane has a map from pubkey to an entry
154 : that tracks the subtraction and addition of an account the lane's
155 : block writes. A pool of LtHash values shared by all lanes keeps the
156 : values of speculative additions that a later write to the account
157 : may need to undo. A block without a reservation for parking its
158 : writable set hashes in band instead and keeps none of this state.
159 :
160 : An entry is inserted the first time the block sees a write to the
161 : account, according to either fd_rdisp_add_txn() or
162 : fd_rdisp_add_writable(). It starts SUB_QUEUED and ADD_NONE and lives
163 : until the lane is released, which also frees its speculation entry.
164 : The state machine implemented for each account is as follows:
165 :
166 : dispatched to execrp execrp done, delta -= value
167 : sub: QUEUED ---------------------> DISPATCHED ----------------------------> DONE
168 :
169 : execrp done, ptxn_idx==PIDX
170 : rdisp_add_pseudo_txn()==PIDX rdisp_get_next_ready()==PIDX delta += value, ptxn_idx=0
171 : speculative: spec_idx acquired and ptxn_idx==PIDX spec[spec_idx] = value, if any
172 : add: NONE -------------------------------> PENDING(PIDX) -----------------------------> DISPATCHED(PIDX) -------------------------------> DONE
173 : ^ | | |
174 : | rewrite rewrite rewrite
175 : | ptxn_idx=0 ptxn_idx=0 delta-=spec[spec_idx]
176 : | spec_idx released, if any spec_idx released, if any spec_idx released
177 : | | | |
178 : | v v v
179 : +------------------------------------------+---------------------------------------------+-------------------------------------------+
180 : PIDX surfaces or returns later and !=ptxn_idx: pseudo transaction completed or hash result dropped without effect
181 :
182 : A rewrite is a new first write to the account or a speculation we
183 : gave up.
184 :
185 : A speculative pseudo transaction is one the scheduler creates before
186 : the block has all its FEC sets, betting that the account's last write
187 : has happened. A speculative addition may need to be undone after it
188 : is applied, so it requires a speculation entry. A pseudo transaction
189 : created once the block is complete is no longer speculative and
190 : acquires no speculation entry, so its addition must not be rewritten
191 : once DONE. This is guaranteed by construction. Such a pseudo
192 : transaction is created only after every transaction of the block has
193 : been inserted, because the fec_eos drain happens after the last FEC
194 : set. So the only writes that can still arrive are registrations,
195 : and those land before the addition can apply. Start-of-block
196 : registrations run before anything else (including a PENDING
197 : addition) in the block gets dispatched. Late ALT registrations come
198 : from serializing transactions. A serializing transaction blocks
199 : every pseudo transaction inserted after it until it completes, and
200 : these pseudo transactions are inserted after every transaction of
201 : the block, so all of them wait on the last serializing transaction.
202 : Each serializing transaction registers at its own completion, no
203 : later than the last serializing transaction's completion. A rewrite
204 : therefore finds such an addition PENDING, which only makes it stale
205 : and needs no undo. */
206 :
207 2889 : #define FD_SCHED_LTHASH_SUB_QUEUED (0)
208 0 : #define FD_SCHED_LTHASH_SUB_DISPATCHED (1)
209 0 : #define FD_SCHED_LTHASH_SUB_DONE (2)
210 :
211 2889 : #define FD_SCHED_LTHASH_ADD_NONE (0)
212 0 : #define FD_SCHED_LTHASH_ADD_PENDING (1) /* Pseudo transaction added to rdisp, but not yet surfaced by fd_rdisp_get_next_ready(). */
213 0 : #define FD_SCHED_LTHASH_ADD_DISPATCHED (2) /* Surfaced and still the addition of record. Might be batched or immediately hashed. */
214 0 : #define FD_SCHED_LTHASH_ADD_DONE (3) /* Applied to the block's LtHash, value cached for potential undo if spec_idx is valid. */
215 :
216 : /* This represents one LtHash account. */
217 : struct fd_sched_lthash_entry {
218 : fd_acct_addr_t acct;
219 : uint map_next;
220 : uint q_next; /* Subtraction queue link while SUB_QUEUED. */
221 : uint spec_idx; /* Speculation pool entry or UINT_MAX. */
222 : uint ptxn_idx; /* Non-zero only while the entry is ADD_PENDING or ADD_DISPATCHED. High bit set if non-zero, like the index returned from fd_rdisp_get_next_ready(). */
223 : uint bank_idx;
224 : uchar sub_state;
225 : uchar add_state;
226 : uchar pad[ 2 ];
227 : };
228 : typedef struct fd_sched_lthash_entry fd_sched_lthash_entry_t;
229 : FD_STATIC_ASSERT( sizeof(fd_sched_lthash_entry_t)==56UL, lthash_entry );
230 :
231 : #define MAP_NAME lthash_map
232 : #define MAP_ELE_T fd_sched_lthash_entry_t
233 : #define MAP_KEY_T fd_acct_addr_t
234 2889 : #define MAP_KEY acct
235 2889 : #define MAP_NEXT map_next
236 15570 : #define MAP_IDX_T uint
237 8667 : #define MAP_KEY_HASH(k,s) fd_hash32( (k)->b, (s) )
238 0 : #define MAP_KEY_EQ(k0,k1) (!memcmp( (k0)->b, (k1)->b, 32UL ))
239 : #include "../../util/tmpl/fd_map_chain.c"
240 :
241 : /* A lane's map holds up to FD_RDISP_MAX_WRITERS_PER_BLOCK accounts.
242 : Entries are handed out in index order and only ever freed together
243 : when the lane is released, so the live entries are the first
244 : entry_cnt of the pool and no free list is needed. */
245 : struct fd_sched_lthash_lane {
246 : fd_sched_lthash_entry_t * entry_pool;
247 : lthash_map_t * map;
248 : uint entry_cnt; /* Live entries. */
249 : ulong bank_idx; /* Block that owns the lane, ULONG_MAX if none. */
250 : };
251 : typedef struct fd_sched_lthash_lane fd_sched_lthash_lane_t;
252 :
253 : /* Half the template's estimate for a full lane, since a lane rarely
254 : holds more than a few thousand accounts. */
255 : static inline ulong
256 16041 : lthash_chain_cnt( void ) {
257 16041 : return fd_ulong_max( lthash_map_chain_cnt_est( FD_RDISP_MAX_WRITERS_PER_BLOCK )>>1, 1UL );
258 16041 : }
259 :
260 : struct fd_sched_block {
261 : ulong slot;
262 : ulong parent_slot;
263 : ulong parent_idx; /* Index of the parent in the pool. */
264 : ulong child_idx; /* Index of the left-child in the pool. */
265 : ulong sibling_idx; /* Index of the right-sibling in the pool. */
266 :
267 : /* Counters. */
268 : uint txn_parsed_cnt;
269 : /* txn_queued_cnt = txn_parsed_cnt-txn_in_flight_cnt-txn_done_cnt */
270 : uint txn_exec_in_flight_cnt;
271 : uint txn_exec_done_cnt;
272 : uint txn_sigverify_in_flight_cnt;
273 : uint txn_sigverify_done_cnt;
274 : uint poh_hashing_in_flight_cnt; /* number of in-flight PoH batch tasks */
275 : uint poh_mblk_in_flight_cnt; /* number of individual in-flight PoH jobs across batches */
276 : uint poh_hashing_done_cnt;
277 : uint poh_hash_cmp_done_cnt; /* poh_hashing_done_cnt==poh_hash_cmp_done_cnt+len(mixin_in_progress) */
278 : uint txn_done_cnt; /* A transaction is considered done when all types of tasks associated with it are done. */
279 : uint shred_cnt;
280 : uint shred_scan_idx; /* First shred boundary not before shred_scan_off. */
281 : uint shred_scan_off; /* Block byte offset at the start of shred_scan_idx. */
282 : uint mblk_cnt; /* Total number of microblocks, including ticks and non ticks.
283 : mblk_cnt==len(unhashed)+poh_mblk_in_flight_cnt+len(mixin_in_progress)+hash_cmp_done_cnt */
284 : uint mblk_tick_cnt; /* Total number of tick microblocks. */
285 : uint mblk_freed_cnt; /* This is ==hash_cmp_done_cnt in most cases, except for aborted
286 : blocks, where the freed cnt will catch up to mblk_cnt and surpass
287 : hash_cmp_done_cnt when the block is reaped. */
288 : uint mblk_unhashed_cnt; /* ==len(unhashed) */
289 : ulong txn_pool_max_popcnt; /* Peak transaction pool occupancy during the time this block was replaying. */
290 : ulong mblk_pool_max_popcnt; /* Peak mblk pool occupancy. */
291 : ulong block_pool_max_popcnt; /* Peak block pool occupancy. */
292 : uint txn_idx_tail; /* Most recently parsed rdisp pool index. */
293 : uint txn_sigverify_next_idx; /* Next rdisp pool index to dispatch for sigverify, or 0 if caught up. */
294 : uint parse_mblk_idx; /* mblk pool index currently receiving parsed transactions. */
295 : uint footer_in_flight_cnt;
296 :
297 : /* Out-of-band LtHash.
298 :
299 : When the block has claim on the lane map, each entry in the map
300 : corresponds to one subtraction for the block.
301 :
302 : entries_cnt==sub_queued_cnt+sub_in_flight_cnt+sub_applied_cnt
303 :
304 : Additions the block owes but has no pseudo transaction for yet are
305 :
306 : add_owed_cnt
307 :
308 : We don't track first writes per block. Each puts an addition on
309 : the owed side, so if we did, they would balance as follows.
310 :
311 : first_write_cnt==add_owed_cnt+add_created_cnt
312 :
313 : Unsurfaced additions that rdisp has been instructed to create but
314 : hasn't handed back to us yet are
315 :
316 : add_created_cnt-add_surfaced_cnt
317 :
318 : In-flight additions, including those staged in a batch and not yet
319 : dispatched, are
320 :
321 : add_surfaced_cnt-add_done_cnt
322 :
323 : Additions completed without a hash, or whose hash was dropped,
324 : because their account was written again within the same block are
325 :
326 : add_stale_cnt==add_done_cnt-add_applied_cnt
327 :
328 : The net number of additions absorbed by the block's delta is
329 :
330 : add_applied_cnt-add_undone_cnt
331 :
332 : At BLOCK_END every entry has been subtracted and added exactly once.
333 :
334 : sub_applied_cnt==add_applied_cnt-add_undone_cnt==entries_cnt */
335 : uint lthash_writable_cnt; /* Up to FD_SCHED_LTHASH_TXN_WRITERS_MAX. */
336 : uint lthash_in_flight_cnt; /* Task count, could be either add or sub. */
337 : uint sub_q_head; /* UINT_MAX if empty. Linked through entry->q_next. */
338 : uint sub_q_tail;
339 : uint sub_queued_cnt;
340 : uint sub_in_flight_cnt;
341 : uint sub_applied_cnt;
342 : uint add_owed_cnt; /* First writes whose addition pseudo transaction is not created yet. */
343 : uint add_created_cnt; /* Addition pseudo transactions that we've instructed rdisp to create and queue up. */
344 : uint add_surfaced_cnt; /* Returned by fd_rdisp_get_next_ready(). Will be either dispatched or completed on the spot. */
345 : uint add_done_cnt; /* fd_rdisp_complete_txn() issued for pseudo transactions, with or without a dispatched hashing. */
346 : uint add_applied_cnt; /* Addition results rolled into the block's delta. */
347 : uint add_undone_cnt; /* Applied additions rolled back due to a subsequent account rewrite. */
348 :
349 : /* PoH verify. */
350 : fd_hash_t poh_hash[ 1 ]; /* running end_hash of last parsed mblk */
351 : int last_mblk_is_tick;
352 : mblk_slist_t mblks_unhashed[ 1 ]; /* A microblock, once parsed out, is in one of these queues. It
353 : progresses from unhashed to mixin. When a microblock is being
354 : hashed/in-flight, it'll be transiently out of any of the queues.
355 : Once a microblock progresses through all stages of work, it'll be
356 : immediately freed. */
357 : mblk_slist_t mblks_mixin_in_progress[ 1 ];
358 : uchar bmtree_mem[ FD_BMTREE_COMMIT_FOOTPRINT(0) ] __attribute__((aligned(FD_BMTREE_COMMIT_ALIGN)));
359 : fd_bmtree_commit_t * bmtree;
360 : ulong tick_height; /* Block is built off of a parent block with this many ticks, ULONG_MAX until known. */
361 : ulong max_tick_height; /* Block should end with precisely this many ticks. */
362 :
363 : /* Parser state. */
364 : ulong mblks_rem; /* Number of microblocks remaining in the current batch. */
365 : ulong txns_rem; /* Number of transactions remaining in the current microblock. */
366 : uint residual_sz; /* Bytes of residual data, at most FD_SCHED_MAX_RESIDUAL_SZ. */
367 : uint fec_buf_boff; /* Byte offset into raw block data of the first residual byte, which is the first byte in the parse buffer during ingest. */
368 : ushort shred_sz[ FD_SCHED_SHRED_SZ_WINDOW ]; /* Payload size of shred i at i%FD_SCHED_SHRED_SZ_WINDOW, for the shreds in [shred_scan_idx, shred_cnt). */
369 : long fec_completed_ns; /* Network arrival (wallclock ns) of the FEC set being ingested; txns
370 : parse out only once the FEC completing their bytes arrives, so this
371 : is stamped on every txn parsed during the current ingest. */
372 : uint fec_eob:1; /* FEC end-of-batch: set if the last FEC set in the batch is being
373 : ingested. */
374 : uint fec_sob:1; /* FEC start-of-batch: set if the parser expects to be parsing out a
375 : batch header. */
376 : uint shred_sz_overflow:1; /* Set once more shreds were outstanding than shred_sz holds. The
377 : shred attribution of the block's remaining transactions saturates. */
378 :
379 : /* Block state. */
380 : uint fec_eos:1; /* FEC end-of-stream: set if the last FEC set in the block has been
381 : ingested. */
382 : uint rooted:1; /* Set if the block is rooted. */
383 : uint dying:1; /* Set if the block has been abandoned and no transactions should be
384 : scheduled from it. */
385 : uint discarded:1; /* Set if the block went down with a discarded (not invalid) lineage:
386 : its own discard or an ancestor's. This field, along with
387 : dead_reason, can be in one of three possible combinations when a
388 : block is dead:
389 :
390 : dead_reason!=DEAD_ANCESTOR, discarded initialized to 0: invalid block due to dead_reason
391 : dead_reason==DEAD_ANCESTOR, discarded==0: invalid lineage
392 : dead_reason==DEAD_ANCESTOR||NONE, discarded==1: discarded lineage */
393 : uint refcnt:1; /* Starts at 1 when the block is added, set to 0 if caller has been
394 : informed to decrement refcnt for sched. */
395 : uint in_sched:1; /* Set if the block is being tracked by the scheduler. */
396 : uint in_rdisp:1; /* Set if the block is being tracked by the dispatcher, either as staged
397 : or unstaged. */
398 : uint block_start_signaled:1; /* Set if the start-of-block sentinel has been dispatched. */
399 : uint block_end_signaled:1; /* Set if the end-of-block sentinel has been dispatched. */
400 : uint block_start_done:1; /* Set if the start-of-block processing has been completed. */
401 : uint block_end_done:1; /* Set if the end-of-block processing has been completed. */
402 : uint staged:1; /* Set if the block is in a dispatcher staging lane; a staged block is
403 : tracked by the dispatcher. */
404 : uint lthash_drained:1; /* Set if fec_eos AND something equivalent to
405 : fd_rdisp_add_all_pseudo_txn() has been done since fec_eos. That is
406 : to say, every writable account has an addition pseudo transaction
407 : that will capture its final value. */
408 : uint lthash_draining:1; /* Counted in sched->lthash_draining_cnt. */
409 : uint lthash_mode_picked:1; /* Set once the block has picked its hashing mode, at its first drain or its
410 : BLOCK_START, whichever comes first. */
411 : uint lthash_inband:1; /* Set if the block hashes in band. The dispatcher tracks no writers for it and the
412 : block owes no out-of-band work. */
413 : uchar lthash_oob_reservation; /* Which out-of-band reservation the block holds, meaningful when
414 : lthash_mode_picked && !lthash_inband. */
415 : ulong staging_lane; /* Ignored if staged==0. */
416 :
417 : /* Slot state tracking */
418 : long state_start_tick; /* Start of the committed state's interval. */
419 : long state_cand_tick; /* Start of the candidate state. */
420 : uchar state; /* Committed state, NONE until the first commit. */
421 : uchar state_cand; /* Candidate state, committed once held for FD_SCHED_SLOT_STATE_DEBOUNCE_NS. */
422 : uchar state_open;
423 : uchar state_conflict; /* Last next_ready found queued txns but none ready. */
424 : ulong luf_depth; /* Depth of longest unstaged fork starting from this node; only
425 : stageable unstaged descendants are counted. */
426 : int dead_reason; /* One of FD_SCHED_DEAD_REASON_*; records the first reason the block
427 : was ruled invalid (set-once via record_dead_reason), NONE otherwise. */
428 : uchar residual[ FD_SCHED_MAX_RESIDUAL_SZ ]; /* Bytes of the previous FEC set that only become parseable once the
429 : next FEC set is ingested, at most a transaction, a block marker or
430 : a microblock or batch header. */
431 :
432 : /* Alpenglow block footer, deserialized out of the marker batch at
433 : parse time. footer_seen is set if the block carries a footer
434 : marker and it is seen by sched; footer is only meaningful in that
435 : case. */
436 : fd_block_footer_t footer;
437 :
438 : /* Alpenglow block structure, mirroring agave's BlockComponentStage
439 : as a set of "seen" flags.
440 : Together with mblk_cnt they determine what may come next:
441 :
442 : agave stage header footer alpentick mblk_cnt
443 : PreParentMarker 0 0 0 0
444 : AcceptingGenesisOrEntries 1 0 0 0 (and no genesis cert yet)
445 : AcceptingEntriesOrFooter 1 0 0 *
446 : AcceptingAlpentick 1 1 0 *
447 : Done 1 1 1 * */
448 : uint header_seen : 1;
449 : uint genesis_cert_seen : 1;
450 : uint update_parent_seen : 1;
451 : uint footer_seen : 1;
452 : uint cert_cnt : 3; /* Footer certs to verify. See fd_alpenglow_footer_certs. */
453 : uint cert_next : 3; /* Footer certs already sent out as tasks. */
454 : uint alpentick_seen : 1;
455 :
456 : /* Replay started the block at its UpdateParent FEC set, so the
457 : UpdateParent stands in for the header as the parent marker. */
458 : uint starts_at_update_parent : 1;
459 : uint flh_active : 1; /* alpenglow_fast_leader_handover is active on the block's bank, set at BLOCK_START */
460 : };
461 : typedef struct fd_sched_block fd_sched_block_t;
462 :
463 : FD_STATIC_ASSERT( sizeof(fd_sched_mblk_t)==104UL, fd_sched_mblk );
464 : FD_STATIC_ASSERT( sizeof(fd_sched_txn_info_t)==184UL, fd_sched_txn_info );
465 : FD_STATIC_ASSERT( sizeof(fd_sched_block_t)==9472UL, fd_sched_block );
466 : FD_STATIC_ASSERT( sizeof(fd_hash_t)==sizeof(((fd_microblock_hdr_t *)0)->hash), unexpected poh hash size );
467 :
468 :
469 : struct fd_sched_metrics {
470 : uint block_added_cnt;
471 : uint block_added_staged_cnt;
472 : uint block_added_unstaged_cnt;
473 : uint block_added_dead_ood_cnt;
474 : uint block_removed_cnt;
475 : uint block_abandoned_cnt;
476 : uint block_bad_cnt;
477 : uint block_promoted_cnt;
478 : uint block_demoted_cnt;
479 : uint deactivate_no_child_cnt;
480 : uint deactivate_no_txn_cnt;
481 : uint deactivate_pruned_cnt;
482 : uint deactivate_abandoned_cnt;
483 : uint lane_switch_cnt;
484 : uint lane_promoted_cnt;
485 : uint lane_demoted_cnt;
486 : uint fork_observed_cnt;
487 : uint alut_success_cnt;
488 : uint alut_serializing_cnt;
489 : uint txn_abandoned_parsed_cnt;
490 : uint txn_abandoned_exec_done_cnt;
491 : uint txn_abandoned_done_cnt;
492 : uint txn_promoted_cnt;
493 : uint txn_max_in_flight_cnt;
494 : ulong txn_weighted_in_flight_cnt;
495 : ulong txn_weighted_in_flight_tickcount;
496 : ulong txn_none_in_flight_tickcount;
497 : ulong txn_parsed_cnt;
498 : ulong txn_exec_done_cnt;
499 : ulong txn_sigverify_done_cnt;
500 : ulong txn_mixin_done_cnt;
501 : ulong txn_done_cnt;
502 : ulong mblk_parsed_cnt;
503 : ulong mblk_poh_hashed_cnt;
504 : ulong mblk_poh_done_cnt;
505 : ulong bytes_ingested_cnt;
506 : ulong bytes_ingested_unparsed_cnt;
507 : ulong bytes_dropped_cnt;
508 : ulong fec_cnt;
509 :
510 : /* Out-of-band LtHash. sum(x) sums per-block counter x over the
511 : blocks in the scheduler. The lthash_discarded_* counters are for
512 : what abandon and demotion throw away. We expect the following
513 : invariants.
514 :
515 : Writes observed, by source, and what the dispatcher made of each.
516 : A given up speculation puts its account back as written, so it is a
517 : source too:
518 : lthash_parsed_writable_cnt+lthash_start_reg_cnt+lthash_late_reg_cnt+lthash_demote_reg_cnt+lthash_spec_given_up_cnt==lthash_first_write_cnt+lthash_coalesced_cnt
519 :
520 : Sometimes rdisp reports a write as the account's first write since
521 : its last addition pseudo transaction. It makes the block owe the
522 : account one addition, which a drain or a speculation will get going
523 : later. Writes in between coalesce into that owed addition, which
524 : is the add-add coalescing rdisp provides:
525 : lthash_first_write_cnt==lthash_add_created_cnt+sum(add_owed_cnt)+lthash_discarded_add_owed_cnt+lthash_inband_add_owed_cnt
526 :
527 : Additions. A surfaced addition sits in sched->lthash_add_batch
528 : until its batch reaches a tile, so the held batch is a term:
529 : lthash_add_created_cnt==lthash_add_surfaced_cnt+sum(add_created_cnt-add_surfaced_cnt)+lthash_discarded_add_pending_cnt
530 : lthash_add_surfaced_cnt==lthash_add_surfaced_stale_cnt+lthash_spec_given_up_cnt+lthash_add_dispatched_cnt+(held additions)+lthash_discarded_add_held_cnt
531 : lthash_add_dispatched_cnt==lthash_add_dispatched_stale_cnt+lthash_add_applied_cnt+(in-flight additions)
532 : lthash_add_applied_cnt-lthash_add_undone_cnt==lthash_frozen_cnt+sum(add_applied_cnt-add_undone_cnt)+lthash_discarded_add_applied_cnt
533 :
534 : Subtractions:
535 : lthash_entry_cnt==lthash_sub_dispatched_cnt+sum(sub_queued_cnt)+lthash_discarded_sub_queued_cnt+lthash_inband_sub_queued_cnt
536 : lthash_sub_dispatched_cnt==lthash_sub_applied_cnt+(in-flight subtractions)
537 : lthash_sub_applied_cnt==lthash_frozen_cnt+sum(sub_applied_cnt)+lthash_discarded_sub_applied_cnt
538 :
539 : Speculation. lthash_spec_cnt of the additions were guesses. Once
540 : the blocks finish, lthash_spec_applied_cnt-lthash_add_undone_cnt of
541 : them survive, lthash_spec_stale_cnt were invalidated before their
542 : hash applied, and lthash_spec_given_up_cnt were withdrawn before
543 : they reached a tile with no write involved. The rest of the stale
544 : counts are drain additions invalidated by a registration that came
545 : after the drain. Draining is a parse time operation, and the
546 : registrations that invalidate it could come from either
547 : start-of-block processing or ALT expansions. The owed, pending and
548 : queued lthash_discarded_* counters come only from abandon, since a
549 : demotable block has nothing queued or unsurfaced and keeps its owed
550 : additions. The two lthash_discarded_*_applied delta counters can
551 : come from either abandon or demotion. */
552 : ulong lthash_parsed_writable_cnt; /* Declared writable accounts of parsed transactions. ALT accounts of a transaction whose tables could not be expanded are not counted here, and instead count as late registrations. */
553 : uint lthash_start_reg_cnt;
554 : uint lthash_late_reg_cnt; /* Declared writable lookup table accounts registered at transaction completion. */
555 : uint lthash_demote_reg_cnt; /* Accounts put back into rdisp when their block was demoted. A later promotion will drain these back out of rdisp. Not really new writes. */
556 : ulong lthash_first_write_cnt; /* Writes rdisp reported as the account's first since its last addition pseudo transaction. */
557 : ulong lthash_coalesced_cnt; /* Writes to an account the block already owed an addition for. */
558 : ulong lthash_entry_cnt; /* Lane head account map entries inserted, each owing one subtraction. */
559 : ulong lthash_sub_dispatched_cnt;
560 : ulong lthash_sub_task_cnt; /* Subtraction tasks dispatched to a tile, each a batch of the accounts lthash_sub_dispatched_cnt counts. */
561 : ulong lthash_sub_applied_cnt;
562 : ulong lthash_add_created_cnt; /* Addition pseudo transactions rdisp handed us. */
563 : ulong lthash_spec_cnt; /* Of those, speculatively created while the block was still receiving FEC sets. */
564 : ulong lthash_add_surfaced_cnt; /* Pseudo transactions returned by fd_rdisp_get_next_ready(). */
565 : ulong lthash_add_surfaced_stale_cnt; /* Of those, no longer the account's addition of record due to a rewrite. We will complete these pseudo transactions without hashing. */
566 : ulong lthash_add_dispatched_cnt;
567 : ulong lthash_add_task_cnt; /* Addition tasks dispatched to a tile, each a batch of the accounts lthash_add_dispatched_cnt counts. */
568 : ulong lthash_add_dispatched_stale_cnt; /* Of those, rewritten while in flight. We will drop the hashing result. */
569 : ulong lthash_add_batch_held_cnt; /* Addition batches held because a transaction surfaced behind them, released by a stale addition completed in the same poll, or because a speculative addition went alone. */
570 : ulong lthash_add_batch_wait_idle_cnt; /* Polls that left an addition batch held because of insufficient idle tile. */
571 : ulong lthash_add_applied_cnt;
572 : ulong lthash_add_undone_cnt; /* Applied additions rolled back because the account was rewritten. */
573 : ulong lthash_spec_applied_cnt; /* Of lthash_add_applied_cnt, speculative. */
574 : ulong lthash_spec_stale_cnt; /* Speculations rewritten while PENDING or DISPATCHED, invalidated before they applied. Applied speculations are not included here. */
575 : uint lthash_spec_given_up_cnt; /* Speculations that surfaced once the block was complete, or when the idleness policy would not let them onto a tile. */
576 : uint lthash_pool_full_cnt; /* Deactivations of a block that still owed additions, because the speculation pool had no free entry. Mutually exclusive with deactivate_no_txn_cnt. Such blocks have a chance to be activated if sufficient entries per exec tile are free again. */
577 : uint lthash_spec_yield_cnt; /* Deactivations of a block that could only speculate, because it's yielding to another lane head that has must-do work. */
578 : ulong lthash_frozen_cnt; /* Accounts in the delta when BLOCK_END completes. End-of-block writers are hashed in band and are not counted. */
579 : uint lthash_discarded_add_owed_cnt; /* Additions owed by abandoned blocks. Never created. */
580 : uint lthash_discarded_add_pending_cnt; /* Pseudo transactions of abandoned blocks. Never surfaced. */
581 : uint lthash_discarded_add_held_cnt; /* Additions of abandoned blocks batched behind a transaction. Never on a tile. */
582 : uint lthash_discarded_add_applied_cnt; /* Additions in the delta of an abandoned or demoted block, net of undone. */
583 : uint lthash_discarded_sub_queued_cnt; /* Subtractions of abandoned blocks. Never dispatched. */
584 : uint lthash_discarded_sub_applied_cnt; /* Subtractions in the delta of an abandoned or demoted block. */
585 : uint lthash_inband_add_owed_cnt; /* Additions owed by blocks that picked in-band hashing. Never created. */
586 : uint lthash_inband_sub_queued_cnt; /* Subtractions of blocks that picked in-band hashing. Never dispatched. */
587 : uint lthash_inband_block_cnt; /* Blocks that picked in-band hashing because every out-of-band reservation was taken. */
588 : ulong lane_wait_cnt;
589 : };
590 : typedef struct fd_sched_metrics fd_sched_metrics_t;
591 :
592 : /* A batch of LtHash entries of one block and one kind, either what a
593 : tile is actively hashing or the additions that have been staged but
594 : not yet dispatched. A pseudo transaction is unreadable once it is
595 : completed, so we capture the account addresses at batching. */
596 : struct fd_sched_lthash_batch {
597 : ulong bank_idx;
598 : ulong cnt; /* In [0, FD_SCHED_LTHASH_BATCH_MAX]. */
599 : uint lane;
600 : uchar kind;
601 : uint entry_idx[ FD_SCHED_LTHASH_BATCH_MAX ]; /* Entries in the lane map. */
602 : uint ptxn_idx [ FD_SCHED_LTHASH_BATCH_MAX ]; /* FD_RDISP_LTHASH_PSEUDO_TXN bit set. 0 for a subtraction. */
603 : fd_acct_addr_t accts [ FD_SCHED_LTHASH_BATCH_MAX ];
604 : };
605 : typedef struct fd_sched_lthash_batch fd_sched_lthash_batch_t;
606 : FD_STATIC_ASSERT( alignof(fd_sched_lthash_batch_t)==alignof(ulong), lthash_batch_align ); /* The layout places the records at alignof(ulong). */
607 :
608 : #define DEQUE_NAME ref_q
609 0 : #define DEQUE_T ulong
610 : #include "../../util/tmpl/fd_deque_dynamic.c"
611 :
612 : struct fd_sched {
613 : fd_acct_addr_t aluts[ 256 ]; /* Resolve ALUT accounts into this buffer for more parallelism. */
614 : char print_buf[ FD_SCHED_MAX_PRINT_BUF_SZ ];
615 : ulong print_buf_sz;
616 : uchar fec_buf[ FD_SCHED_MAX_FEC_BUF_SZ ]; /* Parse buffer. It holds the residual followed by the new FEC set. */
617 : uint fec_buf_sz; /* Size of the fec_buf in bytes. */
618 : uint fec_buf_soff; /* Starting offset into fec_buf for unparsed transactions. */
619 : uchar txn[ FD_TXN_MAX_SZ ] __attribute__((aligned(alignof(fd_txn_t)))); /* The transaction being parsed out. */
620 : ulong max_shreds_per_block; /* Immutable. */
621 : ulong max_txn_per_slot; /* Immutable. */
622 : ulong max_mblk_per_slot; /* Immutable, FD_SCHED_MAX_MBLK_PER_SLOT scaled with max_shreds_per_block. */
623 : ulong lthash_txn_writers_max; /* FD_SCHED_LTHASH_TXN_WRITERS_MAX unless lowered by a test. */
624 : ulong lthash_oob_reserved; /* Bit i set if out-of-band reservation i is held by a block. */
625 : ulong lthash_oob_block_max; /* FD_SCHED_LTHASH_OOB_BLOCK_MAX unless lowered by a test. */
626 : fd_chkdup_t chkdup[ 1 ];
627 : fd_sched_metrics_t metrics[ 1 ];
628 : ulong canary; /* == FD_SCHED_MAGIC */
629 : ulong depth; /* Immutable. Pool depth. */
630 : ulong block_cnt_max; /* Immutable. */
631 : ulong exec_cnt; /* Immutable. */
632 : int bypass_poh_verify; /* Test/fuzz: skip the PoH end_hash compare in maybe_mixin. */
633 : int bypass_alut_resolution; /* Test/fuzz: skip ALUT resolution (no accdb). */
634 : long txn_in_flight_last_tick;
635 : long next_ready_last_tick;
636 : ulong next_ready_last_bank_idx;
637 : ulong root_idx;
638 : fd_rdisp_t * rdisp;
639 : ulong txn_exec_ready_bitset[ 1 ];
640 : ulong sigverify_ready_bitset[ 1 ];
641 : ulong poh_ready_bitset[ 1 ];
642 : ulong footer_ready_bitset[ 1 ];
643 : ulong lthash_ready_bitset[ 1 ];
644 : ulong active_bank_idx; /* Index of the actively replayed block, or ULONG_MAX if no block is
645 : actively replayed; has to have a transaction to dispatch; staged
646 : blocks that have no transactions to dispatch are not eligible for
647 : being active. */
648 : ulong last_active_bank_idx;
649 : ulong staged_bitset; /* Bit i set if staging lane i is occupied. */
650 : ulong staged_head_bank_idx[ FD_SCHED_MAX_STAGING_LANES ]; /* Head of the linear chain in each staging lane, ignored if bit i is
651 : not set in the bitset. */
652 : ulong staged_popcnt_wmk;
653 : ulong txn_pool_free_cnt;
654 : fd_txn_p_t * txn_pool; /* Just a flat array. */
655 : fd_sched_txn_info_t * txn_info_pool; /* Just a flat array. */
656 : fd_sched_mblk_t * mblk_pool; /* Just a flat array. */
657 : ulong mblk_pool_free_cnt;
658 : uint mblk_pool_free_head;
659 : ulong tile_to_bank_idx[ FD_SCHED_MAX_EXEC_TILE_CNT ]; /* Index of the bank that the exec tile is executing against. */
660 : fd_sched_poh_hash_t poh_inflight[ FD_SCHED_MAX_EXEC_TILE_CNT ]; /* PoH dispatch metadata, indexed by exec tile. */
661 : fd_sched_lthash_batch_t * lthash_inflight; /* What each tile is hashing, FD_SCHED_MAX_EXEC_TILE_CNT records indexed by exec tile. */
662 : fd_sched_lthash_batch_t * lthash_add_batch; /* Surfaced additions not yet put on a tile, cnt 0 if none. */
663 : fd_sched_lthash_lane_t lthash_lane[ FD_SCHED_MAX_STAGING_LANES ];
664 : fd_lthash_value_t * lthash_spec_pool; /* FD_SCHED_LTHASH_SPEC_MAX values of speculative additions. */
665 : uint * lthash_spec_free; /* Stack of the free lthash_spec_pool indices, lthash_spec_free[ lthash_spec_free_cnt-1 ] on top. */
666 : ulong lthash_spec_free_cnt;
667 : fd_lthash_value_t lthash_delta[ FD_SCHED_LTHASH_OOB_BLOCK_MAX ]; /* Delta of the block holding out-of-band reservation i. */
668 : uint lthash_draining_cnt;
669 : txn_bitset_t exec_done_set[ txn_bitset_word_cnt ]; /* Indexed by txn_idx. */
670 : txn_bitset_t sigverify_done_set[ txn_bitset_word_cnt ]; /* Indexed by txn_idx. */
671 : txn_bitset_t poh_mixin_done_set[ txn_bitset_word_cnt ]; /* Indexed by txn_idx. */
672 : fd_sched_block_t * block_pool; /* Just a flat array. */
673 : ulong block_pool_popcnt;
674 : ulong * ref_q;
675 :
676 : long state_debounce_ticks;
677 : ulong state_bank_idx; /* Block whose slot state is open, ULONG_MAX if none. */
678 : ulong state_cnt;
679 : ulong state_drop_cnt;
680 : fd_sched_slot_state_t state_buf[ FD_SCHED_SLOT_STATE_MAX ];
681 : };
682 : typedef struct fd_sched fd_sched_t;
683 :
684 :
685 : /* Internal helpers. */
686 :
687 : static int
688 : verify_ticks_eager( fd_sched_block_t * block );
689 :
690 : static int
691 : verify_ticks_final( fd_sched_block_t * block );
692 :
693 : static int
694 : on_final( fd_sched_block_t * block );
695 :
696 : static void
697 : add_block( fd_sched_t * sched,
698 : ulong bank_idx,
699 : ulong parent_bank_idx );
700 :
701 : FD_WARN_UNUSED static int
702 : fd_sched_parse( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx );
703 :
704 : FD_WARN_UNUSED static int
705 : fd_sched_parse_txn( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx );
706 :
707 : static void
708 : dispatch_sigverify( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, fd_sched_task_t * out );
709 :
710 : static uint
711 : shred_split( fd_sched_block_t * block,
712 : uint query );
713 :
714 : static void
715 : dispatch_poh( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, fd_sched_task_t * out );
716 :
717 : static int
718 : poh_retire_mblk( fd_sched_t * sched, fd_sched_block_t * block, uint mblk_idx );
719 :
720 : FD_WARN_UNUSED static int
721 : maybe_mixin( fd_sched_t * sched, fd_sched_block_t * block );
722 :
723 : static void
724 0 : free_mblk( fd_sched_t * sched, fd_sched_block_t * block, uint mblk_idx ) {
725 0 : sched->mblk_pool[ mblk_idx ].next = sched->mblk_pool_free_head;
726 0 : sched->mblk_pool_free_head = mblk_idx;
727 0 : sched->mblk_pool_free_cnt++;
728 0 : block->mblk_freed_cnt++;
729 0 : }
730 :
731 : static void
732 0 : free_mblk_slist( fd_sched_t * sched, fd_sched_block_t * block, mblk_slist_t * list ) {
733 0 : while( !mblk_slist_is_empty( list, sched->mblk_pool ) ) {
734 0 : uint idx = (uint)mblk_slist_idx_pop_head( list, sched->mblk_pool );
735 0 : free_mblk( sched, block, idx );
736 0 : }
737 0 : }
738 :
739 : static void
740 : try_activate_block( fd_sched_t * sched );
741 :
742 : static void
743 : check_or_set_active_block( fd_sched_t * sched );
744 :
745 : static void
746 : subtree_abandon( fd_sched_t * sched, fd_sched_block_t * block );
747 :
748 : static void
749 : subtree_release_refcnt( fd_sched_t * sched, fd_sched_block_t * block );
750 :
751 : static void
752 : subtree_prune( fd_sched_t * sched, ulong bank_idx, ulong except_idx );
753 :
754 : static void
755 : maybe_switch_block( fd_sched_t * sched, ulong bank_idx );
756 :
757 : static ulong
758 : compute_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx );
759 :
760 : static ulong
761 : promote_from_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx, int lane_idx );
762 :
763 : static int
764 : lane_is_demotable( fd_sched_t * sched, int lane_idx );
765 :
766 : static void
767 : demote_lane( fd_sched_t * sched, int lane_idx );
768 :
769 : static inline fd_sched_block_t *
770 29288 : block_pool_ele( fd_sched_t * sched, ulong idx ) {
771 29288 : FD_TEST( idx<sched->block_cnt_max || idx==ULONG_MAX );
772 29288 : return idx==ULONG_MAX ? NULL : sched->block_pool+idx;
773 29288 : }
774 :
775 : FD_FN_UNUSED static inline int
776 0 : block_is_void( fd_sched_block_t * block ) {
777 0 : /* We've seen everything in the block and no transaction got parsed
778 0 : out. */
779 0 : return block->fec_eos && block->txn_parsed_cnt==0;
780 0 : }
781 :
782 : /* Assumes that every serializing transaction has completed, so every
783 : late registration has happened. */
784 : static inline int
785 0 : block_lthash_settled( fd_sched_block_t * block ) {
786 0 : return block->lthash_drained && !block->sub_queued_cnt && !block->sub_in_flight_cnt && block->add_created_cnt==block->add_done_cnt;
787 0 : }
788 :
789 : static inline int
790 0 : block_should_signal_end( fd_sched_block_t * block ) {
791 : /* Under the current policy of eager synchronous PoH mixin, hashing
792 : done plus fec_eos imply that all mixins have been done. */
793 0 : if( FD_UNLIKELY( !( !block->fec_eos || ((block->mblk_cnt==block->poh_hashing_done_cnt&&block->mblk_cnt==block->poh_hash_cmp_done_cnt)||block->mblk_cnt!=block->poh_hashing_done_cnt) ) ) ) FD_LOG_CRIT(( "invariant violation: slot %lu fec_eos %d mblk_cnt %u poh_hashing_done_cnt %u poh_hash_cmp_done_cnt %u", block->slot, block->fec_eos, block->mblk_cnt, block->poh_hashing_done_cnt, block->poh_hash_cmp_done_cnt ));
794 0 : return block->fec_eos && block->txn_parsed_cnt==block->txn_done_cnt && block->mblk_cnt==block->poh_hashing_done_cnt && block->block_start_done && !block->block_end_signaled && block_lthash_settled( block );
795 0 : }
796 :
797 : static inline int
798 0 : block_will_signal_end( fd_sched_block_t * block ) {
799 0 : return block->fec_eos && !block->block_end_signaled;
800 0 : }
801 :
802 : /* Is there something known to be dispatchable in the block? This is an
803 : important liveness property. A block that doesn't contain any known
804 : dispatchable tasks will be deactivated or demoted. The block
805 : boundary tests come first because they settle a complete or an
806 : unstarted block with one load each. The replay tile asks this
807 : question of every staged block on every loop while a first FEC set
808 : waits for a lane, so this needs to be fast. */
809 : static inline int
810 4555 : block_is_dispatchable( fd_sched_block_t * block ) {
811 4555 : return !block->block_start_signaled ||
812 4555 : block_will_signal_end( block ) ||
813 4555 : block->txn_parsed_cnt-block->txn_exec_in_flight_cnt-block->txn_exec_done_cnt>0U || /* exec_queued_cnt */
814 4555 : block->txn_parsed_cnt-block->txn_sigverify_in_flight_cnt-block->txn_sigverify_done_cnt>0U || /* sigverify_queued_cnt */
815 4555 : block->mblk_cnt-block->poh_mblk_in_flight_cnt-block->poh_hashing_done_cnt>0U || /* poh_queued_cnt */
816 4555 : block->sub_queued_cnt>0U ||
817 4555 : block->add_created_cnt>block->add_surfaced_cnt;
818 4555 : }
819 :
820 : /* Tasks of the block on exec tiles, footer tasks included. */
821 : static inline ulong
822 0 : block_in_flight_cnt( fd_sched_block_t * block ) {
823 0 : return (ulong)block->txn_exec_in_flight_cnt + (ulong)block->txn_sigverify_in_flight_cnt + (ulong)block->poh_hashing_in_flight_cnt + (ulong)block->footer_in_flight_cnt + (ulong)block->lthash_in_flight_cnt;
824 0 : }
825 :
826 : /* Unlike block_in_flight_cnt, this goes by the LtHash account counters,
827 : so an addition batch staged but not yet dispatched keeps the block in
828 : flight. */
829 : static inline int
830 4 : block_is_in_flight( fd_sched_block_t * block ) {
831 4 : return block->txn_exec_in_flight_cnt || block->txn_sigverify_in_flight_cnt || block->poh_hashing_in_flight_cnt || block->footer_in_flight_cnt || (block->block_end_signaled && !block->block_end_done)
832 4 : || block->sub_in_flight_cnt || block->add_surfaced_cnt>block->add_done_cnt;
833 4 : }
834 :
835 : static inline int
836 6560 : block_is_done( fd_sched_block_t * block ) {
837 6560 : return block->fec_eos && block->txn_parsed_cnt==block->txn_done_cnt && block->mblk_cnt==block->poh_hash_cmp_done_cnt && block->block_start_done && block->block_end_done && !block->footer_in_flight_cnt;
838 6560 : }
839 :
840 : static inline int
841 5560 : block_is_stageable( fd_sched_block_t * block ) {
842 5560 : int rv = !block_is_done( block ) && !block->dying;
843 5560 : if( FD_UNLIKELY( rv && !block->in_rdisp ) ) {
844 : /* Invariant: stageable blocks may be currently staged or unstaged,
845 : but must be in the dispatcher either way. When a block
846 : transitions to DONE, it will be immediately removed from the
847 : dispatcher. When a block transitions to DYING, it will be
848 : eventually abandoned from the dispatcher. */
849 0 : FD_LOG_CRIT(( "invariant violation: stageable block->in_rdisp==0, txn_parsed_cnt %u, txn_done_cnt %u, fec_eos %u,, slot %lu, parent slot %lu",
850 0 : block->txn_parsed_cnt, block->txn_done_cnt, (uint)block->fec_eos, block->slot, block->parent_slot ));
851 0 : }
852 5560 : return rv;
853 5560 : }
854 :
855 : static inline int
856 4 : block_is_promotable( fd_sched_block_t * block ) {
857 4 : return block_is_stageable( block ) && block_is_dispatchable( block ) && !block->staged;
858 4 : }
859 :
860 : static inline int
861 0 : block_is_demotable( fd_sched_block_t * block ) {
862 : /* A block can only be demoted from rdisp if it is empty, meaning no
863 : PENDING, READY, or DISPATCHED transactions. This is equivalent to
864 : having no in-flight transactions (DISPATCHED) and no queued
865 : transactions (PENDING or READY). This function actually implements
866 : a stronger requirement. We consider a block demotable only if
867 : there are no in-flight or queued tasks of any kind. */
868 0 : return block->staged && !block_is_dispatchable( block ) && !block_is_in_flight( block );
869 0 : }
870 :
871 : static inline int
872 4555 : block_is_activatable( fd_sched_block_t * block ) {
873 4555 : return block_is_stageable( block ) && block_is_dispatchable( block ) && block->staged;
874 4555 : }
875 :
876 : static inline ulong
877 6449 : block_to_idx( fd_sched_t * sched, fd_sched_block_t * block ) { return (ulong)(block-sched->block_pool); }
878 :
879 : static inline int
880 1001 : block_is_lane_head( fd_sched_t * sched, fd_sched_block_t * block ) {
881 1001 : return block->staged && sched->staged_head_bank_idx[ block->staging_lane ]==block_to_idx( sched, block );
882 1001 : }
883 :
884 : FD_FN_UNUSED static inline int
885 0 : block_is_lane_tail( fd_sched_t * sched, fd_sched_block_t * block ) {
886 0 : if( !block->staged ) return 0;
887 0 : fd_rdisp_staging_lane_info_t lane_info[ FD_SCHED_MAX_STAGING_LANES ];
888 0 : ulong occupied = fd_rdisp_staging_lane_info( sched->rdisp, lane_info );
889 0 : return fd_ulong_extract_bit( occupied, (int)block->staging_lane ) && lane_info[ block->staging_lane ].insert_ready_block==block_to_idx( sched, block );
890 0 : }
891 :
892 : /* A block with absolutely nothing to dispatch may still speculate on
893 : additions when it holds a lane, is still receiving FEC sets, owes an
894 : addition the dispatcher offers as ready (meaning every writer of the
895 : account so far has completed), and the speculation pool has at least
896 : spec_free_min free entries. Only the active block speculates. An
897 : already speculating block keeps going until the last free entry. In
898 : contrast, deactivated blocks are activated for speculation only if a
899 : higher free entry threshold is reached, so a trickle of freed entries
900 : does not bounce blocks in and out of the active state. */
901 : static inline int
902 0 : block_may_speculate( fd_sched_t * sched, fd_sched_block_t * block, ulong spec_free_min ) {
903 0 : if( block->fec_eos || !block->add_owed_cnt ) return 0;
904 0 : if( sched->lthash_spec_free_cnt<spec_free_min ) return 0;
905 : /* No need to check for tail because we have one block per lane. */
906 0 : if( !block_is_lane_head( sched, block ) ) return 0;
907 0 : return fd_rdisp_lane_ready_acct_cnt( sched->rdisp, block->staging_lane )>0UL;
908 0 : }
909 :
910 : static inline int
911 0 : other_lane_has_real_work( fd_sched_t * sched, fd_sched_block_t * block ) {
912 0 : ulong staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, (int)block->staging_lane );
913 0 : while( staged_bitset ) {
914 0 : int lane_idx = fd_ulong_find_lsb( staged_bitset );
915 0 : staged_bitset = fd_ulong_pop_lsb( staged_bitset );
916 0 : fd_sched_block_t * head = block_pool_ele( sched, sched->staged_head_bank_idx[ lane_idx ] );
917 0 : if( block_is_done( block_pool_ele( sched, head->parent_idx ) ) && block_is_activatable( head ) ) return 1;
918 0 : }
919 0 : return 0;
920 0 : }
921 :
922 : static inline int
923 3554 : block_should_deactivate( fd_sched_t * sched, fd_sched_block_t * block ) {
924 : /* We allow a grace period, during which a block has nothing to
925 : dispatch, but has something in-flight. The block is allowed to
926 : stay activated and ingest FEC sets during this time. The block
927 : will be deactivated if there's still nothing to dispatch by the
928 : time all in-flight tasks are completed, unless it can spend the
929 : idle time speculating on LtHash additions. */
930 3554 : return !block_is_activatable( block ) && !block_is_in_flight( block ) && !block_may_speculate( sched, block, 1UL );
931 3554 : }
932 :
933 : static inline int
934 2 : block_is_prunable( fd_sched_block_t * block ) {
935 2 : return !block->in_rdisp && !block_is_in_flight( block );
936 2 : }
937 :
938 : static void
939 0 : slot_state_push( fd_sched_t * sched, fd_sched_block_t * block, long end_tick ) {
940 0 : if( FD_UNLIKELY( sched->state_cnt>=FD_SCHED_SLOT_STATE_MAX ) ) { sched->state_drop_cnt++; return; }
941 0 : sched->state_buf[ sched->state_cnt++ ] = (fd_sched_slot_state_t){
942 0 : .slot = block->slot,
943 0 : .start_tick = block->state_start_tick,
944 0 : .end_tick = end_tick,
945 0 : .state = block->state
946 0 : };
947 0 : }
948 :
949 : static void
950 0 : slot_state_maybe_commit( fd_sched_t * sched, fd_sched_block_t * block, long now ) {
951 0 : if( block->state_cand==block->state || now-block->state_cand_tick<sched->state_debounce_ticks ) return;
952 : /* The first committed state starts at the window start. */
953 0 : if( FD_LIKELY( block->state!=FD_SCHED_SLOT_STATE_NONE ) ) {
954 0 : slot_state_push( sched, block, block->state_cand_tick );
955 0 : block->state_start_tick = block->state_cand_tick;
956 0 : }
957 0 : block->state = block->state_cand;
958 0 : }
959 :
960 : static void
961 4554 : slot_state_eval( fd_sched_t * sched, fd_sched_block_t * block ) {
962 4554 : if( FD_LIKELY( !block->state_open ) ) return;
963 :
964 1 : ulong queued_cnt = block->txn_parsed_cnt-block->txn_exec_in_flight_cnt-block->txn_exec_done_cnt;
965 1 : ulong free_tiles = sched->txn_exec_ready_bitset[ 0 ] & sched->sigverify_ready_bitset[ 0 ] & sched->poh_ready_bitset[ 0 ] & sched->footer_ready_bitset[ 0 ] & sched->lthash_ready_bitset[ 0 ];
966 1 : uchar next;
967 1 : if( block->block_end_signaled ) next = FD_SCHED_SLOT_STATE_FINISH;
968 1 : else if( queued_cnt && !free_tiles ) next = FD_SCHED_SLOT_STATE_BLOCKED_TILES;
969 1 : else if( queued_cnt && block->state_conflict ) next = FD_SCHED_SLOT_STATE_BLOCKED_CONFLICTS;
970 1 : else if( queued_cnt ) next = block->state_cand; /* ready txn and free tile, dispatch is imminent */
971 1 : else if( !block->fec_eos ) next = FD_SCHED_SLOT_STATE_BLOCKED_NETWORK;
972 1 : else next = FD_SCHED_SLOT_STATE_FINISH;
973 :
974 1 : long now = fd_tickcount();
975 1 : slot_state_maybe_commit( sched, block, now );
976 1 : if( next!=block->state_cand ) {
977 0 : block->state_cand = next;
978 0 : block->state_cand_tick = now;
979 0 : }
980 1 : }
981 :
982 : static void
983 : slot_state_close( fd_sched_t * sched, fd_sched_block_t * block );
984 :
985 : static void
986 0 : slot_state_open( fd_sched_t * sched, fd_sched_block_t * block ) {
987 0 : ulong bank_idx = block_to_idx( sched, block );
988 0 : if( FD_UNLIKELY( sched->state_bank_idx!=ULONG_MAX && sched->state_bank_idx!=bank_idx ) ) {
989 0 : slot_state_close( sched, block_pool_ele( sched, sched->state_bank_idx ) );
990 0 : }
991 0 : sched->state_bank_idx = bank_idx;
992 0 : block->state_open = 1;
993 0 : block->state_conflict = 0;
994 0 : block->state = FD_SCHED_SLOT_STATE_NONE;
995 0 : block->state_cand = FD_SCHED_SLOT_STATE_NONE;
996 0 : block->state_start_tick = block->state_cand_tick = fd_tickcount();
997 0 : }
998 :
999 : static void
1000 2 : slot_state_close( fd_sched_t * sched, fd_sched_block_t * block ) {
1001 2 : if( FD_LIKELY( !block->state_open ) ) return;
1002 0 : block->state_open = 0;
1003 0 : sched->state_bank_idx = ULONG_MAX;
1004 0 : long now = fd_tickcount();
1005 0 : slot_state_maybe_commit( sched, block, now );
1006 0 : if( FD_UNLIKELY( block->state==FD_SCHED_SLOT_STATE_NONE ) ) block->state = block->state_cand; /* window shorter than the debounce */
1007 0 : if( FD_UNLIKELY( block->state==FD_SCHED_SLOT_STATE_NONE ) ) return;
1008 0 : slot_state_push( sched, block, now );
1009 0 : }
1010 :
1011 : /* active_bank_set changes the active block. Slot states are tracked
1012 : for at most one block at a time. */
1013 :
1014 : static void
1015 1003 : active_bank_set( fd_sched_t * sched, ulong bank_idx ) {
1016 1003 : sched->active_bank_idx = bank_idx;
1017 1003 : if( FD_UNLIKELY( bank_idx==ULONG_MAX ) ) return;
1018 1001 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
1019 1001 : if( FD_LIKELY( block->block_start_signaled && !block->block_end_done && !block->dying && !block->state_open ) ) slot_state_open( sched, block );
1020 1001 : }
1021 :
1022 : __attribute__((format(printf,2,3)))
1023 : static void
1024 : fd_sched_printf( fd_sched_t * sched,
1025 : char const * fmt,
1026 12 : ... ) {
1027 12 : va_list ap;
1028 12 : ulong len;
1029 12 : va_start( ap, fmt );
1030 12 : int ret = vsnprintf( sched->print_buf+sched->print_buf_sz,
1031 12 : FD_SCHED_MAX_PRINT_BUF_SZ-sched->print_buf_sz,
1032 12 : fmt, ap );
1033 12 : va_end( ap );
1034 12 : len = fd_ulong_if( ret<0, 0UL, fd_ulong_min( (ulong)ret, FD_SCHED_MAX_PRINT_BUF_SZ-sched->print_buf_sz-1UL ) );
1035 12 : sched->print_buf[ sched->print_buf_sz+len ] = '\0';
1036 12 : sched->print_buf_sz += len;
1037 12 : }
1038 :
1039 : FD_FN_UNUSED static void
1040 0 : print_histogram( fd_sched_t * sched, fd_histf_t * hist, ulong converter, char * title ) {
1041 0 : fd_sched_printf( sched, " +---------------------+----------------------+--------------+\n" );
1042 0 : fd_sched_printf( sched, " | %-19s | | Count |\n", title );
1043 0 : fd_sched_printf( sched, " +---------------------+----------------------+--------------+\n" );
1044 0 :
1045 0 : ulong total_count = 0;
1046 0 : for( ulong i=0UL; i<fd_histf_bucket_cnt( hist ); i++ ) {
1047 0 : total_count += fd_histf_cnt( hist, i );
1048 0 : }
1049 0 :
1050 0 : for( ulong i=0UL; i< fd_histf_bucket_cnt( hist ); i++ ) {
1051 0 : ulong bucket_count = fd_histf_cnt( hist, i );
1052 0 :
1053 0 : char * lt_str;
1054 0 : char lt_buf[ 64 ];
1055 0 : if( FD_UNLIKELY( i==fd_histf_bucket_cnt( hist )-1UL ) ) {
1056 0 : lt_str = "+Inf";
1057 0 : } else {
1058 0 : ulong edge = fd_histf_right( hist, i );
1059 0 : if( converter==FD_METRICS_CONVERTER_NANOSECONDS ) {
1060 0 : edge = fd_metrics_convert_ticks_to_nanoseconds( edge-1UL );
1061 0 : FD_TEST( fd_cstr_printf_check( lt_buf, sizeof( lt_buf ), NULL, "<= %lu nanos", edge ) );
1062 0 : } else if( converter==FD_METRICS_CONVERTER_NONE ) {
1063 0 : FD_TEST( fd_cstr_printf_check( lt_buf, sizeof( lt_buf ), NULL, "<= %lu", edge-1UL ) );
1064 0 : }
1065 0 : lt_str = lt_buf;
1066 0 : }
1067 0 :
1068 0 : /* Create visual bar - scale to max 20 characters. */
1069 0 : char bar_buf[ 22 ];
1070 0 : if( bucket_count>0UL && total_count>0UL ) {
1071 0 : ulong bar_length = (bucket_count*20UL)/total_count;
1072 0 : if( !bar_length ) bar_length = 1;
1073 0 : for( ulong j=0UL; j<bar_length; j++ ) { bar_buf[ j ] = '*'; }
1074 0 : bar_buf[ bar_length ] = '\0';
1075 0 : } else {
1076 0 : bar_buf[ 0 ] = '\0';
1077 0 : }
1078 0 :
1079 0 : fd_sched_printf( sched, " | %19s | %-20s | %12lu |\n", lt_str, bar_buf, bucket_count );
1080 0 : }
1081 0 : }
1082 :
1083 : FD_FN_UNUSED static void
1084 0 : print_block_metrics( fd_sched_t * sched, fd_sched_block_t * block ) {
1085 0 : fd_sched_printf( sched, "block idx %lu, block slot %lu, parent_slot %lu, txn_parsed_cnt %u, txn_exec_done_cnt %u, txn_sigverify_done_cnt %u, poh_hashing_done_cnt %u, poh_hash_cmp_done_cnt %u, txn_done_cnt %u, shred_cnt %u, mblk_cnt %u, mblk_tick_cnt %u, mblk_freed_cnt %u, mblk_unhashed_cnt %u, txn_pool_max_popcnt %lu/%lu, mblk_pool_max_popcnt %lu/%lu, block_pool_max_popcnt %lu/%lu, mblks_rem %lu, txns_rem %lu, residual_sz %u, fec_buf_boff %u, fec_eob %d, fec_sob %d, shred_sz_overflow %d, fec_eos %d, rooted %d\n",
1086 0 : block_to_idx( sched, block ), block->slot, block->parent_slot, block->txn_parsed_cnt, block->txn_exec_done_cnt, block->txn_sigverify_done_cnt, block->poh_hashing_done_cnt, block->poh_hash_cmp_done_cnt, block->txn_done_cnt, block->shred_cnt, block->mblk_cnt, block->mblk_tick_cnt, block->mblk_freed_cnt, block->mblk_unhashed_cnt, block->txn_pool_max_popcnt, sched->depth, block->mblk_pool_max_popcnt, sched->depth, block->block_pool_max_popcnt, sched->block_cnt_max, block->mblks_rem, block->txns_rem, block->residual_sz, block->fec_buf_boff, block->fec_eob, block->fec_sob, block->shred_sz_overflow, block->fec_eos, block->rooted );
1087 0 : }
1088 :
1089 : FD_FN_UNUSED static void
1090 8 : print_block_debug( fd_sched_t * sched, fd_sched_block_t * block ) {
1091 8 : fd_sched_printf( sched, "block idx %lu, block slot %lu, parent_slot %lu, txn_parsed_cnt %u, txn_exec_in_flight_cnt %u, txn_exec_done_cnt %u, txn_sigverify_in_flight_cnt %u, txn_sigverify_done_cnt %u, poh_hashing_in_flight_cnt %u, poh_mblk_in_flight_cnt %u, poh_hashing_done_cnt %u, poh_hash_cmp_done_cnt %u, footer_in_flight_cnt %u, txn_done_cnt %u, shred_cnt %u, mblk_cnt %u, mblk_tick_cnt %u, mblk_freed_cnt %u, mblk_unhashed_cnt %u, txn_pool_max_popcnt %lu/%lu, mblk_pool_max_popcnt %lu/%lu, block_pool_max_popcnt %lu/%lu, lthash_writable_cnt %u, lthash_in_flight_cnt %u, sub_queued_cnt %u, sub_in_flight_cnt %u, sub_applied_cnt %u, add_owed_cnt %u, add_created_cnt %u, add_surfaced_cnt %u, add_done_cnt %u, add_applied_cnt %u, add_undone_cnt %u, mblks_rem %lu, txns_rem %lu, residual_sz %u, fec_buf_boff %u, fec_eob %d, fec_sob %d, shred_sz_overflow %d, fec_eos %d, rooted %d, dying %d, in_rdisp %d, block_start_signaled %d, block_end_signaled %d, block_start_done %d, block_end_done %d, staged %d, lthash_drained %d, lthash_draining %d, lthash_mode_picked %d, lthash_inband %d, lthash_oob_reservation %u, staging_lane %lu\n",
1092 8 : block_to_idx( sched, block ), block->slot, block->parent_slot, block->txn_parsed_cnt, block->txn_exec_in_flight_cnt, block->txn_exec_done_cnt, block->txn_sigverify_in_flight_cnt, block->txn_sigverify_done_cnt, block->poh_hashing_in_flight_cnt, block->poh_mblk_in_flight_cnt, block->poh_hashing_done_cnt, block->poh_hash_cmp_done_cnt, block->footer_in_flight_cnt, block->txn_done_cnt, block->shred_cnt, block->mblk_cnt, block->mblk_tick_cnt, block->mblk_freed_cnt, block->mblk_unhashed_cnt, block->txn_pool_max_popcnt, sched->depth, block->mblk_pool_max_popcnt, sched->depth, block->block_pool_max_popcnt, sched->block_cnt_max, block->lthash_writable_cnt, block->lthash_in_flight_cnt, block->sub_queued_cnt, block->sub_in_flight_cnt, block->sub_applied_cnt, block->add_owed_cnt, block->add_created_cnt, block->add_surfaced_cnt, block->add_done_cnt, block->add_applied_cnt, block->add_undone_cnt, block->mblks_rem, block->txns_rem, block->residual_sz, block->fec_buf_boff, block->fec_eob, block->fec_sob, block->shred_sz_overflow, block->fec_eos, block->rooted, block->dying, block->in_rdisp, block->block_start_signaled, block->block_end_signaled, block->block_start_done, block->block_end_done, block->staged, block->lthash_drained, block->lthash_draining, block->lthash_mode_picked, block->lthash_inband, (uint)block->lthash_oob_reservation, block->staging_lane );
1093 8 : }
1094 :
1095 : FD_FN_UNUSED static void
1096 2 : print_block_and_parent( fd_sched_t * sched, fd_sched_block_t * block ) {
1097 2 : print_block_debug( sched, block );
1098 2 : fd_sched_block_t * parent = block_pool_ele( sched, block->parent_idx );
1099 2 : if( FD_LIKELY( parent ) ) print_block_debug( sched, parent );
1100 2 : }
1101 :
1102 : FD_FN_UNUSED static void
1103 2 : print_metrics( fd_sched_t * sched ) {
1104 2 : fd_sched_printf( sched, "metrics: block_added_cnt %u, block_added_staged_cnt %u, block_added_unstaged_cnt %u, block_added_dead_ood_cnt %u, block_removed_cnt %u, block_abandoned_cnt %u, block_bad_cnt %u, block_promoted_cnt %u, block_demoted_cnt %u, deactivate_no_child_cnt %u, deactivate_no_txn_cnt %u, deactivate_pruned_cnt %u, deactivate_abandoned_cnt %u, lane_switch_cnt %u, lane_promoted_cnt %u, lane_demoted_cnt %u, fork_observed_cnt %u, alut_success_cnt %u, alut_serializing_cnt %u, txn_abandoned_parsed_cnt %u, txn_abandoned_exec_done_cnt %u, txn_abandoned_done_cnt %u, txn_promoted_cnt %u, txn_max_in_flight_cnt %u, txn_weighted_in_flight_cnt %lu, txn_weighted_in_flight_tickcount %lu, txn_none_in_flight_tickcount %lu, txn_parsed_cnt %lu, txn_exec_done_cnt %lu, txn_sigverify_done_cnt %lu, txn_mixin_done_cnt %lu, txn_done_cnt %lu, mblk_parsed_cnt %lu, mblk_poh_hashed_cnt %lu, mblk_poh_done_cnt %lu, bytes_ingested_cnt %lu, bytes_ingested_unparsed_cnt %lu, bytes_dropped_cnt %lu, fec_cnt %lu, lthash_parsed_writable_cnt %lu, lthash_start_reg_cnt %u, lthash_late_reg_cnt %u, lthash_demote_reg_cnt %u, lthash_first_write_cnt %lu, lthash_coalesced_cnt %lu, lthash_entry_cnt %lu, lthash_sub_dispatched_cnt %lu, lthash_sub_task_cnt %lu, lthash_sub_applied_cnt %lu, lthash_add_created_cnt %lu, lthash_spec_cnt %lu, lthash_add_surfaced_cnt %lu, lthash_add_surfaced_stale_cnt %lu, lthash_add_dispatched_cnt %lu, lthash_add_task_cnt %lu, lthash_add_dispatched_stale_cnt %lu, lthash_add_batch_held_cnt %lu, lthash_add_batch_wait_idle_cnt %lu, lthash_add_applied_cnt %lu, lthash_add_undone_cnt %lu, lthash_spec_applied_cnt %lu, lthash_spec_stale_cnt %lu, lthash_spec_given_up_cnt %u, lthash_pool_full_cnt %u, lthash_spec_yield_cnt %u, lthash_frozen_cnt %lu, lthash_discarded_add_owed_cnt %u, lthash_discarded_add_pending_cnt %u, lthash_discarded_add_held_cnt %u, lthash_discarded_add_applied_cnt %u, lthash_discarded_sub_queued_cnt %u, lthash_discarded_sub_applied_cnt %u, lthash_inband_add_owed_cnt %u, lthash_inband_sub_queued_cnt %u, lthash_inband_block_cnt %u, lane_wait_cnt %lu\n",
1105 2 : sched->metrics->block_added_cnt, sched->metrics->block_added_staged_cnt, sched->metrics->block_added_unstaged_cnt, sched->metrics->block_added_dead_ood_cnt, sched->metrics->block_removed_cnt, sched->metrics->block_abandoned_cnt, sched->metrics->block_bad_cnt, sched->metrics->block_promoted_cnt, sched->metrics->block_demoted_cnt, sched->metrics->deactivate_no_child_cnt, sched->metrics->deactivate_no_txn_cnt, sched->metrics->deactivate_pruned_cnt, sched->metrics->deactivate_abandoned_cnt, sched->metrics->lane_switch_cnt, sched->metrics->lane_promoted_cnt, sched->metrics->lane_demoted_cnt, sched->metrics->fork_observed_cnt, sched->metrics->alut_success_cnt, sched->metrics->alut_serializing_cnt, sched->metrics->txn_abandoned_parsed_cnt, sched->metrics->txn_abandoned_exec_done_cnt, sched->metrics->txn_abandoned_done_cnt, sched->metrics->txn_promoted_cnt, sched->metrics->txn_max_in_flight_cnt, sched->metrics->txn_weighted_in_flight_cnt, sched->metrics->txn_weighted_in_flight_tickcount, sched->metrics->txn_none_in_flight_tickcount, sched->metrics->txn_parsed_cnt, sched->metrics->txn_exec_done_cnt, sched->metrics->txn_sigverify_done_cnt, sched->metrics->txn_mixin_done_cnt, sched->metrics->txn_done_cnt, sched->metrics->mblk_parsed_cnt, sched->metrics->mblk_poh_hashed_cnt, sched->metrics->mblk_poh_done_cnt, sched->metrics->bytes_ingested_cnt, sched->metrics->bytes_ingested_unparsed_cnt, sched->metrics->bytes_dropped_cnt, sched->metrics->fec_cnt, sched->metrics->lthash_parsed_writable_cnt, sched->metrics->lthash_start_reg_cnt, sched->metrics->lthash_late_reg_cnt, sched->metrics->lthash_demote_reg_cnt, sched->metrics->lthash_first_write_cnt, sched->metrics->lthash_coalesced_cnt, sched->metrics->lthash_entry_cnt, sched->metrics->lthash_sub_dispatched_cnt, sched->metrics->lthash_sub_task_cnt, sched->metrics->lthash_sub_applied_cnt, sched->metrics->lthash_add_created_cnt, sched->metrics->lthash_spec_cnt, sched->metrics->lthash_add_surfaced_cnt, sched->metrics->lthash_add_surfaced_stale_cnt, sched->metrics->lthash_add_dispatched_cnt, sched->metrics->lthash_add_task_cnt, sched->metrics->lthash_add_dispatched_stale_cnt, sched->metrics->lthash_add_batch_held_cnt, sched->metrics->lthash_add_batch_wait_idle_cnt, sched->metrics->lthash_add_applied_cnt, sched->metrics->lthash_add_undone_cnt, sched->metrics->lthash_spec_applied_cnt, sched->metrics->lthash_spec_stale_cnt, sched->metrics->lthash_spec_given_up_cnt, sched->metrics->lthash_pool_full_cnt, sched->metrics->lthash_spec_yield_cnt, sched->metrics->lthash_frozen_cnt, sched->metrics->lthash_discarded_add_owed_cnt, sched->metrics->lthash_discarded_add_pending_cnt, sched->metrics->lthash_discarded_add_held_cnt, sched->metrics->lthash_discarded_add_applied_cnt, sched->metrics->lthash_discarded_sub_queued_cnt, sched->metrics->lthash_discarded_sub_applied_cnt, sched->metrics->lthash_inband_add_owed_cnt, sched->metrics->lthash_inband_sub_queued_cnt, sched->metrics->lthash_inband_block_cnt, sched->metrics->lane_wait_cnt );
1106 2 : }
1107 :
1108 : FD_FN_UNUSED static void
1109 2 : print_sched( fd_sched_t * sched ) {
1110 2 : fd_sched_printf( sched, "sched canary 0x%lx, exec_cnt %lu, root_idx %lu, txn_exec_ready_bitset[ 0 ] 0x%lx, sigverify_ready_bitset[ 0 ] 0x%lx, poh_ready_bitset[ 0 ] 0x%lx, footer_ready_bitset[ 0 ] 0x%lx, lthash_ready_bitset[ 0 ] 0x%lx, active_idx %lu, staged_bitset %lu, staged_head_idx[0] %lu, staged_head_idx[1] %lu, staged_head_idx[2] %lu, staged_head_idx[3] %lu, staged_popcnt_wmk %lu, txn_pool_free_cnt %lu/%lu, lthash_add_batch_cnt %lu, lthash_draining_cnt %u, block_pool_popcnt %lu/%lu\n",
1111 2 : sched->canary, sched->exec_cnt, sched->root_idx, sched->txn_exec_ready_bitset[ 0 ], sched->sigverify_ready_bitset[ 0 ], sched->poh_ready_bitset[ 0 ], sched->footer_ready_bitset[ 0 ], sched->lthash_ready_bitset[ 0 ], sched->active_bank_idx, sched->staged_bitset, sched->staged_head_bank_idx[ 0 ], sched->staged_head_bank_idx[ 1 ], sched->staged_head_bank_idx[ 2 ], sched->staged_head_bank_idx[ 3 ], sched->staged_popcnt_wmk, sched->txn_pool_free_cnt, sched->depth, sched->lthash_add_batch->cnt, sched->lthash_draining_cnt, sched->block_pool_popcnt, sched->block_cnt_max );
1112 2 : fd_sched_block_t * active_block = block_pool_ele( sched, sched->active_bank_idx );
1113 2 : if( active_block ) print_block_debug( sched, active_block );
1114 10 : for( int l=0; l<(int)FD_SCHED_MAX_STAGING_LANES; l++ ) {
1115 8 : if( fd_ulong_extract_bit( sched->staged_bitset, l ) ) {
1116 2 : fd_sched_block_t * block = block_pool_ele( sched, sched->staged_head_bank_idx[ l ] );
1117 2 : print_block_debug( sched, block );
1118 2 : }
1119 8 : }
1120 2 : }
1121 :
1122 : FD_FN_UNUSED static void
1123 2 : print_all( fd_sched_t * sched, fd_sched_block_t * block ) {
1124 2 : print_metrics( sched );
1125 2 : print_sched( sched );
1126 2 : print_block_and_parent( sched, block );
1127 2 : }
1128 :
1129 : /* How many idle exec tiles a poll leaves free for the critical path
1130 : when it hands out anything but a transaction. We leave one idle
1131 : while the block may still get transactions and has none in flight,
1132 : so that a transaction that becomes ready dispatches right away.
1133 : Once the block has all its FEC sets, a queued transaction only
1134 : becomes ready when an in-flight one completes and frees its own
1135 : tile, so there is nothing to reserve for. The same holds while a
1136 : transaction is in flight. A single exec tile is kept busy
1137 : regardless. */
1138 : static inline int
1139 0 : idle_reserve( fd_sched_t const * sched, fd_sched_block_t const * block ) {
1140 0 : return !(block->fec_eos || block->txn_exec_in_flight_cnt>0U || sched->exec_cnt==1UL);
1141 0 : }
1142 :
1143 : /* The reserve for an LtHash batch. An LtHash batch may hold a tile for
1144 : longer than a sigverify, since its accounts may be large or cold, so
1145 : it always leaves one tile idle, except with a single exec tile. */
1146 : static inline int
1147 0 : lthash_idle_reserve( fd_sched_t const * sched ) {
1148 0 : return sched->exec_cnt>1UL;
1149 0 : }
1150 :
1151 : /* Each tile runs at most one task at a time, so no tile may be busy
1152 : in two bitsets. */
1153 : static inline void
1154 0 : exec_ready_bitsets_check( fd_sched_t * sched ) {
1155 0 : ulong mask = fd_ulong_mask_lsb( (int)sched->exec_cnt );
1156 0 : ulong exec_busy = (~sched->txn_exec_ready_bitset [ 0 ])&mask;
1157 0 : ulong sigverify_busy = (~sched->sigverify_ready_bitset[ 0 ])&mask;
1158 0 : ulong poh_busy = (~sched->poh_ready_bitset [ 0 ])&mask;
1159 0 : ulong footer_busy = (~sched->footer_ready_bitset [ 0 ])&mask;
1160 0 : ulong lthash_busy = (~sched->lthash_ready_bitset [ 0 ])&mask;
1161 0 : if( FD_UNLIKELY( (exec_busy&(sigverify_busy|poh_busy|footer_busy|lthash_busy)) | (sigverify_busy&(poh_busy|footer_busy|lthash_busy)) | (poh_busy&(footer_busy|lthash_busy)) | (footer_busy<hash_busy) ) ) {
1162 0 : FD_LOG_CRIT(( "invariant violation: txn_exec_ready_bitset 0x%lx sigverify_ready_bitset 0x%lx poh_ready_bitset 0x%lx footer_ready_bitset 0x%lx lthash_ready_bitset 0x%lx", sched->txn_exec_ready_bitset[ 0 ], sched->sigverify_ready_bitset[ 0 ], sched->poh_ready_bitset[ 0 ], sched->footer_ready_bitset[ 0 ], sched->lthash_ready_bitset[ 0 ] ));
1163 0 : }
1164 0 : }
1165 :
1166 : /* Every busy tile should be accounted for by an in-flight task of a
1167 : staged lane head, which could be an actively dispatching block, or a
1168 : dying block waiting for in-flight tasks to drain. */
1169 : static void
1170 0 : exec_busy_check( fd_sched_t * sched, fd_sched_block_t * block ) {
1171 0 : exec_ready_bitsets_check( sched );
1172 0 : ulong total_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ]&sched->sigverify_ready_bitset[ 0 ]&sched->poh_ready_bitset[ 0 ]&sched->footer_ready_bitset[ 0 ]&sched->lthash_ready_bitset[ 0 ] );
1173 0 : if( FD_UNLIKELY( block_in_flight_cnt( block )!=total_exec_busy_cnt ) ) {
1174 : /* Ideally we'd simply assert that the two sides of the equation are
1175 : equal. But abandoned blocks throw a wrench into this. We allow
1176 : abandoned blocks to have in-flight transactions that are
1177 : naturally drained while we try to dispatch from another block.
1178 : In such cases, the total number of in-flight transactions should
1179 : include the abandoned blocks too. The contract is that blocks
1180 : with in-flight transactions cannot be abandoned or demoted from
1181 : rdisp. So a dying block has to be the head of one of the staging
1182 : lanes. */
1183 : // FIXME This contract no longer true if we implement immediate
1184 : // demotion of abandoned blocks.
1185 0 : ulong total_in_flight = 0UL;
1186 0 : for( int l=0; l<(int)FD_SCHED_MAX_STAGING_LANES; l++ ) {
1187 0 : if( fd_ulong_extract_bit( sched->staged_bitset, l ) ) {
1188 0 : fd_sched_block_t * staged_block = block_pool_ele( sched, sched->staged_head_bank_idx[ l ] );
1189 0 : if( FD_UNLIKELY( block_is_in_flight( staged_block )&&!(staged_block==block||staged_block->dying) ) ) {
1190 0 : sched->print_buf_sz = 0UL;
1191 0 : print_all( sched, staged_block );
1192 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1193 0 : FD_LOG_CRIT(( "invariant violation: in-flight block is neither active nor dying" ));
1194 0 : }
1195 0 : total_in_flight += block_in_flight_cnt( staged_block );
1196 0 : }
1197 0 : }
1198 0 : if( FD_UNLIKELY( total_in_flight!=total_exec_busy_cnt ) ) {
1199 0 : sched->print_buf_sz = 0UL;
1200 0 : print_all( sched, block );
1201 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1202 0 : FD_LOG_CRIT(( "invariant violation: total_in_flight %lu != total_exec_busy_cnt %lu", total_in_flight, total_exec_busy_cnt ));
1203 0 : }
1204 0 : FD_LOG_DEBUG(( "exec_busy_cnt %lu checks out", total_exec_busy_cnt ));
1205 0 : }
1206 0 : }
1207 :
1208 : static inline void
1209 : record_dead_reason( fd_sched_block_t * block,
1210 2 : int dead_reason ) {
1211 2 : if( FD_LIKELY( block->dead_reason==FD_SCHED_DEAD_REASON_NONE ) ) {
1212 2 : block->dead_reason = dead_reason;
1213 2 : block->discarded = 0;
1214 2 : }
1215 2 : }
1216 :
1217 : /* Record that the block goes down with its lineage rather than for a
1218 : defect of its own, aka DEAD_ANCESTOR. Also record the parent's
1219 : deadness flavor so we can tell a discarded (but not invalid) lineage
1220 : from an invalid one.
1221 :
1222 : Gated on the block having no dead reason of its own, so a verdict
1223 : sched already reached is never overwritten. That gate neither bounds
1224 : how often the inherit runs nor sees a ruling the replay tile made,
1225 : which leaves dead_reason NONE. Callers are responsible for invoking
1226 : this only on a block that is newly going down. */
1227 : static inline void
1228 0 : record_lineage_death( fd_sched_block_t * block, fd_sched_block_t const * parent ) {
1229 0 : if( FD_LIKELY( block->dead_reason==FD_SCHED_DEAD_REASON_NONE ) ) {
1230 0 : record_dead_reason( block, FD_SCHED_DEAD_REASON_DEAD_ANCESTOR );
1231 0 : block->discarded = parent->discarded;
1232 0 : }
1233 0 : }
1234 :
1235 : static void
1236 2 : handle_bad_block( fd_sched_t * sched, fd_sched_block_t * block, int dead_reason ) {
1237 2 : record_dead_reason( block, dead_reason );
1238 2 : sched->print_buf_sz = 0UL;
1239 2 : print_all( sched, block );
1240 2 : FD_LOG_DEBUG(( "%s", sched->print_buf ));
1241 2 : subtree_abandon( sched, block );
1242 2 : sched->metrics->block_bad_cnt++;
1243 2 : check_or_set_active_block( sched );
1244 2 : }
1245 :
1246 : /* Returns the number of shred boundaries strictly before query, or
1247 : UINT_MAX once the block's shred sizes have overflowed their window.
1248 : Transaction offsets are visited in nondecreasing order, so each shred
1249 : length is scanned at most once per block. */
1250 : static uint
1251 : shred_split( fd_sched_block_t * block,
1252 7638 : uint query ) {
1253 7638 : if( FD_UNLIKELY( block->shred_sz_overflow ) ) return UINT_MAX;
1254 7638 : FD_TEST( block->shred_scan_idx<=block->shred_cnt );
1255 7638 : FD_TEST( block->shred_scan_off<=query );
1256 121334 : while( block->shred_scan_idx<block->shred_cnt ) {
1257 121334 : uint next_off = block->shred_scan_off + (uint)block->shred_sz[ block->shred_scan_idx&(uint)(FD_SCHED_SHRED_SZ_WINDOW-1UL) ];
1258 121334 : if( next_off>=query ) break;
1259 113696 : block->shred_scan_off = next_off;
1260 113696 : block->shred_scan_idx++;
1261 113696 : }
1262 7638 : return block->shred_scan_idx;
1263 7638 : }
1264 :
1265 :
1266 : /* Out-of-band LtHash helpers. */
1267 :
1268 : static inline int
1269 : block_owns_lane_map( fd_sched_t * sched,
1270 4433 : fd_sched_block_t * block ) {
1271 4433 : return block->staged && sched->lthash_lane[ block->staging_lane ].bank_idx==block_to_idx( sched, block );
1272 4433 : }
1273 :
1274 : static inline fd_lthash_value_t *
1275 : block_lthash_delta( fd_sched_t * sched,
1276 0 : fd_sched_block_t * block ) {
1277 0 : FD_TEST( block->lthash_mode_picked && !block->lthash_inband );
1278 0 : return sched->lthash_delta+block->lthash_oob_reservation;
1279 0 : }
1280 :
1281 : static inline fd_sched_lthash_entry_t *
1282 : lthash_lane_query( fd_sched_t * sched,
1283 : ulong lane,
1284 2889 : fd_acct_addr_t const * acct ) {
1285 2889 : fd_sched_lthash_lane_t * lane_map = sched->lthash_lane+lane;
1286 2889 : return lthash_map_ele_query( lane_map->map, acct, NULL, lane_map->entry_pool );
1287 2889 : }
1288 :
1289 : static void
1290 : lthash_spec_release( fd_sched_t * sched,
1291 0 : uint idx ) {
1292 0 : if( FD_UNLIKELY( (ulong)idx>=FD_SCHED_LTHASH_SPEC_MAX || sched->lthash_spec_free_cnt>=FD_SCHED_LTHASH_SPEC_MAX ) ) {
1293 0 : FD_LOG_CRIT(( "invariant violation: release of speculation entry %u, free cnt %lu", idx, sched->lthash_spec_free_cnt ));
1294 0 : }
1295 0 : sched->lthash_spec_free[ sched->lthash_spec_free_cnt++ ] = idx;
1296 0 : }
1297 :
1298 : /* Releases the lane's map. The entries' speculation entries go back
1299 : to the pool and the map forgets the accounts. Pointers to and
1300 : indices of its entries are invalid afterwards. */
1301 : static void
1302 : lthash_release_lane_map( fd_sched_t * sched,
1303 2 : ulong lane ) {
1304 2 : fd_sched_lthash_lane_t * lane_map = sched->lthash_lane+lane;
1305 2 : if( FD_UNLIKELY( lane_map->bank_idx==ULONG_MAX ) ) FD_LOG_CRIT(( "invariant violation: releasing lane %lu, which no bank owns", lane ));
1306 2 : for( ulong i=0UL; i<lane_map->entry_cnt; i++ ) {
1307 0 : if( lane_map->entry_pool[ i ].spec_idx!=UINT_MAX ) lthash_spec_release( sched, lane_map->entry_pool[ i ].spec_idx );
1308 0 : }
1309 2 : lthash_map_reset( lane_map->map );
1310 2 : lane_map->entry_cnt = 0U;
1311 2 : lane_map->bank_idx = ULONG_MAX;
1312 2 : }
1313 :
1314 : /* A new entry for acct in the block's lane map. The account owes its
1315 : one subtraction, so the entry goes on the block's sub queue. */
1316 : static fd_sched_lthash_entry_t *
1317 : lthash_entry_new( fd_sched_t * sched,
1318 : fd_sched_block_t * block,
1319 2889 : fd_acct_addr_t const * acct ) {
1320 : /* A block claims its lane map upon being staged. */
1321 2889 : if( FD_UNLIKELY( !block_owns_lane_map( sched, block ) ) ) FD_LOG_CRIT(( "invariant violation: block %lu:%lu owns no lane map", block->slot, block_to_idx( sched, block ) ));
1322 2889 : ulong lane = block->staging_lane;
1323 2889 : fd_sched_lthash_lane_t * lane_map = sched->lthash_lane+lane;
1324 2889 : if( FD_UNLIKELY( lane_map->entry_cnt==FD_RDISP_MAX_WRITERS_PER_BLOCK ) ) FD_LOG_CRIT(( "lane %lu has no free entry", lane ));
1325 2889 : if( FD_UNLIKELY( lthash_map_ele_query_const( lane_map->map, acct, NULL, lane_map->entry_pool ) ) ) {
1326 0 : FD_BASE58_ENCODE_32_BYTES( acct->b, acct_str );
1327 0 : FD_LOG_CRIT(( "invariant violation: account %s already in lane %lu", acct_str, lane ));
1328 0 : }
1329 :
1330 2889 : uint idx = lane_map->entry_cnt++;
1331 2889 : fd_sched_lthash_entry_t * e = lane_map->entry_pool+idx;
1332 2889 : e->acct = *acct;
1333 2889 : e->q_next = UINT_MAX;
1334 2889 : e->spec_idx = UINT_MAX;
1335 2889 : e->ptxn_idx = 0U;
1336 2889 : e->bank_idx = (uint)lane_map->bank_idx;
1337 2889 : e->sub_state = (uchar)FD_SCHED_LTHASH_SUB_QUEUED;
1338 2889 : e->add_state = (uchar)FD_SCHED_LTHASH_ADD_NONE;
1339 2889 : lthash_map_idx_insert( lane_map->map, idx, lane_map->entry_pool );
1340 2889 : sched->metrics->lthash_entry_cnt++;
1341 2889 : if( FD_LIKELY( block->sub_q_tail==UINT_MAX ) ) block->sub_q_head = idx;
1342 1919 : else lane_map->entry_pool[ block->sub_q_tail ].q_next = idx;
1343 2889 : block->sub_q_tail = idx;
1344 2889 : block->sub_queued_cnt++;
1345 2889 : return e;
1346 2889 : }
1347 :
1348 : /* Applies the rewrite rule to the entry. If its addition was applied,
1349 : the value in its speculation entry is subtracted from block_lthash,
1350 : the block's delta. Then the speculation entry is freed, the pseudo
1351 : transaction of record is cleared and the entry becomes ADD_NONE, so
1352 : the stale pseudo transaction is completed on the spot when it
1353 : surfaces, or is ignored when its in-flight hashing finishes.
1354 : sub_state is untouched. */
1355 : static void
1356 : lthash_entry_clear_add( fd_sched_t * sched,
1357 : fd_sched_lthash_entry_t * e,
1358 0 : fd_lthash_value_t * block_lthash ) {
1359 0 : uint spec_idx = e->spec_idx;
1360 0 : if( e->add_state==FD_SCHED_LTHASH_ADD_DONE ) {
1361 0 : if( FD_UNLIKELY( spec_idx==UINT_MAX ) ) FD_LOG_CRIT(( "invariant violation: rewrite of an applied addition with no entry to undo it, bank idx %u", e->bank_idx ));
1362 0 : fd_lthash_sub( block_lthash, sched->lthash_spec_pool+spec_idx );
1363 0 : }
1364 0 : if( spec_idx!=UINT_MAX ) {
1365 0 : lthash_spec_release( sched, spec_idx );
1366 0 : e->spec_idx = UINT_MAX;
1367 0 : }
1368 0 : e->ptxn_idx = 0U;
1369 0 : e->add_state = (uchar)FD_SCHED_LTHASH_ADD_NONE;
1370 0 : }
1371 :
1372 : /* A write to the entry's account arrived after its addition was
1373 : created, which undoes the addition. */
1374 : static void
1375 : lthash_entry_rewrite( fd_sched_t * sched,
1376 : fd_sched_block_t * block,
1377 0 : fd_sched_lthash_entry_t * e ) {
1378 0 : int undone = e->add_state==FD_SCHED_LTHASH_ADD_DONE;
1379 0 : sched->metrics->lthash_add_undone_cnt += (ulong)undone;
1380 0 : sched->metrics->lthash_spec_stale_cnt += (ulong)(!undone && e->spec_idx!=UINT_MAX);
1381 0 : block->add_undone_cnt += (uint)undone;
1382 0 : lthash_entry_clear_add( sched, e, block_lthash_delta( sched, block ) );
1383 0 : }
1384 :
1385 : static void
1386 : lthash_entry_new_write_observed( fd_sched_t * sched,
1387 : fd_sched_block_t * block,
1388 2889 : fd_acct_addr_t const * acct ) {
1389 2889 : fd_sched_lthash_entry_t * e = lthash_lane_query( sched, block->staging_lane, acct );
1390 2889 : if( FD_LIKELY( !e ) ) lthash_entry_new( sched, block, acct );
1391 0 : else lthash_entry_rewrite( sched, block, e );
1392 2889 : }
1393 :
1394 : /* The dispatcher created addition pseudo transaction ptxn_idx for the
1395 : block. The pseudo transaction's account gets stamped with ptxn_idx
1396 : as its addition of record. A speculative addition reserves a
1397 : speculation entry so the addition can be undone later if needed.
1398 : Caller is responsible for ensuring a speculation entry is free. */
1399 : static void
1400 : lthash_add_created( fd_sched_t * sched,
1401 : fd_sched_block_t * block,
1402 : ulong ptxn_idx,
1403 0 : int speculative ) {
1404 0 : FD_TEST( ptxn_idx & FD_RDISP_LTHASH_PSEUDO_TXN );
1405 0 : block->add_created_cnt++;
1406 0 : sched->metrics->lthash_add_created_cnt++;
1407 0 : fd_acct_addr_t acct[ 1 ];
1408 0 : fd_rdisp_pseudo_txn_to_addr( sched->rdisp, ptxn_idx, acct );
1409 0 : fd_sched_lthash_entry_t * e = lthash_lane_query( sched, block->staging_lane, acct );
1410 0 : if( FD_LIKELY( !e ) ) {
1411 : /* The account may have no entry yet, which happens when the block
1412 : was not the lane's owner while its transactions parsed, and then
1413 : one is created now. */
1414 0 : e = lthash_entry_new( sched, block, acct );
1415 0 : } else if( FD_UNLIKELY( e->add_state!=FD_SCHED_LTHASH_ADD_NONE || e->ptxn_idx || e->spec_idx!=UINT_MAX ) ) {
1416 : /* Otherwise the entry holds no addition of record. The dispatcher
1417 : only creates an addition for an account whose last reference in
1418 : the block is a write, and there are three ways an account gets
1419 : one. A parsed transaction writes it, start-of-block or late
1420 : registration writes it, or a given up speculation hands it back
1421 : as written. Each of those cleared the entry's addition on the
1422 : way. */
1423 0 : FD_LOG_CRIT(( "invariant violation: addition created for an account with an addition of record, block %lu:%lu, ptxn_idx 0x%lx, add_state %u, entry ptxn_idx 0x%x, spec_idx %u", block->slot, block_to_idx( sched, block ), ptxn_idx, (uint)e->add_state, e->ptxn_idx, e->spec_idx ));
1424 0 : }
1425 0 : e->add_state = (uchar)FD_SCHED_LTHASH_ADD_PENDING;
1426 0 : e->ptxn_idx = (uint)ptxn_idx;
1427 0 : if( FD_LIKELY( speculative ) ) {
1428 0 : FD_TEST( sched->lthash_spec_free_cnt );
1429 0 : e->spec_idx = sched->lthash_spec_free[ --sched->lthash_spec_free_cnt ];
1430 0 : }
1431 0 : }
1432 :
1433 : /* The block was just staged and claims its lane's map. A lane the
1434 : block already owns is left alone. */
1435 : static void
1436 : lthash_claim_lane_map( fd_sched_t * sched,
1437 1001 : fd_sched_block_t * block ) {
1438 1001 : FD_TEST( block_is_lane_head( sched, block ) );
1439 1001 : fd_sched_lthash_lane_t * lane_map = sched->lthash_lane+block->staging_lane;
1440 1001 : ulong bank_idx = block_to_idx( sched, block );
1441 1001 : if( lane_map->bank_idx==bank_idx ) return;
1442 1001 : if( FD_UNLIKELY( lane_map->bank_idx!=ULONG_MAX ) ) FD_LOG_CRIT(( "invariant violation: claiming lane %lu for bank %lu, owned by bank %lu", block->staging_lane, bank_idx, lane_map->bank_idx ));
1443 1001 : FD_TEST( bank_idx<UINT_MAX && !lane_map->entry_cnt );
1444 1001 : lane_map->bank_idx = bank_idx;
1445 1001 : }
1446 :
1447 : /* Which hashing mode does a block get? We would love to hash out of
1448 : band wherever we can, and the one thing that stands in the way is
1449 : demotion. A demoted block parks its writable set in the dispatcher's
1450 : unstaged entries, and that parking lot has limited capacity. We
1451 : don't wanna be stuck in a scenario where we have to demote but we
1452 : can't due to limited parking space. So a block reserves space when
1453 : it picks out-of-band hashing mode. Mode picking happens the first
1454 : time a block would do out-of-band work beyond the dispatcher tracking
1455 : its writers, which is its first drain or its BLOCK_START, whichever
1456 : comes first. A child starts only once its parent is done, so the
1457 : blocks holding reservations are the executing frontier of each fork
1458 : plus whatever demoted blocks still wait to be promoted. Wide forking
1459 : is the only way to run out.
1460 :
1461 : If a reservation cannot be made, the block commits to in-band
1462 : hashing. Such a block parks nothing when demoted, so every demotion
1463 : fits and a lane can always be given up. So replay keeps making
1464 : progress, albeit a tad bit slower. */
1465 : static void
1466 : lthash_pick_mode( fd_sched_t * sched,
1467 0 : fd_sched_block_t * block ) {
1468 0 : FD_TEST( !block->lthash_mode_picked );
1469 0 : FD_TEST( !block->lthash_in_flight_cnt && !block->sub_in_flight_cnt && !block->add_created_cnt );
1470 0 : block->lthash_mode_picked = 1;
1471 0 : ulong free_reservations = (~sched->lthash_oob_reserved)&fd_ulong_mask_lsb( (int)sched->lthash_oob_block_max );
1472 0 : if( FD_LIKELY( free_reservations ) ) {
1473 0 : int reservation = fd_ulong_find_lsb( free_reservations );
1474 0 : FD_TEST( fd_lthash_is_zero( sched->lthash_delta+reservation ) );
1475 0 : sched->lthash_oob_reserved = fd_ulong_set_bit( sched->lthash_oob_reserved, reservation );
1476 0 : block->lthash_oob_reservation = (uchar)reservation;
1477 0 : return;
1478 0 : }
1479 0 : ulong bank_idx = block_to_idx( sched, block );
1480 0 : block->lthash_inband = 1;
1481 0 : sched->metrics->lthash_inband_block_cnt++;
1482 0 : sched->metrics->lthash_inband_add_owed_cnt += block->add_owed_cnt;
1483 0 : sched->metrics->lthash_inband_sub_queued_cnt += block->sub_queued_cnt;
1484 0 : block->add_owed_cnt = 0U;
1485 0 : block->sub_queued_cnt = 0U;
1486 0 : block->sub_q_head = UINT_MAX;
1487 0 : block->sub_q_tail = UINT_MAX;
1488 0 : if( FD_LIKELY( block_owns_lane_map( sched, block ) ) ) lthash_release_lane_map( sched, block->staging_lane );
1489 0 : if( FD_UNLIKELY( fd_rdisp_clear_writable_set( sched->rdisp, bank_idx ) ) ) FD_LOG_CRIT(( "fd_rdisp_clear_writable_set failed for block %lu:%lu", block->slot, bank_idx ));
1490 0 : }
1491 :
1492 : static void
1493 : lthash_release_reservation( fd_sched_t * sched,
1494 2 : fd_sched_block_t * block ) {
1495 2 : if( block->lthash_mode_picked && !block->lthash_inband ) sched->lthash_oob_reserved = fd_ulong_clear_bit( sched->lthash_oob_reserved, (int)block->lthash_oob_reservation );
1496 2 : block->lthash_mode_picked = 0;
1497 2 : block->lthash_inband = 0;
1498 2 : }
1499 :
1500 : /* Marks a staged fec_eos block as draining. Polls that find nothing to
1501 : dispatch then drain it a chunk at a time. Creating pseudo
1502 : transactions costs replay tile time, so the drain yields to
1503 : everything the block must do and overlaps with exec tile work. */
1504 : static void
1505 : lthash_drain_later( fd_sched_t * sched,
1506 992 : fd_sched_block_t * block ) {
1507 992 : if( !block->staged || !block->fec_eos || block->lthash_drained || block->lthash_draining ) return;
1508 992 : sched->lthash_draining_cnt++;
1509 992 : block->lthash_draining = 1;
1510 992 : }
1511 :
1512 : /* Pops up to max of the accounts that rdisp is still tracking for us,
1513 : such that each gets an addition pseudo transaction. The block must
1514 : be insert ready. This is a no-op for an unstaged block, which drains
1515 : when promoted. An in-band block has nothing to drain and counts as
1516 : drained. Returns the number of pseudo transactions created. */
1517 : static ulong
1518 : lthash_maybe_drain( fd_sched_t * sched,
1519 : fd_sched_block_t * block,
1520 0 : ulong max ) {
1521 0 : if( !block->staged || !block->fec_eos || block->lthash_drained ) return 0UL;
1522 0 : if( FD_UNLIKELY( !block->lthash_mode_picked ) ) lthash_pick_mode( sched, block );
1523 0 : if( FD_UNLIKELY( block->lthash_inband ) ) {
1524 0 : sched->lthash_draining_cnt -= (uint)block->lthash_draining;
1525 0 : block->lthash_draining = 0;
1526 0 : block->lthash_drained = 1;
1527 0 : return 0UL;
1528 0 : }
1529 0 : ulong bank_idx = block_to_idx( sched, block );
1530 : /* fd_rdisp_add_pseudo_txn() returns 0 for a block that is not insert
1531 : ready, which we would mistake for a finished drain. A staged block
1532 : is alone on its lane, so it is insert ready by construction, as
1533 : long as it in fact holds its lane. So we assert being head, which
1534 : implies being tail. */
1535 0 : FD_TEST( block_is_lane_head( sched, block ) );
1536 0 : ulong pop_cnt = 0UL;
1537 0 : while( pop_cnt<max ) {
1538 0 : ulong ptxn_idx = fd_rdisp_add_pseudo_txn( sched->rdisp, bank_idx );
1539 0 : if( FD_UNLIKELY( !ptxn_idx ) ) break;
1540 0 : block->add_owed_cnt--;
1541 0 : lthash_add_created( sched, block, ptxn_idx, 0 );
1542 0 : pop_cnt++;
1543 0 : }
1544 0 : if( FD_UNLIKELY( pop_cnt==max ) ) {
1545 : /* The set may hold more, so the next call continues. */
1546 0 : sched->lthash_draining_cnt += (uint)!block->lthash_draining;
1547 0 : block->lthash_draining = 1;
1548 0 : return pop_cnt;
1549 0 : }
1550 0 : sched->lthash_draining_cnt -= (uint)block->lthash_draining;
1551 0 : block->lthash_draining = 0;
1552 0 : block->lthash_drained = 1;
1553 0 : return pop_cnt;
1554 0 : }
1555 :
1556 : /* Returns the draining block we should work on next. We prioritize
1557 : draining the active block, otherwise we work on the draining block on
1558 : the lowest lane. A block that is not active cannot run its additions
1559 : until it's activated, so draining it ahead of the active block only
1560 : delays the active block's end. Assumes lthash_draining_cnt is
1561 : non-zero. */
1562 : static fd_sched_block_t *
1563 0 : lthash_find_draining_block( fd_sched_t * sched ) {
1564 0 : fd_sched_block_t * active = block_pool_ele( sched, sched->active_bank_idx );
1565 0 : if( FD_LIKELY( active && active->lthash_draining ) ) return active;
1566 0 : for( ulong lanes=sched->staged_bitset; lanes; lanes=fd_ulong_pop_lsb( lanes ) ) {
1567 0 : fd_sched_block_t * block = block_pool_ele( sched, sched->staged_head_bank_idx[ fd_ulong_find_lsb( lanes ) ] );
1568 0 : if( block->lthash_draining ) return block;
1569 0 : }
1570 0 : FD_LOG_CRIT(( "invariant violation: %u blocks draining but no staged block is", sched->lthash_draining_cnt ));
1571 0 : }
1572 :
1573 : /* Paced at up to once per fd_sched_task_next_ready(), from the points
1574 : where it has nothing left to hand out. */
1575 : static inline ulong
1576 0 : lthash_drain_chunk( fd_sched_t * sched ) {
1577 0 : if( FD_LIKELY( !sched->lthash_draining_cnt ) ) return 0UL;
1578 0 : return lthash_maybe_drain( sched, lthash_find_draining_block( sched ), FD_SCHED_LTHASH_DRAIN_CHUNK );
1579 0 : }
1580 :
1581 : /* Returns the lowest lane whose head can be readily demoted, or -1 if
1582 : there is none.
1583 :
1584 : If we ever want to, we could instead pick the lane with the fewest
1585 : lane map entries to make demotion cheaper. */
1586 : static int
1587 0 : pick_idle_lane( fd_sched_t * sched ) {
1588 0 : for( ulong lanes=sched->staged_bitset; lanes; lanes=fd_ulong_pop_lsb( lanes ) ) {
1589 0 : int lane_idx = fd_ulong_find_lsb( lanes );
1590 0 : if( sched->staged_head_bank_idx[ lane_idx ]==sched->active_bank_idx ) continue;
1591 0 : if( lane_is_demotable( sched, lane_idx ) ) return lane_idx;
1592 0 : }
1593 0 : return -1;
1594 0 : }
1595 :
1596 : /* Should a FEC set wait for a staging lane? Recall that our aim is to
1597 : maximize replay throughput. Lane switching is cheap. So as long as
1598 : some lane has work queued up, we'd be able to keep exec tiles busy.
1599 : In that case, we shouldn't let the FEC set in if it needs a lane and
1600 : we don't have one, because it would just enter as unstaged, and later
1601 : down the road we would incur a promotion stall. Recall that the
1602 : scheduler's job is not being the biggest buffer holding all received
1603 : transactions. We're buffering transactions only to uncover the
1604 : critical path, and to bridge a bursty FEC producer with multiple exec
1605 : tiles consuming tasks at different rates. So the only reason we'd
1606 : ingest a FEC as unstaged is it could potentially unblock other FEC
1607 : sets queued behind it. Some of those held up FEC sets might complete
1608 : a staged block and thus free a lane. However, recall that the chain
1609 : rarely forks wide. Usually, it's one or two forks that may run for a
1610 : while. In that case, with the lanes we have, we'd be able to easily
1611 : switch between the few forks and overlap execution on one fork with
1612 : ingestion on another fork, without incurring any promotion or
1613 : demotion.
1614 :
1615 : With all that being said, in the common case where the lanes are just
1616 : full for holding blocks down one or two forks, there's always replay
1617 : progress to be made, and so we ask the FEC to wait if it needs a
1618 : lane. Only when no progress is to be made with any of the lanes we
1619 : have, which again happens only under wide forking, do we allow the
1620 : FEC in as potentially unstaged. The FEC would then actually trigger
1621 : a demotion upon entering, and take the lane of the demoted block.
1622 : Note that this implies demotion really only happens under wide
1623 : forking combined with incomplete frontiers. Also note that we really
1624 : do have to demote something in this case, otherwise replay would just
1625 : wedge, because the incomplete frontiers might be blocked behind this
1626 : upcoming FEC set.
1627 :
1628 : Finally, what do we consider to be a FEC set that needs a lane?
1629 : There are some cases that disqualifies a FEC from being given a lane.
1630 : Beyond that, a block's first FEC set needs a lane. */
1631 : static int
1632 : fec_waits_for_lane( fd_sched_t * sched,
1633 : ulong parent_bank_idx,
1634 0 : int is_first_in_block ) {
1635 0 : if( FD_LIKELY( !is_first_in_block ) ) return 0;
1636 0 : if( FD_LIKELY( sched->staged_bitset!=fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) return 0;
1637 : /* A ULONG_MAX parent index is only used for backfill, where at the
1638 : time of capacity check, the target ancestor is not known yet. So
1639 : we just assume we'll need a lane in that case. Any other parent
1640 : index should be valid. */
1641 0 : if( FD_LIKELY( parent_bank_idx<sched->block_cnt_max ) ) {
1642 0 : fd_sched_block_t * parent = block_pool_ele( sched, parent_bank_idx );
1643 0 : FD_TEST( parent->in_sched );
1644 : /* A child of a dying block is dead on arrival and doesn't need a
1645 : lane. */
1646 0 : if( FD_UNLIKELY( parent->dying ) ) return 0;
1647 : /* A child of an unstaged stageable block enters unstaged. */
1648 0 : if( !parent->staged && block_is_stageable( parent ) ) return 0;
1649 0 : }
1650 0 : for( ulong lanes=sched->staged_bitset; lanes; lanes=fd_ulong_pop_lsb( lanes ) ) {
1651 0 : fd_sched_block_t * head = block_pool_ele( sched, sched->staged_head_bank_idx[ fd_ulong_find_lsb( lanes ) ] );
1652 0 : if( block_is_dispatchable( head ) || block_is_in_flight( head ) ) return 1;
1653 0 : }
1654 0 : return 0;
1655 0 : }
1656 :
1657 : /* Returns the lowest free lane. Assumes at least one lane is free. */
1658 : static int
1659 1001 : pick_free_lane( fd_sched_t * sched ) {
1660 1001 : ulong free_lanes = (~sched->staged_bitset)&fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES );
1661 1001 : FD_TEST( free_lanes );
1662 1001 : return fd_ulong_find_lsb( free_lanes );
1663 1001 : }
1664 :
1665 : /* The block owes the addition of an account, a write that rdisp did not
1666 : see through a transaction at parse time and was just told of through
1667 : fd_rdisp_add_writable(), whose return value is rv. The caller has
1668 : brought the account's entry up to date. See to it that the addition
1669 : gets created whether or not the block has drained. */
1670 : static void
1671 : lthash_owe_addition( fd_sched_t * sched,
1672 : fd_sched_block_t * block,
1673 0 : int rv ) {
1674 0 : ulong bank_idx = block_to_idx( sched, block );
1675 0 : if( FD_LIKELY( rv==1 ) ) {
1676 0 : sched->metrics->lthash_first_write_cnt++;
1677 0 : block->add_owed_cnt++;
1678 0 : if( block->lthash_drained ) {
1679 : /* Re-drain. We might not get another chance to create a pseudo
1680 : transaction for the account, if this were for example a late
1681 : ALT expansion for an already fec_eos block. A block still
1682 : draining pops the account with the rest. */
1683 0 : block->lthash_drained = 0;
1684 0 : lthash_drain_later( sched, block );
1685 0 : }
1686 0 : return;
1687 0 : }
1688 0 : if( FD_LIKELY( rv==0 ) ) {
1689 : /* We got lucky. This account is already tracked by rdisp, which
1690 : essentially deduped the addition for us. We can later instruct
1691 : rdisp to create a pseudo transaction for the account, and it'll
1692 : capture the final value of the account. */
1693 0 : sched->metrics->lthash_coalesced_cnt++;
1694 0 : return;
1695 0 : }
1696 :
1697 : /* A staged block is alone on its lane and an unstaged block is insert
1698 : ready as well, so the dispatcher should never refuse a write. */
1699 0 : FD_LOG_CRIT(( "fd_rdisp_add_writable returned %d for block %lu:%lu", rv, block->slot, bank_idx ));
1700 0 : }
1701 :
1702 : /* For registering account writes that rdisp did not see through a
1703 : transaction at parse time. Typically, that would come from
1704 : start-of-block processing or an ALT expansion failure. */
1705 : static void
1706 : lthash_register( fd_sched_t * sched,
1707 : fd_sched_block_t * block,
1708 0 : fd_acct_addr_t const * addr ) {
1709 0 : FD_TEST( block_is_lane_head( sched, block ) );
1710 0 : int rv = fd_rdisp_add_writable( sched->rdisp, block_to_idx( sched, block ), addr );
1711 0 : if( FD_LIKELY( rv==1 ) ) lthash_entry_new_write_observed( sched, block, addr );
1712 0 : lthash_owe_addition( sched, block, rv );
1713 0 : }
1714 :
1715 : static void
1716 129026 : lthash_block_init( fd_sched_block_t * block ) {
1717 129026 : block->lthash_writable_cnt = 0U;
1718 129026 : block->lthash_in_flight_cnt = 0U;
1719 129026 : block->add_owed_cnt = 0U;
1720 129026 : block->sub_q_head = UINT_MAX;
1721 129026 : block->sub_q_tail = UINT_MAX;
1722 129026 : block->sub_queued_cnt = 0U;
1723 129026 : block->sub_in_flight_cnt = 0U;
1724 129026 : block->sub_applied_cnt = 0U;
1725 129026 : block->add_created_cnt = 0U;
1726 129026 : block->add_surfaced_cnt = 0U;
1727 129026 : block->add_done_cnt = 0U;
1728 129026 : block->add_applied_cnt = 0U;
1729 129026 : block->add_undone_cnt = 0U;
1730 129026 : block->lthash_drained = 0;
1731 129026 : block->lthash_draining = 0;
1732 129026 : }
1733 :
1734 : /* Forgets the block's LtHash progress when it leaves the pipeline:
1735 : finished, abandoned, or demoted. A demoted block re-derives
1736 : everything if promoted. An abandoned block may still be draining.
1737 : The dispatcher forgets what is left of its set when the lane empties.
1738 : The delta is found through the block's reservation, so this needs to
1739 : come before the reservation is released. */
1740 : static void
1741 : lthash_block_reset( fd_sched_t * sched,
1742 2 : fd_sched_block_t * block ) {
1743 2 : sched->lthash_draining_cnt -= (uint)block->lthash_draining;
1744 2 : int owns = block_owns_lane_map( sched, block );
1745 2 : lthash_block_init( block );
1746 2 : if( FD_LIKELY( owns ) ) lthash_release_lane_map( sched, block->staging_lane );
1747 2 : if( block->lthash_mode_picked && !block->lthash_inband ) fd_lthash_zero( sched->lthash_delta+block->lthash_oob_reservation );
1748 2 : }
1749 :
1750 : static void
1751 0 : lthash_metrics_check( fd_sched_t * sched ) {
1752 0 : fd_sched_metrics_t const * m = sched->metrics;
1753 0 : ulong add_owed_cnt = 0UL, add_pending_cnt = 0UL, add_delta_cnt = 0UL, sub_queued_cnt = 0UL, sub_in_flight_cnt = 0UL, sub_delta_cnt = 0UL, task_in_flight_cnt = 0UL;
1754 0 : for( ulong i=0UL; i<sched->block_cnt_max; i++ ) {
1755 0 : fd_sched_block_t const * block = sched->block_pool+i;
1756 0 : if( !block->in_sched ) continue;
1757 0 : add_owed_cnt += block->add_owed_cnt;
1758 0 : add_pending_cnt += block->add_created_cnt-block->add_surfaced_cnt;
1759 0 : add_delta_cnt += block->add_applied_cnt-block->add_undone_cnt;
1760 0 : sub_queued_cnt += block->sub_queued_cnt;
1761 0 : sub_in_flight_cnt += block->sub_in_flight_cnt;
1762 0 : sub_delta_cnt += block->sub_applied_cnt;
1763 0 : task_in_flight_cnt += block->lthash_in_flight_cnt;
1764 0 : }
1765 0 : ulong add_on_tile_cnt = 0UL, sub_on_tile_cnt = 0UL, task_on_tile_cnt = 0UL;
1766 0 : for( ulong busy = (~sched->lthash_ready_bitset[ 0 ])&fd_ulong_mask_lsb( (int)sched->exec_cnt ); busy; busy = fd_ulong_pop_lsb( busy ) ) {
1767 0 : fd_sched_lthash_batch_t const * rec = sched->lthash_inflight+fd_ulong_find_lsb( busy );
1768 0 : if( rec->kind==FD_SCHED_TT_LTHASH_ADD ) add_on_tile_cnt += rec->cnt;
1769 0 : else sub_on_tile_cnt += rec->cnt;
1770 0 : task_on_tile_cnt++;
1771 0 : }
1772 0 : ulong add_held_cnt = sched->lthash_add_batch->cnt;
1773 0 : FD_TEST( task_in_flight_cnt==task_on_tile_cnt );
1774 0 : FD_TEST( sub_in_flight_cnt==sub_on_tile_cnt );
1775 0 : FD_TEST( m->lthash_parsed_writable_cnt+m->lthash_start_reg_cnt+m->lthash_late_reg_cnt+m->lthash_demote_reg_cnt+m->lthash_spec_given_up_cnt==m->lthash_first_write_cnt+m->lthash_coalesced_cnt );
1776 0 : FD_TEST( m->lthash_first_write_cnt==m->lthash_add_created_cnt+add_owed_cnt+m->lthash_discarded_add_owed_cnt+m->lthash_inband_add_owed_cnt );
1777 0 : FD_TEST( m->lthash_add_created_cnt==m->lthash_add_surfaced_cnt+add_pending_cnt+m->lthash_discarded_add_pending_cnt );
1778 0 : FD_TEST( m->lthash_add_surfaced_cnt==m->lthash_add_surfaced_stale_cnt+m->lthash_spec_given_up_cnt+m->lthash_add_dispatched_cnt+add_held_cnt+m->lthash_discarded_add_held_cnt );
1779 0 : FD_TEST( m->lthash_add_dispatched_cnt==m->lthash_add_dispatched_stale_cnt+m->lthash_add_applied_cnt+add_on_tile_cnt );
1780 0 : FD_TEST( m->lthash_add_applied_cnt-m->lthash_add_undone_cnt==m->lthash_frozen_cnt+add_delta_cnt+m->lthash_discarded_add_applied_cnt );
1781 0 : FD_TEST( m->lthash_entry_cnt==m->lthash_sub_dispatched_cnt+sub_queued_cnt+m->lthash_discarded_sub_queued_cnt+m->lthash_inband_sub_queued_cnt );
1782 0 : FD_TEST( m->lthash_sub_dispatched_cnt==m->lthash_sub_applied_cnt+sub_on_tile_cnt );
1783 0 : FD_TEST( m->lthash_sub_applied_cnt==m->lthash_frozen_cnt+sub_delta_cnt+m->lthash_discarded_sub_applied_cnt );
1784 0 : }
1785 :
1786 : /* Rolls the block's LtHash progress back before it leaves its lane.
1787 : Each account goes back into rdisp, to be re-derived with exactly one
1788 : subtraction and one addition when we later promote the block.
1789 : Rehashing a demoted block's progress is the price of keeping the lane
1790 : map per lane. */
1791 : static void
1792 : lthash_demote( fd_sched_t * sched,
1793 0 : fd_sched_block_t * block ) {
1794 0 : ulong bank_idx = block_to_idx( sched, block );
1795 0 : ulong lane = block->staging_lane;
1796 0 : FD_TEST( !block->sub_queued_cnt && !block->sub_in_flight_cnt && !block->lthash_in_flight_cnt && block->add_created_cnt==block->add_done_cnt );
1797 : /* A draining block has fec_eos set and has not signaled BLOCK_END,
1798 : so block_will_signal_end keeps it dispatchable and it is never
1799 : demotable. */
1800 0 : FD_TEST( !block->lthash_draining );
1801 :
1802 : /* Owed additions are respected by rdisp demotion. In addition, we
1803 : need to stick the writable accounts that we are tracking back into
1804 : rdisp. */
1805 0 : uint add_owed_cnt = block->add_owed_cnt;
1806 0 : if( block_owns_lane_map( sched, block ) ) {
1807 0 : fd_sched_lthash_lane_t * lane_map = sched->lthash_lane+lane;
1808 0 : for( ulong i=0UL; i<lane_map->entry_cnt; i++ ) {
1809 0 : fd_sched_lthash_entry_t * e = lane_map->entry_pool+i;
1810 : /* A demotable block has nothing queued or in flight, so every
1811 : entry should be SUB_DONE with no pending or dispatched
1812 : addition. */
1813 0 : FD_TEST( e->sub_state==FD_SCHED_LTHASH_SUB_DONE && (e->add_state==FD_SCHED_LTHASH_ADD_NONE || e->add_state==FD_SCHED_LTHASH_ADD_DONE) );
1814 0 : int rv = fd_rdisp_add_writable( sched->rdisp, bank_idx, &e->acct );
1815 : /* A staged block is alone on its lane, so it is insert ready. */
1816 0 : if( FD_UNLIKELY( rv<0 ) ) FD_LOG_CRIT(( "fd_rdisp_add_writable failed for demoted block %lu:%lu", block->slot, bank_idx ));
1817 : /* A 0 means rdisp already tracks the account and owes it an
1818 : addition. We don't count that as a coalesce, so
1819 : lthash_coalesced_cnt only counts coalescing under normal
1820 : operation. */
1821 0 : sched->metrics->lthash_demote_reg_cnt += (uint)(rv==1);
1822 0 : sched->metrics->lthash_first_write_cnt += (ulong)(rv==1);
1823 0 : add_owed_cnt += (uint)(rv==1);
1824 0 : }
1825 0 : }
1826 : /* Of the six discarded counters only the two delta counters below
1827 : apply here. The check above ensures nothing queued and nothing
1828 : unsurfaced. The owed additions are not lost, since rdisp will
1829 : carry them through demotion, and they stay in the live counter. */
1830 0 : sched->metrics->lthash_discarded_sub_applied_cnt += block->sub_applied_cnt;
1831 0 : sched->metrics->lthash_discarded_add_applied_cnt += block->add_applied_cnt-block->add_undone_cnt;
1832 0 : lthash_block_reset( sched, block );
1833 0 : block->add_owed_cnt = add_owed_cnt;
1834 0 : }
1835 :
1836 : static inline void
1837 : lthash_batch_init( fd_sched_block_t * block,
1838 : fd_sched_lthash_batch_t * batch,
1839 : ulong bank_idx,
1840 0 : ulong kind ) {
1841 0 : batch->bank_idx = bank_idx;
1842 0 : batch->cnt = 0UL;
1843 0 : batch->lane = (uint)block->staging_lane;
1844 0 : batch->kind = (uchar)kind;
1845 0 : }
1846 :
1847 : static inline void
1848 : lthash_batch_append( fd_sched_t * sched,
1849 : fd_sched_lthash_batch_t * batch,
1850 : fd_sched_lthash_entry_t * e,
1851 0 : ulong ptxn_idx ) {
1852 0 : FD_TEST( batch->cnt<FD_SCHED_LTHASH_BATCH_MAX );
1853 0 : ulong i = batch->cnt++;
1854 0 : batch->entry_idx[ i ] = (uint)(e-sched->lthash_lane[ batch->lane ].entry_pool);
1855 0 : batch->ptxn_idx [ i ] = (uint)ptxn_idx;
1856 0 : batch->accts [ i ] = e->acct;
1857 0 : }
1858 :
1859 : static void
1860 : lthash_dispatch( fd_sched_t * sched,
1861 : fd_sched_block_t * block,
1862 : fd_sched_lthash_batch_t const * batch,
1863 : int exec_tile_idx,
1864 0 : fd_sched_task_t * out ) {
1865 0 : FD_TEST( batch->cnt && batch->cnt<=FD_SCHED_LTHASH_BATCH_MAX );
1866 0 : fd_sched_lthash_batch_t * rec = sched->lthash_inflight+exec_tile_idx;
1867 0 : if( rec!=batch ) { /* FIXME optimize for end-of-block additions */
1868 0 : rec->bank_idx = batch->bank_idx;
1869 0 : rec->cnt = batch->cnt;
1870 0 : rec->lane = batch->lane;
1871 0 : rec->kind = batch->kind;
1872 0 : fd_memcpy( rec->entry_idx, batch->entry_idx, batch->cnt*sizeof(uint) );
1873 0 : fd_memcpy( rec->ptxn_idx, batch->ptxn_idx, batch->cnt*sizeof(uint) );
1874 0 : fd_memcpy( rec->accts, batch->accts, batch->cnt*sizeof(fd_acct_addr_t) );
1875 0 : }
1876 :
1877 0 : out->task_type = rec->kind;
1878 0 : out->lthash->bank_idx = rec->bank_idx;
1879 0 : out->lthash->exec_idx = (ulong)exec_tile_idx;
1880 0 : out->lthash->cnt = rec->cnt;
1881 0 : out->lthash->accts = rec->accts;
1882 :
1883 0 : sched->lthash_ready_bitset[ 0 ] = fd_ulong_clear_bit( sched->lthash_ready_bitset[ 0 ], exec_tile_idx );
1884 0 : sched->tile_to_bank_idx[ exec_tile_idx ] = rec->bank_idx;
1885 0 : block->lthash_in_flight_cnt++;
1886 0 : if( rec->kind==FD_SCHED_TT_LTHASH_ADD ) {
1887 0 : sched->metrics->lthash_add_dispatched_cnt += rec->cnt;
1888 0 : sched->metrics->lthash_add_task_cnt++;
1889 0 : } else {
1890 0 : sched->metrics->lthash_sub_dispatched_cnt += rec->cnt;
1891 0 : sched->metrics->lthash_sub_task_cnt++;
1892 0 : }
1893 0 : exec_busy_check( sched, block );
1894 0 : }
1895 :
1896 : /* A pseudo transaction surfaced from fd_rdisp_get_next_ready() for the
1897 : active block. If it is still the account's pseudo transaction of
1898 : record, mark its entry ADD_DISPATCHED, meaning it has left the
1899 : dispatcher whether or not it will be staged for batching, and return
1900 : the entry. Otherwise the account has a later rewrite, so complete
1901 : the pseudo transaction without hashing and return NULL. */
1902 : static fd_sched_lthash_entry_t *
1903 : lthash_surface( fd_sched_t * sched,
1904 : fd_sched_block_t * block,
1905 : ulong bank_idx,
1906 0 : ulong ptxn_idx ) {
1907 0 : block->add_surfaced_cnt++;
1908 0 : sched->metrics->lthash_add_surfaced_cnt++;
1909 0 : fd_acct_addr_t acct[ 1 ];
1910 0 : fd_rdisp_pseudo_txn_to_addr( sched->rdisp, ptxn_idx, acct );
1911 0 : fd_sched_lthash_entry_t * e = lthash_lane_query( sched, block->staging_lane, acct );
1912 0 : if( FD_UNLIKELY( !e ) ) {
1913 : /* Every pseudo transaction of a head block must have an entry. */
1914 0 : sched->print_buf_sz = 0UL;
1915 0 : print_all( sched, block );
1916 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1917 0 : FD_LOG_CRIT(( "invariant violation: pseudo transaction 0x%lx of block %lu:%lu has no entry", ptxn_idx, block->slot, bank_idx ));
1918 0 : }
1919 0 : FD_TEST( e->bank_idx==(uint)bank_idx );
1920 0 : if( FD_UNLIKELY( e->ptxn_idx!=(uint)ptxn_idx ) ) {
1921 0 : sched->metrics->lthash_add_surfaced_stale_cnt++;
1922 0 : block->add_done_cnt++;
1923 0 : fd_rdisp_complete_txn( sched->rdisp, ptxn_idx, 1 );
1924 0 : return NULL;
1925 0 : }
1926 0 : FD_TEST( e->add_state==FD_SCHED_LTHASH_ADD_PENDING );
1927 0 : e->add_state = (uchar)FD_SCHED_LTHASH_ADD_DISPATCHED;
1928 0 : return e;
1929 0 : }
1930 :
1931 : static void
1932 : lthash_dispatch_one_add( fd_sched_t * sched,
1933 : fd_sched_block_t * block,
1934 : ulong bank_idx,
1935 : fd_sched_lthash_entry_t * e,
1936 : ulong ptxn_idx,
1937 : int exec_tile_idx,
1938 0 : fd_sched_task_t * out ) {
1939 0 : fd_sched_lthash_batch_t * rec = sched->lthash_inflight+exec_tile_idx;
1940 0 : lthash_batch_init( block, rec, bank_idx, FD_SCHED_TT_LTHASH_ADD );
1941 0 : lthash_batch_append( sched, rec, e, ptxn_idx );
1942 0 : lthash_dispatch( sched, block, rec, exec_tile_idx, out );
1943 0 : }
1944 :
1945 : /* A speculative addition that surfaced when the block may no longer
1946 : speculate on it, or when the idleness policy would not let it onto a
1947 : tile, is given up. The surfaced pseudo transaction is completed
1948 : without hashing. Its entry is rewritten, which frees the speculation
1949 : entry, and the account goes back to the dispatcher as written, the
1950 : way any write does, so the block owes the addition again. No
1951 : transaction actually rewrote the account, so this is neither a stale
1952 : addition nor an invalidated speculation for metric purposes. */
1953 : static void
1954 : lthash_spec_give_up( fd_sched_t * sched,
1955 : fd_sched_block_t * block,
1956 : ulong bank_idx,
1957 : fd_sched_lthash_entry_t * e,
1958 0 : ulong ptxn_idx ) {
1959 0 : sched->metrics->lthash_spec_given_up_cnt++;
1960 : /* Complete the pseudo transaction first, so that it is no longer the
1961 : account's last reference in rdisp if a re-drain below creates the
1962 : new one. */
1963 0 : block->add_done_cnt++;
1964 0 : fd_rdisp_complete_txn( sched->rdisp, ptxn_idx, 1 );
1965 0 : lthash_entry_clear_add( sched, e, block_lthash_delta( sched, block ) );
1966 0 : lthash_owe_addition( sched, block, fd_rdisp_add_writable( sched->rdisp, bank_idx, &e->acct ) );
1967 0 : }
1968 :
1969 : static void
1970 : lthash_dispatch_held_batch( fd_sched_t * sched,
1971 : fd_sched_block_t * block,
1972 : ulong bank_idx,
1973 : int exec_tile_idx,
1974 0 : fd_sched_task_t * out ) {
1975 0 : fd_sched_lthash_batch_t * batch = sched->lthash_add_batch;
1976 : /* The held batch belongs to the active block. Its surfaced,
1977 : unfinished additions keep the block in flight, so it is not
1978 : deactivated, demoted or pruned and does not signal BLOCK_END, and
1979 : no other block is activated behind the batch. The one way out is
1980 : abandon, which releases the batch where it marks the block dying.
1981 : So a held batch is either dispatched or released before its block
1982 : goes anywhere. */
1983 0 : FD_TEST( batch->cnt && batch->kind==FD_SCHED_TT_LTHASH_ADD && batch->bank_idx==bank_idx && batch->lane==(uint)block->staging_lane );
1984 :
1985 0 : for( ulong i=0UL; i<batch->cnt; i++ ) {
1986 0 : fd_sched_lthash_entry_t * e = sched->lthash_lane[ batch->lane ].entry_pool+batch->entry_idx[ i ];
1987 0 : if( FD_UNLIKELY( e->ptxn_idx!=batch->ptxn_idx[ i ] || e->add_state!=FD_SCHED_LTHASH_ADD_DISPATCHED ) ) {
1988 : /* A non-speculative addition cannot become stale after it has
1989 : surfaced. See the account state machine. So no batched pseudo
1990 : transaction can actually be invalidated by a rewrite. */
1991 0 : sched->print_buf_sz = 0UL;
1992 0 : print_all( sched, block );
1993 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
1994 0 : FD_LOG_CRIT(( "invariant violation: held addition %lu of %lu of block %lu:%lu was rewritten, ticket 0x%x, entry 0x%x state %u", i, batch->cnt, block->slot, bank_idx, batch->ptxn_idx[ i ], e->ptxn_idx, (uint)e->add_state ));
1995 0 : }
1996 0 : }
1997 0 : lthash_dispatch( sched, block, batch, exec_tile_idx, out );
1998 0 : batch->cnt = 0UL;
1999 0 : }
2000 :
2001 : static void
2002 : lthash_release_held_batch( fd_sched_t * sched,
2003 2 : fd_sched_block_t * block ) {
2004 2 : fd_sched_lthash_batch_t * batch = sched->lthash_add_batch;
2005 2 : if( FD_LIKELY( !batch->cnt || batch->bank_idx!=block_to_idx( sched, block ) ) ) return;
2006 0 : for( ulong i=0UL; i<batch->cnt; i++ ) {
2007 0 : fd_sched_lthash_entry_t * e = sched->lthash_lane[ batch->lane ].entry_pool+batch->entry_idx[ i ];
2008 0 : if( FD_LIKELY( e->ptxn_idx==batch->ptxn_idx[ i ] ) ) {
2009 0 : e->ptxn_idx = 0U;
2010 0 : e->add_state = (uchar)FD_SCHED_LTHASH_ADD_NONE;
2011 0 : }
2012 0 : block->add_done_cnt++;
2013 0 : fd_rdisp_complete_txn( sched->rdisp, (ulong)batch->ptxn_idx[ i ], 1 );
2014 0 : }
2015 0 : sched->metrics->lthash_discarded_add_held_cnt += (uint)batch->cnt;
2016 0 : batch->cnt = 0UL;
2017 0 : }
2018 :
2019 : /* Dispatches up to cap of the block's oldest queued subtractions as one
2020 : task to the highest idle tile in ready_bitset, if more than threshold
2021 : tiles are idle. cap is in [1, FD_SCHED_LTHASH_BATCH_MAX]. Returns 1
2022 : if a task was written to out, 0 otherwise. */
2023 : static int
2024 : lthash_dispatch_sub( fd_sched_t * sched,
2025 : fd_sched_block_t * block,
2026 : ulong bank_idx,
2027 : ulong ready_bitset,
2028 : int threshold,
2029 : ulong cap,
2030 0 : fd_sched_task_t * out ) {
2031 0 : if( FD_UNLIKELY( !block->sub_queued_cnt ) ) return 0;
2032 0 : if( FD_UNLIKELY( fd_ulong_popcnt( ready_bitset )<=threshold ) ) return 0;
2033 :
2034 0 : int exec_tile_idx = fd_ulong_find_msb( ready_bitset );
2035 0 : ulong lane = block->staging_lane;
2036 0 : fd_sched_lthash_batch_t * batch = sched->lthash_inflight+exec_tile_idx;
2037 0 : lthash_batch_init( block, batch, bank_idx, FD_SCHED_TT_LTHASH_SUB );
2038 0 : while( block->sub_queued_cnt && batch->cnt<cap ) {
2039 0 : uint entry_idx = block->sub_q_head;
2040 0 : FD_TEST( entry_idx!=UINT_MAX );
2041 0 : fd_sched_lthash_entry_t * e = sched->lthash_lane[ lane ].entry_pool+entry_idx;
2042 0 : FD_TEST( e->sub_state==FD_SCHED_LTHASH_SUB_QUEUED && e->bank_idx==(uint)bank_idx );
2043 :
2044 0 : block->sub_q_head = e->q_next;
2045 0 : if( FD_UNLIKELY( block->sub_q_head==UINT_MAX ) ) block->sub_q_tail = UINT_MAX;
2046 0 : e->q_next = UINT_MAX;
2047 0 : e->sub_state = (uchar)FD_SCHED_LTHASH_SUB_DISPATCHED;
2048 0 : block->sub_queued_cnt--;
2049 0 : block->sub_in_flight_cnt++;
2050 0 : lthash_batch_append( sched, batch, e, 0UL );
2051 0 : }
2052 0 : lthash_dispatch( sched, block, batch, exec_tile_idx, out );
2053 0 : return 1;
2054 0 : }
2055 :
2056 : /* The block is assumed to be active, and nothing else of it could go
2057 : out this poll, either because nothing is queued or because every
2058 : queued transaction is blocked by something in-flight. The block is
2059 : also assumed to satisfy block_may_speculate(). The dispatcher hands
2060 : us an account whose writers have all completed, so the guess is READY
2061 : at once and goes straight to a tile. The one exception is a
2062 : serializing transaction still outstanding, which every pseudo
2063 : transaction created after it waits behind. Speculation is paused for
2064 : a moment until that serializing transaction completes.
2065 :
2066 : Returns 1 with a task in out, or 0 when a pseudo transaction was
2067 : created but waits on a serializing transaction, so the caller returns
2068 : without a task, or -1 when speculation does not apply and the caller
2069 : proceeds. */
2070 : static inline int
2071 : lthash_speculate( fd_sched_t * sched,
2072 : fd_sched_block_t * block,
2073 : ulong bank_idx,
2074 : ulong ready_bitset,
2075 0 : fd_sched_task_t * out ) {
2076 : /* Any unsurfaced pseudo transaction is a speculation, since with
2077 : fec_eos clear there has been no drain. */
2078 0 : uint spec_pending = block->add_created_cnt-block->add_surfaced_cnt;
2079 0 : if( spec_pending ) {
2080 : /* We speculate one addition at a time. */
2081 0 : FD_TEST( spec_pending==1U );
2082 0 : return -1;
2083 0 : }
2084 :
2085 0 : if( fd_ulong_popcnt( ready_bitset )<=idle_reserve( sched, block ) ) return -1;
2086 :
2087 0 : ulong ptxn_idx = fd_rdisp_add_ready_pseudo_txn( sched->rdisp, bank_idx );
2088 0 : if( FD_UNLIKELY( !ptxn_idx ) ) FD_LOG_CRIT(( "invariant violation: block %lu:%lu owes %u additions and has a ready account but fd_rdisp_add_ready_pseudo_txn returned none", block->slot, bank_idx, block->add_owed_cnt ));
2089 0 : sched->metrics->lthash_spec_cnt++;
2090 0 : block->add_owed_cnt--;
2091 0 : lthash_add_created( sched, block, ptxn_idx, 1 );
2092 :
2093 : /* We expect to get either the only pseudo transaction, the one we
2094 : just instructed rdisp to create, or nothing because it is blocked
2095 : by a serializing transaction. There should otherwise be no task
2096 : queued, especially no other pseudo transaction queued. */
2097 0 : ulong idx = fd_rdisp_get_next_ready( sched->rdisp, bank_idx );
2098 0 : if( FD_LIKELY( idx==ptxn_idx ) ) {
2099 0 : fd_sched_lthash_entry_t * e = lthash_surface( sched, block, bank_idx, idx );
2100 0 : FD_TEST( e );
2101 0 : lthash_dispatch_one_add( sched, block, bank_idx, e, idx, fd_ulong_find_msb( ready_bitset ), out );
2102 0 : return 1;
2103 0 : }
2104 0 : if( FD_UNLIKELY( idx ) ) FD_LOG_CRIT(( "invariant violation: fd_rdisp_get_next_ready returned 0x%lx instead of pseudo transaction 0x%lx of block %lu:%lu", idx, ptxn_idx, block->slot, bank_idx ));
2105 :
2106 : /* Pseudo transaction pending behind a serializing transaction. It'll
2107 : be eligible for surfacing once that transaction completes. */
2108 0 : return 0;
2109 0 : }
2110 :
2111 :
2112 : /* Public functions. */
2113 :
2114 : ulong
2115 12023 : fd_sched_align( void ) {
2116 12023 : return fd_ulong_max( alignof(fd_sched_t),
2117 12023 : fd_ulong_max( fd_rdisp_align(),
2118 12023 : fd_ulong_max( alignof(fd_sched_block_t), 64UL ))); /* Minimally cache line aligned. */
2119 12023 : }
2120 :
2121 : static inline ulong
2122 4021 : sched_lthash_batch_sz( void ) {
2123 : /* One batch per exec tile, plus one staging area for the addition
2124 : batch being assembled or held. */
2125 4021 : return (FD_SCHED_MAX_EXEC_TILE_CNT+1UL)*sizeof(fd_sched_lthash_batch_t);
2126 4021 : }
2127 :
2128 : ulong
2129 : fd_sched_footprint( ulong depth,
2130 : ulong block_cnt_max,
2131 : ulong max_shreds_per_block,
2132 1005 : ulong max_txn_per_slot ) {
2133 1005 : if( FD_UNLIKELY( depth<FD_SCHED_MIN_DEPTH || depth>FD_SCHED_MAX_DEPTH ) ) return 0UL; /* bad depth */
2134 1005 : if( FD_UNLIKELY( !block_cnt_max ) ) return 0UL; /* bad block_cnt_max */
2135 1005 : if( FD_UNLIKELY( depth>UINT_MAX-1UL ) ) return 0UL; /* mblk_pool use uint as pointers */
2136 1005 : if( FD_UNLIKELY( !max_shreds_per_block || max_shreds_per_block>UINT_MAX ) ) return 0UL; /* shred_cnt is uint */
2137 1005 : if( FD_UNLIKELY( !max_txn_per_slot || max_txn_per_slot >UINT_MAX ) ) return 0UL; /* txn_parsed_cnt is uint */
2138 :
2139 1005 : ulong rdisp_footprint = fd_rdisp_footprint( depth, block_cnt_max, FD_SCHED_LTHASH_ACCT_EXTRA );
2140 1005 : if( FD_UNLIKELY( !rdisp_footprint ) ) return 0UL;
2141 :
2142 1005 : ulong l = FD_LAYOUT_INIT;
2143 1005 : l = FD_LAYOUT_APPEND( l, fd_sched_align(), sizeof(fd_sched_t) );
2144 1005 : l = FD_LAYOUT_APPEND( l, fd_rdisp_align(), rdisp_footprint ); /* dispatcher */
2145 1005 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_block_t), block_cnt_max*sizeof(fd_sched_block_t) ); /* block pool */
2146 1005 : l = FD_LAYOUT_APPEND( l, ref_q_align(), ref_q_footprint( block_cnt_max ) );
2147 1005 : l = FD_LAYOUT_APPEND( l, alignof(fd_txn_p_t), depth*sizeof(fd_txn_p_t) ); /* txn_pool */
2148 1005 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_txn_info_t), depth*sizeof(fd_sched_txn_info_t) ); /* txn_info_pool */
2149 1005 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_mblk_t), depth*sizeof(fd_sched_mblk_t) ); /* mblk_pool */
2150 5017 : for( ulong i=0UL; i<FD_SCHED_MAX_STAGING_LANES; i++ ) {
2151 4012 : l = FD_LAYOUT_APPEND( l, alignof(fd_sched_lthash_entry_t), FD_RDISP_MAX_WRITERS_PER_BLOCK*sizeof(fd_sched_lthash_entry_t) );
2152 4012 : l = FD_LAYOUT_APPEND( l, lthash_map_align(), lthash_map_footprint( lthash_chain_cnt() ) );
2153 4012 : }
2154 1005 : l = FD_LAYOUT_APPEND( l, alignof(fd_lthash_value_t), FD_SCHED_LTHASH_SPEC_MAX*sizeof(fd_lthash_value_t) ); /* lthash_spec_pool */
2155 1005 : l = FD_LAYOUT_APPEND( l, alignof(uint), FD_SCHED_LTHASH_SPEC_MAX*sizeof(uint) ); /* lthash_spec_free */
2156 1005 : l = FD_LAYOUT_APPEND( l, alignof(ulong), sched_lthash_batch_sz() ); /* lthash_inflight, lthash_add_batch */
2157 1005 : return FD_LAYOUT_FINI( l, fd_sched_align() );
2158 1005 : }
2159 :
2160 : void *
2161 : fd_sched_new( void * mem,
2162 : fd_rng_t * rng,
2163 : ulong depth,
2164 : ulong block_cnt_max,
2165 : ulong max_shreds_per_block,
2166 : ulong max_txn_per_slot,
2167 1000 : ulong exec_cnt ) {
2168 :
2169 1000 : if( FD_UNLIKELY( !mem ) ) {
2170 0 : FD_LOG_WARNING(( "NULL mem" ));
2171 0 : return NULL;
2172 0 : }
2173 :
2174 1000 : if( FD_UNLIKELY( !rng ) ) {
2175 0 : FD_LOG_WARNING(( "NULL rng" ));
2176 0 : return NULL;
2177 0 : }
2178 :
2179 1000 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_sched_align() ) ) ) {
2180 0 : FD_LOG_WARNING(( "misaligned mem (%p)", mem ));
2181 0 : return NULL;
2182 0 : }
2183 :
2184 1000 : if( FD_UNLIKELY( depth<FD_SCHED_MIN_DEPTH || depth>FD_SCHED_MAX_DEPTH ) ) {
2185 0 : FD_LOG_WARNING(( "bad depth (%lu)", depth ));
2186 0 : return NULL;
2187 0 : }
2188 :
2189 1000 : if( FD_UNLIKELY( !block_cnt_max ) ) {
2190 0 : FD_LOG_WARNING(( "bad block_cnt_max (%lu)", block_cnt_max ));
2191 0 : return NULL;
2192 0 : }
2193 :
2194 1000 : if( FD_UNLIKELY( depth>UINT_MAX-1UL ) ) {
2195 0 : FD_LOG_WARNING(( "bad depth (%lu)", depth ));
2196 0 : return NULL;
2197 0 : }
2198 :
2199 1000 : if( FD_UNLIKELY( !max_shreds_per_block || max_shreds_per_block>UINT_MAX ) ) {
2200 0 : FD_LOG_WARNING(( "bad max_shreds_per_block (%lu)", max_shreds_per_block ));
2201 0 : return NULL;
2202 0 : }
2203 :
2204 1000 : if( FD_UNLIKELY( !max_txn_per_slot || max_txn_per_slot>UINT_MAX ) ) {
2205 0 : FD_LOG_WARNING(( "bad max_txn_per_slot (%lu)", max_txn_per_slot ));
2206 0 : return NULL;
2207 0 : }
2208 :
2209 1000 : if( FD_UNLIKELY( !exec_cnt || exec_cnt>FD_SCHED_MAX_EXEC_TILE_CNT ) ) {
2210 0 : FD_LOG_WARNING(( "bad exec_cnt (%lu)", exec_cnt ));
2211 0 : return NULL;
2212 0 : }
2213 :
2214 1000 : FD_SCRATCH_ALLOC_INIT( l, mem );
2215 1000 : fd_sched_t * sched = FD_SCRATCH_ALLOC_APPEND( l, fd_sched_align(), sizeof(fd_sched_t) );
2216 1000 : void * _rdisp = FD_SCRATCH_ALLOC_APPEND( l, fd_rdisp_align(), fd_rdisp_footprint( depth, block_cnt_max, FD_SCHED_LTHASH_ACCT_EXTRA ) );
2217 1000 : void * _bpool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_block_t), block_cnt_max*sizeof(fd_sched_block_t) );
2218 1000 : void * _ref_q = FD_SCRATCH_ALLOC_APPEND( l, ref_q_align(), ref_q_footprint( block_cnt_max ) );
2219 1000 : fd_txn_p_t * _txn_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txn_p_t), depth*sizeof(fd_txn_p_t) );
2220 1000 : fd_sched_txn_info_t * _txn_info_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_txn_info_t), depth*sizeof(fd_sched_txn_info_t) );
2221 1000 : fd_sched_mblk_t * _mblk_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_mblk_t), depth*sizeof(fd_sched_mblk_t) );
2222 5015 : for( ulong i=0UL; i<FD_SCHED_MAX_STAGING_LANES; i++ ) {
2223 4015 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_lthash_entry_t), FD_RDISP_MAX_WRITERS_PER_BLOCK*sizeof(fd_sched_lthash_entry_t) );
2224 4015 : void * _lane_map = FD_SCRATCH_ALLOC_APPEND( l, lthash_map_align(), lthash_map_footprint( lthash_chain_cnt() ) );
2225 4015 : FD_TEST( lthash_map_new( _lane_map, lthash_chain_cnt(), fd_rng_ulong( rng ) ) );
2226 4015 : sched->lthash_lane[ i ].entry_cnt = 0U;
2227 4015 : sched->lthash_lane[ i ].bank_idx = ULONG_MAX;
2228 4015 : }
2229 2000 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_lthash_value_t), FD_SCHED_LTHASH_SPEC_MAX*sizeof(fd_lthash_value_t) );
2230 1000 : uint * _spec_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), FD_SCHED_LTHASH_SPEC_MAX*sizeof(uint) );
2231 1000 : void * _lthash_batch = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), sched_lthash_batch_sz() );
2232 1000 : FD_SCRATCH_ALLOC_FINI( l, fd_sched_align() );
2233 :
2234 1000 : sched->txn_pool = _txn_pool;
2235 1000 : sched->txn_info_pool = _txn_info_pool;
2236 1000 : sched->mblk_pool = _mblk_pool;
2237 :
2238 1000 : FD_TEST( fd_rdisp_new( _rdisp, depth, block_cnt_max, FD_SCHED_LTHASH_ACCT_EXTRA, fd_rng_ulong( rng ) ) );
2239 :
2240 : /* Acquire is LIFO, so this hands out speculation entries 0, 1, 2, ... */
2241 8258536 : for( ulong i=0UL; i<FD_SCHED_LTHASH_SPEC_MAX; i++ ) _spec_free[ i ] = (uint)(FD_SCHED_LTHASH_SPEC_MAX-1UL-i);
2242 1000 : sched->lthash_spec_free_cnt = FD_SCHED_LTHASH_SPEC_MAX;
2243 1000 : fd_memset( sched->lthash_delta, 0, sizeof(sched->lthash_delta) );
2244 1000 : fd_memset( _lthash_batch, 0, sched_lthash_batch_sz() );
2245 :
2246 1000 : fd_sched_block_t * bpool = (fd_sched_block_t *)_bpool;
2247 130024 : for( ulong i=0; i<block_cnt_max; i++ ) {
2248 129024 : bpool[ i ].in_sched = 0;
2249 129024 : lthash_block_init( bpool+i );
2250 129024 : mblk_slist_new( bpool[ i ].mblks_unhashed );
2251 129024 : mblk_slist_new( bpool[ i ].mblks_mixin_in_progress );
2252 129024 : }
2253 :
2254 1000 : FD_TEST( fd_chkdup_new( sched->chkdup, rng ) );
2255 :
2256 1000 : fd_memset( sched->metrics, 0, sizeof(fd_sched_metrics_t) );
2257 1000 : sched->txn_in_flight_last_tick = fd_tickcount(); /* boot counts as idle */
2258 1000 : sched->next_ready_last_tick = LONG_MAX;
2259 1000 : sched->next_ready_last_bank_idx = ULONG_MAX;
2260 1000 : sched->state_debounce_ticks = (long)( (double)FD_SCHED_SLOT_STATE_DEBOUNCE_NS*fd_tempo_tick_per_ns( NULL ) );
2261 1000 : sched->state_bank_idx = ULONG_MAX;
2262 1000 : sched->state_cnt = 0UL;
2263 1000 : sched->state_drop_cnt = 0UL;
2264 :
2265 1000 : sched->canary = FD_SCHED_MAGIC;
2266 1000 : sched->depth = depth;
2267 1000 : sched->lthash_draining_cnt = 0U;
2268 1000 : sched->block_cnt_max = block_cnt_max;
2269 1000 : sched->max_shreds_per_block = max_shreds_per_block;
2270 1000 : sched->max_txn_per_slot = max_txn_per_slot;
2271 1000 : sched->max_mblk_per_slot = fd_ulong_min( FD_SCHED_MAX_MBLK_PER_SLOT*max_shreds_per_block/FD_SHRED_BLK_MAX, UINT_MAX ); /* mblk_cnt is uint */
2272 1000 : sched->exec_cnt = exec_cnt;
2273 1000 : sched->bypass_poh_verify = 0;
2274 1000 : sched->bypass_alut_resolution = 0;
2275 1000 : sched->lthash_txn_writers_max = FD_SCHED_LTHASH_TXN_WRITERS_MAX;
2276 1000 : sched->lthash_oob_reserved = 0UL;
2277 1000 : sched->lthash_oob_block_max = FD_SCHED_LTHASH_OOB_BLOCK_MAX;
2278 1000 : sched->root_idx = ULONG_MAX;
2279 1000 : sched->active_bank_idx = ULONG_MAX;
2280 1000 : sched->last_active_bank_idx = ULONG_MAX;
2281 1000 : sched->staged_bitset = 0UL;
2282 5032 : for( ulong l=0UL; l<FD_SCHED_MAX_STAGING_LANES; l++ ) sched->staged_head_bank_idx[ l ] = ULONG_MAX;
2283 1000 : sched->staged_popcnt_wmk = 0UL;
2284 :
2285 1000 : sched->txn_exec_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
2286 1000 : sched->sigverify_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
2287 1000 : sched->poh_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
2288 1000 : sched->footer_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
2289 1000 : sched->lthash_ready_bitset[ 0 ] = fd_ulong_mask_lsb( (int)exec_cnt );
2290 :
2291 1000 : sched->txn_pool_free_cnt = depth-1UL; /* -1 because index 0 is unusable as a sentinel reserved by the dispatcher */
2292 :
2293 4128760 : for( ulong i=0UL; i<depth-1UL; i++ ) sched->mblk_pool[ i ].next = (uint)(i+1UL);
2294 1000 : sched->mblk_pool[ depth-1UL ].next = UINT_MAX;
2295 1000 : sched->mblk_pool_free_head = 0U;
2296 1000 : sched->mblk_pool_free_cnt = depth;
2297 :
2298 1000 : txn_bitset_new( sched->exec_done_set );
2299 1000 : txn_bitset_new( sched->sigverify_done_set );
2300 1000 : txn_bitset_new( sched->poh_mixin_done_set );
2301 :
2302 1000 : sched->block_pool_popcnt = 0UL;
2303 :
2304 1000 : ref_q_new( _ref_q, block_cnt_max );
2305 :
2306 1000 : return sched;
2307 1000 : }
2308 :
2309 : fd_sched_t *
2310 1008 : fd_sched_join( void * mem ) {
2311 :
2312 1008 : if( FD_UNLIKELY( !mem ) ) {
2313 0 : FD_LOG_WARNING(( "NULL mem" ));
2314 0 : return NULL;
2315 0 : }
2316 :
2317 1008 : fd_sched_t * sched = (fd_sched_t *)mem;
2318 1008 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2319 1008 : ulong depth = sched->depth;
2320 1008 : ulong block_cnt_max = sched->block_cnt_max;
2321 :
2322 1008 : FD_SCRATCH_ALLOC_INIT( l, mem );
2323 1008 : /* */ FD_SCRATCH_ALLOC_APPEND( l, fd_sched_align(), sizeof(fd_sched_t) );
2324 1008 : void * _rdisp = FD_SCRATCH_ALLOC_APPEND( l, fd_rdisp_align(), fd_rdisp_footprint( depth, block_cnt_max, FD_SCHED_LTHASH_ACCT_EXTRA ) );
2325 1008 : void * _bpool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_block_t), block_cnt_max*sizeof(fd_sched_block_t) );
2326 1008 : void * _ref_q = FD_SCRATCH_ALLOC_APPEND( l, ref_q_align(), ref_q_footprint( block_cnt_max ) );
2327 1008 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txn_p_t), depth*sizeof(fd_txn_p_t) );
2328 1008 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_txn_info_t), depth*sizeof(fd_sched_txn_info_t) );
2329 1008 : /* */ FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_mblk_t), depth*sizeof(fd_sched_mblk_t) );
2330 5040 : for( ulong i=0UL; i<FD_SCHED_MAX_STAGING_LANES; i++ ) {
2331 4032 : void * _pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sched_lthash_entry_t), FD_RDISP_MAX_WRITERS_PER_BLOCK*sizeof(fd_sched_lthash_entry_t) );
2332 4032 : void * _map = FD_SCRATCH_ALLOC_APPEND( l, lthash_map_align(), lthash_map_footprint( lthash_chain_cnt() ) );
2333 0 : sched->lthash_lane[ i ].entry_pool = (fd_sched_lthash_entry_t *)_pool;
2334 4032 : sched->lthash_lane[ i ].map = lthash_map_join( _map );
2335 4032 : FD_TEST( sched->lthash_lane[ i ].map );
2336 4032 : }
2337 1008 : sched->lthash_spec_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_lthash_value_t), FD_SCHED_LTHASH_SPEC_MAX*sizeof(fd_lthash_value_t) );
2338 1008 : sched->lthash_spec_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), FD_SCHED_LTHASH_SPEC_MAX*sizeof(uint) );
2339 1008 : void * _batch = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), sched_lthash_batch_sz() );
2340 1008 : FD_SCRATCH_ALLOC_FINI( l, fd_sched_align() );
2341 :
2342 1008 : sched->rdisp = fd_rdisp_join( _rdisp );
2343 1008 : sched->ref_q = ref_q_join( _ref_q );
2344 1008 : sched->block_pool = _bpool;
2345 1008 : sched->lthash_inflight = (fd_sched_lthash_batch_t *)_batch;
2346 1008 : sched->lthash_add_batch = sched->lthash_inflight+FD_SCHED_MAX_EXEC_TILE_CNT;
2347 :
2348 130032 : for( ulong i=0; i<block_cnt_max; i++ ) {
2349 129024 : mblk_slist_join( sched->block_pool[ i ].mblks_unhashed );
2350 129024 : mblk_slist_join( sched->block_pool[ i ].mblks_mixin_in_progress );
2351 129024 : }
2352 :
2353 1008 : txn_bitset_join( sched->exec_done_set );
2354 1008 : txn_bitset_join( sched->sigverify_done_set );
2355 1008 : txn_bitset_join( sched->poh_mixin_done_set );
2356 :
2357 1008 : return sched;
2358 1008 : }
2359 :
2360 : int
2361 4557 : fd_sched_fec_can_ingest( fd_sched_t * sched, fd_sched_fec_t * fec ) {
2362 4557 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2363 4557 : FD_TEST( fec->bank_idx<sched->block_cnt_max );
2364 4557 : FD_TEST( fec->parent_bank_idx<sched->block_cnt_max );
2365 :
2366 4557 : if( FD_UNLIKELY( fec->fec->data_sz>FD_SCHED_MAX_PAYLOAD_PER_FEC ) ) {
2367 0 : sched->print_buf_sz = 0UL;
2368 0 : print_metrics( sched );
2369 0 : print_sched( sched );
2370 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2371 0 : FD_LOG_CRIT(( "invalid FEC set: fec->data_sz %u, slot %lu, parent slot %lu", fec->fec->data_sz, fec->slot, fec->parent_slot ));
2372 0 : }
2373 :
2374 4557 : ulong fec_buf_sz = 0UL;
2375 4557 : fd_sched_block_t * block = block_pool_ele( sched, fec->bank_idx );
2376 4557 : if( FD_LIKELY( !fec->is_first_in_block ) ) {
2377 3556 : fec_buf_sz += block->residual_sz;
2378 3556 : } else {
2379 : /* No residual data as this is a fresh new block. */
2380 1001 : }
2381 : /* Addition is safe and won't overflow because we checked the FEC set
2382 : size above. */
2383 4557 : fec_buf_sz += fec->fec->data_sz;
2384 : /* Assuming every transaction is min size, do we have enough free
2385 : entries in the txn pool? For a more precise txn count, we would
2386 : have to do some parsing. */
2387 4557 : return sched->txn_pool_free_cnt>=fec_buf_sz/FD_TXN_MIN_SERIALIZED_SZ && sched->mblk_pool_free_cnt>=fec_buf_sz/sizeof(fd_microblock_hdr_t);
2388 4557 : }
2389 :
2390 : ulong
2391 0 : fd_sched_can_ingest_cnt( fd_sched_t * sched ) {
2392 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2393 : /* Worst case, we need one byte from the incoming data to extract a
2394 : transaction out of the residual data, and the rest of the incoming
2395 : data contributes toward min sized transactions. */
2396 0 : return fd_ulong_min( sched->txn_pool_free_cnt/FD_SCHED_MAX_TXN_PER_FEC, sched->mblk_pool_free_cnt/FD_SCHED_MAX_MBLK_PER_FEC );
2397 0 : }
2398 :
2399 : int
2400 : fd_sched_fec_waits_for_lane( fd_sched_t * sched,
2401 : ulong parent_bank_idx,
2402 0 : int is_first_in_block ) {
2403 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2404 0 : if( FD_LIKELY( !fec_waits_for_lane( sched, parent_bank_idx, is_first_in_block ) ) ) return 0;
2405 0 : sched->metrics->lane_wait_cnt++;
2406 0 : return 1;
2407 0 : }
2408 :
2409 : int
2410 0 : fd_sched_is_drained( fd_sched_t * sched ) {
2411 0 : int nothing_inflight = sched->exec_cnt==(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ]&sched->sigverify_ready_bitset[ 0 ]&sched->poh_ready_bitset[ 0 ]&sched->footer_ready_bitset[ 0 ]&sched->lthash_ready_bitset[ 0 ] );
2412 0 : int nothing_queued = sched->active_bank_idx==ULONG_MAX;
2413 0 : return nothing_inflight && nothing_queued;
2414 0 : }
2415 :
2416 : ulong
2417 : fd_sched_footer_dispatch( fd_sched_t * sched,
2418 : ulong bank_idx,
2419 0 : ulong cnt ) {
2420 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2421 0 : FD_TEST( bank_idx==sched->active_bank_idx );
2422 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2423 0 : FD_TEST( block->block_end_signaled && !block->block_end_done );
2424 0 : ulong idle = sched->txn_exec_ready_bitset[ 0 ]&sched->sigverify_ready_bitset[ 0 ]&sched->poh_ready_bitset[ 0 ]&sched->footer_ready_bitset[ 0 ]&sched->lthash_ready_bitset[ 0 ];
2425 0 : ulong claimed = 0UL;
2426 0 : for( ; idle && cnt; cnt--, idle=fd_ulong_pop_lsb( idle ) ) {
2427 0 : int t = fd_ulong_find_lsb( idle );
2428 0 : claimed = fd_ulong_set_bit( claimed, t );
2429 0 : sched->tile_to_bank_idx[ t ] = bank_idx;
2430 0 : block->footer_in_flight_cnt++;
2431 0 : }
2432 0 : sched->footer_ready_bitset[ 0 ] &= ~claimed;
2433 0 : return claimed;
2434 0 : }
2435 :
2436 : FD_WARN_UNUSED int
2437 : fd_sched_fec_ingest( fd_sched_t * sched,
2438 4556 : fd_sched_fec_t * fec ) {
2439 4556 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2440 4556 : FD_TEST( fec->bank_idx<sched->block_cnt_max );
2441 4556 : FD_TEST( fec->parent_bank_idx<sched->block_cnt_max );
2442 4556 : FD_TEST( ref_q_empty( sched->ref_q ) );
2443 :
2444 4556 : fd_sched_block_t * block = block_pool_ele( sched, fec->bank_idx );
2445 :
2446 4556 : if( FD_UNLIKELY( fec->fec->data_sz>FD_SCHED_MAX_PAYLOAD_PER_FEC ) ) {
2447 0 : sched->print_buf_sz = 0UL;
2448 0 : print_all( sched, block );
2449 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2450 0 : FD_LOG_CRIT(( "invalid FEC set: fec->data_sz %u, slot %lu, parent slot %lu", fec->fec->data_sz, fec->slot, fec->parent_slot ));
2451 0 : }
2452 :
2453 4556 : sched->metrics->fec_cnt++;
2454 :
2455 4556 : if( FD_UNLIKELY( fec->is_first_in_block ) ) {
2456 : /* This is a new block. */
2457 1000 : add_block( sched, fec->bank_idx, fec->parent_bank_idx );
2458 1000 : block->slot = fec->slot;
2459 1000 : block->parent_slot = fec->parent_slot;
2460 1000 : block->starts_at_update_parent = fec->starts_at_update_parent;
2461 :
2462 1000 : if( FD_UNLIKELY( block->dying ) ) {
2463 : /* The child of a dead block is also dead. We added it to our
2464 : fork tree just so we could track an entire lineage of dead
2465 : children and propagate the dead property to the entire lineage,
2466 : in case there were frags for more than one dead children
2467 : in-flight at the time the parent was abandoned. That being
2468 : said, we shouldn't need to add the dead child to the
2469 : dispatcher. */
2470 0 : sched->metrics->block_added_dead_ood_cnt++;
2471 :
2472 : /* Release the refcnt right away since we don't intend to replay
2473 : it at all. We do this so its bank does not linger and stall
2474 : root advancement until the next root notify. */
2475 0 : subtree_release_refcnt( sched, block );
2476 :
2477 : /* Ignore the FEC set for a dead block. */
2478 0 : sched->metrics->bytes_dropped_cnt += fec->fec->data_sz;
2479 0 : return 0;
2480 0 : }
2481 :
2482 : /* A block whose parent is staged or done takes a free lane of its
2483 : own. A child of an unstaged stageable parent will be unstaged
2484 : too. This is not only lazy, sparing a promotion of the parent
2485 : right now, but an invariant that keeps a lane shortage from
2486 : deadlocking replay, since a staged block then never has an
2487 : unstaged parent that still has to run. See the lane eviction
2488 : comments in try_activate_block(). */
2489 1000 : fd_sched_block_t * parent_block = block_pool_ele( sched, fec->parent_bank_idx );
2490 1001 : int needs_lane = parent_block->staged || !block_is_stageable( parent_block );
2491 1000 : if( FD_UNLIKELY( needs_lane && sched->staged_bitset==fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) {
2492 : /* Every lane is taken. A demotable block is empty and waiting
2493 : for FEC sets and loses nothing by waiting unstaged, so we give
2494 : its lane to the new block. This is how the replay tile's wait
2495 : for a lane ends when no lane voluntarily frees up. A caller
2496 : that chooses to ingest while there are no demotable lanes will
2497 : get the new block unstaged. */
2498 0 : int lane_idx = pick_idle_lane( sched );
2499 0 : if( FD_LIKELY( lane_idx>=0 ) ) {
2500 0 : demote_lane( sched, lane_idx );
2501 0 : sched->metrics->lane_demoted_cnt++;
2502 0 : }
2503 0 : }
2504 1001 : if( FD_LIKELY( needs_lane && sched->staged_bitset!=fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) {
2505 : /* A lane is free. */
2506 1001 : int lane_idx = pick_free_lane( sched );
2507 1001 : sched->staged_bitset = fd_ulong_set_bit( sched->staged_bitset, lane_idx );
2508 1001 : sched->staged_head_bank_idx[ lane_idx ] = fec->bank_idx;
2509 1001 : sched->staged_popcnt_wmk = fd_ulong_max( sched->staged_popcnt_wmk, (ulong)fd_ulong_popcnt( sched->staged_bitset ) );
2510 1001 : block->in_rdisp = 1;
2511 1001 : block->staged = 1;
2512 1001 : block->staging_lane = (ulong)lane_idx;
2513 1001 : fd_rdisp_add_block( sched->rdisp, fec->bank_idx, block->staging_lane );
2514 1001 : lthash_claim_lane_map( sched, block );
2515 1001 : sched->metrics->block_added_cnt++;
2516 1001 : sched->metrics->block_added_staged_cnt++;
2517 1001 : FD_LOG_DEBUG(( "block %lu:%lu entered lane %lu: add", block->slot, fec->bank_idx, block->staging_lane ));
2518 >1844*10^16 : } else {
2519 : /* The parent is unstaged, or no lane is free. The block will be
2520 : unstaged until it is promoted. */
2521 >1844*10^16 : block->in_rdisp = 1;
2522 >1844*10^16 : block->staged = 0;
2523 >1844*10^16 : fd_rdisp_add_block( sched->rdisp, fec->bank_idx, FD_RDISP_UNSTAGED );
2524 >1844*10^16 : sched->metrics->block_added_cnt++;
2525 >1844*10^16 : sched->metrics->block_added_unstaged_cnt++;
2526 >1844*10^16 : FD_LOG_DEBUG(( "block %lu:%lu entered lane unstaged: add", block->slot, fec->bank_idx ));
2527 >1844*10^16 : }
2528 1000 : }
2529 :
2530 4556 : block->txn_pool_max_popcnt = fd_ulong_max( block->txn_pool_max_popcnt, sched->depth - sched->txn_pool_free_cnt - 1UL );
2531 4556 : block->mblk_pool_max_popcnt = fd_ulong_max( block->mblk_pool_max_popcnt, sched->depth - sched->mblk_pool_free_cnt );
2532 4556 : block->block_pool_max_popcnt = fd_ulong_max( block->block_pool_max_popcnt, sched->block_pool_popcnt );
2533 :
2534 4556 : if( FD_UNLIKELY( block->dying ) ) {
2535 : /* Ignore the FEC set for a dead block. */
2536 0 : sched->metrics->bytes_dropped_cnt += fec->fec->data_sz;
2537 0 : return 1;
2538 0 : }
2539 :
2540 4556 : if( FD_UNLIKELY( !block->in_rdisp ) ) {
2541 : /* Invariant: block must be in the dispatcher at this point. */
2542 0 : sched->print_buf_sz = 0UL;
2543 0 : print_all( sched, block );
2544 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2545 0 : FD_LOG_CRIT(( "invariant violation: block->in_rdisp==0, slot %lu, parent slot %lu",
2546 0 : block->slot, block->parent_slot ));
2547 0 : }
2548 :
2549 4556 : if( FD_UNLIKELY( block->fec_eos ) ) {
2550 : /* This means something is wrong upstream. We're getting more FEC
2551 : sets for a block that has already ended, or so we were told. */
2552 0 : sched->print_buf_sz = 0UL;
2553 0 : print_all( sched, block );
2554 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2555 0 : FD_LOG_CRIT(( "invariant violation: block->fec_eos set but getting more FEC sets, slot %lu, parent slot %lu", fec->slot, fec->parent_slot ));
2556 0 : }
2557 4556 : if( FD_UNLIKELY( block->fec_eob ) ) {
2558 : /* fec_eob is only ever set from is_last_in_batch, and the parser is
2559 : expected to finish a batch in full upon getting the FEC set that
2560 : ends the batch. An incomplete parse is caught eagerly below,
2561 : which marks the block dying and is caught above. Getting here
2562 : means the eager check below regressed. */
2563 0 : sched->print_buf_sz = 0UL;
2564 0 : print_all( sched, block );
2565 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2566 0 : FD_LOG_CRIT(( "invariant violation: block->fec_eob set but getting more FEC sets, slot %lu, parent slot %lu", fec->slot, fec->parent_slot ));
2567 0 : }
2568 4556 : if( FD_UNLIKELY( block->child_idx!=ULONG_MAX ) ) {
2569 : /* This means something is wrong upstream. FEC sets are not being
2570 : delivered in replay order. We got a child block FEC set before
2571 : this block was completely delivered. */
2572 0 : sched->print_buf_sz = 0UL;
2573 0 : print_all( sched, block );
2574 0 : fd_sched_block_t * child_block = block_pool_ele( sched, block->child_idx );
2575 0 : print_block_debug( sched, child_block );
2576 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2577 0 : FD_LOG_CRIT(( "invariant violation: block->child_idx %lu, slot %lu, parent slot %lu", block->child_idx, fec->slot, fec->parent_slot ));
2578 0 : }
2579 :
2580 : /* The parse buffer takes the residual from the previous FEC set
2581 : within the same batch, followed by the new FEC set. Both are
2582 : bounded, so this always fits. */
2583 4556 : FD_TEST( block->residual_sz<=FD_SCHED_MAX_RESIDUAL_SZ );
2584 4556 : fd_memcpy( sched->fec_buf, block->residual, block->residual_sz );
2585 4556 : fd_memcpy( sched->fec_buf+block->residual_sz, fec->data, fec->fec->data_sz );
2586 4556 : sched->fec_buf_sz = block->residual_sz+(uint)fec->fec->data_sz;
2587 4556 : sched->fec_buf_soff = 0U;
2588 4556 : sched->metrics->bytes_ingested_cnt += fec->fec->data_sz;
2589 :
2590 4556 : block->fec_eob = fec->is_last_in_batch;
2591 4556 : block->fec_eos = fec->is_last_in_block;
2592 :
2593 4556 : ulong tracked_shred_cnt = fd_ulong_min( fec->shred_cnt, FD_FEC_SHRED_CNT );
2594 4556 : uint tracked_data_sz = 0U;
2595 150316 : for( ulong i=0UL; i<tracked_shred_cnt; i++ ) tracked_data_sz += fec->fec->shred_sz[ i ];
2596 4556 : if( FD_LIKELY( fec->shred_cnt<=FD_FEC_SHRED_CNT ) ) FD_TEST( tracked_data_sz==fec->fec->data_sz );
2597 1 : else FD_TEST( tracked_data_sz<=fec->fec->data_sz );
2598 :
2599 4556 : uint prev_shred_off = 0U;
2600 150288 : for( ulong i=0; i<fec->shred_cnt; i++ ) {
2601 145732 : FD_TEST( block->shred_cnt<sched->max_shreds_per_block );
2602 145732 : uint shred_off;
2603 145733 : if( FD_LIKELY( i<FD_FEC_SHRED_CNT ) ) {
2604 145733 : shred_off = prev_shred_off + fec->fec->shred_sz[ i ];
2605 >1844*10^16 : } else if( FD_UNLIKELY( i!=fec->shred_cnt-1UL ) ) {
2606 : /* We don't track shred boundaries after 32 shreds, assume they're
2607 : sized uniformly */
2608 0 : ulong num_overflow_shreds = fec->shred_cnt-FD_FEC_SHRED_CNT;
2609 0 : ulong overflow_idx = i-FD_FEC_SHRED_CNT;
2610 0 : ulong overflow_data_sz = (ulong)fec->fec->data_sz-tracked_data_sz;
2611 0 : shred_off = tracked_data_sz + (uint)(overflow_data_sz / num_overflow_shreds * (overflow_idx + 1UL));
2612 >1844*10^16 : } else {
2613 >1844*10^16 : shred_off = (uint)fec->fec->data_sz;
2614 >1844*10^16 : }
2615 145732 : FD_TEST( shred_off>=prev_shred_off && shred_off-prev_shred_off<=USHORT_MAX );
2616 145766 : if( FD_LIKELY( block->shred_cnt-block->shred_scan_idx<FD_SCHED_SHRED_SZ_WINDOW ) ) {
2617 145766 : block->shred_sz[ block->shred_cnt&(uint)(FD_SCHED_SHRED_SZ_WINDOW-1UL) ] = (ushort)(shred_off-prev_shred_off);
2618 >1844*10^16 : } else {
2619 >1844*10^16 : block->shred_sz_overflow = 1;
2620 >1844*10^16 : }
2621 145732 : block->shred_cnt++;
2622 145732 : prev_shred_off = shred_off;
2623 145732 : }
2624 :
2625 4556 : block->fec_completed_ns = fec->completed_ns;
2626 :
2627 4556 : int err = fd_sched_parse( sched, block, fec->alut_ctx );
2628 :
2629 4556 : if( FD_UNLIKELY( err!=FD_SCHED_DEAD_REASON_NONE ) ) {
2630 0 : handle_bad_block( sched, block, err );
2631 0 : sched->metrics->bytes_dropped_cnt += sched->fec_buf_sz-sched->fec_buf_soff;
2632 0 : return 0;
2633 0 : }
2634 :
2635 4556 : if( FD_UNLIKELY( (fec->is_last_in_batch||fec->is_last_in_block) && (block->txns_rem||block->mblks_rem||block->fec_eob) ) ) {
2636 : /* A malformed block that fails to parse out exactly as many
2637 : transactions and microblocks as it should.
2638 :
2639 : Upon getting a last-in-batch FEC, everything in the ongoing batch
2640 : should completely parse, and the eob flag should be reset. There
2641 : should be no go around for a last-in-batch FEC. */
2642 2 : FD_LOG_INFO(( "bad block: SHORT_BLOCK, bytes_rem %u, txns_rem %lu, mblks_rem %lu, fec_eob %d, slot %lu, parent slot %lu", sched->fec_buf_sz-sched->fec_buf_soff, block->txns_rem, block->mblks_rem, block->fec_eob, block->slot, block->parent_slot ));
2643 2 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_SHORT_BLOCK );
2644 2 : return 0;
2645 2 : }
2646 :
2647 4554 : if( FD_UNLIKELY( block->fec_eos && !block->last_mblk_is_tick ) ) {
2648 : /* The last microblock should be a tick.
2649 :
2650 : Note that this early parse-time detection could cause us to throw
2651 : a slightly different error from Agave, in the case that there are
2652 : too few ticks, since the tick count check precedes the trailing
2653 : entry check in Agave. That being said, ultimately a
2654 : TRAILING_ENTRY renders a block invalid, regardless of anything
2655 : else. */
2656 0 : FD_LOG_INFO(( "bad block: TRAILING_ENTRY, slot %lu, parent slot %lu, mblk_cnt %u", block->slot, block->parent_slot, block->mblk_cnt ));
2657 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_TRAILING_ENTRY );
2658 0 : return 0;
2659 0 : }
2660 :
2661 4554 : if( FD_UNLIKELY( block->fec_eos ) ) {
2662 991 : int dead_reason = on_final( block );
2663 991 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
2664 0 : handle_bad_block( sched, block, dead_reason );
2665 0 : return 0;
2666 0 : }
2667 991 : }
2668 :
2669 : /* Park the residual until the next FEC set. The parser stops only on
2670 : a parseable unit it has not fully received. A parseable unit
2671 : includes a transaction, a block marker, or a microblock or batch
2672 : header. The parser drops the trailing bytes of a finished batch.
2673 : So a residual larger than any of those parseable units is malformed
2674 : content. Instead of crashing, we consider the block invalid. */
2675 4554 : uint residual_sz = sched->fec_buf_sz-sched->fec_buf_soff;
2676 4554 : if( FD_UNLIKELY( residual_sz>FD_SCHED_MAX_RESIDUAL_SZ ) ) {
2677 0 : FD_LOG_INFO(( "bad block: UNPARSEABLE_CONTENT, residual_sz %u, slot %lu, parent slot %lu", residual_sz, fec->slot, fec->parent_slot ));
2678 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_UNPARSEABLE_CONTENT );
2679 0 : sched->metrics->bytes_dropped_cnt += residual_sz;
2680 0 : return 0;
2681 0 : }
2682 4554 : fd_memcpy( block->residual, sched->fec_buf+sched->fec_buf_soff, residual_sz );
2683 4554 : block->fec_buf_boff += sched->fec_buf_soff;
2684 4554 : block->residual_sz = residual_sz;
2685 : /* Pass the shreds the next transaction cannot start in, so that few
2686 : shred sizes stay outstanding between FEC sets. */
2687 4554 : shred_split( block, block->fec_buf_boff );
2688 :
2689 : /* We just received a FEC set, which may have made all transactions in
2690 : a partially parsed microblock available. If this were a malformed
2691 : block that ends in a non-tick microblock, there's not going to be a
2692 : hashing task from the missing ending tick to drain the mixin queue.
2693 : So we try to drain the mixin queue right here. Another option is
2694 : to drain it at dispatch time, when we are about to dispatch the end
2695 : of block signal, right before the check for whether block should
2696 : end. */
2697 4554 : int mixin_res;
2698 4554 : while( (mixin_res=maybe_mixin( sched, block )) ) {
2699 0 : if( FD_UNLIKELY( mixin_res==-1 ) ) {
2700 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_ENTRY_HASH_MISMATCH_INGEST );
2701 0 : return 0;
2702 0 : }
2703 0 : FD_TEST( mixin_res==1||mixin_res==2 );
2704 0 : }
2705 :
2706 : /* Every account the block intends to write has now been observed.
2707 : Each gets a pseudo transaction for its LtHash addition, created by
2708 : polls that find nothing to dispatch. */
2709 4554 : if( FD_UNLIKELY( fec->is_last_in_block ) ) lthash_drain_later( sched, block );
2710 :
2711 4554 : slot_state_eval( sched, block );
2712 :
2713 : /* Check if we need to set the active block. */
2714 4554 : check_or_set_active_block( sched );
2715 :
2716 4554 : return 1;
2717 4554 : }
2718 :
2719 : /* A cert holds a tile for about a millisecond. So footer cert verify
2720 : tasks follow the LtHash batch policy. They go out only when no
2721 : transaction is ready, leave the hash batch reserve idle, and are
2722 : offered from the footer's parse until BLOCK_END. Like a hash batch
2723 : it takes the highest idle tile. Returns 1 if a task was written to
2724 : out, 0 otherwise. */
2725 : static int
2726 : cert_dispatch( fd_sched_t * sched,
2727 : fd_sched_block_t * block,
2728 : ulong bank_idx,
2729 : ulong ready_bitset,
2730 : int reserve,
2731 0 : fd_sched_task_t * out ) {
2732 0 : if( FD_LIKELY( !block->footer_seen || block->cert_next>=block->cert_cnt || block->block_end_signaled ) ) return 0;
2733 0 : if( FD_UNLIKELY( (ulong)fd_ulong_popcnt( ready_bitset )<=(ulong)reserve ) ) return 0;
2734 0 : int t = fd_ulong_find_msb( ready_bitset );
2735 0 : sched->footer_ready_bitset[ 0 ] = fd_ulong_clear_bit( sched->footer_ready_bitset[ 0 ], t );
2736 0 : sched->tile_to_bank_idx[ t ] = bank_idx;
2737 0 : block->footer_in_flight_cnt++;
2738 0 : out->task_type = FD_SCHED_TT_CERT_VERIFY;
2739 0 : out->cert_verify->bank_idx = bank_idx;
2740 0 : out->cert_verify->cert_idx = (ulong)block->cert_next++;
2741 0 : out->cert_verify->exec_idx = (ulong)t;
2742 0 : return 1;
2743 0 : }
2744 :
2745 : /* If may_drain is set, allow one drain chunk once nothing else can be
2746 : dispatched. The re-entry the chunk earns passes 0, so a poll drains
2747 : at most once. */
2748 : static ulong
2749 0 : task_next_ready( fd_sched_t * sched, fd_sched_task_t * out, int may_drain ) {
2750 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
2751 0 : FD_TEST( ref_q_empty( sched->ref_q ) );
2752 :
2753 0 : ulong exec_ready_bitset0 = sched->txn_exec_ready_bitset[ 0 ];
2754 0 : ulong exec_fully_ready_bitset = sched->sigverify_ready_bitset[ 0 ] & sched->poh_ready_bitset[ 0 ] & sched->footer_ready_bitset[ 0 ] & sched->lthash_ready_bitset[ 0 ] & exec_ready_bitset0;
2755 0 : if( FD_UNLIKELY( !exec_fully_ready_bitset ) ) {
2756 : /* Early exit if no exec tiles available. Also let's make some
2757 : progress on a pending drain. */
2758 0 : lthash_drain_chunk( sched );
2759 0 : return 0UL;
2760 0 : }
2761 :
2762 0 : if( FD_UNLIKELY( sched->active_bank_idx==ULONG_MAX ) ) {
2763 : /* No need to try activating a block. If we're in this state,
2764 : there's truly nothing to execute. We will activate something
2765 : when we ingest a FEC set with transactions. */
2766 0 : return 0UL;
2767 0 : }
2768 :
2769 0 : out->task_type = FD_SCHED_TT_NULL;
2770 :
2771 : /* We could in theory reevaluate staging lane allocation here and do
2772 : promotion/demotion as needed. It's a policy decision to minimize
2773 : fork churn for now and just execute down the same active fork. */
2774 :
2775 0 : ulong bank_idx = sched->active_bank_idx;
2776 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
2777 0 : if( FD_UNLIKELY( block_should_deactivate( sched, block ) ) ) {
2778 0 : sched->print_buf_sz = 0UL;
2779 0 : print_all( sched, block );
2780 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2781 0 : FD_LOG_CRIT(( "invariant violation: active block %lu:%lu is not activatable nor has anything in-flight", block->slot, sched->active_bank_idx ));
2782 0 : }
2783 :
2784 0 : block->txn_pool_max_popcnt = fd_ulong_max( block->txn_pool_max_popcnt, sched->depth - sched->txn_pool_free_cnt - 1UL );
2785 0 : block->mblk_pool_max_popcnt = fd_ulong_max( block->mblk_pool_max_popcnt, sched->depth - sched->mblk_pool_free_cnt );
2786 0 : block->block_pool_max_popcnt = fd_ulong_max( block->block_pool_max_popcnt, sched->block_pool_popcnt );
2787 :
2788 0 : if( FD_UNLIKELY( !block->block_start_signaled ) ) {
2789 0 : if( !block->lthash_mode_picked ) lthash_pick_mode( sched, block );
2790 0 : out->task_type = FD_SCHED_TT_BLOCK_START;
2791 0 : out->block_start->bank_idx = bank_idx;
2792 0 : out->block_start->parent_bank_idx = block->parent_idx;
2793 0 : out->block_start->slot = block->slot;
2794 0 : out->block_start->lthash_inband = block->lthash_inband;
2795 0 : block->block_start_signaled = 1;
2796 0 : slot_state_open( sched, block );
2797 0 : sched->next_ready_last_tick = fd_tickcount();
2798 0 : sched->next_ready_last_bank_idx = bank_idx;
2799 0 : return 1UL;
2800 0 : }
2801 :
2802 : /* Transactions, PoH and sigverify take the lowest idle tile. LtHash
2803 : batches take the highest. */
2804 0 : ulong exec_tile_idx0 = fd_ulong_if( !!exec_fully_ready_bitset, (ulong)fd_ulong_find_lsb( exec_fully_ready_bitset ), ULONG_MAX );
2805 0 : ulong exec_tile_idx1 = fd_ulong_if( !!exec_fully_ready_bitset, (ulong)fd_ulong_find_msb( exec_fully_ready_bitset ), ULONG_MAX );
2806 0 : ulong idle_cnt = (ulong)fd_ulong_popcnt( exec_fully_ready_bitset );
2807 :
2808 : /* Every task but a transaction leaves reserve tiles idle, a hash
2809 : batch lthash_reserve. A hash batch carries at most lthash_cap
2810 : accounts. */
2811 0 : int reserve = idle_reserve( sched, block );
2812 0 : int lthash_reserve = lthash_idle_reserve( sched );
2813 0 : ulong lthash_cap = fd_ulong_if( block->txn_exec_in_flight_cnt>0U, FD_SCHED_LTHASH_BATCH_BUSY_MAX, FD_SCHED_LTHASH_BATCH_MAX );
2814 :
2815 0 : ulong exec_queued_cnt = block->txn_parsed_cnt-block->txn_exec_in_flight_cnt-block->txn_exec_done_cnt;
2816 0 : int add_unsurfaced = block->add_created_cnt>block->add_surfaced_cnt; /* Addition pseudo transactions are queued in rdisp along with real transactions. */
2817 0 : int add_held = sched->lthash_add_batch->cnt>0UL;
2818 0 : ulong txn_idx = 0UL;
2819 0 : if( FD_LIKELY( (exec_queued_cnt>0UL || add_unsurfaced || add_held) && idle_cnt ) ) { /* Optimize for no fork switching. */
2820 : /* Transaction execution has the highest priority. Current mainnet
2821 : block times are very much dominated by critical path transaction
2822 : execution. To achieve the fastest block replay speed, we can't
2823 : afford to make any mistake in critical path dispatching. Any
2824 : deviation from perfect critical path dispatching is basically
2825 : irrecoverable. As such, we try to keep all the exec tiles busy
2826 : with transaction execution, but we allow at most one transaction
2827 : to be in-flight per exec tile. This is to ensure that whenever a
2828 : critical path transaction completes, we have at least one exec
2829 : tile, e.g. the one that just completed said transaction, readily
2830 : available to continue executing down the critical path. */
2831 0 : ulong appended_cnt = 0UL; /* Additions this poll batched. */
2832 0 : txn_idx = fd_rdisp_get_next_ready( sched->rdisp, bank_idx );
2833 0 : while( FD_UNLIKELY( txn_idx & FD_RDISP_LTHASH_PSEUDO_TXN ) ) {
2834 : /* A pseudo transaction surfaced. By rdisp's scoring policy we
2835 : know that this implies no real transaction is ready. */
2836 0 : block->state_conflict = (uchar)( exec_queued_cnt>0UL );
2837 0 : fd_sched_lthash_entry_t * e = lthash_surface( sched, block, bank_idx, txn_idx );
2838 0 : if( FD_LIKELY( e ) ) {
2839 : /* ... and the addition is not stale. */
2840 0 : fd_sched_lthash_batch_t * batch = sched->lthash_add_batch;
2841 0 : if( FD_UNLIKELY( e->spec_idx!=UINT_MAX ) ) {
2842 : /* This is a speculative addition that waited behind a
2843 : serializing transaction. The speculation goes out under
2844 : the same idleness policy as lthash_speculate(), and could
2845 : be given up rather than forced onto the last idle tile.
2846 : Once the block is complete it is given up outright. Giving
2847 : up creates an addition sequenced behind every real
2848 : transaction, as desired. If we did a speculation here,
2849 : there's a small chance it might be undone. */
2850 0 : if( FD_LIKELY( !block->fec_eos && idle_cnt>(ulong)reserve ) ) {
2851 0 : lthash_dispatch_one_add( sched, block, bank_idx, e, txn_idx, (int)exec_tile_idx1, out );
2852 0 : sched->metrics->lthash_add_batch_held_cnt += (ulong)(batch->cnt>0UL);
2853 0 : sched->next_ready_last_tick = fd_tickcount();
2854 0 : sched->next_ready_last_bank_idx = bank_idx;
2855 0 : return 1UL;
2856 0 : }
2857 0 : lthash_spec_give_up( sched, block, bank_idx, e, txn_idx );
2858 0 : } else {
2859 0 : if( FD_UNLIKELY( !batch->cnt ) ) lthash_batch_init( block, batch, bank_idx, FD_SCHED_TT_LTHASH_ADD );
2860 0 : lthash_batch_append( sched, batch, e, txn_idx );
2861 0 : appended_cnt++;
2862 0 : if( FD_UNLIKELY( batch->cnt==FD_SCHED_LTHASH_BATCH_MAX ) ) {
2863 : /* A full batch goes out even if we're down to the last idle
2864 : tile, since the next poll could give us an addition and
2865 : we'll have no space to batch it. */
2866 0 : lthash_dispatch_held_batch( sched, block, bank_idx, (int)exec_tile_idx1, out );
2867 0 : sched->next_ready_last_tick = fd_tickcount();
2868 0 : sched->next_ready_last_bank_idx = bank_idx;
2869 0 : return 1UL;
2870 0 : }
2871 0 : if( FD_UNLIKELY( batch->cnt>=lthash_cap ) ) {
2872 : /* The batch is big enough to go out. The addition just
2873 : appended implies no transaction is ready, otherwise rdisp
2874 : would have returned the transaction instead of the
2875 : addition. So we're not leaving critical path work on the
2876 : table. */
2877 0 : txn_idx = 0UL;
2878 0 : break;
2879 0 : }
2880 0 : }
2881 0 : }
2882 0 : txn_idx = fd_rdisp_get_next_ready( sched->rdisp, bank_idx );
2883 0 : }
2884 0 : block->state_conflict = (uchar)( !txn_idx && exec_queued_cnt>0UL );
2885 0 : if( FD_UNLIKELY( sched->lthash_add_batch->cnt ) ) {
2886 0 : if( FD_LIKELY( !txn_idx ) ) {
2887 : /* Nothing is ready behind the batch, or the batch is at its
2888 : cap. */
2889 0 : if( FD_LIKELY( idle_cnt>(ulong)lthash_reserve ) ) {
2890 0 : lthash_dispatch_held_batch( sched, block, bank_idx, (int)exec_tile_idx1, out );
2891 0 : sched->next_ready_last_tick = fd_tickcount();
2892 0 : sched->next_ready_last_bank_idx = bank_idx;
2893 0 : return 1UL;
2894 0 : }
2895 : /* Too few idle tiles for a hash batch. The batch stays held
2896 : and goes out from a later call, once a poll finds nothing
2897 : ready and enough tiles are idle. */
2898 0 : sched->metrics->lthash_add_batch_wait_idle_cnt++;
2899 0 : } else if( appended_cnt ) {
2900 : /* A real transaction surfaced after we've batched some
2901 : additions in this poll. */
2902 0 : sched->metrics->lthash_add_batch_held_cnt++;
2903 0 : }
2904 0 : }
2905 0 : if( FD_UNLIKELY( txn_idx==0UL && exec_queued_cnt>0UL ) ) {
2906 : /* There are transactions queued but none ready for execution.
2907 : This implies that there must be in-flight transactions on whose
2908 : completion the queued transactions depend. A queued
2909 : transaction may also be waiting on a dispatched speculative
2910 : addition. */
2911 0 : if( FD_UNLIKELY( !block->txn_exec_in_flight_cnt && block->add_surfaced_cnt==block->add_done_cnt ) ) {
2912 0 : sched->print_buf_sz = 0UL;
2913 0 : print_all( sched, block );
2914 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
2915 0 : FD_LOG_CRIT(( "invariant violation: no ready transaction found but block->txn_exec_in_flight_cnt==0" ));
2916 0 : }
2917 :
2918 : /* Next up are PoH tasks. Same dispatching policy as sigverify
2919 : tasks. */
2920 0 : ulong poh_ready_bitset = exec_fully_ready_bitset;
2921 0 : ulong poh_hashing_queued_cnt = block->mblk_cnt-block->poh_mblk_in_flight_cnt-block->poh_hashing_done_cnt;
2922 0 : int poh_threshold = reserve;
2923 0 : if( FD_LIKELY( poh_hashing_queued_cnt>0UL && fd_ulong_popcnt( poh_ready_bitset )>poh_threshold ) ) {
2924 0 : dispatch_poh( sched, block, bank_idx, fd_ulong_find_lsb( poh_ready_bitset ), out );
2925 0 : sched->next_ready_last_tick = fd_tickcount();
2926 0 : sched->next_ready_last_bank_idx = bank_idx;
2927 0 : return 1UL;
2928 0 : }
2929 :
2930 : /* Dispatch more sigverify tasks to completely idle tiles over the
2931 : reserve, at most one in flight per tile. */
2932 0 : ulong sigverify_ready_bitset = exec_fully_ready_bitset;
2933 0 : ulong sigverify_queued_cnt = block->txn_parsed_cnt-block->txn_sigverify_in_flight_cnt-block->txn_sigverify_done_cnt;
2934 0 : if( FD_LIKELY( sigverify_queued_cnt>0UL && fd_ulong_popcnt( sigverify_ready_bitset )>reserve ) ) {
2935 0 : dispatch_sigverify( sched, block, bank_idx, fd_ulong_find_lsb( sigverify_ready_bitset ), out );
2936 0 : sched->next_ready_last_tick = sched->txn_info_pool[ out->txn_sigverify->txn_idx ].tick_sigverify_disp = fd_tickcount();
2937 0 : sched->next_ready_last_bank_idx = bank_idx;
2938 0 : return 1UL;
2939 0 : }
2940 :
2941 : /* Then footer certs and LtHash subtractions, under the hash batch
2942 : reserve. */
2943 0 : if( FD_UNLIKELY( cert_dispatch( sched, block, bank_idx, exec_fully_ready_bitset, lthash_reserve, out ) ) ) {
2944 0 : sched->next_ready_last_tick = fd_tickcount();
2945 0 : sched->next_ready_last_bank_idx = bank_idx;
2946 0 : return 1UL;
2947 0 : }
2948 0 : if( FD_UNLIKELY( lthash_dispatch_sub( sched, block, bank_idx, exec_fully_ready_bitset, lthash_reserve, lthash_cap, out ) ) ) {
2949 0 : sched->next_ready_last_tick = fd_tickcount();
2950 0 : sched->next_ready_last_bank_idx = bank_idx;
2951 0 : return 1UL;
2952 0 : }
2953 : /* Nothing the block must do could go out, so let's make some
2954 : progress on a pending drain. A pseudo transaction the drain
2955 : created may be ready, so poll once more. */
2956 0 : if( may_drain && lthash_drain_chunk( sched ) ) return task_next_ready( sched, out, 0 );
2957 :
2958 : /* Now try to dispatch a speculative addition, the same way we do
2959 : in the branch where the transaction queue is empty. Every
2960 : account we select as a speculation candidate has seen its last
2961 : parsed writer finish, so the speculation doesn't block any
2962 : queued transactions. While having lthash additions in flight
2963 : could in theory slightly delay transaction dispatch when the
2964 : blocker transaction returns, the delay should be minimal. FECs
2965 : also tend to end on a long serial transaction path, so there's
2966 : typically not a wide DAG phase being starved by lthash
2967 : additions. More often, not allowing lthash speculations to go
2968 : out in this branch causes them to be clustered at the end of a
2969 : mid-slot FEC, after the last transaction of the FEC has been
2970 : dispatched. This has a good chance to lead to lthash
2971 : speculations finishing well after the last transaction of the
2972 : FEC has returned, which is not ideal. */
2973 0 : if( FD_LIKELY( block_may_speculate( sched, block, 1UL ) ) ) {
2974 0 : int spec = lthash_speculate( sched, block, bank_idx, exec_fully_ready_bitset, out );
2975 0 : if( FD_UNLIKELY( spec>=0 ) ) {
2976 0 : if( spec ) {
2977 0 : sched->next_ready_last_tick = fd_tickcount();
2978 0 : sched->next_ready_last_bank_idx = bank_idx;
2979 0 : }
2980 0 : return (ulong)spec;
2981 0 : }
2982 0 : }
2983 0 : return 0UL;
2984 0 : }
2985 0 : }
2986 0 : if( FD_LIKELY( txn_idx ) ) {
2987 0 : out->task_type = FD_SCHED_TT_TXN_EXEC;
2988 0 : out->txn_exec->txn_idx = txn_idx;
2989 0 : out->txn_exec->bank_idx = bank_idx;
2990 0 : out->txn_exec->slot = block->slot;
2991 0 : out->txn_exec->exec_idx = exec_tile_idx0;
2992 0 : FD_TEST( out->txn_exec->exec_idx!=ULONG_MAX );
2993 :
2994 0 : long now = fd_tickcount();
2995 0 : ulong delta = (ulong)(now-sched->txn_in_flight_last_tick);
2996 0 : ulong txn_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( exec_ready_bitset0 );
2997 0 : sched->metrics->txn_none_in_flight_tickcount += fd_ulong_if( txn_exec_busy_cnt==0UL, delta, 0UL );
2998 0 : sched->metrics->txn_weighted_in_flight_tickcount += fd_ulong_if( txn_exec_busy_cnt!=0UL, delta, 0UL );
2999 0 : sched->metrics->txn_weighted_in_flight_cnt += delta*txn_exec_busy_cnt;
3000 0 : sched->txn_in_flight_last_tick = now;
3001 :
3002 0 : sched->txn_info_pool[ out->txn_exec->txn_idx ].tick_exec_disp = now;
3003 0 : sched->txn_info_pool[ out->txn_exec->txn_idx ].exec_tile_idx = exec_tile_idx0;
3004 :
3005 0 : sched->txn_exec_ready_bitset[ 0 ] = fd_ulong_clear_bit( exec_ready_bitset0, (int)exec_tile_idx0);
3006 0 : sched->tile_to_bank_idx[ exec_tile_idx0 ] = bank_idx;
3007 :
3008 0 : block->txn_exec_in_flight_cnt++;
3009 0 : sched->metrics->txn_max_in_flight_cnt = fd_uint_max( sched->metrics->txn_max_in_flight_cnt, block->txn_exec_in_flight_cnt );
3010 :
3011 0 : exec_busy_check( sched, block );
3012 0 : sched->next_ready_last_tick = now;
3013 0 : sched->next_ready_last_bank_idx = bank_idx;
3014 0 : return 1UL;
3015 0 : }
3016 :
3017 : /* At this point txn_queued_cnt==0. Any unsurfaced addition waits on
3018 : an in-flight writer or a serializing transaction. */
3019 :
3020 : /* Next up are PoH tasks. Same dispatching policy as sigverify. */
3021 0 : ulong poh_ready_bitset = exec_fully_ready_bitset;
3022 0 : ulong poh_hashing_queued_cnt = block->mblk_cnt-block->poh_mblk_in_flight_cnt-block->poh_hashing_done_cnt;
3023 0 : int poh_threshold = reserve;
3024 0 : if( FD_LIKELY( poh_hashing_queued_cnt>0UL && fd_ulong_popcnt( poh_ready_bitset )>poh_threshold ) ) {
3025 0 : dispatch_poh( sched, block, bank_idx, fd_ulong_find_lsb( poh_ready_bitset ), out );
3026 0 : sched->next_ready_last_tick = fd_tickcount();
3027 0 : sched->next_ready_last_bank_idx = bank_idx;
3028 0 : return 1UL;
3029 0 : }
3030 :
3031 : /* Try to dispatch a sigverify task to a qualified idle tile. */
3032 0 : ulong sigverify_ready_bitset = exec_fully_ready_bitset;
3033 0 : ulong sigverify_queued_cnt = block->txn_parsed_cnt-block->txn_sigverify_in_flight_cnt-block->txn_sigverify_done_cnt;
3034 0 : if( FD_LIKELY( sigverify_queued_cnt>0UL && fd_ulong_popcnt( sigverify_ready_bitset )>reserve ) ) {
3035 0 : dispatch_sigverify( sched, block, bank_idx, fd_ulong_find_lsb( sigverify_ready_bitset ), out );
3036 0 : sched->next_ready_last_tick = sched->txn_info_pool[ out->txn_sigverify->txn_idx ].tick_sigverify_disp = fd_tickcount();
3037 0 : sched->next_ready_last_bank_idx = bank_idx;
3038 0 : return 1UL;
3039 0 : }
3040 :
3041 : /* Then footer certs and LtHash subtractions, under the hash batch
3042 : reserve. */
3043 0 : if( FD_UNLIKELY( cert_dispatch( sched, block, bank_idx, exec_fully_ready_bitset, lthash_reserve, out ) ) ) {
3044 0 : sched->next_ready_last_tick = fd_tickcount();
3045 0 : sched->next_ready_last_bank_idx = bank_idx;
3046 0 : return 1UL;
3047 0 : }
3048 0 : if( FD_UNLIKELY( lthash_dispatch_sub( sched, block, bank_idx, exec_fully_ready_bitset, lthash_reserve, lthash_cap, out ) ) ) {
3049 0 : sched->next_ready_last_tick = fd_tickcount();
3050 0 : sched->next_ready_last_bank_idx = bank_idx;
3051 0 : return 1UL;
3052 0 : }
3053 : /* Same as above, make some progress on a pending drain and poll once
3054 : more for what the chunk created. */
3055 0 : if( may_drain && lthash_drain_chunk( sched ) ) return task_next_ready( sched, out, 0 );
3056 :
3057 : /* Absolutely nothing else left to dispatch. If the block is still
3058 : receiving FEC sets and owes additions, we will spend the idle tiles
3059 : on speculative additions. Speculation is the lowest priority work
3060 : there is, though. So if nothing of the block is in flight and
3061 : another lane head is ready with must-do work, yield. */
3062 0 : int may_speculate = block_may_speculate( sched, block, 1UL );
3063 0 : if( FD_LIKELY( may_speculate ) ) {
3064 0 : if( FD_UNLIKELY( !block_is_in_flight( block ) && other_lane_has_real_work( sched, block ) ) ) {
3065 0 : FD_LOG_DEBUG(( "reset active_bank_idx %lu: speculation yields", sched->active_bank_idx ));
3066 0 : sched->last_active_bank_idx = sched->active_bank_idx;
3067 0 : sched->active_bank_idx = ULONG_MAX;
3068 0 : sched->metrics->lthash_spec_yield_cnt++;
3069 0 : try_activate_block( sched );
3070 0 : return 0UL;
3071 0 : }
3072 0 : int spec = lthash_speculate( sched, block, bank_idx, exec_fully_ready_bitset, out );
3073 0 : if( FD_UNLIKELY( spec>=0 ) ) {
3074 0 : if( spec ) {
3075 0 : sched->next_ready_last_tick = fd_tickcount();
3076 0 : sched->next_ready_last_bank_idx = bank_idx;
3077 0 : }
3078 0 : return (ulong)spec;
3079 0 : }
3080 0 : }
3081 :
3082 : /* A complete block may still have additions to hand to the
3083 : dispatcher, which we do a chunk at a time. So there could be
3084 : moments when every task we handed out has come back, nothing is
3085 : running, and yet the block cannot end, because we still owe
3086 : additions. Another poll will surface them. Return empty handed,
3087 : since we know the block cannot end yet, and since the liveness
3088 : check at the end of this function would otherwise mistake a block
3089 : with unsurfaced additions for being stuck and abort. */
3090 0 : if( FD_UNLIKELY( block->lthash_draining || block->add_created_cnt>block->add_surfaced_cnt ) ) return 0UL;
3091 :
3092 0 : if( FD_UNLIKELY( block_should_signal_end( block ) ) ) {
3093 0 : FD_TEST( block->block_start_signaled );
3094 0 : if( FD_UNLIKELY( block->add_owed_cnt ) ) {
3095 0 : sched->print_buf_sz = 0UL;
3096 0 : print_all( sched, block );
3097 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
3098 0 : FD_LOG_CRIT(( "invariant violation: block %lu:%lu ends owing %u additions", block->slot, bank_idx, block->add_owed_cnt ));
3099 0 : }
3100 0 : int tick_reason = verify_ticks_final( block );
3101 0 : if( FD_UNLIKELY( tick_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
3102 : /* Tick verification can't be done at parse time (except for
3103 : TRAILING_ENTRY), because we may not know the expected number of
3104 : hashes yet. It can't be driven by transaction dispatch or
3105 : completion, because the block may be empty. Similarly, it can't
3106 : be driven by PoH hashing, because a bad block may simply not
3107 : have any microblocks. */
3108 0 : handle_bad_block( sched, block, tick_reason );
3109 0 : out->task_type = FD_SCHED_TT_MARK_DEAD;
3110 0 : out->mark_dead->bank_idx = bank_idx;
3111 0 : sched->next_ready_last_tick = fd_tickcount();
3112 0 : sched->next_ready_last_bank_idx = bank_idx;
3113 0 : return 1UL;
3114 0 : }
3115 0 : out->task_type = FD_SCHED_TT_BLOCK_END;
3116 0 : out->block_end->bank_idx = bank_idx;
3117 0 : block->block_end_signaled = 1;
3118 0 : FD_TEST( block->refcnt );
3119 0 : block->refcnt = 0;
3120 0 : FD_TEST( ref_q_avail( sched->ref_q ) );
3121 0 : ref_q_push_tail( sched->ref_q, bank_idx );
3122 0 : sched->next_ready_last_tick = fd_tickcount();
3123 0 : sched->next_ready_last_bank_idx = bank_idx;
3124 0 : return 1UL;
3125 0 : }
3126 :
3127 : /* Nothing queued for the active block. If we haven't received all
3128 : the FEC sets for it, then return and wait for more FEC sets, while
3129 : there are in-flight transactions. This is a policy decision to
3130 : minimize fork churn and allow for executing down the current fork
3131 : as much as we can. If we have received all the FEC sets for it,
3132 : then we'd still like to return and wait for the in-flight
3133 : transactions to retire, before switching to a different block.
3134 :
3135 : Either way, there should be in-flight transactions. We deactivate
3136 : the active block the moment we exhausted transactions from it.
3137 :
3138 : We don't assert block_is_in_flight() here because there might be
3139 : in-flight tasks from dying blocks that are preventing us from
3140 : dispatching anything momentarily. So we assert the global exec
3141 : tile busy count. This doesn't lose too much assertion coverage
3142 : since dying blocks are few and far between. */
3143 0 : ulong total_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( exec_fully_ready_bitset );
3144 0 : if( FD_UNLIKELY( !total_exec_busy_cnt ) ) {
3145 0 : sched->print_buf_sz = 0UL;
3146 0 : print_all( sched, block );
3147 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
3148 0 : FD_LOG_CRIT(( "invariant violation: expected in-flight tasks but none found" ));
3149 0 : }
3150 :
3151 0 : return 0UL;
3152 0 : }
3153 :
3154 : ulong
3155 0 : fd_sched_task_next_ready( fd_sched_t * sched, fd_sched_task_t * out ) {
3156 0 : ulong bank_idx = sched->active_bank_idx;
3157 0 : ulong ret = task_next_ready( sched, out, 1 );
3158 0 : if( FD_LIKELY( bank_idx!=ULONG_MAX ) ) slot_state_eval( sched, block_pool_ele( sched, bank_idx ) );
3159 0 : return ret;
3160 0 : }
3161 :
3162 : int
3163 0 : fd_sched_task_done( fd_sched_t * sched, ulong task_type, ulong txn_idx, ulong exec_idx, void * data ) {
3164 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3165 :
3166 0 : ulong bank_idx = ULONG_MAX;
3167 0 : switch( task_type ) {
3168 0 : case FD_SCHED_TT_BLOCK_START:
3169 0 : case FD_SCHED_TT_BLOCK_END: {
3170 0 : (void)txn_idx;
3171 0 : (void)data;
3172 0 : bank_idx = sched->active_bank_idx;
3173 0 : break;
3174 0 : }
3175 0 : case FD_SCHED_TT_TXN_EXEC:
3176 0 : case FD_SCHED_TT_TXN_SIGVERIFY: {
3177 0 : (void)data;
3178 0 : FD_TEST( txn_idx < sched->depth );
3179 0 : bank_idx = sched->tile_to_bank_idx[ exec_idx ];
3180 0 : break;
3181 0 : }
3182 0 : case FD_SCHED_TT_POH_HASH:
3183 0 : case FD_SCHED_TT_CERT_VERIFY:
3184 0 : case FD_SCHED_TT_REWARDS: {
3185 0 : (void)txn_idx;
3186 0 : bank_idx = sched->tile_to_bank_idx[ exec_idx ];
3187 0 : break;
3188 0 : }
3189 0 : case FD_SCHED_TT_LTHASH_SUB:
3190 0 : case FD_SCHED_TT_LTHASH_ADD: {
3191 : /* Like any other tile task, an LtHash task keeps its block in the
3192 : scheduler and the dispatcher until it lands. block_is_in_flight
3193 : counts it, which gates abandon, demotion and pruning, and
3194 : BLOCK_END waits on it, so the block checks below hold for a
3195 : result of either kind. */
3196 0 : FD_TEST( exec_idx<sched->exec_cnt );
3197 0 : FD_TEST( data );
3198 0 : bank_idx = sched->tile_to_bank_idx[ exec_idx ];
3199 0 : break;
3200 0 : }
3201 0 : default: FD_LOG_CRIT(( "unsupported task_type %lu", task_type ));
3202 0 : }
3203 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3204 :
3205 0 : if( FD_UNLIKELY( !block->in_sched ) ) {
3206 0 : FD_LOG_CRIT(( "invariant violation: block->in_sched==0, block %lu:%lu, parent slot %lu",
3207 0 : block->slot, bank_idx, block->parent_slot ));
3208 0 : }
3209 0 : if( FD_UNLIKELY( !block->staged ) ) {
3210 : /* Invariant: only staged blocks can have in-flight transactions. */
3211 0 : FD_LOG_CRIT(( "invariant violation: block->staged==0, block %lu:%lu, parent slot %lu",
3212 0 : block->slot, bank_idx, block->parent_slot ));
3213 0 : }
3214 0 : if( FD_UNLIKELY( !block->in_rdisp ) ) {
3215 : /* Invariant: staged blocks must be in the dispatcher. */
3216 0 : FD_LOG_CRIT(( "invariant violation: block->in_rdisp==0, block %lu:%lu, parent slot %lu",
3217 0 : block->slot, bank_idx, block->parent_slot ));
3218 0 : }
3219 :
3220 0 : block->txn_pool_max_popcnt = fd_ulong_max( block->txn_pool_max_popcnt, sched->depth - sched->txn_pool_free_cnt - 1UL );
3221 0 : block->mblk_pool_max_popcnt = fd_ulong_max( block->mblk_pool_max_popcnt, sched->depth - sched->mblk_pool_free_cnt );
3222 0 : block->block_pool_max_popcnt = fd_ulong_max( block->block_pool_max_popcnt, sched->block_pool_popcnt );
3223 :
3224 0 : int exec_tile_idx = (int)exec_idx;
3225 :
3226 0 : switch( task_type ) {
3227 0 : case FD_SCHED_TT_BLOCK_START: {
3228 0 : FD_TEST( !block->block_start_done );
3229 0 : block->block_start_done = 1;
3230 0 : if( FD_UNLIKELY( block->starts_at_update_parent && !block->flh_active ) ) {
3231 : /* Without the feature an initial UpdateParent is unexpected, as
3232 : agave rules it. The UpdateParent was parsed at ingest, but
3233 : the feature is only known once BLOCK_START cloned the bank. */
3234 0 : FD_LOG_INFO(( "bad block: SPURIOUS_UPDATE_PARENT, slot %lu, parent slot %lu, started at its UpdateParent without alpenglow_fast_leader_handover", block->slot, block->parent_slot ));
3235 0 : handle_bad_block( sched, block, FD_SCHED_DEAD_REASON_SPURIOUS_UPDATE_PARENT );
3236 0 : return FD_SCHED_DEAD_REASON_SPURIOUS_UPDATE_PARENT;
3237 0 : }
3238 0 : break;
3239 0 : }
3240 0 : case FD_SCHED_TT_BLOCK_END: {
3241 : /* It may seem redundant to be invoking task_done() on these
3242 : somewhat fake tasks. But these are necessary to drive state
3243 : transition for empty blocks or slow blocks. */
3244 0 : FD_TEST( !block->block_end_done );
3245 0 : block->block_end_done = 1;
3246 0 : if( FD_LIKELY( block_is_done( block ) ) ) slot_state_close( sched, block ); /* else on the last cert task done */
3247 0 : sched->print_buf_sz = 0UL;
3248 0 : print_block_metrics( sched, block );
3249 0 : FD_LOG_DEBUG(( "block %lu:%lu replayed fully: %s", block->slot, bank_idx, sched->print_buf ));
3250 0 : break;
3251 0 : }
3252 0 : case FD_SCHED_TT_TXN_EXEC: {
3253 0 : long now = fd_tickcount();
3254 0 : ulong delta = (ulong)(now-sched->txn_in_flight_last_tick);
3255 0 : ulong txn_exec_busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ] );
3256 0 : sched->metrics->txn_weighted_in_flight_tickcount += delta;
3257 0 : sched->metrics->txn_weighted_in_flight_cnt += delta*txn_exec_busy_cnt;
3258 0 : sched->txn_in_flight_last_tick = now;
3259 :
3260 0 : sched->txn_info_pool[ txn_idx ].tick_exec_done = now;
3261 :
3262 0 : block->txn_exec_done_cnt++;
3263 0 : block->txn_exec_in_flight_cnt--;
3264 0 : FD_TEST( !fd_ulong_extract_bit( sched->txn_exec_ready_bitset[ 0 ], exec_tile_idx ) );
3265 0 : sched->txn_exec_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->txn_exec_ready_bitset[ 0 ], exec_tile_idx );
3266 0 : sched->metrics->txn_exec_done_cnt++;
3267 0 : txn_bitset_insert( sched->exec_done_set, txn_idx );
3268 0 : sched->txn_info_pool[ txn_idx ].flags |= FD_SCHED_TXN_EXEC_DONE;
3269 0 : int dead_reason = FD_SCHED_DEAD_REASON_NONE;
3270 0 : if( FD_UNLIKELY( (sched->txn_info_pool[ txn_idx ].flags&FD_SCHED_TXN_ALT_UNRESOLVED) && data ) ) {
3271 0 : fd_sched_txn_alts_t const * alts = fd_type_pun_const( data );
3272 0 : if( FD_UNLIKELY( block->lthash_writable_cnt+alts->cnt>sched->lthash_txn_writers_max ) ) {
3273 0 : FD_LOG_INFO(( "bad block: TOO_MANY_WRITABLE_ACCOUNTS, slot %lu, parent slot %lu, txn_idx %lu, lthash_writable_cnt %u, late_reg_cnt %lu", block->slot, block->parent_slot, txn_idx, block->lthash_writable_cnt, alts->cnt ));
3274 0 : dead_reason = FD_SCHED_DEAD_REASON_TOO_MANY_WRITABLE_ACCOUNTS;
3275 0 : } else {
3276 0 : block->lthash_writable_cnt += (uint)alts->cnt;
3277 0 : if( FD_LIKELY( !block->lthash_inband ) ) {
3278 0 : for( ulong i=0UL; i<alts->cnt; i++ ) lthash_register( sched, block, alts->addrs+i );
3279 0 : sched->metrics->lthash_late_reg_cnt += (uint)alts->cnt;
3280 0 : }
3281 0 : }
3282 0 : }
3283 0 : if( txn_bitset_test( sched->sigverify_done_set, txn_idx ) && txn_bitset_test( sched->poh_mixin_done_set, txn_idx ) ) {
3284 : /* Release the txn_idx if all tasks on it are done. This is
3285 : guaranteed to only happen once per transaction because
3286 : whichever one completed first would not release. */
3287 0 : if( FD_UNLIKELY( block->txn_idx_tail==(uint)txn_idx ) ) block->txn_idx_tail = 0U;
3288 0 : fd_rdisp_complete_txn( sched->rdisp, txn_idx, 1 );
3289 0 : sched->txn_pool_free_cnt++;
3290 0 : block->txn_done_cnt++;
3291 0 : sched->metrics->txn_done_cnt++;
3292 0 : } else {
3293 0 : fd_rdisp_complete_txn( sched->rdisp, txn_idx, 0 );
3294 0 : }
3295 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE && !block->dying ) ) {
3296 0 : handle_bad_block( sched, block, dead_reason );
3297 0 : return dead_reason;
3298 0 : }
3299 0 : break;
3300 0 : }
3301 0 : case FD_SCHED_TT_TXN_SIGVERIFY: {
3302 0 : sched->txn_info_pool[ txn_idx ].tick_sigverify_done = fd_tickcount();
3303 0 : block->txn_sigverify_done_cnt++;
3304 0 : block->txn_sigverify_in_flight_cnt--;
3305 0 : FD_TEST( !fd_ulong_extract_bit( sched->sigverify_ready_bitset[ 0 ], exec_tile_idx ) );
3306 0 : sched->sigverify_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->sigverify_ready_bitset[ 0 ], exec_tile_idx );
3307 0 : sched->metrics->txn_sigverify_done_cnt++;
3308 0 : txn_bitset_insert( sched->sigverify_done_set, txn_idx );
3309 0 : sched->txn_info_pool[ txn_idx ].flags |= FD_SCHED_TXN_SIGVERIFY_DONE;
3310 0 : if( txn_bitset_test( sched->exec_done_set, txn_idx ) && txn_bitset_test( sched->poh_mixin_done_set, txn_idx ) ) {
3311 : /* Release the txn_idx if all tasks on it are done. This is
3312 : guaranteed to only happen once per transaction because
3313 : whichever one completed first would not release. */
3314 0 : if( FD_UNLIKELY( block->txn_idx_tail==(uint)txn_idx ) ) block->txn_idx_tail = 0U;
3315 0 : fd_rdisp_complete_txn( sched->rdisp, txn_idx, 1 );
3316 0 : sched->txn_pool_free_cnt++;
3317 0 : block->txn_done_cnt++;
3318 0 : sched->metrics->txn_done_cnt++;
3319 0 : }
3320 0 : break;
3321 0 : }
3322 0 : case FD_SCHED_TT_POH_HASH: {
3323 0 : block->poh_hashing_in_flight_cnt--;
3324 0 : FD_TEST( !fd_ulong_extract_bit( sched->poh_ready_bitset[ 0 ], exec_tile_idx ) );
3325 0 : sched->poh_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->poh_ready_bitset[ 0 ], exec_tile_idx );
3326 0 : fd_execrp_poh_hash_done_msg_t * msg = fd_type_pun( data );
3327 0 : FD_TEST( msg->cnt && msg->cnt<=FD_EXECRP_POH_PARA );
3328 0 : fd_sched_poh_hash_t const * task = sched->poh_inflight+exec_tile_idx;
3329 0 : FD_TEST( msg->cnt==task->cnt );
3330 :
3331 : /* Once the block is found to be bad, the remaining elements are
3332 : freed without retiring. */
3333 0 : int dead_reason = FD_SCHED_DEAD_REASON_NONE;
3334 0 : for( ulong i=0UL; i<msg->cnt; i++ ) {
3335 0 : block->poh_mblk_in_flight_cnt--;
3336 0 : uint mblk_idx = (uint)task->mblk_idx[ i ];
3337 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
3338 0 : memcpy( mblk->curr_hash, msg->hash+i, sizeof(fd_hash_t) );
3339 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
3340 0 : free_mblk( sched, block, mblk_idx );
3341 0 : continue;
3342 0 : }
3343 0 : dead_reason = poh_retire_mblk( sched, block, mblk_idx );
3344 0 : }
3345 0 : if( FD_LIKELY( dead_reason==FD_SCHED_DEAD_REASON_NONE ) ) dead_reason = verify_ticks_eager( block );
3346 0 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) {
3347 0 : handle_bad_block( sched, block, dead_reason );
3348 0 : return dead_reason;
3349 0 : }
3350 0 : break;
3351 0 : }
3352 0 : case FD_SCHED_TT_CERT_VERIFY:
3353 0 : case FD_SCHED_TT_REWARDS: {
3354 0 : FD_TEST( block->footer_in_flight_cnt );
3355 0 : block->footer_in_flight_cnt--;
3356 0 : FD_TEST( !fd_ulong_extract_bit( sched->footer_ready_bitset[ 0 ], exec_tile_idx ) );
3357 0 : sched->footer_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->footer_ready_bitset[ 0 ], exec_tile_idx );
3358 0 : if( FD_UNLIKELY( block_is_done( block ) ) ) slot_state_close( sched, block );
3359 0 : break;
3360 0 : }
3361 0 : case FD_SCHED_TT_LTHASH_SUB: {
3362 : /* A subtraction is never invalidated. The parent is frozen, so
3363 : the value does not depend on when it ran. */
3364 0 : fd_sched_lthash_batch_t const * rec = sched->lthash_inflight+exec_tile_idx;
3365 0 : FD_TEST( rec->kind==FD_SCHED_TT_LTHASH_SUB && rec->bank_idx==bank_idx && rec->cnt && rec->cnt<=FD_SCHED_LTHASH_BATCH_MAX );
3366 0 : for( ulong i=0UL; i<rec->cnt; i++ ) {
3367 0 : fd_sched_lthash_entry_t * e = sched->lthash_lane[ rec->lane ].entry_pool+rec->entry_idx[ i ];
3368 0 : FD_TEST( e->sub_state==FD_SCHED_LTHASH_SUB_DISPATCHED && e->bank_idx==(uint)bank_idx );
3369 0 : e->sub_state = (uchar)FD_SCHED_LTHASH_SUB_DONE;
3370 0 : }
3371 0 : fd_lthash_sub( block_lthash_delta( sched, block ), fd_type_pun_const( data ) );
3372 0 : block->sub_in_flight_cnt -= (uint)rec->cnt;
3373 0 : block->sub_applied_cnt += (uint)rec->cnt;
3374 0 : block->lthash_in_flight_cnt--;
3375 0 : sched->metrics->lthash_sub_applied_cnt += rec->cnt;
3376 0 : FD_TEST( !fd_ulong_extract_bit( sched->lthash_ready_bitset[ 0 ], exec_tile_idx ) );
3377 0 : sched->lthash_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->lthash_ready_bitset[ 0 ], exec_tile_idx );
3378 0 : break;
3379 0 : }
3380 0 : case FD_SCHED_TT_LTHASH_ADD: {
3381 0 : fd_sched_lthash_batch_t const * rec = sched->lthash_inflight+exec_tile_idx;
3382 0 : FD_TEST( rec->kind==FD_SCHED_TT_LTHASH_ADD && rec->bank_idx==bank_idx && rec->cnt && rec->cnt<=FD_SCHED_LTHASH_BATCH_MAX );
3383 : /* Entries live until the lane is reset, so the record's entries
3384 : are still these accounts'. */
3385 0 : ulong stale_cnt = 0UL;
3386 0 : for( ulong i=0UL; i<rec->cnt; i++ ) {
3387 0 : fd_sched_lthash_entry_t * e = sched->lthash_lane[ rec->lane ].entry_pool+rec->entry_idx[ i ];
3388 0 : FD_TEST( e->bank_idx==(uint)bank_idx );
3389 : /* A later write to the account was observed while the batch
3390 : was on the tile, so its value is not final. The rewrite
3391 : freed its speculation entry, if any. */
3392 0 : stale_cnt += (ulong)(rec->ptxn_idx[ i ]!=e->ptxn_idx);
3393 0 : }
3394 0 : if( FD_UNLIKELY( stale_cnt ) ) {
3395 : /* Stale hash. Drop it. */
3396 0 : if( FD_UNLIKELY( rec->cnt>1UL ) ) {
3397 : /* We're given one sum for a whole batch, so a stale entry
3398 : spoils the batch. We don't need to handle this, because
3399 : only a speculative addition can be stale after it has
3400 : surfaced, and a speculative addition goes out alone. See
3401 : the account state machine. So a stale batch is an
3402 : invariant violation. A stale non-speculative batch of one
3403 : account is silently dropped here, though it ought to be an
3404 : invariant violation too. */
3405 0 : sched->print_buf_sz = 0UL;
3406 0 : print_all( sched, block );
3407 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
3408 0 : FD_LOG_CRIT(( "invariant violation: %lu of %lu batched additions of block %lu:%lu were rewritten on the tile", stale_cnt, rec->cnt, block->slot, bank_idx ));
3409 0 : }
3410 0 : sched->metrics->lthash_add_dispatched_stale_cnt++;
3411 0 : } else {
3412 : /* Good hash. Fold it in. */
3413 0 : fd_lthash_add( block_lthash_delta( sched, block ), fd_type_pun_const( data ) );
3414 0 : for( ulong i=0UL; i<rec->cnt; i++ ) {
3415 0 : fd_sched_lthash_entry_t * e = sched->lthash_lane[ rec->lane ].entry_pool+rec->entry_idx[ i ];
3416 0 : FD_TEST( e->add_state==FD_SCHED_LTHASH_ADD_DISPATCHED );
3417 0 : if( e->spec_idx!=UINT_MAX ) {
3418 : /* A speculative addition is always sent individually. */
3419 0 : FD_TEST( rec->cnt==1UL );
3420 0 : fd_memcpy( sched->lthash_spec_pool+e->spec_idx, data, sizeof(fd_lthash_value_t) );
3421 0 : sched->metrics->lthash_spec_applied_cnt++;
3422 0 : }
3423 0 : e->add_state = (uchar)FD_SCHED_LTHASH_ADD_DONE;
3424 0 : e->ptxn_idx = 0U;
3425 0 : }
3426 0 : block->add_applied_cnt += (uint)rec->cnt;
3427 0 : sched->metrics->lthash_add_applied_cnt += rec->cnt;
3428 0 : }
3429 0 : block->add_done_cnt += (uint)rec->cnt;
3430 0 : for( ulong i=0UL; i<rec->cnt; i++ ) fd_rdisp_complete_txn( sched->rdisp, (ulong)rec->ptxn_idx[ i ], 1 );
3431 0 : block->lthash_in_flight_cnt--;
3432 0 : FD_TEST( !fd_ulong_extract_bit( sched->lthash_ready_bitset[ 0 ], exec_tile_idx ) );
3433 0 : sched->lthash_ready_bitset[ 0 ] = fd_ulong_set_bit( sched->lthash_ready_bitset[ 0 ], exec_tile_idx );
3434 0 : break;
3435 0 : }
3436 0 : }
3437 :
3438 0 : if( FD_UNLIKELY( block->dying && !block_is_in_flight( block ) ) ) {
3439 0 : if( FD_UNLIKELY( sched->active_bank_idx==bank_idx ) ) {
3440 0 : FD_LOG_CRIT(( "invariant violation: active block %lu:%lu shouldn't be dying, parent slot %lu",
3441 0 : block->slot, bank_idx, block->parent_slot ));
3442 0 : }
3443 0 : FD_LOG_DEBUG(( "dying block %lu:%lu drained", block->slot, bank_idx ));
3444 0 : subtree_abandon( sched, block );
3445 0 : try_activate_block( sched );
3446 0 : return 0;
3447 0 : }
3448 :
3449 0 : slot_state_eval( sched, block );
3450 :
3451 0 : if( FD_UNLIKELY( !block->dying && sched->active_bank_idx!=bank_idx ) ) {
3452 : /* Block is not dead. So we should be actively replaying it. */
3453 0 : fd_sched_block_t * active_block = block_pool_ele( sched, sched->active_bank_idx );
3454 0 : FD_LOG_CRIT(( "invariant violation: sched->active_bank_idx %lu, slot %lu, parent slot %lu, bank_idx %lu, slot %lu, parent slot %lu",
3455 0 : sched->active_bank_idx, active_block->slot, active_block->parent_slot,
3456 0 : bank_idx, block->slot, block->parent_slot ));
3457 0 : }
3458 :
3459 0 : maybe_switch_block( sched, bank_idx );
3460 :
3461 0 : return 0;
3462 0 : }
3463 :
3464 : void
3465 0 : fd_sched_block_abandon( fd_sched_t * sched, ulong bank_idx, int cause ) {
3466 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3467 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3468 :
3469 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3470 0 : if( FD_UNLIKELY( !block->in_sched ) ) {
3471 0 : FD_LOG_CRIT(( "invariant violation: block->in_sched==0, block %lu:%lu, parent slot %lu",
3472 0 : block->slot, bank_idx, block->parent_slot ));
3473 0 : }
3474 :
3475 0 : if( FD_LIKELY( cause==FD_SCHED_ABANDON_DISCARDED && !block->dying ) ) block->discarded = 1;
3476 :
3477 0 : FD_LOG_INFO(( "abandoning block %lu:%lu", block->slot, bank_idx ));
3478 0 : sched->print_buf_sz = 0UL;
3479 0 : print_all( sched, block );
3480 0 : FD_LOG_DEBUG(( "%s", sched->print_buf ));
3481 :
3482 0 : subtree_abandon( sched, block );
3483 0 : try_activate_block( sched );
3484 0 : }
3485 :
3486 : int
3487 0 : fd_sched_get_dead_reason( fd_sched_t * sched, ulong bank_idx ) {
3488 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3489 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3490 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3491 0 : if( FD_UNLIKELY( !block->in_sched ) ) return FD_SCHED_DEAD_REASON_NONE;
3492 0 : return block->dead_reason;
3493 0 : }
3494 :
3495 : int
3496 0 : fd_sched_block_is_discarded( fd_sched_t * sched, ulong bank_idx ) {
3497 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3498 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3499 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3500 0 : if( FD_UNLIKELY( !block->in_sched ) ) return 0;
3501 0 : return (int)block->discarded;
3502 0 : }
3503 :
3504 : void
3505 0 : fd_sched_cancel( fd_sched_t * sched, ulong bank_idx ) {
3506 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3507 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3508 :
3509 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3510 0 : if( FD_UNLIKELY( !block->in_sched ) ) return;
3511 0 : FD_LOG_INFO(( "canceling subtree at block %lu:%lu", block->slot, bank_idx ));
3512 0 : fd_sched_block_t * parent = block_pool_ele( sched, block->parent_idx );
3513 0 : if( FD_LIKELY( parent ) ) {
3514 : /* Splice the block out of its parent's children list. */
3515 0 : ulong block_idx = block_to_idx( sched, block );
3516 0 : ulong * idx_p = &parent->child_idx;
3517 0 : while( *idx_p!=block_idx ) {
3518 0 : idx_p = &(block_pool_ele( sched, *idx_p )->sibling_idx);
3519 0 : }
3520 0 : *idx_p = block->sibling_idx;
3521 0 : }
3522 0 : subtree_prune( sched, bank_idx, ULONG_MAX );
3523 0 : }
3524 :
3525 : void
3526 1008 : fd_sched_block_add_done( fd_sched_t * sched, ulong bank_idx, ulong parent_bank_idx, ulong slot ) {
3527 1008 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3528 1008 : FD_TEST( bank_idx<sched->block_cnt_max );
3529 :
3530 1008 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3531 1008 : add_block( sched, bank_idx, parent_bank_idx );
3532 1008 : block->slot = slot;
3533 1008 : block->fec_eos = 1;
3534 1008 : block->block_start_signaled = 1;
3535 1008 : block->block_end_signaled = 1;
3536 1008 : block->block_start_done = 1;
3537 1008 : block->block_end_done = 1;
3538 1008 : block->refcnt = 0;
3539 1008 : if( FD_LIKELY( parent_bank_idx!=ULONG_MAX ) ) {
3540 0 : fd_sched_block_t * parent_block = block_pool_ele( sched, parent_bank_idx );
3541 0 : block->parent_slot = parent_block->slot;
3542 0 : }
3543 1008 : if( FD_UNLIKELY( parent_bank_idx==ULONG_MAX ) ) {
3544 : /* Assumes that a NULL parent implies the snapshot slot. */
3545 1008 : block->parent_slot = ULONG_MAX;
3546 1008 : block->rooted = 1;
3547 1008 : sched->root_idx = bank_idx;
3548 1008 : }
3549 1008 : }
3550 :
3551 : void
3552 1008 : fd_sched_advance_root( fd_sched_t * sched, ulong root_idx ) {
3553 1008 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3554 1008 : FD_TEST( root_idx<sched->block_cnt_max );
3555 1008 : FD_TEST( sched->root_idx<sched->block_cnt_max );
3556 1008 : FD_TEST( ref_q_empty( sched->ref_q ) );
3557 :
3558 1008 : fd_sched_block_t * new_root = block_pool_ele( sched, root_idx );
3559 1008 : fd_sched_block_t * old_root = block_pool_ele( sched, sched->root_idx );
3560 1008 : if( FD_UNLIKELY( !old_root->rooted ) ) {
3561 0 : FD_LOG_CRIT(( "invariant violation: old_root is not rooted, slot %lu, parent slot %lu",
3562 0 : old_root->slot, old_root->parent_slot ));
3563 0 : }
3564 :
3565 : /* Early exit if the new root is the same as the old root. */
3566 1008 : if( FD_UNLIKELY( root_idx==sched->root_idx ) ) {
3567 1008 : FD_LOG_INFO(( "new root is the same as the old root, slot %lu, parent slot %lu",
3568 1008 : new_root->slot, new_root->parent_slot ));
3569 1008 : return;
3570 1008 : }
3571 :
3572 0 : subtree_prune( sched, sched->root_idx, root_idx );
3573 :
3574 0 : new_root->parent_idx = ULONG_MAX;
3575 0 : sched->root_idx = root_idx;
3576 0 : }
3577 :
3578 : void
3579 1008 : fd_sched_root_notify( fd_sched_t * sched, ulong root_idx ) {
3580 1008 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3581 1008 : FD_TEST( root_idx<sched->block_cnt_max );
3582 1008 : FD_TEST( sched->root_idx<sched->block_cnt_max );
3583 1008 : FD_TEST( ref_q_empty( sched->ref_q ) );
3584 :
3585 1008 : fd_sched_block_t * block = block_pool_ele( sched, root_idx );
3586 1008 : fd_sched_block_t * old_root = block_pool_ele( sched, sched->root_idx );
3587 1008 : if( FD_UNLIKELY( !old_root->rooted ) ) {
3588 0 : FD_LOG_CRIT(( "invariant violation: old_root is not rooted, slot %lu, parent slot %lu",
3589 0 : old_root->slot, old_root->parent_slot ));
3590 0 : }
3591 :
3592 : /* Early exit if the new root is the same as the old root. */
3593 1008 : if( FD_UNLIKELY( root_idx==sched->root_idx ) ) {
3594 1008 : FD_LOG_INFO(( "new root is the same as the old root, slot %lu, parent slot %lu",
3595 1008 : block->slot, block->parent_slot ));
3596 1008 : return;
3597 1008 : }
3598 :
3599 : /* Mark every node from the new root up through its parents to the
3600 : old root as being rooted. */
3601 0 : fd_sched_block_t * curr = block;
3602 0 : fd_sched_block_t * prev = NULL;
3603 0 : while( curr ) {
3604 0 : if( FD_UNLIKELY( !block_is_done( curr ) ) ) {
3605 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is not done, slot %lu, parent slot %lu",
3606 0 : curr->slot, curr->parent_slot ));
3607 0 : }
3608 0 : if( FD_UNLIKELY( curr->dying ) ) {
3609 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is dying, slot %lu, parent slot %lu",
3610 0 : curr->slot, curr->parent_slot ));
3611 0 : }
3612 0 : if( FD_UNLIKELY( curr->staged ) ) {
3613 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is staged, slot %lu, parent slot %lu",
3614 0 : curr->slot, curr->parent_slot ));
3615 0 : }
3616 0 : if( FD_UNLIKELY( curr->in_rdisp ) ) {
3617 0 : FD_LOG_CRIT(( "invariant violation: rooting a block that is in the dispatcher, slot %lu, parent slot %lu",
3618 0 : curr->slot, curr->parent_slot ));
3619 0 : }
3620 0 : curr->rooted = 1;
3621 0 : prev = curr;
3622 0 : curr = block_pool_ele( sched, curr->parent_idx );
3623 0 : }
3624 :
3625 : /* If we didn't reach the old root, the new root is not a descendant. */
3626 0 : if( FD_UNLIKELY( prev!=old_root ) ) {
3627 0 : FD_LOG_CRIT(( "invariant violation: new root is not a descendant of old root, new root slot %lu, parent slot %lu, old root slot %lu, parent slot %lu",
3628 0 : block->slot, block->parent_slot, old_root->slot, old_root->parent_slot ));
3629 0 : }
3630 :
3631 0 : ulong old_active_bank_idx = sched->active_bank_idx;
3632 0 : uint old_deactivate_abandon = sched->metrics->deactivate_abandoned_cnt;
3633 :
3634 : /* Now traverse from old root towards new root, and abandon all
3635 : minority forks. */
3636 0 : curr = old_root;
3637 0 : while( curr && curr->rooted && curr!=block ) { /* curr!=block to avoid abandoning good forks. */
3638 0 : fd_sched_block_t * rooted_child_block = NULL;
3639 0 : ulong child_idx = curr->child_idx;
3640 0 : while( child_idx!=ULONG_MAX ) {
3641 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
3642 0 : child_idx = child->sibling_idx;
3643 0 : if( child->rooted ) {
3644 0 : rooted_child_block = child;
3645 0 : } else {
3646 : /* This is a minority fork. Consensus converged elsewhere, so
3647 : the fork is discarded rather than invalid. Mark the subtree
3648 : root as discarded so subtree_abandon() propagates the flag
3649 : down.
3650 :
3651 : Gated on the block not already going down. A block that is
3652 : already dying was abandoned for a reason of its own, either
3653 : sched's own verdict or runtime's verdict on its validity, and
3654 : losing the fork choice race afterwards must not re-label it
3655 : discarded. */
3656 0 : if( FD_LIKELY( !child->dying ) ) child->discarded = 1;
3657 :
3658 0 : ulong abandoned_cnt = sched->metrics->block_abandoned_cnt;
3659 0 : subtree_abandon( sched, child );
3660 0 : abandoned_cnt = sched->metrics->block_abandoned_cnt-abandoned_cnt;
3661 0 : if( FD_UNLIKELY( abandoned_cnt ) ) FD_LOG_DEBUG(( "abandoned %lu blocks on minority fork starting at block %lu:%lu", abandoned_cnt, child->slot, block_to_idx( sched, child ) ));
3662 0 : }
3663 0 : }
3664 0 : curr = rooted_child_block;
3665 0 : }
3666 :
3667 : /* If the active block got abandoned, we need to reset it. */
3668 0 : if( sched->active_bank_idx==ULONG_MAX ) {
3669 : /* Count a minority-fork deactivation once, as MinorityFork only */
3670 0 : sched->metrics->deactivate_abandoned_cnt = old_deactivate_abandon;
3671 0 : sched->metrics->deactivate_pruned_cnt += fd_uint_if( old_active_bank_idx!=ULONG_MAX, 1U, 0U );
3672 0 : try_activate_block( sched );
3673 0 : }
3674 0 : }
3675 :
3676 : ulong
3677 0 : fd_sched_slot_states_drain( fd_sched_t * sched, fd_sched_slot_state_t * out ) {
3678 0 : ulong cnt = sched->state_cnt;
3679 0 : if( FD_UNLIKELY( sched->state_drop_cnt ) ) {
3680 0 : FD_LOG_WARNING(( "dropped %lu slot state intervals", sched->state_drop_cnt ));
3681 0 : sched->state_drop_cnt = 0UL;
3682 0 : }
3683 0 : fd_memcpy( out, sched->state_buf, cnt*sizeof(fd_sched_slot_state_t) );
3684 0 : sched->state_cnt = 0UL;
3685 0 : return cnt;
3686 0 : }
3687 :
3688 : ulong
3689 4557 : fd_sched_pruned_block_next( fd_sched_t * sched ) {
3690 4557 : if( !ref_q_empty( sched->ref_q ) ) {
3691 0 : ulong bank_idx = ref_q_pop_head( sched->ref_q );
3692 0 : return bank_idx;
3693 0 : }
3694 4557 : return ULONG_MAX;
3695 4557 : }
3696 :
3697 : void
3698 0 : fd_sched_set_poh_params( fd_sched_t * sched, ulong bank_idx, ulong tick_height, ulong max_tick_height, fd_hash_t const * start_poh ) {
3699 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3700 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3701 0 : FD_TEST( max_tick_height>tick_height );
3702 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3703 0 : block->tick_height = tick_height;
3704 0 : block->max_tick_height = max_tick_height;
3705 : #if FD_SCHED_SKIP_POH
3706 : /* No-op. */
3707 : (void)start_poh;
3708 : #else
3709 0 : if( FD_LIKELY( block->mblk_cnt ) ) {
3710 : /* Fix up the first mblk's curr_hash. */
3711 0 : FD_TEST( block->mblk_unhashed_cnt );
3712 0 : FD_TEST( !mblk_slist_is_empty( block->mblks_unhashed, sched->mblk_pool ) );
3713 0 : FD_TEST( !block->mblk_freed_cnt );
3714 0 : fd_sched_mblk_t * first_mblk = sched->mblk_pool + mblk_slist_idx_peek_head( block->mblks_unhashed, sched->mblk_pool );
3715 0 : memcpy( first_mblk->curr_hash, start_poh, sizeof(fd_hash_t) );
3716 0 : } else {
3717 0 : memcpy( block->poh_hash, start_poh, sizeof(fd_hash_t) );
3718 0 : }
3719 0 : #endif
3720 0 : }
3721 :
3722 : void
3723 0 : fd_sched_set_flh_active( fd_sched_t * sched, ulong bank_idx, int flh_active ) {
3724 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3725 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3726 0 : block_pool_ele( sched, bank_idx )->flh_active = !!flh_active;
3727 0 : }
3728 :
3729 : void
3730 1008 : fd_sched_set_bypass_poh_verify( fd_sched_t * sched, int bypass_poh_verify ) {
3731 1008 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3732 1008 : sched->bypass_poh_verify = !!bypass_poh_verify;
3733 1008 : }
3734 :
3735 : void
3736 1008 : fd_sched_set_bypass_alut_resolution( fd_sched_t * sched, int bypass_alut_resolution ) {
3737 1008 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3738 1008 : sched->bypass_alut_resolution = !!bypass_alut_resolution;
3739 1008 : }
3740 :
3741 : void
3742 0 : fd_sched_set_lthash_txn_writers_max( fd_sched_t * sched, ulong lthash_txn_writers_max ) {
3743 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3744 0 : FD_TEST( lthash_txn_writers_max<=FD_SCHED_LTHASH_TXN_WRITERS_MAX );
3745 0 : sched->lthash_txn_writers_max = lthash_txn_writers_max;
3746 0 : }
3747 :
3748 : void
3749 0 : fd_sched_set_lthash_oob_block_max( fd_sched_t * sched, ulong lthash_oob_block_max ) {
3750 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3751 0 : FD_TEST( lthash_oob_block_max<=FD_SCHED_LTHASH_OOB_BLOCK_MAX );
3752 0 : sched->lthash_oob_block_max = lthash_oob_block_max;
3753 0 : }
3754 :
3755 : fd_txn_p_t *
3756 0 : fd_sched_get_txn( fd_sched_t * sched, ulong txn_idx ) {
3757 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3758 0 : if( FD_UNLIKELY( txn_idx>=sched->depth ) ) {
3759 0 : return NULL;
3760 0 : }
3761 0 : return sched->txn_pool+txn_idx;
3762 0 : }
3763 :
3764 : fd_sched_txn_info_t *
3765 0 : fd_sched_get_txn_info( fd_sched_t * sched, ulong txn_idx ) {
3766 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3767 0 : if( FD_UNLIKELY( txn_idx>=sched->depth ) ) {
3768 0 : return NULL;
3769 0 : }
3770 0 : return sched->txn_info_pool+txn_idx;
3771 0 : }
3772 :
3773 : fd_hash_t *
3774 0 : fd_sched_get_poh( fd_sched_t * sched, ulong bank_idx ) {
3775 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3776 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3777 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3778 0 : FD_TEST( block->fec_eos );
3779 0 : FD_TEST( block->mblk_cnt );
3780 0 : return block->poh_hash;
3781 0 : }
3782 :
3783 : uint
3784 0 : fd_sched_get_shred_cnt( fd_sched_t * sched, ulong bank_idx ) {
3785 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3786 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3787 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3788 0 : return block->shred_cnt;
3789 0 : }
3790 :
3791 : fd_block_footer_t const *
3792 0 : fd_sched_get_footer( fd_sched_t * sched, ulong bank_idx ) {
3793 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3794 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3795 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3796 0 : return block->footer_seen ? &block->footer : NULL;
3797 0 : }
3798 :
3799 : fd_lthash_value_t const *
3800 0 : fd_sched_lthash_delta( fd_sched_t * sched, ulong bank_idx ) {
3801 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3802 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3803 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3804 : /* The delta accumulates from the block's first LtHash task on and is
3805 : final once BLOCK_END is signaled. Completing BLOCK_END zeroes it. */
3806 0 : FD_TEST( block->in_sched && !block->block_end_done );
3807 0 : return block_lthash_delta( sched, block );
3808 0 : }
3809 :
3810 : void
3811 0 : fd_sched_block_start_writes( fd_sched_t * sched, ulong bank_idx, fd_acct_addr_t const * addrs, ulong cnt ) {
3812 0 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
3813 0 : FD_TEST( bank_idx<sched->block_cnt_max );
3814 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3815 0 : FD_TEST( block->in_sched && block->staged && block->in_rdisp );
3816 0 : FD_TEST( !block->lthash_inband );
3817 : /* We expect this to run during BLOCK_START processing, before any
3818 : real transaction of the block is dispatched. */
3819 0 : FD_TEST( block->block_start_signaled && !block->block_start_done );
3820 0 : FD_TEST( !block->txn_exec_in_flight_cnt && !block->txn_exec_done_cnt );
3821 : /* Exceeding the start-of-block writable account budget is a bug, not
3822 : an invalid block. */
3823 0 : if( FD_UNLIKELY( cnt>FD_SCHED_LTHASH_START_WRITES_MAX ) ) FD_LOG_CRIT(( "start-of-block processing wrote %lu accounts, over the budget of %lu, block %lu:%lu", cnt, FD_SCHED_LTHASH_START_WRITES_MAX, block->slot, bank_idx ));
3824 0 : for( ulong i=0UL; i<cnt; i++ ) lthash_register( sched, block, addrs+i );
3825 0 : sched->metrics->lthash_start_reg_cnt += (uint)cnt;
3826 0 : }
3827 :
3828 : void
3829 : fd_sched_metrics_write( fd_sched_t * sched,
3830 0 : fd_clock_tile_t const * clock ) {
3831 0 : FD_MGAUGE_SET( REPLAY, SCHED_ACTIVE_BANK_INDEX, sched->active_bank_idx );
3832 0 : FD_MGAUGE_SET( REPLAY, SCHED_LAST_DISPATCH_BANK_INDEX, sched->next_ready_last_bank_idx );
3833 0 : FD_MGAUGE_SET( REPLAY, SCHED_LAST_DISPATCH_TIME, sched->next_ready_last_tick==LONG_MAX ? 0UL : (ulong)fd_clock_tile_tickcount_to_wallclock( clock, sched->next_ready_last_tick ) );
3834 0 : FD_MGAUGE_SET( REPLAY, SCHED_STAGING_LANE_USED, (ulong)fd_ulong_popcnt( sched->staged_bitset ) );
3835 0 : FD_MGAUGE_SET( REPLAY, SCHED_STAGING_LANE_PEAK, sched->staged_popcnt_wmk );
3836 0 : ulong staging_lane_head_bank_idx[ FD_METRICS_ENUM_STAGING_LANE_CNT ];
3837 0 : for( ulong lane=0UL; lane<FD_METRICS_ENUM_STAGING_LANE_CNT; lane++ ) {
3838 0 : staging_lane_head_bank_idx[ lane ] = fd_ulong_if( fd_ulong_extract_bit( sched->staged_bitset, (int)lane ), sched->staged_head_bank_idx[ lane ], ULONG_MAX );
3839 0 : }
3840 0 : FD_MGAUGE_ENUM_COPY( REPLAY, SCHED_STAGING_LANE_HEAD_BANK_INDEX, staging_lane_head_bank_idx );
3841 0 : FD_MGAUGE_SET( REPLAY, SCHED_TXN_POOL_USED, sched->depth-sched->txn_pool_free_cnt-1UL );
3842 0 : FD_MGAUGE_SET( REPLAY, SCHED_TXN_POOL_CAPACITY, sched->depth-1UL );
3843 0 : FD_MGAUGE_SET( REPLAY, SCHED_MICROBLOCK_POOL_USED, sched->depth-sched->mblk_pool_free_cnt );
3844 0 : FD_MGAUGE_SET( REPLAY, SCHED_MICROBLOCK_POOL_CAPACITY, sched->depth );
3845 0 : FD_MGAUGE_SET( REPLAY, SCHED_BLOCK_POOL_USED, sched->block_pool_popcnt );
3846 0 : FD_MGAUGE_SET( REPLAY, SCHED_BLOCK_POOL_CAPACITY, sched->block_cnt_max );
3847 :
3848 0 : fd_sched_metrics_t const * m = sched->metrics;
3849 :
3850 0 : ulong sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_CNT ];
3851 0 : sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_V_STAGED_IDX ] = m->block_added_staged_cnt;
3852 0 : sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_V_UNSTAGED_IDX ] = m->block_added_unstaged_cnt;
3853 0 : sched_block_added[ FD_METRICS_ENUM_SCHED_BLOCK_STAGING_STATE_V_DEAD_IDX ] = m->block_added_dead_ood_cnt;
3854 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_BLOCK_ADDED, sched_block_added );
3855 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_REPLAYED, m->block_removed_cnt );
3856 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_ABANDONED, m->block_abandoned_cnt );
3857 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_REJECTED, m->block_bad_cnt );
3858 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_PROMOTED, m->block_promoted_cnt );
3859 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_DEMOTED, m->block_demoted_cnt );
3860 0 : ulong sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_CNT ];
3861 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_NO_CHILD_IDX ] = m->deactivate_no_child_cnt;
3862 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_NO_WORK_IDX ] = m->deactivate_no_txn_cnt;
3863 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_ABANDONED_IDX ] = m->deactivate_abandoned_cnt;
3864 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_MINORITY_FORK_IDX ] = m->deactivate_pruned_cnt;
3865 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_SPEC_YIELD_IDX ] = m->lthash_spec_yield_cnt;
3866 0 : sched_deactivate[ FD_METRICS_ENUM_SCHED_DEACTIVATE_REASON_V_SPEC_POOL_FULL_IDX ] = m->lthash_pool_full_cnt;
3867 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_BLOCK_DEACTIVATED, sched_deactivate );
3868 0 : FD_MCNT_SET( REPLAY, SCHED_LANE_SWITCHED, m->lane_switch_cnt );
3869 0 : FD_MCNT_SET( REPLAY, SCHED_LANE_PROMOTED, m->lane_promoted_cnt );
3870 0 : FD_MCNT_SET( REPLAY, SCHED_LANE_DEMOTED, m->lane_demoted_cnt );
3871 0 : FD_MCNT_SET( REPLAY, SCHED_BLOCK_INBAND_PICKED, m->lthash_inband_block_cnt );
3872 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_PROMOTED, m->txn_promoted_cnt );
3873 0 : FD_MCNT_SET( REPLAY, SCHED_FORK_OBSERVED, m->fork_observed_cnt );
3874 0 : ulong sched_alut[ FD_METRICS_ENUM_SCHED_LUT_RESOLVE_RESULT_CNT ];
3875 0 : sched_alut[ FD_METRICS_ENUM_SCHED_LUT_RESOLVE_RESULT_V_SUCCESS_IDX ] = m->alut_success_cnt;
3876 0 : sched_alut[ FD_METRICS_ENUM_SCHED_LUT_RESOLVE_RESULT_V_FAILED_IDX ] = m->alut_serializing_cnt;
3877 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_TXN_LUT_RESOLVE_ATTEMPTED, sched_alut );
3878 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_PARSED_ABANDONED, m->txn_abandoned_parsed_cnt );
3879 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_EXECUTED_ABANDONED, m->txn_abandoned_exec_done_cnt );
3880 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_DONE_ABANDONED, m->txn_abandoned_done_cnt );
3881 : /* Fold in the open interval, which the next dispatch or completion credits */
3882 0 : ulong open_ticks = (ulong)( fd_tickcount()-sched->txn_in_flight_last_tick );
3883 0 : ulong busy_cnt = sched->exec_cnt-(ulong)fd_ulong_popcnt( sched->txn_exec_ready_bitset[ 0 ] );
3884 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_IN_FLIGHT_INTEGRAL_NANOS, m->txn_weighted_in_flight_cnt+open_ticks*busy_cnt );
3885 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_BUSY_DURATION_NANOS, m->txn_weighted_in_flight_tickcount+fd_ulong_if( !!busy_cnt, open_ticks, 0UL ) );
3886 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_IDLE_DURATION_NANOS, m->txn_none_in_flight_tickcount+fd_ulong_if( !busy_cnt, open_ticks, 0UL ) );
3887 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_PARSED, m->txn_parsed_cnt );
3888 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_EXECUTED, m->txn_exec_done_cnt );
3889 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_SIGNATURE_VERIFIED, m->txn_sigverify_done_cnt );
3890 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_POH_MIXED, m->txn_mixin_done_cnt );
3891 0 : FD_MCNT_SET( REPLAY, SCHED_TXN_DONE, m->txn_done_cnt );
3892 0 : FD_MCNT_SET( REPLAY, SCHED_MICROBLOCK_PARSED, m->mblk_parsed_cnt );
3893 0 : FD_MCNT_SET( REPLAY, SCHED_MICROBLOCK_HASHED, m->mblk_poh_hashed_cnt );
3894 0 : FD_MCNT_SET( REPLAY, SCHED_MICROBLOCK_DONE, m->mblk_poh_done_cnt );
3895 0 : FD_MCNT_SET( REPLAY, SCHED_INGESTED_BYTES, m->bytes_ingested_cnt );
3896 0 : FD_MCNT_SET( REPLAY, SCHED_TRAILING_SKIPPED_BYTES, m->bytes_ingested_unparsed_cnt );
3897 0 : FD_MCNT_SET( REPLAY, SCHED_DROPPED_BYTES, m->bytes_dropped_cnt );
3898 0 : FD_MCNT_SET( REPLAY, SCHED_FEC_INGESTED, m->fec_cnt );
3899 :
3900 0 : ulong lthash_write[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_TYPE_CNT ];
3901 0 : lthash_write[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_TYPE_V_PARSED_IDX ] = m->lthash_parsed_writable_cnt;
3902 0 : lthash_write[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_TYPE_V_BLOCK_START_IDX ] = m->lthash_start_reg_cnt;
3903 0 : lthash_write[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_TYPE_V_LATE_LUT_IDX ] = m->lthash_late_reg_cnt;
3904 0 : lthash_write[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_TYPE_V_DEMOTED_IDX ] = m->lthash_demote_reg_cnt;
3905 0 : lthash_write[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_TYPE_V_SPEC_GIVEN_UP_IDX ] = m->lthash_spec_given_up_cnt;
3906 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_LTHASH_WRITE_OBSERVED_BY_TYPE, lthash_write );
3907 0 : ulong lthash_handled[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_RESULT_CNT ];
3908 0 : lthash_handled[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_RESULT_V_FIRST_WRITE_IDX ] = m->lthash_first_write_cnt;
3909 0 : lthash_handled[ FD_METRICS_ENUM_SCHED_LTHASH_WRITE_RESULT_V_COALESCED_IDX ] = m->lthash_coalesced_cnt;
3910 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_LTHASH_WRITE_OBSERVED, lthash_handled );
3911 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_CREATED, m->lthash_add_created_cnt );
3912 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_SURFACED, m->lthash_add_surfaced_cnt );
3913 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_STALE_SURFACED, m->lthash_add_surfaced_stale_cnt );
3914 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_DISPATCHED, m->lthash_add_dispatched_cnt );
3915 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_STALE_DROPPED, m->lthash_add_dispatched_stale_cnt );
3916 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_APPLIED, m->lthash_add_applied_cnt );
3917 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ADD_UNDONE, m->lthash_add_undone_cnt );
3918 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_SPEC_CREATED, m->lthash_spec_cnt );
3919 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_SPEC_APPLIED, m->lthash_spec_applied_cnt );
3920 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_SPEC_INVALIDATED, m->lthash_spec_stale_cnt );
3921 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_SUB_OWED, m->lthash_entry_cnt );
3922 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_SUB_DISPATCHED, m->lthash_sub_dispatched_cnt );
3923 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_SUB_APPLIED, m->lthash_sub_applied_cnt );
3924 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_ACCOUNT_FROZEN, m->lthash_frozen_cnt );
3925 0 : ulong lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_CNT ];
3926 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_ADD_OWED_IDX ] = m->lthash_discarded_add_owed_cnt;
3927 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_ADD_PENDING_IDX ] = m->lthash_discarded_add_pending_cnt;
3928 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_ADD_APPLIED_IDX ] = m->lthash_discarded_add_applied_cnt;
3929 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_SUB_QUEUED_IDX ] = m->lthash_discarded_sub_queued_cnt;
3930 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_SUB_APPLIED_IDX ] = m->lthash_discarded_sub_applied_cnt;
3931 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_ADD_HELD_IDX ] = m->lthash_discarded_add_held_cnt;
3932 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_INBAND_ADD_OWED_IDX ] = m->lthash_inband_add_owed_cnt;
3933 0 : lthash_dropped[ FD_METRICS_ENUM_SCHED_LTHASH_WORK_TYPE_V_INBAND_SUB_QUEUED_IDX ] = m->lthash_inband_sub_queued_cnt;
3934 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_LTHASH_WORK_DROPPED, lthash_dropped );
3935 0 : ulong lthash_task[ FD_METRICS_ENUM_SCHED_LTHASH_TASK_TYPE_CNT ];
3936 0 : lthash_task[ FD_METRICS_ENUM_SCHED_LTHASH_TASK_TYPE_V_SUB_IDX ] = m->lthash_sub_task_cnt;
3937 0 : lthash_task[ FD_METRICS_ENUM_SCHED_LTHASH_TASK_TYPE_V_ADD_IDX ] = m->lthash_add_task_cnt;
3938 0 : FD_MCNT_ENUM_COPY( REPLAY, SCHED_LTHASH_TASK_DISPATCHED, lthash_task );
3939 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_BATCH_HELD, m->lthash_add_batch_held_cnt );
3940 0 : FD_MCNT_SET( REPLAY, SCHED_LTHASH_BATCH_IDLE_WAITED, m->lthash_add_batch_wait_idle_cnt );
3941 0 : }
3942 :
3943 : char *
3944 0 : fd_sched_get_state_cstr( fd_sched_t * sched ) {
3945 0 : lthash_metrics_check( sched );
3946 0 : sched->print_buf_sz = 0UL;
3947 0 : print_metrics( sched );
3948 0 : print_sched( sched );
3949 0 : return sched->print_buf;
3950 0 : }
3951 :
3952 0 : void * fd_sched_leave ( fd_sched_t * sched ) { return sched; }
3953 0 : void * fd_sched_delete( void * mem ) { return mem; }
3954 :
3955 :
3956 : /* Internal helpers. */
3957 :
3958 : static void
3959 : add_block( fd_sched_t * sched,
3960 : ulong bank_idx,
3961 2009 : ulong parent_bank_idx ) {
3962 2009 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
3963 2009 : FD_TEST( !block->in_sched );
3964 2009 : sched->block_pool_popcnt++;
3965 :
3966 2009 : block->txn_parsed_cnt = 0U;
3967 2009 : block->txn_exec_in_flight_cnt = 0U;
3968 2009 : block->txn_exec_done_cnt = 0U;
3969 2009 : block->txn_sigverify_in_flight_cnt = 0U;
3970 2009 : block->txn_sigverify_done_cnt = 0U;
3971 2009 : block->poh_hashing_in_flight_cnt = 0U;
3972 2009 : block->poh_mblk_in_flight_cnt = 0U;
3973 2009 : block->poh_hashing_done_cnt = 0U;
3974 2009 : block->poh_hash_cmp_done_cnt = 0U;
3975 2009 : block->footer_in_flight_cnt = 0U;
3976 2009 : block->txn_done_cnt = 0U;
3977 2009 : block->shred_cnt = 0U;
3978 2009 : block->shred_scan_idx = 0U;
3979 2009 : block->shred_scan_off = 0U;
3980 2009 : block->shred_sz_overflow = 0;
3981 2009 : block->mblk_cnt = 0U;
3982 2009 : block->mblk_freed_cnt = 0U;
3983 2009 : block->mblk_tick_cnt = 0U;
3984 2009 : block->mblk_unhashed_cnt = 0U;
3985 2009 : block->dead_reason = FD_SCHED_DEAD_REASON_NONE;
3986 2009 : block->txn_pool_max_popcnt = sched->depth - sched->txn_pool_free_cnt - 1UL;
3987 2009 : block->mblk_pool_max_popcnt = sched->depth - sched->mblk_pool_free_cnt;
3988 2009 : block->block_pool_max_popcnt = sched->block_pool_popcnt;
3989 2009 : block->txn_idx_tail = 0U;
3990 2009 : block->txn_sigverify_next_idx = 0U;
3991 2009 : block->parse_mblk_idx = UINT_MAX;
3992 :
3993 2009 : mblk_slist_remove_all( block->mblks_unhashed, sched->mblk_pool );
3994 2009 : mblk_slist_remove_all( block->mblks_mixin_in_progress, sched->mblk_pool );
3995 2009 : block->last_mblk_is_tick = 0;
3996 2009 : block->tick_height = ULONG_MAX;
3997 2009 : block->max_tick_height = ULONG_MAX;
3998 :
3999 2009 : block->header_seen = 0;
4000 2009 : block->genesis_cert_seen = 0;
4001 2009 : block->update_parent_seen = 0;
4002 2009 : block->footer_seen = 0;
4003 2009 : block->cert_cnt = 0U;
4004 2009 : block->cert_next = 0U;
4005 2009 : block->alpentick_seen = 0;
4006 2009 : block->starts_at_update_parent = 0;
4007 2009 : block->flh_active = 0;
4008 :
4009 2009 : block->mblks_rem = 0UL;
4010 2009 : block->txns_rem = 0UL;
4011 2009 : block->residual_sz = 0U;
4012 2009 : block->fec_buf_boff = 0U;
4013 2009 : block->fec_eob = 0;
4014 2009 : block->fec_sob = 1;
4015 :
4016 2009 : block->fec_eos = 0;
4017 2009 : block->rooted = 0;
4018 2009 : block->dying = 0;
4019 2009 : block->discarded = 0;
4020 2009 : block->fec_completed_ns = 0L;
4021 2009 : block->refcnt = 1;
4022 2009 : block->in_sched = 1;
4023 2009 : block->in_rdisp = 0;
4024 2009 : block->block_start_signaled = 0;
4025 2009 : block->block_end_signaled = 0;
4026 2009 : block->block_start_done = 0;
4027 2009 : block->block_end_done = 0;
4028 2009 : block->staged = 0;
4029 2009 : block->state_open = 0;
4030 2009 : block->lthash_mode_picked = 0;
4031 2009 : block->lthash_inband = 0;
4032 2009 : FD_TEST( !block->add_owed_cnt && !block->add_created_cnt && !block->sub_queued_cnt && !block->sub_in_flight_cnt && !block->sub_applied_cnt && !block->lthash_in_flight_cnt && !block->lthash_drained && !block->lthash_draining );
4033 :
4034 2009 : block->luf_depth = 0UL;
4035 :
4036 : /* New leaf node, no child, no sibling. */
4037 2009 : block->child_idx = ULONG_MAX;
4038 2009 : block->sibling_idx = ULONG_MAX;
4039 2009 : block->parent_idx = ULONG_MAX;
4040 :
4041 2009 : if( FD_UNLIKELY( parent_bank_idx==ULONG_MAX ) ) {
4042 1008 : return;
4043 1008 : }
4044 :
4045 : /* node->parent link */
4046 1001 : fd_sched_block_t * parent_block = block_pool_ele( sched, parent_bank_idx );
4047 1001 : block->parent_idx = parent_bank_idx;
4048 :
4049 : /* parent->node and sibling->node links */
4050 1001 : ulong child_idx = bank_idx;
4051 1001 : if( FD_LIKELY( parent_block->child_idx==ULONG_MAX ) ) { /* Optimize for no forking. */
4052 1001 : parent_block->child_idx = child_idx;
4053 1001 : } else {
4054 0 : fd_sched_block_t * curr_block = block_pool_ele( sched, parent_block->child_idx );
4055 0 : while( curr_block->sibling_idx!=ULONG_MAX ) {
4056 0 : curr_block = block_pool_ele( sched, curr_block->sibling_idx );
4057 0 : }
4058 0 : curr_block->sibling_idx = child_idx;
4059 0 : sched->metrics->fork_observed_cnt++;
4060 0 : }
4061 :
4062 1001 : if( FD_UNLIKELY( parent_block->dying ) ) {
4063 0 : record_lineage_death( block, parent_block );
4064 0 : block->dying = 1;
4065 0 : }
4066 1001 : }
4067 :
4068 : /* Alpenglow block structure. agave's BlockComponentProcessor rules
4069 : an Alpenglow block invalid unless its components are laid out as
4070 :
4071 : header | [genesis cert] | entries* | [update parent] | entries* | footer | alpentick
4072 :
4073 : with exactly one header and one footer, and the alpentick as the
4074 : final component. The helpers below drive the same state machine off
4075 : the batches sched parses. */
4076 :
4077 : static int
4078 : on_marker( fd_sched_t * sched,
4079 : fd_sched_block_t * block,
4080 1993 : fd_block_marker_t const * marker ) {
4081 1993 : (void)sched;
4082 1993 : switch( marker->kind ) {
4083 :
4084 1001 : case FD_BLOCK_MARKER_KIND_HEADER:
4085 1001 : if( FD_UNLIKELY( block->header_seen ) ) {
4086 0 : FD_LOG_INFO(( "bad block: MULTIPLE_BLOCK_HEADERS, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
4087 0 : return FD_SCHED_DEAD_REASON_MULTIPLE_BLOCK_HEADERS;
4088 0 : }
4089 : /* Checking for shred header and parent_slot mismatch occurs at the
4090 : rotor level, so it does not need to be checked in sched. We do
4091 : this because currently replay tile never sees shred headers. */
4092 1001 : block->header_seen = 1;
4093 1001 : return FD_SCHED_DEAD_REASON_NONE;
4094 :
4095 0 : case FD_BLOCK_MARKER_KIND_GENESIS_CERT:
4096 0 : if( FD_UNLIKELY( !block->header_seen ) ) {
4097 0 : FD_LOG_INFO(( "bad block: MISSING_PARENT_MARKER, slot %lu, parent slot %lu, genesis cert before header", block->slot, block->parent_slot ));
4098 0 : return FD_SCHED_DEAD_REASON_MISSING_PARENT_MARKER;
4099 0 : }
4100 0 : if( FD_UNLIKELY( block->mblk_cnt || block->genesis_cert_seen || block->footer_seen ) ) {
4101 0 : FD_LOG_INFO(( "bad block: GENESIS_CERT_OUT_OF_ORDER, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
4102 0 : return FD_SCHED_DEAD_REASON_GENESIS_CERT_OUT_OF_ORDER;
4103 0 : }
4104 0 : block->genesis_cert_seen = 1;
4105 0 : return FD_SCHED_DEAD_REASON_NONE;
4106 :
4107 992 : case FD_BLOCK_MARKER_KIND_FOOTER:
4108 992 : if( FD_UNLIKELY( !block->header_seen ) ) {
4109 0 : FD_LOG_INFO(( "bad block: MISSING_PARENT_MARKER, slot %lu, parent slot %lu, footer before header", block->slot, block->parent_slot ));
4110 0 : return FD_SCHED_DEAD_REASON_MISSING_PARENT_MARKER;
4111 0 : }
4112 992 : if( FD_UNLIKELY( block->footer_seen ) ) {
4113 0 : FD_LOG_INFO(( "bad block: MULTIPLE_BLOCK_FOOTERS, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
4114 0 : return FD_SCHED_DEAD_REASON_MULTIPLE_BLOCK_FOOTERS;
4115 0 : }
4116 992 : block->footer = marker->footer;
4117 992 : block->footer_seen = 1;
4118 992 : fd_block_footer_cert_t const * cert [ FD_ALPENGLOW_FOOTER_CERT_MAX ];
4119 992 : uint tag [ FD_ALPENGLOW_FOOTER_CERT_MAX ];
4120 992 : ulong quorum[ FD_ALPENGLOW_FOOTER_CERT_MAX ];
4121 992 : ulong cert_cnt = fd_alpenglow_footer_certs( &block->footer, cert, tag, quorum );
4122 992 : FD_TEST( cert_cnt<=FD_ALPENGLOW_FOOTER_CERT_MAX );
4123 992 : block->cert_cnt = (uint)cert_cnt & 7U; /* The test is the bound. The mask only quiets the conversion warning. */
4124 992 : return FD_SCHED_DEAD_REASON_NONE;
4125 :
4126 0 : case FD_BLOCK_MARKER_KIND_UPDATE_PARENT:
4127 0 : if( FD_UNLIKELY( block->update_parent_seen ) ) {
4128 0 : FD_LOG_INFO(( "bad block: MULTIPLE_UPDATE_PARENTS, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
4129 0 : return FD_SCHED_DEAD_REASON_MULTIPLE_UPDATE_PARENTS;
4130 0 : }
4131 0 : if( FD_LIKELY( !block->header_seen ) ) {
4132 : /* Replay started the block here because rotor saw the
4133 : UpdateParent, so it is the parent marker in place of the
4134 : header and must name the parent replay built the block on.
4135 : Anything before it already died as MISSING_PARENT_MARKER. */
4136 0 : if( FD_UNLIKELY( !block->starts_at_update_parent ) ) {
4137 0 : FD_LOG_INFO(( "bad block: SPURIOUS_UPDATE_PARENT, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
4138 0 : return FD_SCHED_DEAD_REASON_SPURIOUS_UPDATE_PARENT;
4139 0 : }
4140 0 : if( FD_UNLIKELY( marker->update_parent.new_parent_slot!=block->parent_slot ) ) {
4141 0 : FD_LOG_INFO(( "bad block: UPDATE_PARENT_MISMATCH, slot %lu, parent slot %lu, update parent names %lu", block->slot, block->parent_slot, marker->update_parent.new_parent_slot ));
4142 0 : return FD_SCHED_DEAD_REASON_UPDATE_PARENT_MISMATCH;
4143 0 : }
4144 0 : block->header_seen = 1; /* the parent marker the remaining rules gate on */
4145 0 : block->update_parent_seen = 1;
4146 0 : return FD_SCHED_DEAD_REASON_NONE;
4147 0 : }
4148 : /* After a header or footer. If after a header, replay should have
4149 : skipped the prefix instead, so reaching here means it was handed
4150 : a prefix it did not know to skip. If after a footer, its a
4151 : misformed block. */
4152 0 : FD_LOG_INFO(( "bad block: SPURIOUS_UPDATE_PARENT, slot %lu, parent slot %lu, update parent after %s", block->slot, block->parent_slot, block->footer_seen ? "footer" : "header" ));
4153 0 : return FD_SCHED_DEAD_REASON_SPURIOUS_UPDATE_PARENT;
4154 :
4155 0 : default:
4156 0 : FD_LOG_INFO(( "bad block: slot %lu, parent slot %lu, unknown block marker kind %u", block->slot, block->parent_slot, marker->kind ));
4157 0 : return FD_SCHED_DEAD_REASON_BAD_BLOCK_MARKER;
4158 1993 : }
4159 1993 : }
4160 :
4161 : /* Called when a batch header declares one or more microblocks. */
4162 :
4163 : static int
4164 2562 : on_entry_batch( fd_sched_block_t * block ) {
4165 2562 : if( FD_UNLIKELY( !block->header_seen ) ) {
4166 0 : FD_LOG_INFO(( "bad block: MISSING_PARENT_MARKER, slot %lu, parent slot %lu, entries before header", block->slot, block->parent_slot ));
4167 0 : return FD_SCHED_DEAD_REASON_MISSING_PARENT_MARKER;
4168 0 : }
4169 2562 : if( FD_UNLIKELY( block->alpentick_seen ) ) {
4170 0 : FD_LOG_INFO(( "bad block: ENTRY_AFTER_BLOCK_FOOTER, slot %lu, parent slot %lu, batch after alpentick", block->slot, block->parent_slot ));
4171 0 : return FD_SCHED_DEAD_REASON_ENTRY_AFTER_BLOCK_FOOTER;
4172 0 : }
4173 : /* Only the alpentick may follow the footer: a batch of exactly one
4174 : microblock which must turn out to be a tick (checked when its
4175 : header parses). */
4176 2562 : if( FD_UNLIKELY( block->footer_seen && block->mblks_rem!=1UL ) ) {
4177 0 : FD_LOG_INFO(( "bad block: ENTRY_AFTER_BLOCK_FOOTER, slot %lu, parent slot %lu, batch of %lu microblocks after footer", block->slot, block->parent_slot, block->mblks_rem ));
4178 0 : return FD_SCHED_DEAD_REASON_ENTRY_AFTER_BLOCK_FOOTER;
4179 0 : }
4180 2562 : return FD_SCHED_DEAD_REASON_NONE;
4181 2562 : }
4182 :
4183 : static int
4184 991 : on_final( fd_sched_block_t * block ) {
4185 991 : if( FD_UNLIKELY( !block->footer_seen ) ) {
4186 0 : FD_LOG_INFO(( "bad block: MISSING_BLOCK_FOOTER, slot %lu, parent slot %lu", block->slot, block->parent_slot ));
4187 0 : return FD_SCHED_DEAD_REASON_MISSING_BLOCK_FOOTER;
4188 0 : }
4189 991 : if( FD_UNLIKELY( !block->alpentick_seen ) ) {
4190 0 : FD_LOG_INFO(( "bad block: INVALID_ALPENTICK_POSITION, slot %lu, parent slot %lu, block ended after footer without alpentick", block->slot, block->parent_slot ));
4191 0 : return FD_SCHED_DEAD_REASON_INVALID_ALPENTICK_POSITION;
4192 0 : }
4193 991 : return FD_SCHED_DEAD_REASON_NONE;
4194 991 : }
4195 :
4196 : /* Bounds the tick count as soon as the tick window is known. Returns
4197 : FD_SCHED_DEAD_REASON_NONE (0) on success, else the
4198 : FD_SCHED_DEAD_REASON_* the block should be ruled invalid for. Does
4199 : not modify the block. */
4200 : static int
4201 4555 : verify_ticks_eager( fd_sched_block_t * block ) {
4202 4555 : FD_TEST( block->tick_height!=ULONG_MAX ); /* PoH params initialized. */
4203 :
4204 4555 : if( FD_UNLIKELY( block->mblk_tick_cnt+block->tick_height>block->max_tick_height ) ) {
4205 1486 : FD_LOG_INFO(( "bad block: TOO_MANY_TICKS, slot %lu, parent slot %lu, tick_cnt %u, tick_height %lu, max_tick_height %lu", block->slot, block->parent_slot, block->mblk_tick_cnt, block->tick_height, block->max_tick_height ));
4206 1486 : return FD_SCHED_DEAD_REASON_TOO_MANY_TICKS;
4207 1486 : }
4208 :
4209 3069 : return FD_SCHED_DEAD_REASON_NONE;
4210 4555 : }
4211 :
4212 : /* https://github.com/anza-xyz/agave/blob/v3.0.6/ledger/src/blockstore_processor.rs#L1057
4213 :
4214 : The only check we don't do here is TRAILING_ENTRY, which can be done
4215 : independently when we parse the final FEC set of a block.
4216 :
4217 : Returns FD_SCHED_DEAD_REASON_NONE (0) on success, else the
4218 : FD_SCHED_DEAD_REASON_* the block should be ruled invalid for. Does
4219 : not modify the block. */
4220 : static int
4221 992 : verify_ticks_final( fd_sched_block_t * block ) {
4222 992 : FD_TEST( block->fec_eos );
4223 :
4224 992 : if( FD_UNLIKELY( block->mblk_tick_cnt+block->tick_height<block->max_tick_height ) ) {
4225 0 : FD_LOG_INFO(( "bad block: TOO_FEW_TICKS, slot %lu, parent slot %lu, tick_cnt %u, tick_height %lu, max_tick_height %lu", block->slot, block->parent_slot, block->mblk_tick_cnt, block->tick_height, block->max_tick_height ));
4226 0 : return FD_SCHED_DEAD_REASON_TOO_FEW_TICKS;
4227 0 : }
4228 :
4229 992 : return verify_ticks_eager( block );
4230 992 : }
4231 :
4232 : int
4233 : fd_sched_block_verify_ticks( fd_sched_t * sched,
4234 : ulong bank_idx,
4235 : ulong tick_height,
4236 4555 : ulong max_tick_height ) {
4237 4555 : FD_TEST( sched->canary==FD_SCHED_MAGIC );
4238 4555 : FD_TEST( bank_idx<sched->block_cnt_max );
4239 4555 : FD_TEST( max_tick_height>tick_height );
4240 4555 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
4241 : /* Set the tick window directly; skip fd_sched_set_poh_params
4242 : PoH fixup, unneeded here and unsafe to repeat per FEC set. */
4243 4555 : block->tick_height = tick_height;
4244 4555 : block->max_tick_height = max_tick_height;
4245 : /* fec_eos => slot fully ingested, so the TOO_FEW check is meaningful
4246 : too. */
4247 4555 : return block->fec_eos ? verify_ticks_final( block ) : verify_ticks_eager( block );
4248 4555 : }
4249 :
4250 28259 : #define CHECK( cond ) do { \
4251 28259 : if( FD_UNLIKELY( !(cond) ) ) { \
4252 2 : return FD_SCHED_DEAD_REASON_NONE; \
4253 2 : } \
4254 28259 : } while( 0 )
4255 :
4256 : /* CHECK that it is safe to read at least n more bytes. */
4257 28259 : #define CHECK_LEFT( n ) CHECK( (n)<=(sched->fec_buf_sz-sched->fec_buf_soff) )
4258 :
4259 : /* Consume as much as possible from the buffer. By the end of this
4260 : function, there could be residual data left over in the buffer. That
4261 : residual has to be either a partially received microblock header or a
4262 : transaction that straddles a FEC set boundary. These bytes are kept
4263 : and will be concatenated with the next FEC set. This residual is
4264 : bounded.
4265 :
4266 : Trailing bytes within a batch are dropped immediately. These
4267 : trailing bytes can span arbitrarily many FEC sets. */
4268 : FD_WARN_UNUSED static int
4269 4557 : fd_sched_parse( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx ) {
4270 32815 : while( 1 ) {
4271 34354 : while( block->txns_rem>0UL ) {
4272 1541 : int err = fd_sched_parse_txn( sched, block, alut_ctx );
4273 1541 : if( FD_UNLIKELY( -1==err ) ) return FD_SCHED_DEAD_REASON_NONE;
4274 1541 : else if( FD_UNLIKELY( err ) ) return err;
4275 1541 : }
4276 :
4277 32813 : if( block->txns_rem==0UL && block->mblks_rem>0UL ) {
4278 23702 : if( FD_UNLIKELY( block->mblk_cnt>=sched->max_mblk_per_slot ) ) {
4279 : /* Microblock count is enforced as invariant
4280 : mblk_cnt+mblks_rem<=max_mblk_per_slot
4281 : when we read the declared count in a batch header. So we
4282 : shouldn't get here. */
4283 0 : sched->print_buf_sz = 0UL;
4284 0 : print_all( sched, block );
4285 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
4286 0 : FD_LOG_CRIT(( "invariant violation: slot %lu, parent slot %lu, mblks_rem %lu, mblk_cnt %u (%u ticks) >= %lu", block->slot, block->parent_slot, block->mblks_rem, block->mblk_cnt, block->mblk_tick_cnt, sched->max_mblk_per_slot ));
4287 0 : }
4288 :
4289 23702 : CHECK_LEFT( sizeof(fd_microblock_hdr_t) );
4290 23702 : fd_microblock_hdr_t * hdr = (fd_microblock_hdr_t *)fd_type_pun( sched->fec_buf+sched->fec_buf_soff );
4291 23702 : sched->fec_buf_soff += (uint)sizeof(fd_microblock_hdr_t);
4292 :
4293 23702 : if( FD_UNLIKELY( hdr->txn_cnt>fd_ulong_sat_sub( sched->max_txn_per_slot, block->txn_parsed_cnt ) ) ) {
4294 0 : FD_LOG_INFO(( "bad block: TOO_MANY_TXNS, microblock header declared too many transactions, slot %lu, parent slot %lu, txn_parsed_cnt %u, hdr->txn_cnt %lu", block->slot, block->parent_slot, block->txn_parsed_cnt, hdr->txn_cnt ));
4295 0 : return FD_SCHED_DEAD_REASON_TOO_MANY_TXNS;
4296 0 : }
4297 23702 : if( FD_UNLIKELY( hdr->hash_cnt!=1UL ) ) {
4298 0 : FD_LOG_INFO(( "bad block: ALPENGLOW_HASH_CNT, slot %lu, parent slot %lu, mblk idx %u declared hash_cnt %lu", block->slot, block->parent_slot, block->mblk_cnt, hdr->hash_cnt ));
4299 0 : return FD_SCHED_DEAD_REASON_ALPENGLOW_HASH_CNT;
4300 0 : }
4301 23702 : if( FD_UNLIKELY( block->footer_seen ) ) {
4302 992 : if( FD_UNLIKELY( hdr->txn_cnt ) ) {
4303 0 : FD_LOG_INFO(( "bad block: ENTRY_AFTER_BLOCK_FOOTER, slot %lu, parent slot %lu, transaction microblock after footer", block->slot, block->parent_slot ));
4304 0 : return FD_SCHED_DEAD_REASON_ENTRY_AFTER_BLOCK_FOOTER;
4305 0 : }
4306 992 : block->alpentick_seen = 1;
4307 992 : }
4308 :
4309 23702 : block->mblks_rem--;
4310 23702 : block->txns_rem = hdr->txn_cnt;
4311 :
4312 23702 : sched->metrics->mblk_parsed_cnt++;
4313 23702 : if( FD_UNLIKELY( !hdr->txn_cnt ) ) {
4314 : /* This is a tick microblock. */
4315 22304 : block->mblk_tick_cnt++;
4316 22304 : }
4317 :
4318 23702 : FD_TEST( sched->mblk_pool_free_cnt ); /* can_ingest should have guaranteed sufficient free capacity. */
4319 23702 : uint mblk_idx = sched->mblk_pool_free_head;
4320 23702 : sched->mblk_pool_free_head = sched->mblk_pool[ mblk_idx ].next;
4321 23702 : sched->mblk_pool_free_cnt--;
4322 :
4323 23702 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
4324 23702 : mblk->start_txn_idx = block->txn_parsed_cnt;
4325 23702 : mblk->end_txn_idx = mblk->start_txn_idx+hdr->txn_cnt;
4326 23702 : mblk->curr_txn_idx = mblk->start_txn_idx;
4327 23702 : mblk->curr_sig_cnt = 0U;
4328 23702 : mblk->mixin_txn_idx = 0U;
4329 23702 : mblk->is_tick = !hdr->txn_cnt;
4330 23702 : memcpy( mblk->end_hash, hdr->hash, sizeof(fd_hash_t) );
4331 23702 : memcpy( mblk->curr_hash, block->poh_hash, sizeof(fd_hash_t) );
4332 23702 : block->parse_mblk_idx = mblk_idx;
4333 :
4334 : /* Update block tracking. */
4335 23702 : memcpy( block->poh_hash, hdr->hash, sizeof(fd_hash_t) );
4336 23702 : block->last_mblk_is_tick = mblk->is_tick;
4337 23702 : block->mblk_cnt++;
4338 :
4339 : #if FD_SCHED_SKIP_POH
4340 : block->poh_hashing_done_cnt++;
4341 : block->poh_hash_cmp_done_cnt++;
4342 : free_mblk( sched, block, mblk_idx );
4343 : #else
4344 23702 : mblk_slist_idx_push_tail( block->mblks_unhashed, mblk_idx, sched->mblk_pool );
4345 23702 : block->mblk_unhashed_cnt++;
4346 23702 : #endif
4347 23702 : continue;
4348 23702 : }
4349 9111 : if( block->txns_rem==0UL && block->mblks_rem==0UL && block->fec_sob ) {
4350 4557 : CHECK_LEFT( sizeof(ulong) );
4351 4555 : FD_TEST( sched->fec_buf_soff==0U );
4352 4555 : if( FD_UNLIKELY( !FD_LOAD( ulong, sched->fec_buf ) && !block->fec_eob ) ) {
4353 : /* Block marker split across FEC sets */
4354 0 : fd_block_marker_t peek[1];
4355 0 : if( FD_UNLIKELY( FD_BLOCK_MARKER_DE_ERR_SZ==fd_block_marker_de( peek, sched->fec_buf, (ulong)sched->fec_buf_sz ) &&
4356 0 : sched->fec_buf_sz<FD_BLOCK_MARKER_SER_MAX ) ) return FD_SCHED_DEAD_REASON_NONE;
4357 0 : }
4358 4555 : block->mblks_rem = FD_LOAD( ulong, sched->fec_buf );
4359 4555 : sched->fec_buf_soff += (uint)sizeof(ulong);
4360 :
4361 4555 : if( FD_UNLIKELY( block->mblks_rem>fd_ulong_sat_sub( sched->max_mblk_per_slot, block->mblk_cnt ) ) ) {
4362 0 : FD_LOG_INFO(( "bad block: TOO_MANY_MICROBLOCKS, slot %lu, parent slot %lu, mblk_cnt %u (%u ticks) + hdr->mblk_cnt %lu >= %lu", block->slot, block->parent_slot, block->mblk_cnt, block->mblk_tick_cnt, block->mblks_rem, sched->max_mblk_per_slot ));
4363 0 : return FD_SCHED_DEAD_REASON_TOO_MANY_MICROBLOCKS;
4364 0 : }
4365 :
4366 4555 : block->fec_sob = 0;
4367 :
4368 4555 : if( FD_UNLIKELY( !block->mblks_rem ) ) {
4369 : /* A zero-microblock batch is a block marker */
4370 1993 : fd_block_marker_t marker[1];
4371 1993 : int err = fd_block_marker_de( marker, sched->fec_buf, (ulong)sched->fec_buf_sz );
4372 1993 : if( FD_UNLIKELY( err ) ) {
4373 0 : FD_LOG_INFO(( "bad block: slot %lu, unable to parse block marker (err %d)", block->slot, err ));
4374 0 : return FD_SCHED_DEAD_REASON_BAD_BLOCK_MARKER;
4375 0 : }
4376 1993 : int dead_reason = on_marker( sched, block, marker );
4377 1993 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) return dead_reason;
4378 1993 : sched->fec_buf_soff = (uint)( FD_BLOCK_MARKER_PREAMBLE_SZ+FD_LOAD( ushort, sched->fec_buf+FD_BLOCK_MARKER_PREAMBLE_SZ-sizeof(ushort) ) ); /* the marker is not trailing bytes */
4379 2562 : } else if( FD_UNLIKELY( block->mblks_rem ) ) {
4380 2562 : int dead_reason = on_entry_batch( block );
4381 2562 : if( FD_UNLIKELY( dead_reason!=FD_SCHED_DEAD_REASON_NONE ) ) return dead_reason;
4382 2562 : }
4383 4555 : continue;
4384 4555 : }
4385 4554 : if( block->txns_rem==0UL && block->mblks_rem==0UL ) {
4386 4553 : break;
4387 4553 : }
4388 4554 : }
4389 4555 : if( !block->fec_sob && block->txns_rem==0UL && block->mblks_rem==0UL ) {
4390 : /* All microblocks announced by the current batch header have been
4391 : parsed out. Anything still in the buffer must be trailing bytes.
4392 : Drop them now because trailing bytes can span many FEC sets and
4393 : accumulating them would overflow the parse buffer. Any followup
4394 : FEC sets within the same batch will simply be discarded
4395 : wholesale. */
4396 4553 : sched->metrics->bytes_ingested_unparsed_cnt += sched->fec_buf_sz-sched->fec_buf_soff;
4397 4553 : block->fec_buf_boff += sched->fec_buf_sz;
4398 4553 : sched->fec_buf_soff = 0U;
4399 4553 : sched->fec_buf_sz = 0U;
4400 4553 : }
4401 4555 : if( block->fec_eob ) {
4402 4552 : block->fec_sob = 1;
4403 4552 : block->fec_eob = 0;
4404 4552 : }
4405 4555 : return FD_SCHED_DEAD_REASON_NONE;
4406 4557 : }
4407 :
4408 : static inline fd_acct_addr_t const *
4409 2889 : get_acct( fd_txn_t const * txn, fd_acct_addr_t const * imms, fd_acct_addr_t const * alts, ushort idx ) {
4410 2889 : if( FD_LIKELY( idx<txn->acct_addr_cnt ) ) return imms+idx;
4411 0 : if( FD_UNLIKELY( !alts ) ) return NULL;
4412 0 : return alts+(idx-txn->acct_addr_cnt);
4413 0 : }
4414 :
4415 : FD_WARN_UNUSED static int
4416 1541 : fd_sched_parse_txn( fd_sched_t * sched, fd_sched_block_t * block, fd_sched_alut_ctx_t * alut_ctx ) {
4417 1541 : fd_txn_t * txn = fd_type_pun( sched->txn );
4418 :
4419 1541 : uchar * payload = sched->fec_buf+sched->fec_buf_soff;
4420 1541 : ulong remaining = sched->fec_buf_sz-sched->fec_buf_soff;
4421 1541 : ulong pay_sz = 0UL;
4422 1541 : ulong txn_sz = fd_txn_parse_core( payload,
4423 1541 : remaining,
4424 1541 : txn,
4425 1541 : NULL,
4426 1541 : &pay_sz );
4427 :
4428 : /* Can't parse out a full transaction yet, EAGAIN. */
4429 1541 : if( FD_UNLIKELY( !pay_sz || !txn_sz ) ) return -1;
4430 :
4431 : /* The pool slots this transaction will land in are cold: warm the
4432 : lines written below (the start of the payload copy, the metadata,
4433 : the start of the parsed txn and the whole info record) while the
4434 : ALUT resolution and the dispatcher insert run. A hint only. */
4435 1541 : ulong next_idx = fd_rdisp_peek_free_txn( sched->rdisp );
4436 1542 : if( FD_LIKELY( next_idx && next_idx<sched->depth ) ) {
4437 1542 : fd_txn_p_t * next_p = sched->txn_pool+next_idx;
4438 1542 : __builtin_prefetch( next_p->payload, 1, 3 );
4439 1542 : __builtin_prefetch( next_p->payload+64UL, 1, 3 );
4440 1542 : __builtin_prefetch( &next_p->payload_sz, 1, 3 );
4441 1542 : __builtin_prefetch( TXN( next_p ), 1, 3 );
4442 1542 : ulong info_lo = fd_ulong_align_dn( (ulong)(sched->txn_info_pool+next_idx ), 64UL );
4443 1542 : ulong info_hi = fd_ulong_align_up( (ulong)(sched->txn_info_pool+next_idx+1UL ), 64UL );
4444 7682 : for( ulong line=info_lo; line<info_hi; line+=64UL ) __builtin_prefetch( (void const *)line, 1, 3 );
4445 1542 : }
4446 :
4447 1541 : if( FD_UNLIKELY( block->txn_parsed_cnt>=sched->max_txn_per_slot ) ) {
4448 : /* Transaction count is enforced as invariant
4449 : txn_parsed_cnt+txns_rem<=max_txn_per_slot
4450 : when we read the declared count in a microblock header. So we
4451 : shouldn't get here. */
4452 0 : sched->print_buf_sz = 0UL;
4453 0 : print_all( sched, block );
4454 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
4455 0 : FD_LOG_CRIT(( "invariant violation: slot %lu, parent slot %lu, txn_parsed_cnt %u", block->slot, block->parent_slot, block->txn_parsed_cnt ));
4456 0 : }
4457 :
4458 1541 : ulong imm_cnt = fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_IMM );
4459 1541 : ulong alt_cnt = fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_ALT );
4460 :
4461 : /* Try to expand ALUTs. */
4462 1541 : int serializing = 0;
4463 1541 : if( alt_cnt>0UL ) {
4464 0 : if( FD_UNLIKELY( sched->bypass_alut_resolution ) ) {
4465 : /* test/fuzz: no accdb to query, so treat ALUT txns as serializing. */
4466 0 : serializing = 1;
4467 0 : } else {
4468 : /* Copy the slot hashes sysvar out and release the read BEFORE
4469 : resolving the ALTs. fd_runtime_load_txn_address_lookup_tables
4470 : issues its own fd_accdb_read_one per lookup table, and the accdb
4471 : acquire state is a single non-nestable flag — holding this read
4472 : open across that call would trip the IDLE assertion in
4473 : fd_accdb_acquire. The slot_hashes view aliases the record data,
4474 : so it must view the copy, not the released record. */
4475 0 : static uchar slot_hashes_buf[ FD_SYSVAR_SLOT_HASHES_BINCODE_SZ ];
4476 0 : ulong slot_hashes_sz = 0UL;
4477 0 : int have_slot_hashes = 0;
4478 0 : fd_acc_t ro = fd_accdb_read_one( alut_ctx->accdb, alut_ctx->fork_id, fd_sysvar_slot_hashes_id.uc );
4479 0 : if( FD_LIKELY( ro.lamports && ro.data_len<=sizeof(slot_hashes_buf) ) ) {
4480 0 : fd_memcpy( slot_hashes_buf, ro.data, ro.data_len );
4481 0 : slot_hashes_sz = ro.data_len;
4482 0 : have_slot_hashes = 1;
4483 0 : }
4484 0 : fd_accdb_unread_one( alut_ctx->accdb, &ro );
4485 :
4486 0 : fd_slot_hashes_t slot_hashes_view[1];
4487 0 : if( FD_LIKELY( have_slot_hashes &&
4488 0 : fd_sysvar_slot_hashes_view( slot_hashes_view, slot_hashes_buf, slot_hashes_sz ) ) ) {
4489 0 : serializing = !!fd_runtime_load_txn_address_lookup_tables( txn, payload, alut_ctx->accdb, alut_ctx->fork_id, alut_ctx->els, slot_hashes_view, sched->aluts );
4490 0 : sched->metrics->alut_success_cnt += (uint)!serializing;
4491 0 : } else {
4492 0 : serializing = 1;
4493 0 : }
4494 0 : }
4495 0 : }
4496 :
4497 : /* Transactions should not have duplicate accounts.
4498 : https://github.com/anza-xyz/agave/blob/v3.1.11/ledger/src/blockstore_processor.rs#L778-L790 */
4499 1541 : fd_acct_addr_t const * imms = fd_txn_get_acct_addrs( txn, payload );
4500 >1844*10^16 : fd_acct_addr_t * alts = (!alt_cnt||serializing) ? NULL : sched->aluts;
4501 1541 : alt_cnt = alts ? alt_cnt : 0UL;
4502 1541 : if( FD_UNLIKELY( fd_chkdup_check( sched->chkdup, imms, imm_cnt, alts, alt_cnt ) ) ) {
4503 0 : FD_LOG_INFO(( "bad block: DUPLICATE_ACCOUNT, slot %lu, parent slot %lu, txn_parsed_cnt %u", block->slot, block->parent_slot, block->txn_parsed_cnt ));
4504 0 : return FD_SCHED_DEAD_REASON_DUPLICATE_ACCOUNT;
4505 0 : }
4506 :
4507 : /* The ALT accounts that could not be resolved right now are accounted
4508 : for when the transaction completes and late registers them. */
4509 1541 : ulong writable_cnt = fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_WRITABLE ) - fd_ulong_if( !alts, fd_txn_account_cnt( txn, FD_TXN_ACCT_CAT_WRITABLE_ALT ), 0UL );
4510 1541 : if( FD_UNLIKELY( block->lthash_writable_cnt+writable_cnt>sched->lthash_txn_writers_max ) ) {
4511 0 : FD_LOG_INFO(( "bad block: TOO_MANY_WRITABLE_ACCOUNTS, slot %lu, parent slot %lu, txn_parsed_cnt %u, lthash_writable_cnt %u, writable_cnt %lu", block->slot, block->parent_slot, block->txn_parsed_cnt, block->lthash_writable_cnt, writable_cnt ));
4512 0 : return FD_SCHED_DEAD_REASON_TOO_MANY_WRITABLE_ACCOUNTS;
4513 0 : }
4514 :
4515 1541 : ulong bank_idx = (ulong)(block-sched->block_pool);
4516 1541 : fd_rdisp_neww_t neww[ fd_rdisp_neww_word_cnt ];
4517 1541 : ulong txn_idx = fd_rdisp_add_txn( sched->rdisp, bank_idx, txn, payload, alts, serializing, block->lthash_inband ? NULL : neww );
4518 1541 : FD_TEST( txn_idx && txn_idx<sched->depth );
4519 1541 : block->lthash_writable_cnt += (uint)writable_cnt;
4520 1542 : if( FD_LIKELY( !block->lthash_inband ) ) {
4521 1542 : ulong first_write_cnt = fd_rdisp_neww_cnt( neww );
4522 1542 : sched->metrics->lthash_parsed_writable_cnt += writable_cnt;
4523 1542 : sched->metrics->lthash_first_write_cnt += first_write_cnt;
4524 1542 : sched->metrics->lthash_coalesced_cnt += writable_cnt-first_write_cnt;
4525 1542 : block->add_owed_cnt += (uint)first_write_cnt;
4526 1542 : }
4527 1542 : if( FD_LIKELY( block_owns_lane_map( sched, block ) ) ) {
4528 4431 : for( ulong iter = fd_rdisp_neww_iter_init( neww ); !fd_rdisp_neww_iter_done( iter ); iter=fd_rdisp_neww_iter_next( neww, iter ) ) {
4529 2889 : fd_acct_addr_t const * acct = get_acct( txn, imms, alts, (ushort)iter );
4530 2889 : FD_TEST( acct );
4531 2889 : lthash_entry_new_write_observed( sched, block, acct );
4532 2889 : }
4533 1542 : }
4534 :
4535 : /* This transaction either needs to be consumed by sigverify or PoH.
4536 : If either of the two are caught up, mark it for consumption.
4537 : Otherwise, link it to the tail of the chain. */
4538 :
4539 1541 : ulong sigverify_dispatched_cnt = (ulong)block->txn_sigverify_done_cnt + (ulong)block->txn_sigverify_in_flight_cnt;
4540 1541 : int sigverify_caught_up = sigverify_dispatched_cnt==block->txn_parsed_cnt;
4541 1541 : #if !FD_SCHED_SKIP_POH
4542 1541 : fd_sched_mblk_t * parse_mblk = sched->mblk_pool + block->parse_mblk_idx;
4543 1541 : int poh_caught_up = parse_mblk->curr_txn_idx==block->txn_parsed_cnt;
4544 1541 : #endif
4545 :
4546 1541 : sched->txn_info_pool[ txn_idx ].next_idx = 0U;
4547 : /* Link to the previous transaction. */
4548 1541 : if( FD_LIKELY( block->txn_idx_tail ) ) {
4549 572 : FD_TEST( block->txn_idx_tail<sched->depth );
4550 572 : sched->txn_info_pool[ block->txn_idx_tail ].next_idx = (uint)txn_idx;
4551 572 : }
4552 : /* If either consumer is caught up, update next pointer to the new
4553 : transaction. */
4554 1541 : if( FD_LIKELY( sigverify_caught_up ) ) block->txn_sigverify_next_idx = (uint)txn_idx;
4555 1541 : #if !FD_SCHED_SKIP_POH
4556 1541 : if( FD_LIKELY( poh_caught_up ) ) parse_mblk->mixin_txn_idx = (uint)txn_idx;
4557 1541 : #endif
4558 1541 : block->txn_idx_tail = (uint)txn_idx;
4559 :
4560 1541 : sched->metrics->txn_parsed_cnt++;
4561 1541 : sched->metrics->alut_serializing_cnt += (uint)serializing;
4562 1541 : sched->txn_pool_free_cnt--;
4563 1541 : fd_txn_p_t * txn_p = sched->txn_pool + txn_idx;
4564 1541 : txn_p->payload_sz = (ushort)pay_sz;
4565 :
4566 1541 : txn_p->start_shred_idx = (ushort)fd_uint_min( shred_split( block, block->fec_buf_boff+sched->fec_buf_soff ), USHORT_MAX ); /* monitoring-only ushorts; saturate under bench shred counts */
4567 1541 : txn_p->start_shred_idx = fd_ushort_if( txn_p->start_shred_idx>0U, (ushort)(txn_p->start_shred_idx-1U), txn_p->start_shred_idx );
4568 1541 : txn_p->end_shred_idx = (ushort)fd_uint_min( shred_split( block, block->fec_buf_boff+sched->fec_buf_soff+(uint)pay_sz ), USHORT_MAX );
4569 :
4570 1541 : fd_memcpy( txn_p->payload, payload, pay_sz );
4571 1541 : fd_memcpy( TXN(txn_p), txn, txn_sz );
4572 1541 : txn_bitset_remove( sched->exec_done_set, txn_idx );
4573 1541 : txn_bitset_remove( sched->sigverify_done_set, txn_idx );
4574 1541 : txn_bitset_remove( sched->poh_mixin_done_set, txn_idx );
4575 1541 : sched->txn_info_pool[ txn_idx ].flags = fd_ulong_if( serializing, FD_SCHED_TXN_ALT_UNRESOLVED, 0UL );
4576 1541 : sched->txn_info_pool[ txn_idx ].received_ns = block->fec_completed_ns;
4577 1541 : sched->txn_info_pool[ txn_idx ].txn_err = 0;
4578 :
4579 1541 : sched->txn_info_pool[ txn_idx ].tick_parsed = fd_tickcount();
4580 1541 : sched->txn_info_pool[ txn_idx ].tick_sigverify_disp = LONG_MAX;
4581 1541 : sched->txn_info_pool[ txn_idx ].tick_sigverify_done = LONG_MAX;
4582 1541 : sched->txn_info_pool[ txn_idx ].tick_exec_disp = LONG_MAX;
4583 1541 : sched->txn_info_pool[ txn_idx ].tick_exec_done = LONG_MAX;
4584 1541 : sched->txn_info_pool[ txn_idx ].tick_load_start = LONG_MAX;
4585 1541 : sched->txn_info_pool[ txn_idx ].tick_check_start = LONG_MAX;
4586 1541 : sched->txn_info_pool[ txn_idx ].tick_exec_start = LONG_MAX;
4587 1541 : sched->txn_info_pool[ txn_idx ].tick_commit_start = LONG_MAX;
4588 1541 : sched->txn_info_pool[ txn_idx ].tick_commit_end = LONG_MAX;
4589 :
4590 1541 : sched->txn_info_pool[ txn_idx ].slot = block->slot;
4591 1541 : sched->txn_info_pool[ txn_idx ].bank_seq = ULONG_MAX;
4592 1541 : sched->txn_info_pool[ txn_idx ].index_in_slot = block->txn_parsed_cnt;
4593 1541 : sched->txn_info_pool[ txn_idx ].exec_tile_idx = ULONG_MAX;
4594 1541 : sched->txn_info_pool[ txn_idx ].sigverify_exec_tile_idx = ULONG_MAX;
4595 1541 : sched->txn_info_pool[ txn_idx ].compute_units_consumed = 0U;
4596 1541 : sched->txn_info_pool[ txn_idx ].max_compute_units = ULONG_MAX;
4597 1541 : sched->txn_info_pool[ txn_idx ].transaction_fee = 0UL;
4598 1541 : sched->txn_info_pool[ txn_idx ].priority_fee = 0UL;
4599 1541 : sched->txn_info_pool[ txn_idx ].tips = 0UL;
4600 1541 : sched->fec_buf_soff += (uint)pay_sz;
4601 1541 : block->txn_parsed_cnt++;
4602 : #if FD_SCHED_SKIP_SIGVERIFY
4603 : txn_bitset_insert( sched->sigverify_done_set, txn_idx );
4604 : block->txn_sigverify_done_cnt++;
4605 : #endif
4606 : #if FD_SCHED_SKIP_POH
4607 : txn_bitset_insert( sched->poh_mixin_done_set, txn_idx );
4608 : #endif
4609 1541 : block->txns_rem--;
4610 1541 : return FD_SCHED_DEAD_REASON_NONE;
4611 1541 : }
4612 :
4613 : #undef CHECK
4614 : #undef CHECK_LEFT
4615 :
4616 : static void
4617 0 : dispatch_sigverify( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, fd_sched_task_t * out ) {
4618 : /* Dispatch transactions for sigverify in parse order. */
4619 0 : uint txn_idx = block->txn_sigverify_next_idx;
4620 0 : FD_TEST( txn_idx && txn_idx<sched->depth );
4621 0 : out->task_type = FD_SCHED_TT_TXN_SIGVERIFY;
4622 0 : out->txn_sigverify->bank_idx = bank_idx;
4623 0 : out->txn_sigverify->txn_idx = txn_idx;
4624 0 : out->txn_sigverify->exec_idx = (ulong)exec_tile_idx;
4625 0 : sched->txn_info_pool[ out->txn_sigverify->txn_idx ].sigverify_exec_tile_idx = (ulong)exec_tile_idx;
4626 0 : sched->sigverify_ready_bitset[ 0 ] = fd_ulong_clear_bit( sched->sigverify_ready_bitset[ 0 ], exec_tile_idx );
4627 0 : sched->tile_to_bank_idx[ exec_tile_idx ] = bank_idx;
4628 0 : block->txn_sigverify_in_flight_cnt++;
4629 0 : block->txn_sigverify_next_idx = sched->txn_info_pool[ txn_idx ].next_idx;
4630 0 : exec_ready_bitsets_check( sched );
4631 0 : }
4632 :
4633 : /* Retires a microblock whose PoH hashing has completed, and eagerly
4634 : does as much mixin as possible. Returns FD_SCHED_DEAD_REASON_NONE on
4635 : success, else the dead reason. */
4636 : static int
4637 0 : poh_retire_mblk( fd_sched_t * sched, fd_sched_block_t * block, uint mblk_idx ) {
4638 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
4639 0 : block->poh_hashing_done_cnt++;
4640 0 : sched->metrics->mblk_poh_hashed_cnt++;
4641 0 : if( FD_LIKELY( !mblk->is_tick ) ) {
4642 0 : mblk_slist_idx_push_tail( block->mblks_mixin_in_progress, mblk_idx, sched->mblk_pool );
4643 0 : } else {
4644 0 : block->poh_hash_cmp_done_cnt++;
4645 0 : sched->metrics->mblk_poh_done_cnt++;
4646 0 : free_mblk( sched, block, mblk_idx );
4647 0 : if( FD_UNLIKELY( memcmp( mblk->curr_hash, mblk->end_hash, sizeof(fd_hash_t) ) ) ) {
4648 0 : FD_BASE58_ENCODE_32_BYTES( mblk->curr_hash->hash, our_str );
4649 0 : FD_BASE58_ENCODE_32_BYTES( mblk->end_hash->hash, ref_str );
4650 0 : FD_LOG_INFO(( "bad block: TICK_HASH_MISMATCH, mblk %u, ours %s, claimed %s, slot %lu, parent slot %lu", mblk_idx, our_str, ref_str, block->slot, block->parent_slot ));
4651 0 : return FD_SCHED_DEAD_REASON_TICK_HASH_MISMATCH;
4652 0 : }
4653 0 : }
4654 0 : int mixin_res;
4655 0 : while( (mixin_res=maybe_mixin( sched, block )) ) {
4656 0 : if( FD_UNLIKELY( mixin_res==-1 ) ) return FD_SCHED_DEAD_REASON_ENTRY_HASH_MISMATCH;
4657 0 : FD_TEST( mixin_res==1||mixin_res==2 );
4658 0 : }
4659 0 : return FD_SCHED_DEAD_REASON_NONE;
4660 0 : }
4661 :
4662 : /* Assembles a PoH task for the given exec tile. Assumes there is a
4663 : PoH task available for dispatching. A tick needs one hash and a
4664 : transaction microblock none (its hash is the mixin), so a batch holds
4665 : only one kind. A transaction microblock still dispatches, as a
4666 : degenerate zero-hashcnt task, so that it retires through
4667 : fd_sched_task_done like any other PoH task. */
4668 : static void
4669 0 : dispatch_poh( fd_sched_t * sched, fd_sched_block_t * block, ulong bank_idx, int exec_tile_idx, fd_sched_task_t * out ) {
4670 0 : fd_sched_poh_hash_t * poh = out->poh_hash;
4671 0 : ulong cnt = 0UL;
4672 0 : ulong hashcnt = 0UL;
4673 0 : while( cnt<FD_SCHED_POH_PARA && block->mblk_unhashed_cnt ) {
4674 0 : uint mblk_idx = (uint)mblk_slist_idx_peek_head( block->mblks_unhashed, sched->mblk_pool );
4675 0 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
4676 0 : ulong mblk_hashcnt = (ulong)mblk->is_tick;
4677 0 : if( FD_UNLIKELY( cnt && mblk_hashcnt!=hashcnt ) ) break;
4678 0 : mblk_slist_idx_pop_head( block->mblks_unhashed, sched->mblk_pool );
4679 0 : block->mblk_unhashed_cnt--;
4680 0 : hashcnt = mblk_hashcnt;
4681 0 : poh->mblk_idx[ cnt ] = mblk_idx;
4682 0 : memcpy( poh->hash+cnt, mblk->curr_hash, sizeof(fd_hash_t) );
4683 0 : cnt++;
4684 0 : block->poh_mblk_in_flight_cnt++;
4685 0 : }
4686 0 : FD_TEST( cnt ); /* poh_hashing_queued_cnt>0 implies at least one queued microblock. */
4687 :
4688 0 : out->task_type = FD_SCHED_TT_POH_HASH;
4689 0 : poh->bank_idx = bank_idx;
4690 0 : poh->exec_idx = (ulong)exec_tile_idx;
4691 0 : poh->cnt = cnt;
4692 0 : poh->hashcnt = hashcnt;
4693 0 : sched->poh_inflight[ exec_tile_idx ] = *poh;
4694 :
4695 0 : sched->poh_ready_bitset[ 0 ] = fd_ulong_clear_bit( sched->poh_ready_bitset[ 0 ], exec_tile_idx );
4696 0 : sched->tile_to_bank_idx[ exec_tile_idx ] = bank_idx;
4697 0 : block->poh_hashing_in_flight_cnt++;
4698 0 : exec_ready_bitsets_check( sched );
4699 0 : }
4700 :
4701 : /* Does up to one transaction mixin. Returns 1 if one mixin was done, 2
4702 : if that mixin also completed a microblock, 0 if no transaction mixin
4703 : was available, -1 if there is a PoH verify error. */
4704 : FD_WARN_UNUSED static int
4705 4555 : maybe_mixin( fd_sched_t * sched, fd_sched_block_t * block ) {
4706 4555 : if( FD_UNLIKELY( mblk_slist_is_empty( block->mblks_mixin_in_progress, sched->mblk_pool ) ) ) return 0;
4707 1 : FD_TEST( block->poh_hashing_done_cnt-block->poh_hash_cmp_done_cnt>0 );
4708 :
4709 : /* The microblock we would like to do mixin on is at the head of the
4710 : queue. It may have had some mixin, it may have never had any
4711 : mixin. In the case of the former, we should continue to mixin the
4712 : same head microblock until it's done, lest the per-block bmtree
4713 : gets clobbered when we start a new one. */
4714 1 : ulong mblk_idx = mblk_slist_idx_pop_head( block->mblks_mixin_in_progress, sched->mblk_pool );
4715 1 : fd_sched_mblk_t * mblk = sched->mblk_pool+mblk_idx;
4716 :
4717 1 : if( FD_UNLIKELY( mblk->end_txn_idx>block->txn_parsed_cnt ) ) {
4718 : /* A partially parsed microblock is by definition at the end of the
4719 : FEC stream. If such a microblock is in progress, there should be
4720 : no other microblock in this block so far that hasn't been
4721 : dispatched, because microblocks are dispatched in parse order. */
4722 0 : if( FD_UNLIKELY( block->mblk_unhashed_cnt ) ) {
4723 0 : sched->print_buf_sz = 0UL;
4724 0 : print_all( sched, block );
4725 0 : FD_LOG_CRIT(( "invariant violation end_txn_idx %lu: %s", mblk->end_txn_idx, sched->print_buf ));
4726 0 : }
4727 :
4728 : /* If we've decided to start mixin on a partially parsed microblock,
4729 : there better be nothing else in-progress. Otherwise, they might
4730 : clobber the per-block bmtree for mixin. */
4731 0 : if( FD_UNLIKELY( mblk->curr_txn_idx!=mblk->start_txn_idx && (block->poh_mblk_in_flight_cnt||!mblk_slist_is_empty( block->mblks_mixin_in_progress, sched->mblk_pool )) ) ) {
4732 0 : sched->print_buf_sz = 0UL;
4733 0 : print_all( sched, block );
4734 0 : FD_LOG_CRIT(( "invariant violation end_txn_idx %lu start_txn_idx %lu curr_txn_idx %lu: %s", mblk->end_txn_idx, mblk->start_txn_idx, mblk->curr_txn_idx, sched->print_buf ));
4735 0 : }
4736 0 : }
4737 :
4738 : /* Very rarely, we've finished hashing, but not all transactions in
4739 : the microblock have been parsed out. This can happen if we haven't
4740 : received all the FEC sets for this microblock. We can't yet fully
4741 : mixin the microblock. So we'll stick it back into the end of the
4742 : queue, and try to see if there's a fully parsed microblock.
4743 : Unless, there's truly nothing else to mixin. Then we would start
4744 : mixin with the partially parsed microblock. We do this because the
4745 : txn pool is meant to be an OOO scheduling window not tied to
4746 : max_live_slots sizing requirements, so there shouldn't be a way for
4747 : external input to tie up txn pool entries for longer than
4748 : necessary. */
4749 1 : if( FD_UNLIKELY( mblk->curr_txn_idx>=block->txn_parsed_cnt || /* Nothing more to mixin for this microblock. */
4750 1 : (mblk->end_txn_idx>block->txn_parsed_cnt && /* There is something to mixin, but the microblock isn't fully parsed yet ... */
4751 1 : mblk->curr_txn_idx==mblk->start_txn_idx && /* ... and we haven't started mixin on it yet ... */
4752 1 : (block->poh_mblk_in_flight_cnt || /* ... and another microblock is in-progress and might preempt this microblock and clobber the bmtree, so we shouldn't start the partial microblock just yet. Counting microblocks rather than tasks matters: a batched PoH task is retired one microblock at a time, and the ones not yet retired are off every list. */
4753 1 : !mblk_slist_is_empty( block->mblks_mixin_in_progress, sched->mblk_pool ))) ) ) {
4754 0 : mblk_slist_idx_push_tail( block->mblks_mixin_in_progress, mblk_idx, sched->mblk_pool );
4755 :
4756 : /* No other microblock in the mixin queue. */
4757 0 : if( FD_UNLIKELY( block->poh_hashing_done_cnt-block->poh_hash_cmp_done_cnt==1 ) ) return 0;
4758 :
4759 : /* At this point, there's at least one more microblock in the mixin
4760 : queue we could try. It's a predecessor (in parse order) that
4761 : finished hashing later than the partially parsed microblock at
4762 : the head of the mixin queue. */
4763 :
4764 : /* It should never clobber the bmtree for a microblock that has had some mixin done on it. */
4765 0 : if( FD_UNLIKELY( mblk->curr_txn_idx!=mblk->start_txn_idx ) ) {
4766 0 : sched->print_buf_sz = 0UL;
4767 0 : print_all( sched, block );
4768 0 : FD_LOG_CRIT(( "invariant violation curr_txn_idx %lu start_txn_idx %lu: %s", mblk->curr_txn_idx, mblk->start_txn_idx, sched->print_buf ));
4769 0 : }
4770 :
4771 0 : mblk_idx = mblk_slist_idx_pop_head( block->mblks_mixin_in_progress, sched->mblk_pool );
4772 0 : mblk = sched->mblk_pool+mblk_idx;
4773 :
4774 : /* It should be a fresh microblock for mixin. */
4775 0 : FD_TEST( mblk->curr_txn_idx==mblk->start_txn_idx );
4776 : /* Invariant: at any given point in time, there can be at most one
4777 : microblock that hasn't been fully parsed yet, due to the nature
4778 : of sequential parsing. So this microblock has to be fully
4779 : parsed. */
4780 0 : FD_TEST( mblk->end_txn_idx<=block->txn_parsed_cnt );
4781 0 : }
4782 :
4783 1 : FD_TEST( mblk->curr_txn_idx<mblk->end_txn_idx );
4784 :
4785 : /* Now mixin. */
4786 1 : if( FD_LIKELY( mblk->curr_txn_idx==mblk->start_txn_idx ) ) block->bmtree = fd_bmtree_commit_init( block->bmtree_mem, 32UL, 1UL, 0UL ); /* Optimize for single-transaction microblocks, which are the majority. */
4787 :
4788 1 : ulong txn_gidx = mblk->mixin_txn_idx;
4789 1 : uint next_txn_pool_idx = sched->txn_info_pool[ txn_gidx ].next_idx;
4790 1 : fd_txn_p_t * _txn = sched->txn_pool+txn_gidx;
4791 1 : fd_txn_t * txn = TXN(_txn);
4792 1 : for( ulong j=0; j<txn->signature_cnt; j++ ) {
4793 0 : fd_bmtree_node_t node[ 1 ];
4794 0 : fd_bmtree_hash_leaf( node, _txn->payload+txn->signature_off+FD_TXN_SIGNATURE_SZ*j, 64UL, 1UL );
4795 0 : fd_bmtree_commit_append( block->bmtree, node, 1UL );
4796 0 : mblk->curr_sig_cnt++;
4797 0 : }
4798 :
4799 : /* Release the txn_idx. */
4800 1 : txn_bitset_insert( sched->poh_mixin_done_set, txn_gidx );
4801 1 : sched->metrics->txn_mixin_done_cnt++;
4802 1 : if( txn_bitset_test( sched->exec_done_set, txn_gidx ) && txn_bitset_test( sched->sigverify_done_set, txn_gidx ) ) {
4803 0 : if( FD_UNLIKELY( block->txn_idx_tail==(uint)txn_gidx ) ) block->txn_idx_tail = 0U;
4804 0 : fd_rdisp_complete_txn( sched->rdisp, txn_gidx, 1 );
4805 0 : sched->txn_pool_free_cnt++;
4806 0 : block->txn_done_cnt++;
4807 0 : sched->metrics->txn_done_cnt++;
4808 0 : }
4809 :
4810 1 : mblk->curr_txn_idx++;
4811 1 : mblk->mixin_txn_idx = next_txn_pool_idx;
4812 1 : int rv = 2;
4813 1 : if( FD_LIKELY( mblk->curr_txn_idx==mblk->end_txn_idx ) ) {
4814 : /* Ready to compute the final hash for this microblock. */
4815 0 : block->poh_hash_cmp_done_cnt++;
4816 0 : sched->metrics->mblk_poh_done_cnt++;
4817 0 : uchar * root = fd_bmtree_commit_fini( block->bmtree );
4818 0 : uchar mixin_buf[ 64 ];
4819 0 : fd_memcpy( mixin_buf, mblk->curr_hash, 32UL );
4820 0 : fd_memcpy( mixin_buf+32UL, root, 32UL );
4821 0 : fd_sha256_hash( mixin_buf, 64UL, mblk->curr_hash );
4822 0 : free_mblk( sched, block, (uint)mblk_idx );
4823 : /* Bypass PoH verification for fuzzing/testing throughput. */
4824 0 : if( FD_UNLIKELY( !sched->bypass_poh_verify && memcmp( mblk->curr_hash, mblk->end_hash, sizeof(fd_hash_t) ) ) ) {
4825 0 : FD_BASE58_ENCODE_32_BYTES( mblk->curr_hash->hash, our_str );
4826 0 : FD_BASE58_ENCODE_32_BYTES( mblk->end_hash->hash, ref_str );
4827 0 : FD_LOG_INFO(( "bad block: poh hash mismatch on mblk %lu, ours %s, claimed %s, txns [%lu,%lu), %u sigs, slot %lu, parent slot %lu", mblk_idx, our_str, ref_str, mblk->start_txn_idx, mblk->end_txn_idx, mblk->curr_sig_cnt, block->slot, block->parent_slot ));
4828 0 : return -1;
4829 0 : }
4830 1 : } else {
4831 : /* There are more transactions to mixin in this microblock. */
4832 1 : mblk_slist_idx_push_head( block->mblks_mixin_in_progress, mblk_idx, sched->mblk_pool );
4833 1 : rv = 1;
4834 1 : }
4835 :
4836 1 : return rv;
4837 1 : }
4838 :
4839 : /* Does activating bank_idx move execution to another fork? A block
4840 : that descends from the last active block continues its fork, even
4841 : across blocks that were added done and never activated, such as a
4842 : leader block. */
4843 : static int
4844 1001 : is_fork_switch( fd_sched_t * sched, ulong bank_idx ) {
4845 1001 : if( FD_UNLIKELY( sched->last_active_bank_idx==ULONG_MAX ) ) return 0;
4846 0 : for( ulong idx=bank_idx; idx!=ULONG_MAX; idx=block_pool_ele( sched, idx )->parent_idx ) {
4847 0 : if( idx==sched->last_active_bank_idx ) return 0;
4848 0 : }
4849 0 : return 1;
4850 0 : }
4851 :
4852 : static void
4853 1003 : try_activate_block( fd_sched_t * sched ) {
4854 : /* Early return if there's already an active block. */
4855 1003 : if( FD_LIKELY( sched->active_bank_idx!=ULONG_MAX ) ) return;
4856 :
4857 : /* See if there are any allocated staging lanes that we can activate
4858 : for scheduling ... */
4859 1003 : ulong staged_bitset = sched->staged_bitset;
4860 1003 : while( staged_bitset ) {
4861 1001 : int lane_idx = fd_ulong_find_lsb( staged_bitset );
4862 1001 : staged_bitset = fd_ulong_pop_lsb( staged_bitset );
4863 :
4864 1001 : ulong head_idx = sched->staged_head_bank_idx[ lane_idx ];
4865 1001 : fd_sched_block_t * head_block = block_pool_ele( sched, head_idx );
4866 1001 : fd_sched_block_t * parent_block = block_pool_ele( sched, head_block->parent_idx );
4867 : //FIXME: restore these invariant checks when we have immediate demotion of dying blocks
4868 : //Today, dying blocks can remain staged if they have in-flight transactions.
4869 : // if( FD_UNLIKELY( parent_block->dying ) ) {
4870 : // /* Invariant: no child of a dying block should be staged. */
4871 : // FD_LOG_CRIT(( "invariant violation: staged_head_bank_idx %lu, slot %lu, parent slot %lu on lane %d has parent_block->dying set, slot %lu, parent slot %lu",
4872 : // head_idx, head_block->slot, head_block->parent_slot, lane_idx, parent_block->slot, parent_block->parent_slot ));
4873 : // }
4874 : // if( FD_UNLIKELY( head_block->dying ) ) {
4875 : // /* Invariant: no dying block should be staged. */
4876 : // FD_LOG_CRIT(( "invariant violation: staged_head_bank_idx %lu, slot %lu, prime %lu on lane %u has head_block->dying set",
4877 : // head_idx, (ulong)head_block->block_id.slot, (ulong)head_block->block_id.prime, lane_idx ));
4878 : // }
4879 1001 : if( block_is_done( parent_block ) && block_is_activatable( head_block ) ) {
4880 : /* ... Yes, on this staging lane the parent block is done. So we
4881 : can activate the staged child. */
4882 1001 : if( FD_UNLIKELY( is_fork_switch( sched, head_idx ) ) ) { /* Forking is rare. */
4883 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: fork switch to lane %d", head_block->slot, head_idx, lane_idx ));
4884 0 : sched->metrics->lane_switch_cnt++;
4885 1001 : } else {
4886 1001 : FD_LOG_DEBUG(( "activating block %lu:%lu on lane %d", head_block->slot, head_idx, lane_idx ));
4887 1001 : }
4888 1001 : active_bank_set( sched, head_idx );
4889 1001 : return;
4890 1001 : }
4891 1001 : }
4892 :
4893 : /* ... No lane head with a done parent has real work. Wake one that
4894 : can speculate on the additions it owes, if the speculation pool has
4895 : room enough to make the switch worthwhile. */
4896 2 : staged_bitset = sched->staged_bitset;
4897 2 : while( staged_bitset ) {
4898 0 : int lane_idx = fd_ulong_find_lsb( staged_bitset );
4899 0 : staged_bitset = fd_ulong_pop_lsb( staged_bitset );
4900 :
4901 0 : ulong head_idx = sched->staged_head_bank_idx[ lane_idx ];
4902 0 : fd_sched_block_t * head_block = block_pool_ele( sched, head_idx );
4903 0 : fd_sched_block_t * parent_block = block_pool_ele( sched, head_block->parent_idx );
4904 0 : if( block_is_done( parent_block ) && block_is_stageable( head_block ) && block_may_speculate( sched, head_block, FD_SCHED_LTHASH_SPEC_THRESHOLD_PER_TILE*sched->exec_cnt ) ) {
4905 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: speculation on lane %d", head_block->slot, head_idx, lane_idx ));
4906 0 : sched->metrics->lane_switch_cnt += (uint)is_fork_switch( sched, head_idx );
4907 0 : active_bank_set( sched, head_idx );
4908 0 : return;
4909 0 : }
4910 0 : }
4911 :
4912 : /* ... No, promote unstaged blocks. */
4913 2 : ulong root_idx = sched->root_idx;
4914 2 : if( FD_UNLIKELY( root_idx==ULONG_MAX ) ) {
4915 0 : FD_LOG_CRIT(( "invariant violation: root_idx==ULONG_MAX indicating fd_sched is uninitialized" ));
4916 0 : }
4917 : /* Find and stage the longest stageable unstaged fork. This is a
4918 : policy decision. */
4919 2 : ulong depth = compute_longest_unstaged_fork( sched, root_idx );
4920 2 : if( FD_LIKELY( depth>0UL ) ) {
4921 0 : if( FD_UNLIKELY( sched->staged_bitset==fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) {
4922 : /* No more staging lanes available. All of them are occupied by
4923 : slow squatters. Only empty blocks can be demoted, and so
4924 : blocks with in-flight transactions, including dying in-flight
4925 : blocks, shouldn't be demoted. We demote all demotable lanes.
4926 : Demotion isn't all that expensive, since demotable blocks have
4927 : no transactions in them. If a demoted block proves to be
4928 : active still, it'll naturally promote back into a staging lane.
4929 :
4930 : In fact, all lanes should be demotable at this point. None of
4931 : the lanes have anything dispatchable, otherwise we would have
4932 : simply activated one of the dispatchable lanes. None of the
4933 : lanes have anything in-flight either, as we allow for a grace
4934 : period while something is in-flight, before we deactivate any
4935 : block. In principle, we could get rid of the grace period and
4936 : deactivate right away. In that case, it's okay if nothing is
4937 : demotable at the moment, as that simply implies that all lanes
4938 : have in-flight tasks. We would get another chance to try to
4939 : demote when the last in-flight task on any lane completes.
4940 :
4941 : A note on why this whole thing does not deadlock:
4942 :
4943 : One might reasonably wonder what happens if all the lanes are
4944 : non-empty, non-dead, but for some reason couldn't be activated
4945 : for dispatching. We would deadlock in this case, as no lane
4946 : dispatches to the point of being demotable, and no unstaged
4947 : block can be promoted. Such is not in fact possible. The only
4948 : way a dispatchable lane can be ineligible for activation is if
4949 : it has a parent block that isn't done yet. So a deadlock
4950 : happens when this parent block, or any of its dispatchable
4951 : ancestors, is unstaged. An important invariant we maintain is
4952 : that a staged block can't have an unstaged stageable parent.
4953 : This invariant, by induction, gives us the guarantee that at
4954 : least one of the lanes can be activated. */
4955 0 : for( int l=0; l<(int)FD_SCHED_MAX_STAGING_LANES; l++ ) {
4956 : /* We would be able to assert that all lanes are demotable,
4957 : except that abandoned blocks are given no grace period for
4958 : deactivation. So there could be lanes transiently occupied
4959 : by dying blocks that are neither demotable (due to in-flight
4960 : tasks) nor activatable (due to being dying). If
4961 : rdisp_demote() supported non-empty blocks, then we could
4962 : probably restore the assertion. */
4963 0 : if( FD_UNLIKELY( !lane_is_demotable( sched, l ) ) ) continue;
4964 0 : demote_lane( sched, l );
4965 0 : sched->metrics->lane_demoted_cnt++;
4966 0 : }
4967 : /* We weren't able to successfully demote anything. This is
4968 : likely because all lanes are occupied by dying blocks with
4969 : in-flight tasks. We would get another chance to try to demote
4970 : when the last in-flight task on any lane completes. */
4971 0 : if( FD_UNLIKELY( sched->staged_bitset==fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) ) ) return;
4972 0 : }
4973 0 : FD_TEST( sched->staged_bitset!=fd_ulong_mask_lsb( FD_SCHED_MAX_STAGING_LANES ) );
4974 0 : int lane_idx = pick_free_lane( sched );
4975 0 : ulong head_bank_idx = promote_from_longest_unstaged_fork( sched, root_idx, lane_idx );
4976 0 : if( FD_UNLIKELY( head_bank_idx==ULONG_MAX ) ) {
4977 : /* We found a promotable fork depth>0. This should not happen. */
4978 0 : FD_LOG_CRIT(( "invariant violation: head_bank_idx==ULONG_MAX" ));
4979 0 : }
4980 : /* We don't bother with promotion unless the block is immediately
4981 : dispatchable. So it's okay to set the active block here. This
4982 : doesn't cause out-of-order block replay because the parent of the
4983 : promoted block must be done. We can prove this by contradiction
4984 : as follows. The promoted block is the shallowest promotable
4985 : block of its fork, so its parent is not promotable. The parent
4986 : is complete, because FEC sets arrive in order along a fork and a
4987 : child of it has arrived. Suppose the parent were not done. It
4988 : is not dying, or the promoted block would be dying too and not
4989 : promotable. It is not unstaged, or it would be promotable, being
4990 : complete, and so dispatchable until it ends. So it would be
4991 : staged and dispatchable, with nothing in flight, since a block
4992 : with tasks in flight is the active block or dying, and no block
4993 : is active here. That makes it activatable, and only a parent of
4994 : its own that is not done kept the first loop above from
4995 : activating it. That parent is staged as well, because a staged
4996 : block never has an unstaged parent that still has to run, and the
4997 : same argument applies to it, and so on up the fork chain until a
4998 : staged block with a done parent is reached, which the first loop
4999 : would have activated. */
5000 0 : fd_sched_block_t * head_block = block_pool_ele( sched, head_bank_idx );
5001 0 : FD_LOG_DEBUG(( "activating block %lu:%lu: unstaged promotion to lane %d", head_block->slot, head_bank_idx, lane_idx ));
5002 0 : sched->metrics->lane_switch_cnt += (uint)is_fork_switch( sched, head_bank_idx );
5003 0 : active_bank_set( sched, head_bank_idx );
5004 0 : return;
5005 0 : }
5006 : /* No unstaged blocks to promote. So we're done. Yay. */
5007 2 : }
5008 :
5009 : static void
5010 4555 : check_or_set_active_block( fd_sched_t * sched ) {
5011 4555 : if( FD_UNLIKELY( sched->active_bank_idx==ULONG_MAX ) ) {
5012 1002 : try_activate_block( sched );
5013 3553 : } else {
5014 3553 : fd_sched_block_t * active_block = block_pool_ele( sched, sched->active_bank_idx );
5015 3553 : if( FD_UNLIKELY( block_should_deactivate( sched, active_block ) ) ) {
5016 0 : sched->print_buf_sz = 0UL;
5017 0 : print_all( sched, active_block );
5018 0 : FD_LOG_NOTICE(( "%s", sched->print_buf ));
5019 0 : FD_LOG_CRIT(( "invariant violation: should have been deactivated" ));
5020 0 : }
5021 3553 : }
5022 4555 : }
5023 :
5024 : /* This function has two main jobs:
5025 : - Mark everything on the fork tree dying.
5026 : - Take blocks out of rdisp if possible. */
5027 : static void
5028 2 : subtree_mark_and_maybe_prune_rdisp( fd_sched_t * sched, fd_sched_block_t * block ) {
5029 2 : if( FD_UNLIKELY( block->rooted ) ) {
5030 0 : FD_LOG_CRIT(( "invariant violation: rooted block should not be abandoned, slot %lu, parent slot %lu",
5031 0 : block->slot, block->parent_slot ));
5032 0 : }
5033 : /* All minority fork nodes pass through this function eventually. So
5034 : this is a good point to check per-node invariants for minority
5035 : forks. */
5036 2 : if( FD_UNLIKELY( block->staged && !block->in_rdisp ) ) {
5037 0 : FD_LOG_CRIT(( "invariant violation: staged block is not in the dispatcher, slot %lu, parent slot %lu",
5038 0 : block->slot, block->parent_slot ));
5039 0 : }
5040 :
5041 2 : fd_sched_block_t * parent = block_pool_ele( sched, block->parent_idx );
5042 2 : if( FD_UNLIKELY( !parent || !parent->in_sched ) ) {
5043 : /* Only the root has no parent. Abandon should never be called on
5044 : the root. So any block we are trying to abandon should have a
5045 : parent. */
5046 0 : FD_LOG_CRIT(( "invariant violation: parent not found slot %lu, parent slot %lu",
5047 0 : block->slot, block->parent_slot ));
5048 0 : }
5049 :
5050 : /* Gated on the parent going down and on this block not already being
5051 : on its way down.
5052 :
5053 : The parent test is because this function also runs on the block the
5054 : public abandon API was called on, whose parent is very much alive.
5055 : Inheriting there would stamp DEAD_ANCESTOR on a block that went
5056 : down on its own, and overwrite the discarded bit the caller just
5057 : set with the live parent's 0.
5058 :
5059 : The block test is because the walk also revisits blocks that
5060 : already went down, and an ancestor abandoned later must not
5061 : re-label them. This cannot be decided based on dead_reason alone
5062 : because a verdict the replay tile made leaves it NONE.
5063 :
5064 : Descendants work fine either way, because dying is set below before
5065 : we recurse, so every child rightfully sees a dying parent. */
5066 2 : if( FD_UNLIKELY( parent->dying && !block->dying ) ) record_lineage_death( block, parent );
5067 :
5068 : /* Setting the flag is non-optional and can happen more than once. */
5069 2 : block->dying = 1;
5070 2 : slot_state_close( sched, block );
5071 2 : lthash_release_held_batch( sched, block );
5072 :
5073 : /* Removal from dispatcher should only happen once. */
5074 2 : if( block->in_rdisp ) {
5075 :
5076 : /* The dispatcher expects blocks to be abandoned in the same order
5077 : that they were added on each lane, which holds trivially with
5078 : every staged block having its own lane. */
5079 2 : if( FD_UNLIKELY( block->staged && sched->active_bank_idx==sched->staged_head_bank_idx[ block->staging_lane ] && sched->active_bank_idx!=ULONG_MAX ) ) {
5080 2 : FD_TEST( block_pool_ele( sched, sched->active_bank_idx )==block );
5081 2 : FD_LOG_DEBUG(( "reset active_bank_idx %lu: abandon", sched->active_bank_idx ));
5082 2 : sched->last_active_bank_idx = sched->active_bank_idx;
5083 2 : active_bank_set( sched, ULONG_MAX );
5084 2 : sched->metrics->deactivate_abandoned_cnt++;
5085 2 : }
5086 :
5087 : /* We inform the dispatcher of an abandon only when there are no
5088 : more in-flight transactions. Otherwise, if the dispatcher
5089 : recycles the same txn_id that was just abandoned, and we receive
5090 : completion of an in-flight transaction whose txn_id was just
5091 : recycled. */
5092 : // FIXME The recycling might be fine now that we no longer use
5093 : // txn_id to index into anything. We might be able to just drop
5094 : // txn_id on abandoned blocks. Though would this leak transaction
5095 : // content if the txn_id is recycled?
5096 : // Note that subtree pruning from sched isn't dependent on the
5097 : // in-flight check being present here, as is_prunable already checks
5098 : // for in-flight==0.
5099 2 : int abandon = !block_is_in_flight( block );
5100 :
5101 2 : if( abandon ) {
5102 2 : block->in_rdisp = 0;
5103 2 : sched->metrics->lthash_discarded_add_owed_cnt += block->add_owed_cnt;
5104 2 : sched->metrics->lthash_discarded_add_pending_cnt += block->add_created_cnt-block->add_surfaced_cnt;
5105 2 : sched->metrics->lthash_discarded_sub_queued_cnt += block->sub_queued_cnt;
5106 2 : sched->metrics->lthash_discarded_sub_applied_cnt += block->sub_applied_cnt;
5107 2 : sched->metrics->lthash_discarded_add_applied_cnt += block->add_applied_cnt-block->add_undone_cnt;
5108 2 : lthash_block_reset( sched, block );
5109 2 : lthash_release_reservation( sched, block );
5110 2 : fd_rdisp_abandon_block( sched->rdisp, (ulong)(block-sched->block_pool) );
5111 2 : block->txn_idx_tail = 0U;
5112 2 : sched->txn_pool_free_cnt += block->txn_parsed_cnt-block->txn_done_cnt; /* in_flight_cnt==0 */
5113 :
5114 2 : sched->metrics->block_abandoned_cnt++;
5115 2 : sched->metrics->txn_abandoned_parsed_cnt += block->txn_parsed_cnt;
5116 2 : sched->metrics->txn_abandoned_exec_done_cnt += block->txn_exec_done_cnt;
5117 2 : sched->metrics->txn_abandoned_done_cnt += block->txn_done_cnt;
5118 :
5119 : //FIXME when demote supports non-empty blocks, we should demote
5120 : //the block from the lane unconditionally and immediately,
5121 : //regardless of whether it's safe to abandon or not. So a block
5122 : //would go immediately from staged to unstaged and eventually to
5123 : //abandoned.
5124 2 : if( FD_LIKELY( block->staged ) ) {
5125 2 : FD_LOG_DEBUG(( "block %lu:%lu exited lane %lu: abandon", block->slot, block_to_idx( sched, block ), block->staging_lane ));
5126 2 : block->staged = 0;
5127 : /* Release the staging lane, which the block had to itself. */
5128 2 : sched->staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, (int)block->staging_lane );
5129 2 : sched->staged_head_bank_idx[ block->staging_lane ] = ULONG_MAX;
5130 2 : }
5131 2 : }
5132 2 : }
5133 :
5134 : /* Abandon the entire fork chaining off of this block. */
5135 2 : ulong child_idx = block->child_idx;
5136 2 : while( child_idx!=ULONG_MAX ) {
5137 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
5138 0 : subtree_mark_and_maybe_prune_rdisp( sched, child );
5139 0 : child_idx = child->sibling_idx;
5140 0 : }
5141 2 : }
5142 :
5143 : /* This function tells sched that the subtree is no longer worth
5144 : replaying. It can happen as a result of one of the following.
5145 : - Bad block at head of lane.
5146 : - Bad block at tail of lane.
5147 : - Root notify.
5148 :
5149 : It's safe to call this function more than once on the same block. */
5150 : static void
5151 2 : subtree_abandon( fd_sched_t * sched, fd_sched_block_t * block ) {
5152 2 : subtree_mark_and_maybe_prune_rdisp( sched, block );
5153 : /* We do not gate refcnt releases on the entire subtree being
5154 : prunable. In the root notify case there can be in-flight tasks in
5155 : the middle of a subtree, and gating on whole-subtree prunability
5156 : would defer the release of an otherwise prunable block, e.g. an
5157 : inactive sibling of an in-flight block, to a later abandon such as
5158 : a subsequent root notify. That could stall bank reclamation.
5159 :
5160 : The sched fork tree still stays in sync with the banks tree. This
5161 : is to prevent attacks targeting lifetime discrepancy. So while we
5162 : know that the subtree can be pruned from sched at this point, we
5163 : only release refcnts here and stop short of actually pruning, which
5164 : remains solely initiated by banks. */
5165 2 : subtree_release_refcnt( sched, block );
5166 2 : }
5167 :
5168 : /* Release the bank refcnt for every block in the subtree that sched is
5169 : done with. This is evaluated per block. A block's refcnt is
5170 : released as soon as that block individually qualifies, independent of
5171 : whether its siblings or descendants still have in-flight tasks. This
5172 : is safe because the actual pruning is initiated separately by banks,
5173 : and gated on the whole subtree's refcnts reaching zero. Releasing
5174 : refcnts eagerly per block get us there sooner. Blocks that are still
5175 : in-flight will be released when their last task drains. This
5176 : function is idempotent. */
5177 : static void
5178 2 : subtree_release_refcnt( fd_sched_t * sched, fd_sched_block_t * block ) {
5179 2 : FD_TEST( block->in_sched );
5180 2 : if( block->refcnt && block_is_prunable( block ) ) {
5181 2 : FD_TEST( block->dying ); /* The happy path releases the refcnt on full replay. Only bad blocks end up here. */
5182 2 : FD_TEST( !block->block_end_done ); /* Implied by an outstanding refcnt. The happy path releases the refcnt on full replay. */
5183 2 : block->refcnt = 0;
5184 2 : FD_TEST( ref_q_avail( sched->ref_q ) );
5185 2 : ref_q_push_tail( sched->ref_q, block_to_idx( sched, block ) );
5186 2 : if( FD_LIKELY( block->block_start_done ) ) FD_LOG_DEBUG(( "block %lu:%lu replayed partially, releasing refcnt without full replay", block->slot, block_to_idx( sched, block ) ));
5187 2 : else FD_LOG_DEBUG(( "block %lu:%lu replayed nothing, releasing refcnt without any replay", block->slot, block_to_idx( sched, block ) ));
5188 2 : }
5189 :
5190 2 : ulong child_idx = block->child_idx;
5191 2 : while( child_idx!=ULONG_MAX ) {
5192 0 : fd_sched_block_t * child_block = block_pool_ele( sched, child_idx );
5193 0 : subtree_release_refcnt( sched, child_block );
5194 0 : child_idx = child_block->sibling_idx;
5195 0 : }
5196 2 : }
5197 :
5198 : static void
5199 0 : subtree_prune( fd_sched_t * sched, ulong bank_idx, ulong except_idx ) {
5200 0 : fd_sched_block_t * head = block_pool_ele( sched, bank_idx );
5201 0 : head->parent_idx = ULONG_MAX;
5202 0 : fd_sched_block_t * tail = head;
5203 :
5204 0 : while( head ) {
5205 0 : FD_TEST( !head->refcnt );
5206 0 : FD_TEST( head->in_sched );
5207 0 : head->in_sched = 0;
5208 0 : slot_state_close( sched, head );
5209 0 : if( FD_UNLIKELY( block_to_idx( sched, head )==sched->last_active_bank_idx ) ) sched->last_active_bank_idx = ULONG_MAX; /* Don't count a lane switch if the last active block has been pruned. */
5210 :
5211 0 : ulong child_idx = head->child_idx;
5212 0 : while( child_idx!=ULONG_MAX ) {
5213 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
5214 : /* Add children to be visited. We abuse the parent_idx field to
5215 : link up the next block to visit. */
5216 0 : if( child_idx!=except_idx ) {
5217 0 : tail->parent_idx = child_idx;
5218 0 : tail = child;
5219 0 : tail->parent_idx = ULONG_MAX;
5220 0 : }
5221 0 : child_idx = child->sibling_idx;
5222 0 : }
5223 :
5224 : /* Prune the current block. We will never publish halfway into a
5225 : staging lane, because anything that we are publishing away should
5226 : be out of the dispatcher at this point, much less staged.
5227 : - Anything on the rooted fork should have finished replaying
5228 : gracefully and be out of the dispatcher.
5229 : - Anything on minority forks should have been marked dying and
5230 : be taken out of the dispatcher.
5231 :
5232 : There should be no more in-flight tasks either. Prunes are
5233 : initiated by banks, and the refcnt on the bank won't drop to zero
5234 : unless all in-flight tasks have been drained. So the fact that
5235 : we're pruning implies that the bank thinks there's nothing more
5236 : in-flight. */
5237 0 : if( FD_UNLIKELY( block_is_in_flight( head ) ) ) {
5238 0 : FD_LOG_CRIT(( "invariant violation: block has tasks in flight (%u exec %u sigverify %u poh %u cert %u lthash sub, %u/%u lthash add surfaced/done, %u lthash tasks), slot %lu, parent slot %lu",
5239 0 : head->txn_exec_in_flight_cnt, head->txn_sigverify_in_flight_cnt, head->poh_hashing_in_flight_cnt, head->footer_in_flight_cnt, head->sub_in_flight_cnt, head->add_surfaced_cnt, head->add_done_cnt, head->lthash_in_flight_cnt, head->slot, head->parent_slot ));
5240 0 : }
5241 0 : if( FD_UNLIKELY( head->in_rdisp ) ) {
5242 : /* We should have removed it from the dispatcher when we were
5243 : notified of the new root, or when in-flight transactions were
5244 : drained. */
5245 0 : FD_LOG_CRIT(( "invariant violation: block is in the dispatcher, slot %lu, parent slot %lu", head->slot, head->parent_slot ));
5246 0 : }
5247 :
5248 : /* Return remaining mblk descriptors to the shared pool. */
5249 0 : free_mblk_slist( sched, head, head->mblks_unhashed );
5250 0 : free_mblk_slist( sched, head, head->mblks_mixin_in_progress );
5251 :
5252 0 : if( FD_UNLIKELY( !head->block_end_done ) ) {
5253 0 : sched->print_buf_sz = 0UL;
5254 0 : print_block_metrics( sched, head );
5255 0 : if( FD_LIKELY( head->block_start_done ) ) FD_LOG_DEBUG(( "block %lu:%lu replayed partially, pruning without full replay: %s", head->slot, block_to_idx( sched, head ), sched->print_buf ));
5256 0 : else FD_LOG_DEBUG(( "block %lu:%lu replayed nothing, pruning without any replay: %s", head->slot, block_to_idx( sched, head ), sched->print_buf ));
5257 0 : }
5258 :
5259 0 : sched->block_pool_popcnt--;
5260 :
5261 0 : fd_sched_block_t * next = block_pool_ele( sched, head->parent_idx );
5262 :
5263 : /* We don't have to clear the indices here since no one should be
5264 : accessing them. Defensive programming. */
5265 0 : head->parent_idx = ULONG_MAX;
5266 0 : head->child_idx = ULONG_MAX;
5267 0 : head->sibling_idx = ULONG_MAX;
5268 :
5269 0 : head = next;
5270 0 : }
5271 0 : }
5272 :
5273 : static void
5274 0 : maybe_switch_block( fd_sched_t * sched, ulong bank_idx ) {
5275 : /* This only happens rarely when there are dying in-flight blocks.
5276 : Early exit and don't let dying blocks affect replay. */
5277 0 : if( FD_UNLIKELY( bank_idx!=sched->active_bank_idx ) ) return;
5278 :
5279 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
5280 0 : if( FD_UNLIKELY( block_is_done( block ) ) ) {
5281 : /* A successfully replayed block has exactly one subtraction and one
5282 : addition per writable account, net of undone speculations. This
5283 : is the culmination of all the out-of-band LtHash activity in the
5284 : block, and a cheap way for us to check count invariants. */
5285 0 : FD_TEST( block->sub_applied_cnt==block->add_applied_cnt-block->add_undone_cnt );
5286 0 : sched->metrics->lthash_frozen_cnt += block->sub_applied_cnt;
5287 0 : lthash_block_reset( sched, block );
5288 0 : lthash_release_reservation( sched, block );
5289 0 : fd_rdisp_remove_block( sched->rdisp, bank_idx );
5290 0 : FD_LOG_DEBUG(( "block %lu:%lu exited lane %lu: remove", block->slot, bank_idx, block->staging_lane ));
5291 0 : block->in_rdisp = 0;
5292 0 : block->staged = 0;
5293 0 : sched->metrics->block_removed_cnt++;
5294 0 : FD_LOG_DEBUG(( "reset active_bank_idx %lu: remove", sched->active_bank_idx ));
5295 0 : sched->last_active_bank_idx = sched->active_bank_idx;
5296 0 : active_bank_set( sched, ULONG_MAX );
5297 :
5298 : /* Every staged block has a lane to itself, including this block.
5299 : Release the lane and look for the next block to run, which is
5300 : typically a child on another lane. */
5301 0 : sched->staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, (int)block->staging_lane );
5302 0 : sched->staged_head_bank_idx[ block->staging_lane ] = ULONG_MAX;
5303 0 : try_activate_block( sched );
5304 : /* The common case activates a child of the block that just
5305 : finished. Count the uncommon cases. */
5306 0 : int child_activated = sched->active_bank_idx!=ULONG_MAX && block_pool_ele( sched, sched->active_bank_idx )->parent_idx==bank_idx;
5307 0 : sched->metrics->deactivate_no_child_cnt += (uint)!child_activated;
5308 0 : } else if( block_should_deactivate( sched, block ) ) {
5309 : /* We exhausted the active block, but it's not fully done yet. We
5310 : are just not getting FEC sets for it fast enough. This could
5311 : happen when the network path is congested, or when the leader
5312 : simply went down. Reset the active block. */
5313 0 : sched->last_active_bank_idx = sched->active_bank_idx;
5314 0 : active_bank_set( sched, ULONG_MAX );
5315 : /* Nothing of the block is in flight, so no owed account is waiting
5316 : on a writer. A block that still owes additions could not
5317 : speculate on them because the speculation pool is out of entries,
5318 : or because every owed account is over the hotness threshold and
5319 : the block has nothing it may run. The remaining reason not to
5320 : speculate, fec_eos, never reaches this branch. A block with
5321 : fec_eos is dispatched to finish, and then it leaves through the
5322 : branch above. */
5323 0 : if( block->add_owed_cnt && !sched->lthash_spec_free_cnt ) sched->metrics->lthash_pool_full_cnt++;
5324 0 : else sched->metrics->deactivate_no_txn_cnt++;
5325 0 : try_activate_block( sched );
5326 0 : }
5327 0 : }
5328 :
5329 : /* Returns length of the longest stageable unstaged fork, if there is
5330 : one, and 0 otherwise. */
5331 : static ulong
5332 4 : compute_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx ) {
5333 4 : if( FD_UNLIKELY( bank_idx==ULONG_MAX ) ) {
5334 0 : FD_LOG_CRIT(( "invariant violation: bank_idx==ULONG_MAX" ));
5335 0 : }
5336 :
5337 4 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
5338 :
5339 4 : ulong max_child_depth = 0UL;
5340 4 : ulong child_idx = block->child_idx;
5341 6 : while( child_idx!=ULONG_MAX ) {
5342 2 : ulong child_depth = compute_longest_unstaged_fork( sched, child_idx );
5343 2 : if( child_depth > max_child_depth ) {
5344 0 : max_child_depth = child_depth;
5345 0 : }
5346 2 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
5347 2 : child_idx = child->sibling_idx;
5348 2 : }
5349 :
5350 4 : block->luf_depth = max_child_depth + fd_ulong_if( block_is_promotable( block ), 1UL, 0UL );
5351 4 : return block->luf_depth;
5352 4 : }
5353 :
5354 : /* Promotes one block onto lane_idx, the shallowest promotable block of
5355 : the longest stageable unstaged fork under bank_idx, and returns its
5356 : index, or ULONG_MAX if there is no such block. Assumes
5357 : compute_longest_unstaged_fork() ran on the fork tree since it last
5358 : changed. */
5359 : static ulong
5360 0 : promote_from_longest_unstaged_fork( fd_sched_t * sched, ulong bank_idx, int lane_idx ) {
5361 0 : while( bank_idx!=ULONG_MAX ) {
5362 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
5363 0 : if( FD_LIKELY( block_is_promotable( block ) ) ) {
5364 0 : block->staged = 1;
5365 0 : block->staging_lane = (ulong)lane_idx;
5366 0 : fd_rdisp_promote_block( sched->rdisp, bank_idx, block->staging_lane );
5367 0 : sched->staged_bitset = fd_ulong_set_bit( sched->staged_bitset, lane_idx );
5368 : /* No need to update staged_popcnt_wmk because the fact that there
5369 : are unstaged blocks implies we already maxed out lanes at one
5370 : point. */
5371 0 : sched->staged_head_bank_idx[ lane_idx ] = bank_idx;
5372 0 : sched->metrics->block_promoted_cnt++;
5373 0 : sched->metrics->lane_promoted_cnt++;
5374 : /* We never dispatch from an unstaged block, so every transaction
5375 : it ingested since it was last staged got promoted. */
5376 0 : sched->metrics->txn_promoted_cnt += block->txn_parsed_cnt-block->txn_done_cnt;
5377 0 : FD_LOG_DEBUG(( "block %lu:%lu entered lane %lu: promote", block->slot, bank_idx, block->staging_lane ));
5378 : /* The lane map is claimed first so that the pseudo transactions
5379 : the drain creates land in it. A block that hit fec_eos while
5380 : unstaged has not drained yet, so the polls to come drain it. */
5381 0 : if( FD_LIKELY( !block->lthash_inband ) ) lthash_claim_lane_map( sched, block );
5382 0 : lthash_drain_later( sched, block );
5383 0 : return bank_idx;
5384 0 : }
5385 :
5386 : /* Descend into the longest stageable unstaged fork. */
5387 0 : ulong max_depth = 0UL;
5388 0 : ulong best_child_idx = ULONG_MAX;
5389 0 : for( ulong child_idx=block->child_idx; child_idx!=ULONG_MAX; ) {
5390 0 : fd_sched_block_t * child = block_pool_ele( sched, child_idx );
5391 0 : if( child->luf_depth>max_depth ) {
5392 0 : max_depth = child->luf_depth;
5393 0 : best_child_idx = child_idx;
5394 0 : }
5395 0 : child_idx = child->sibling_idx;
5396 0 : }
5397 0 : bank_idx = best_child_idx;
5398 0 : }
5399 0 : return ULONG_MAX;
5400 0 : }
5401 :
5402 : static int
5403 0 : lane_is_demotable( fd_sched_t * sched, int lane_idx ) {
5404 0 : fd_sched_block_t * block = block_pool_ele( sched, sched->staged_head_bank_idx[ lane_idx ] );
5405 0 : FD_TEST( block->staged );
5406 0 : FD_TEST( block->staging_lane==(ulong)lane_idx );
5407 0 : return block_is_demotable( block );
5408 0 : }
5409 :
5410 : static void
5411 0 : demote_lane( fd_sched_t * sched, int lane_idx ) {
5412 0 : ulong bank_idx = sched->staged_head_bank_idx[ lane_idx ];
5413 0 : fd_sched_block_t * block = block_pool_ele( sched, bank_idx );
5414 0 : FD_TEST( block->staged );
5415 0 : FD_TEST( block->staging_lane==(ulong)lane_idx );
5416 :
5417 0 : lthash_demote( sched, block );
5418 0 : int ret = fd_rdisp_demote_block( sched->rdisp, bank_idx );
5419 0 : if( FD_UNLIKELY( ret!=0 ) ) {
5420 0 : FD_LOG_CRIT(( "fd_rdisp_demote_block failed for block %lu:%lu, lane %d", block->slot, bank_idx, lane_idx ));
5421 0 : }
5422 0 : FD_LOG_DEBUG(( "block %lu:%lu exited lane %lu: demote", block->slot, bank_idx, block->staging_lane ));
5423 0 : block->staged = 0;
5424 :
5425 0 : sched->staged_bitset = fd_ulong_clear_bit( sched->staged_bitset, lane_idx );
5426 0 : sched->staged_head_bank_idx[ lane_idx ] = ULONG_MAX;
5427 0 : sched->metrics->block_demoted_cnt++;
5428 0 : }
|
