Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_accdb.h"
3 : #include "fd_accdb_shmem.h"
4 : #include "fd_accdb_private.h"
5 : #include "../../util/fd_hash32.h"
6 :
7 : #if FD_TMPL_USE_HANDHOLDING
8 : #include "../../ballet/txn/fd_txn.h"
9 : #include "../../ballet/base58/fd_base58.h"
10 : #endif
11 : #include "../../util/racesan/fd_racesan_target.h"
12 :
13 : #include "../../disco/events/generated/fd_event_gen.h"
14 : #include "../../disco/sleep/fd_sleep.h"
15 : #include "../runtime/fd_runtime_const.h" /* FD_RUNTIME_ACC_SZ_MAX */
16 :
17 : FD_STATIC_ASSERT( sizeof(fd_accdb_cache_line_t)==FD_ACCDB_CACHE_META_SZ, cache_meta_sz );
18 :
19 0 : #define FD_ACCDB_BOUNCE_SZ (16UL<<20)
20 : FD_STATIC_ASSERT( FD_ACCDB_BOUNCE_SZ>=FD_RUNTIME_ACC_SZ_MAX+sizeof(fd_accdb_disk_meta_t), bounce_fits_any_record );
21 :
22 : /* Preevict batches up to PREEVICT_BATCH_SZ bytes, but stages a larger
23 : record alone, so its buffer holds one record of any size. */
24 0 : #define PREEVICT_BATCH_SZ (256UL<<10)
25 : #define PREEVICT_STAGE_SZ (FD_RUNTIME_ACC_SZ_MAX+sizeof(fd_accdb_disk_meta_t))
26 : FD_STATIC_ASSERT( PREEVICT_BATCH_SZ<=PREEVICT_STAGE_SZ, batch_fits_stage );
27 :
28 : /* Minimum ticks (tens of microseconds) between a join's wakeups of the
29 : accdb tile (cache_free_ring), and the default between its
30 : background_preevict watermark scans. */
31 12 : #define FD_ACCDB_BG_RATE_TICKS (100000L)
32 :
33 : #if FD_HAS_RACESAN
34 : /* Test-only telemetry: background_compact publishes the pubkey + dest
35 : offset of the record it is about to relocation-CAS at the
36 : accdb_compact:pre_offset_cas hook, so test_accdb_racesan can PROVE the
37 : parked relocation is the account it set up (avoiding a vacuous test).
38 : Zero-cost / absent in production (racesan off). */
39 : uchar fd_accdb_dbg_reloc_pubkey[ 32UL ];
40 : ulong fd_accdb_dbg_reloc_dest;
41 : ulong fd_accdb_dbg_reloc_cnt;
42 : ulong fd_accdb_dbg_compact_scanned; /* bytes compaction stepped over */
43 : #endif
44 :
45 : #include <stddef.h>
46 : #include <unistd.h>
47 : #include <fcntl.h>
48 : #include <errno.h>
49 : #include <sys/uio.h>
50 :
51 : struct fd_accdb_fork {
52 : fd_accdb_fork_shmem_t * shmem;
53 : descends_set_t * descends;
54 : };
55 :
56 : typedef struct fd_accdb_fork fd_accdb_fork_t;
57 :
58 543702 : #define FD_ACCDB_ACQUIRE_STATE_IDLE (0)
59 7291 : #define FD_ACCDB_ACQUIRE_STATE_PHASE_A (1)
60 540422 : #define FD_ACCDB_ACQUIRE_STATE_OPEN (2)
61 :
62 : struct __attribute__((aligned(FD_ACCDB_ALIGN))) fd_accdb_private {
63 : int fd;
64 :
65 : int acquire_state;
66 :
67 : fd_accdb_shmem_t * shmem;
68 :
69 : /* Doorbell for the parked accdb tile, NULL when nobody parks */
70 : fd_sleep_t * sleep;
71 : ulong sleep_tile_id;
72 :
73 : fd_accdb_fork_t * fork_pool;
74 : fork_pool_t fork_shmem_pool[1];
75 :
76 : fd_accdb_accmeta_t * acc_pool;
77 : acc_pool_t acc_pool_join[1];
78 : uint * acc_map;
79 :
80 : uchar * cache [ FD_ACCDB_CACHE_CLASS_CNT ];
81 :
82 : fd_accdb_partition_t * partition_pool;
83 : compaction_dlist_t * compaction_dlist[ FD_ACCDB_COMPACTION_LAYER_CNT ];
84 : deferred_free_dlist_t * deferred_free_dlist;
85 :
86 : txn_pool_t txn_pool[1];
87 :
88 : /* Pointer into shmem->joiner_epochs[ my_slot ].val for writer
89 : joiners, or into a private per-tile fseq for read-only joiners.
90 : Set to the current global epoch on entry to an epoch-protected
91 : operation, and ULONG_MAX on exit. Used to determine when
92 : deferred frees are safe. */
93 : ulong * my_epoch_slot;
94 :
95 : /* Read-only pointers to external epoch slots (e.g. fseqs owned by
96 : RO consumer tiles like the rpc tile). Scanned in addition to
97 : shmem->joiner_epochs[] by compaction's deferred-free
98 : reclamation. Borrowed; the caller of fd_accdb_new owns the
99 : storage. */
100 : ulong const * const * external_epoch_slots;
101 : ulong external_epoch_cnt;
102 :
103 : /* Side buffer of acc pool indices that have been CAS-unlinked from
104 : their hash chains but cannot be released back to acc_pool yet,
105 : because concurrent readers (acquire / compact) may still be
106 : traversing the removed nodes via map.next. The batch is released
107 : once all joiner_epochs exceed shmem->deferred_epoch. Indices
108 : are written here (not into pool.next) until after the epoch drain
109 : because pool.next is union-aliased to cache_idx, which a concurrent
110 : cold_load_acc may still write through a captured pointer. Backed
111 : by shmem->deferred_acc_buf_off; cnt and epoch live in shmem too. */
112 : uint * deferred_acc_buf;
113 :
114 : fd_accdb_metrics_t metrics[1];
115 :
116 : ulong prealloc_failed_at;
117 :
118 : /* Compaction bounce buffer. */
119 : uchar * bounce;
120 : ulong carry_off;
121 : ulong carry_sz;
122 : ulong carry_layer;
123 :
124 : /* Preevict write staging, PREEVICT_STAGE_SZ bytes. */
125 : uchar * stage;
126 :
127 : /* This join's index into the per-join shmem arrays (joiner_epochs,
128 : cache_free, cache_free_cnt, join_cred, cache_free_hungry bits), and
129 : the last time it woke the accdb tile (cache_free_ring). */
130 : ulong join_idx;
131 : long free_ring_ticks;
132 :
133 : /* Last time background_preevict scanned the watermarks (it reads
134 : every join's free-list counts, so it is rate-limited), and whether
135 : that scan left a class short of its target. Throttled calls keep
136 : reporting busy while a refill is pending, so the accdb tile does
137 : not park mid-refill. */
138 : long preevict_ticks;
139 : long preevict_interval;
140 : int preevict_pending;
141 :
142 : /* Reservation state (res_*), constant after join. res_limit[c] is
143 : ULONG_MAX if class c is untracked, so the hot path never loads the
144 : shared counter just to learn that. cred points at this join's
145 : shmem->join_cred line. */
146 : ulong res_limit[ FD_ACCDB_CACHE_CLASS_CNT ];
147 : ulong cred_max [ FD_ACCDB_CACHE_CLASS_CNT ];
148 : ulong * cred;
149 :
150 : /* Track account addresses changed since a full snap.
151 : Used to determine which accounts should be packed into a full
152 : snapshot (including tombstones for accounts no longer present in
153 : accdb, but present in the full snapshot). */
154 : struct {
155 : uint * chains;
156 : fd_accdb_delta_t * pool;
157 : struct {
158 : uchar const * pubkey;
159 : uint chain;
160 : } scratch[ FD_ACCDB_MAX_ACQUIRE_CNT ];
161 : } delta;
162 :
163 : /* Batch metric updates to avoid shared atomics per write. */
164 : struct {
165 : ulong bytes; /* bytes reserved on partition_idx */
166 : ulong num_ops; /* reservations behind those bytes */
167 : ulong partition_idx; /* set while num_ops>0 */
168 : ulong disk_used_added;
169 : ulong disk_used_removed;
170 : ulong accounts_total_added;
171 : } write_stats __attribute__((aligned(64)));
172 : };
173 :
174 : static inline fd_accdb_cache_line_t *
175 : cache_line( fd_accdb_t * accdb,
176 : ulong cls,
177 2945472 : ulong idx ) {
178 2945472 : return (fd_accdb_cache_line_t *)( accdb->cache[ cls ] + idx * fd_accdb_cache_slot_sz[ cls ] );
179 2945472 : }
180 :
181 : /* Bump the per-partition read counters for the partition that contains
182 : file_offset. Called at preadv2 sites. Writes are counted at
183 : allocate time (see fd_accdb_partition_write_bump) so that they reflect
184 : bytes committed to a partition rather than syscalls — the snapshot
185 : loader bypasses pwritev2 entirely, but every write still goes through
186 : reserve_next_write. */
187 : static inline void
188 : fd_accdb_partition_read_bump( fd_accdb_t * accdb,
189 : ulong file_offset,
190 0 : ulong bytes ) {
191 0 : if( FD_UNLIKELY( !bytes ) ) return;
192 : /* Readonly joiners have no partition_pool join (see
193 : fd_accdb_join_readonly) and do not contribute to per-partition
194 : read telemetry today; their disk reads still show up in the
195 : joiner-local fd_accdb_metrics_t bytes_read/read_ops. */
196 0 : if( FD_UNLIKELY( !accdb->partition_pool ) ) return;
197 0 : ulong partition_idx = file_offset / accdb->shmem->partition_sz;
198 0 : ulong partition_cnt = partition_pool_max( accdb->partition_pool );
199 0 : FD_CHECK_CRIT( partition_idx<partition_cnt, "read partition index out of range" );
200 0 : fd_accdb_partition_t * p = partition_pool_ele( accdb->partition_pool, partition_idx );
201 0 : FD_ATOMIC_FETCH_AND_ADD( &p->bytes_read, bytes );
202 0 : FD_ATOMIC_FETCH_AND_ADD( &p->read_ops, 1UL );
203 0 : }
204 :
205 : /* Bump the per-partition write counters. bytes is how much landed on
206 : this partition, over num_ops reservations. */
207 : static inline void
208 : fd_accdb_partition_write_bump( fd_accdb_t * accdb,
209 : ulong partition_idx,
210 : ulong bytes,
211 0 : ulong num_ops ) {
212 0 : if( FD_UNLIKELY( !bytes ) ) return;
213 0 : ulong partition_cnt = partition_pool_max( accdb->partition_pool );
214 0 : FD_CHECK_CRIT( partition_idx<partition_cnt, "write partition index out of range" );
215 0 : fd_accdb_partition_t * p = partition_pool_ele( accdb->partition_pool, partition_idx );
216 0 : FD_ATOMIC_FETCH_AND_ADD( &p->bytes_written, bytes );
217 0 : FD_ATOMIC_FETCH_AND_ADD( &p->write_ops, num_ops );
218 0 : }
219 :
220 : void
221 0 : fd_accdb_flush_metrics( fd_accdb_t * accdb ) {
222 0 : ulong bytes = accdb->write_stats.bytes;
223 0 : ulong num_ops = accdb->write_stats.num_ops;
224 0 : ulong part_idx = accdb->write_stats.partition_idx;
225 0 : ulong disk_used_added = accdb->write_stats.disk_used_added;
226 0 : ulong disk_used_removed = accdb->write_stats.disk_used_removed;
227 0 : ulong accounts_total_added = accdb->write_stats.accounts_total_added;
228 :
229 0 : memset( &accdb->write_stats, 0, sizeof(accdb->write_stats) );
230 :
231 0 : if( num_ops ) {
232 0 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_current_bytes, bytes );
233 0 : fd_accdb_partition_write_bump( accdb, part_idx, bytes, num_ops );
234 0 : }
235 :
236 0 : if( disk_used_added ) FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_used_bytes, disk_used_added );
237 0 : if( disk_used_removed ) FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, disk_used_removed );
238 0 : if( accounts_total_added ) FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->accounts_total, accounts_total_added );
239 0 : }
240 :
241 : static inline ulong
242 : cache_line_idx( fd_accdb_t * accdb,
243 : ulong cls,
244 2065567 : fd_accdb_cache_line_t const * line ) {
245 2065567 : return (ulong)( (uchar const *)line - accdb->cache[ cls ] ) / fd_accdb_cache_slot_sz[ cls ];
246 2065567 : }
247 :
248 : #if FD_TMPL_USE_HANDHOLDING
249 : static inline int
250 : fd_accdb_ptr_in_region( fd_accdb_t const * accdb,
251 : ulong cls,
252 : void const * ptr ) {
253 : if( FD_UNLIKELY( cls>=FD_ACCDB_CACHE_CLASS_CNT ) ) return 0;
254 :
255 : uchar const * base = accdb->cache[ cls ];
256 : if( FD_UNLIKELY( !base ) ) return 0;
257 :
258 : ulong slot_sz = fd_accdb_cache_slot_sz[ cls ];
259 : ulong region_sz = accdb->shmem->cache_class_max[ cls ] * slot_sz;
260 : uchar const * p = (uchar const *)ptr;
261 :
262 : if( FD_UNLIKELY( p<base || p>=base+region_sz ) ) return 0;
263 : return ( (ulong)( p - base ) % slot_sz )==FD_ACCDB_CACHE_META_SZ;
264 : }
265 : #endif
266 :
267 : /* res_init sets the join's reservation state (res_* below), given
268 : accdb->join_idx. */
269 :
270 : static void
271 : res_init( fd_accdb_t * accdb,
272 12 : fd_accdb_shmem_t * shmem ) {
273 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
274 96 : ulong max = shmem->cache_class_max[ c ];
275 96 : int tracked = FD_VOLATILE_CONST( shmem->cache_class_used[ c ].val )!=ULONG_MAX;
276 96 : accdb->res_limit[ c ] = tracked ? max : ULONG_MAX;
277 96 : accdb->cred_max [ c ] = tracked ? fd_accdb_cred_max( max, shmem->joiner_cnt_max ) : 0UL;
278 96 : }
279 12 : accdb->cred = shmem->join_cred[ accdb->join_idx ].val;
280 12 : }
281 :
282 : FD_FN_CONST ulong
283 48 : fd_accdb_align( void ) {
284 48 : return FD_ACCDB_ALIGN;
285 48 : }
286 :
287 : FD_FN_CONST ulong
288 : fd_accdb_footprint( ulong max_live_slots,
289 12 : int compaction ) {
290 12 : ulong l;
291 12 : l = FD_LAYOUT_INIT;
292 12 : l = FD_LAYOUT_APPEND( l, FD_ACCDB_ALIGN, sizeof(fd_accdb_t) );
293 12 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_fork_t), max_live_slots*sizeof(fd_accdb_fork_t) );
294 12 : if( compaction ) l = FD_LAYOUT_APPEND( l, FD_ACCDB_ALIGN, FD_ACCDB_BOUNCE_SZ+PREEVICT_STAGE_SZ );
295 12 : return FD_LAYOUT_FINI( l, FD_ACCDB_ALIGN );
296 12 : }
297 :
298 : void *
299 : fd_accdb_new( void * ljoin,
300 : fd_accdb_shmem_t * shmem,
301 : int fd,
302 : ulong external_epoch_cnt,
303 : ulong const ** external_epoch_slots,
304 : fd_sleep_t * sleep,
305 : ulong sleep_tile_id,
306 12 : int compaction ) {
307 12 : if( FD_UNLIKELY( !ljoin ) ) {
308 0 : FD_LOG_WARNING(( "NULL ljoin" ));
309 0 : return NULL;
310 0 : }
311 :
312 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)ljoin, fd_accdb_align() ) ) ) {
313 0 : FD_LOG_WARNING(( "misaligned ljoin" ));
314 0 : return NULL;
315 0 : }
316 :
317 12 : if( FD_UNLIKELY( fd<0 ) ) {
318 0 : FD_LOG_WARNING(( "fd must be a valid file descriptor" ));
319 0 : return NULL;
320 0 : }
321 :
322 12 : ulong max_live_slots = shmem->max_live_slots;
323 12 : ulong max_accounts = shmem->max_accounts;
324 12 : ulong max_account_writes_per_slot = shmem->max_account_writes_per_slot;
325 12 : ulong partition_cnt = shmem->partition_cnt;
326 :
327 12 : ulong chain_cnt = fd_ulong_pow2_up( (max_accounts>>1) + (max_accounts&1UL) );
328 12 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
329 :
330 12 : FD_SCRATCH_ALLOC_INIT( l, shmem );
331 12 : FD_SCRATCH_ALLOC_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
332 12 : void * _fork_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
333 12 : void * _descends_sets = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
334 12 : void * _acc_map = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
335 12 : void * _acc_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
336 12 : void * _txn_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_txn_t), txn_max*sizeof(fd_accdb_txn_t) );
337 12 : void * _partition_pool = FD_SCRATCH_ALLOC_APPEND( l, partition_pool_align(), partition_pool_footprint( partition_cnt ) );
338 0 : void * _compaction_dlists[ FD_ACCDB_COMPACTION_LAYER_CNT ];
339 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
340 36 : _compaction_dlists[ k ] = FD_SCRATCH_ALLOC_APPEND( l, compaction_dlist_align(), compaction_dlist_footprint() );
341 36 : }
342 12 : void * _deferred_free_dlist = FD_SCRATCH_ALLOC_APPEND( l, deferred_free_dlist_align(), deferred_free_dlist_footprint() );
343 :
344 12 : FD_SCRATCH_ALLOC_INIT( l2, ljoin );
345 12 : fd_accdb_t * accdb = FD_SCRATCH_ALLOC_APPEND( l2, fd_accdb_align(), sizeof(fd_accdb_t) );
346 12 : void * _local_fork_pool = FD_SCRATCH_ALLOC_APPEND( l2, alignof(fd_accdb_fork_t), max_live_slots*sizeof(fd_accdb_fork_t) );
347 12 : uchar * _bounce = compaction ? FD_SCRATCH_ALLOC_APPEND( l2, FD_ACCDB_ALIGN, FD_ACCDB_BOUNCE_SZ+PREEVICT_STAGE_SZ ) : NULL;
348 :
349 0 : accdb->fd = fd;
350 12 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_IDLE;
351 12 : accdb->sleep = sleep;
352 12 : accdb->sleep_tile_id = sleep_tile_id;
353 :
354 12 : accdb->shmem = (fd_accdb_shmem_t *)shmem;
355 12 : FD_TEST( acc_pool_join( accdb->acc_pool_join, shmem->acc_pool, _acc_pool_ele, max_accounts ) );
356 12 : accdb->acc_pool = accdb->acc_pool_join->ele;
357 12 : accdb->acc_map = _acc_map;
358 12 : FD_TEST( txn_pool_join( accdb->txn_pool, shmem->txn_pool, _txn_pool_ele, txn_max ) );
359 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache[ c ] = (uchar *)shmem + shmem->cache_region_off[ c ];
360 12 : accdb->partition_pool = partition_pool_join( _partition_pool );
361 12 : FD_TEST( accdb->partition_pool );
362 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
363 36 : accdb->compaction_dlist[ k ] = compaction_dlist_join( _compaction_dlists[ k ] );
364 36 : FD_TEST( accdb->compaction_dlist[ k ] );
365 36 : }
366 12 : accdb->deferred_free_dlist = deferred_free_dlist_join( _deferred_free_dlist );
367 12 : FD_TEST( accdb->deferred_free_dlist );
368 :
369 12 : FD_TEST( fork_pool_join( accdb->fork_shmem_pool, shmem->fork_pool, _fork_pool_ele, max_live_slots ) );
370 12 : accdb->fork_pool = _local_fork_pool;
371 204 : for( ulong i=0UL; i<max_live_slots; i++ ) {
372 192 : fd_accdb_fork_t * fork = &accdb->fork_pool[ i ];
373 192 : fork->shmem = fork_pool_ele( accdb->fork_shmem_pool, i );
374 192 : fork->descends = descends_set_join( (uchar *)_descends_sets + i*descends_set_footprint( max_live_slots ) );
375 192 : FD_TEST( fork->shmem );
376 192 : FD_TEST( fork->descends );
377 192 : }
378 :
379 12 : ulong epoch_idx = FD_ATOMIC_FETCH_AND_ADD( &shmem->joiner_cnt, 1UL );
380 12 : FD_TEST( epoch_idx<shmem->joiner_cnt_max );
381 12 : accdb->my_epoch_slot = &shmem->joiner_epochs[ epoch_idx ].val;
382 12 : accdb->join_idx = epoch_idx;
383 12 : accdb->free_ring_ticks = 0L;
384 12 : res_init( accdb, shmem );
385 12 : accdb->preevict_ticks = 0L;
386 12 : accdb->preevict_interval = FD_ACCDB_BG_RATE_TICKS;
387 12 : accdb->preevict_pending = 0;
388 :
389 12 : accdb->external_epoch_slots = external_epoch_slots;
390 12 : accdb->external_epoch_cnt = external_epoch_cnt;
391 :
392 12 : accdb->bounce = _bounce;
393 12 : accdb->carry_off = 0UL;
394 12 : accdb->carry_sz = 0UL;
395 12 : accdb->carry_layer = 0UL;
396 12 : accdb->stage = _bounce ? _bounce+FD_ACCDB_BOUNCE_SZ : NULL;
397 :
398 12 : accdb->deferred_acc_buf = (uint *)( (uchar *)shmem + shmem->deferred_acc_buf_off );
399 :
400 12 : accdb->delta.chains = (uint *) ( (uchar *)shmem + shmem->delta.chain_off );
401 12 : accdb->delta.pool = (fd_accdb_delta_t *) ( (uchar *)shmem + shmem->delta.ele_off );
402 :
403 12 : accdb->prealloc_failed_at = ULONG_MAX;
404 :
405 12 : memset( accdb->metrics, 0, sizeof(fd_accdb_metrics_t) );
406 12 : memset( &accdb->write_stats, 0, sizeof(accdb->write_stats) );
407 :
408 12 : return accdb;
409 12 : }
410 :
411 : static void delta_reset( fd_accdb_t * accdb );
412 :
413 : /* wait_idle waits until the accdb tile has applied every published
414 : root / purge request and released everything they retired. The
415 : accdb tile then does not touch the fork, acc or txn pools until the
416 : next request. */
417 :
418 : static void
419 0 : wait_idle( fd_accdb_t * accdb ) {
420 0 : while( fd_accdb_requests_pending( accdb ) ) {
421 0 : fd_racesan_hook( "accdb_wait_idle:spin" );
422 0 : FD_SPIN_PAUSE();
423 0 : }
424 0 : FD_COMPILER_MFENCE();
425 0 : }
426 :
427 : /* publish_req publishes a new request word, after the fork tree
428 : updates it describes. */
429 :
430 : static void
431 : publish_req( fd_accdb_t * accdb,
432 64019 : fd_accdb_req_t req ) {
433 64019 : FD_COMPILER_MFENCE();
434 64019 : FD_VOLATILE( accdb->shmem->req.u ) = req.u;
435 64019 : FD_COMPILER_MFENCE();
436 64019 : if( FD_UNLIKELY( accdb->sleep ) ) fd_sleep_ring( accdb->sleep, accdb->sleep_tile_id );
437 64019 : }
438 :
439 : void
440 0 : fd_accdb_reset( fd_accdb_t * accdb ) {
441 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
442 :
443 : /* Once idle, the accdb tile holds nothing that refers to the pools
444 : rebuilt below. Compaction keeps a carried tail between steps, so
445 : it must not be running either. Partitions are only queued for
446 : compaction once a snapshot load ends, and snapin resets only before
447 : loading a full snapshot. */
448 0 : wait_idle( accdb );
449 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
450 0 : FD_CHECK_CRIT( compaction_dlist_is_empty( accdb->compaction_dlist[ k ], accdb->partition_pool ),
451 0 : "accdb reset while compaction is queued" );
452 0 : }
453 :
454 : /* Reset pools through the joiner's existing pointers. acc_pool and
455 : txn_pool use POOL_LAZY=1 so reset is O(1). fork_pool and
456 : partition_pool rebuild their free lists in O(max_live_slots) and
457 : O(partition_cnt), both small. */
458 0 : spin_lock_acquire( &shmem->grow_lock );
459 0 : acc_pool_reset( accdb->acc_pool_join );
460 0 : txn_pool_reset( accdb->txn_pool );
461 0 : fork_pool_reset( accdb->fork_shmem_pool );
462 0 : partition_pool_reset( accdb->partition_pool );
463 :
464 0 : delta_reset( accdb );
465 :
466 : /* Clear hash chains */
467 0 : fd_memset( accdb->acc_map, 0xFF, shmem->chain_cnt*sizeof(uint) );
468 :
469 : /* Empty dlists */
470 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
471 0 : compaction_dlist_remove_all( accdb->compaction_dlist[ k ], accdb->partition_pool );
472 0 : }
473 0 : deferred_free_dlist_remove_all( accdb->deferred_free_dlist, accdb->partition_pool );
474 :
475 : /* Null descends_sets. */
476 0 : for( ulong i=0UL; i<shmem->max_live_slots; i++ ) {
477 0 : descends_set_null( accdb->fork_pool[ i ].descends );
478 0 : }
479 :
480 : /* Reset shmem scalar fields. */
481 : /* generation keeps counting. req and done name roots by generation,
482 : so reusing generations would make a new root look already applied. */
483 0 : shmem->root_fork_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
484 0 : shmem->partition_lock = 0;
485 0 : shmem->partition_max = 0UL;
486 0 : shmem->partition_fallocated = 0UL;
487 0 : spin_lock_release( &shmem->grow_lock );
488 :
489 : /* Write heads: sentinel values that force partition-switch on first
490 : write. */
491 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
492 0 : shmem->whead[ k ] = accdb_offset( shmem->partition_cnt, shmem->partition_sz );
493 0 : shmem->has_partition[ k ] = 0;
494 0 : }
495 :
496 : /* Cache state */
497 0 : fd_accdb_shmem_cache_alloc_reset( shmem );
498 :
499 : /* Reset every cache slot's metadata to empty sentinels. */
500 0 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
501 0 : ulong slot_sz = fd_accdb_cache_slot_sz[ c ];
502 0 : for( ulong i=0UL; i<shmem->cache_class_max[ c ]; i++ ) {
503 0 : fd_accdb_cache_line_t * line = (fd_accdb_cache_line_t *)( accdb->cache[ c ] + i*slot_sz );
504 0 : line->key.generation = UINT_MAX;
505 0 : line->acc_idx = UINT_MAX;
506 0 : line->refcnt = 0U;
507 0 : line->referenced = 0;
508 0 : line->persisted = 1;
509 0 : line->seq = 0U;
510 0 : }
511 0 : }
512 :
513 : /* Epoch system: reset epoch and all slot values to idle, but
514 : preserve joiner_cnt and each tile's my_epoch_slot pointer so that
515 : tiles which joined during init keep their original slot indices. */
516 0 : shmem->epoch = 1UL;
517 0 : for( ulong i=0UL; i<FD_ACCDB_MAX_JOINERS; i++ ) shmem->joiner_epochs[ i ].val = ULONG_MAX;
518 :
519 : /* Deferred acc buffer. */
520 0 : shmem->deferred_acc_buf_cnt = 0UL;
521 0 : shmem->deferred_epoch = 0UL;
522 0 : shmem->deferred_fork_head = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
523 0 : shmem->deferred_fork_tail = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
524 :
525 : /* Shared metrics: zero gauges that reflect current state (now empty)
526 : but preserve counters and accounts_capacity. */
527 0 : shmem->shmetrics->accounts_total = 0UL;
528 0 : shmem->shmetrics->disk_allocated_bytes = 0UL;
529 0 : shmem->shmetrics->disk_current_bytes = 0UL;
530 0 : shmem->shmetrics->disk_used_bytes = 0UL;
531 0 : shmem->shmetrics->in_compaction = 0;
532 :
533 0 : shmem->snapshot_loading = 0;
534 :
535 0 : FD_COMPILER_MFENCE();
536 :
537 : /* Reset local state */
538 0 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_IDLE;
539 0 : memset( &accdb->write_stats, 0, sizeof(accdb->write_stats) );
540 0 : }
541 :
542 : void
543 0 : fd_accdb_snapshot_load_begin( fd_accdb_t * accdb ) {
544 0 : FD_CHECK_CRIT( fd_accdb_snapshot_sync_state( &accdb->shmem->snapshot_sync )==FD_ACCDB_SNAPSHOT_SYNC_IDLE,
545 0 : "snapshot load started while snapshot production active" );
546 0 : FD_VOLATILE( accdb->shmem->snapshot_loading ) = 1;
547 0 : }
548 :
549 : static inline void
550 : change_partition( fd_accdb_t * accdb,
551 : accdb_offset_t const * offset_before,
552 : accdb_offset_t * out_offset,
553 : int * has_partition,
554 : uchar layer );
555 :
556 : void
557 0 : fd_accdb_snapshot_load_end( fd_accdb_t * accdb ) {
558 0 : fd_accdb_flush_metrics( accdb );
559 :
560 : /* Once the snapshot is done loading, the cold partition we were
561 : writing into is left open as the layer 2 head, and we switch the
562 : layer 0 head to a new hot partition. */
563 0 : ulong const cold = FD_ACCDB_COMPACTION_LAYER_CNT-1UL;
564 0 : if( FD_LIKELY( accdb->shmem->has_partition[ 0 ] ) ) {
565 0 : if( FD_LIKELY( !accdb->shmem->has_partition[ cold ] ) ) {
566 0 : accdb_offset_t snap_head = accdb->shmem->whead[ 0 ];
567 0 : int no_head = 0;
568 0 : change_partition( accdb, &snap_head, &accdb->shmem->whead[ 0 ], &no_head, 0 );
569 0 : spin_lock_acquire( &accdb->shmem->partition_lock );
570 0 : accdb->shmem->whead[ cold ] = snap_head;
571 0 : accdb->shmem->has_partition[ cold ] = 1;
572 0 : spin_lock_release( &accdb->shmem->partition_lock );
573 0 : } else {
574 0 : change_partition( accdb, &accdb->shmem->whead[ 0 ], &accdb->shmem->whead[ 0 ], &accdb->shmem->has_partition[ 0 ], 0 );
575 0 : }
576 0 : }
577 :
578 0 : spin_lock_acquire( &accdb->shmem->partition_lock );
579 0 : if( FD_LIKELY( accdb->shmem->has_partition[ 0 ] ) ) {
580 0 : ulong new_idx = packed_partition_idx( &accdb->shmem->whead[ 0 ] );
581 0 : fd_accdb_partition_t * newp = partition_pool_ele( accdb->partition_pool, new_idx );
582 0 : FD_VOLATILE( newp->layer ) = 0;
583 0 : }
584 :
585 0 : FD_VOLATILE( accdb->shmem->snapshot_loading ) = 0;
586 :
587 : /* Sweep all partitions written during the load — any that crossed
588 : the fragmentation threshold while enqueue was suppressed are
589 : re-checked now and pushed onto the compaction queue. */
590 0 : ulong partition_max = accdb->shmem->partition_max;
591 0 : for( ulong p=0UL; p<partition_max; p++ ) {
592 0 : fd_accdb_shmem_try_enqueue_compaction( accdb->shmem, p );
593 0 : }
594 :
595 0 : spin_lock_release( &accdb->shmem->partition_lock );
596 0 : }
597 :
598 : static inline uint
599 : delta_chain( fd_accdb_shmem_t const * accdb,
600 0 : uchar const pubkey[ 32 ] ) {
601 0 : uint hash = (uint)fd_hash32( pubkey, accdb->delta.seed );
602 0 : return hash & accdb->delta.chain_mask;
603 0 : }
604 :
605 : static int
606 : delta_insert( fd_accdb_t * accdb,
607 0 : uchar const pubkey[ 32 ] ) {
608 : /* FIXME consider batch inserting */
609 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
610 0 : if( FD_UNLIKELY( shmem->delta.head >= shmem->delta.ele_max ) ) return 0;
611 :
612 0 : uint * chains = accdb->delta.chains;
613 0 : uint * chain = &chains[ delta_chain( shmem, pubkey ) ];
614 0 : fd_accdb_delta_t * pool = accdb->delta.pool;
615 :
616 0 : uint head = *chain;
617 0 : for( uint cur=head; cur!=UINT_MAX; cur=pool[ cur ].next ) {
618 0 : if( FD_UNLIKELY( !memcmp( pool[ cur ].pubkey, pubkey, 32UL ) ) ) return 1;
619 0 : }
620 :
621 0 : ulong idx = shmem->delta.head++;
622 0 : fd_accdb_delta_t * delta = &pool[ idx ];
623 0 : delta->next = head;
624 0 : memcpy( delta->pubkey, pubkey, 32UL );
625 0 : *chain = (uint)idx;
626 0 : return 1;
627 0 : }
628 :
629 : int
630 : fd_accdb_snapshot_recover_delta( fd_accdb_t * accdb,
631 0 : fd_accdb_fork_id_t fork_id ) {
632 0 : if( FD_UNLIKELY( fork_id.val>=fork_pool_ele_max( accdb->fork_shmem_pool ) ) ) {
633 0 : FD_LOG_CRIT(( "fd_accdb_snapshot_recover_delta: invalid fork id %u (capacity %lu)",
634 0 : (uint)fork_id.val, fork_pool_ele_max( accdb->fork_shmem_pool ) ));
635 0 : }
636 :
637 0 : uint txn_idx = accdb->fork_pool[ fork_id.val ].shmem->txn_head;
638 0 : while( txn_idx!=UINT_MAX ) {
639 0 : fd_accdb_txn_t const * txn = txn_pool_ele( accdb->txn_pool, (ulong)txn_idx );
640 0 : fd_accdb_accmeta_t const * acc = &accdb->acc_pool[ txn->acc_pool_idx ];
641 0 : if( FD_UNLIKELY( !delta_insert( accdb, acc->key.pubkey ) ) ) return -1;
642 0 : txn_idx = txn->fork.next;
643 0 : }
644 0 : return 0;
645 0 : }
646 :
647 : void
648 : fd_accdb_snapshot_save_whead( fd_accdb_t * accdb,
649 0 : fd_accdb_snapshot_recovery_t * out ) {
650 : /* Flush metrics to update disk_current_bytes. */
651 0 : fd_accdb_flush_metrics( accdb );
652 :
653 0 : out->whead_val = FD_VOLATILE_CONST( accdb->shmem->whead[ 0 ].val );
654 0 : out->has_partition = FD_VOLATILE_CONST( accdb->shmem->has_partition[ 0 ] );
655 0 : out->partition_max = FD_VOLATILE_CONST( accdb->shmem->partition_max );
656 0 : out->disk_current_bytes = FD_VOLATILE_CONST( accdb->shmem->shmetrics->disk_current_bytes );
657 :
658 0 : if( out->has_partition ) {
659 0 : accdb_offset_t whead = { .val = out->whead_val };
660 0 : ulong idx = packed_partition_idx( &whead );
661 0 : fd_accdb_partition_t * part = partition_pool_ele( accdb->partition_pool, idx );
662 0 : out->savepoint_bytes_freed = FD_VOLATILE_CONST( part->bytes_freed );
663 0 : } else {
664 0 : out->savepoint_bytes_freed = 0UL;
665 0 : }
666 0 : }
667 :
668 : void
669 : fd_accdb_snapshot_revert_whead( fd_accdb_t * accdb,
670 0 : fd_accdb_snapshot_recovery_t const * recover ) {
671 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
672 :
673 : /* Partitions are about to be released, so flush metrics first. */
674 0 : fd_accdb_flush_metrics( accdb );
675 :
676 : /* The incremental fork purge must be applied, and its accounts
677 : released, before its partitions are. */
678 0 : wait_idle( accdb );
679 :
680 0 : ulong cur_partition_max = shmem->partition_max;
681 :
682 : /* Materialize the active partition's write_offset from the whead
683 : before releasing. Closed partitions have write_offset set by
684 : change_partition, but the last active partition still has
685 : write_offset == 0 from its initialization. The real byte offset
686 : is encoded in whead[0]. */
687 0 : if( shmem->has_partition[ 0 ] && cur_partition_max>recover->partition_max ) {
688 0 : ulong active_idx = packed_partition_idx( &shmem->whead[ 0 ] );
689 0 : if( active_idx>=recover->partition_max && active_idx<cur_partition_max ) {
690 0 : fd_accdb_partition_t * active = partition_pool_ele( accdb->partition_pool, active_idx );
691 0 : active->write_offset = packed_partition_offset( &shmem->whead[ 0 ] );
692 0 : }
693 0 : }
694 :
695 : /* Release partitions that have been previously allocated. Must hold
696 : partition_lock because partition_pool_ele_release mutates the
697 : pool free list. Before releasing, unlink any partition that sits
698 : on a compaction dlist (queued flag).
699 :
700 : Release in descending index order so that the LIFO free list
701 : re-acquires them in ascending order (P, P+1, P+2, ...). This
702 : preserves the ascending partition index order expected when the
703 : released suffix is allocated again. */
704 0 : spin_lock_acquire( &shmem->partition_lock );
705 0 : for( ulong p=cur_partition_max; p>recover->partition_max; p-- ) {
706 0 : fd_accdb_partition_t * part = partition_pool_ele( accdb->partition_pool, p-1UL );
707 0 : if( FD_UNLIKELY( part->queued ) ) {
708 0 : compaction_dlist_ele_remove( accdb->compaction_dlist[ part->layer ], part, accdb->partition_pool );
709 0 : }
710 0 : FD_VOLATILE( part->write_offset ) = 0UL;
711 0 : partition_pool_ele_release( accdb->partition_pool, part );
712 0 : }
713 :
714 0 : shmem->whead[ 0 ].val = recover->whead_val;
715 0 : shmem->has_partition[ 0 ] = recover->has_partition;
716 0 : shmem->partition_max = recover->partition_max;
717 :
718 : /* disk_used_bytes is NOT saved/restored here. It is implicitly
719 : reverted by purge_inner -> acc_unlink, which decrements
720 : disk_used_bytes for each unlinked entry. The caller must
721 : complete the purge before calling revert_whead. */
722 :
723 0 : shmem->shmetrics->disk_current_bytes = recover->disk_current_bytes;
724 :
725 0 : if( recover->has_partition ) {
726 0 : accdb_offset_t sp_off = (accdb_offset_t){ .val = recover->whead_val };
727 0 : ulong sp_idx = packed_partition_idx( &sp_off );
728 0 : fd_accdb_partition_t * sp = partition_pool_ele( accdb->partition_pool, sp_idx );
729 0 : sp->bytes_freed = recover->savepoint_bytes_freed;
730 0 : sp->write_offset = 0UL;
731 0 : }
732 :
733 0 : spin_lock_release( &shmem->partition_lock );
734 0 : }
735 :
736 : fd_accdb_t *
737 12 : fd_accdb_join( void * shaccdb ) {
738 12 : if( FD_UNLIKELY( !shaccdb ) ) {
739 0 : FD_LOG_WARNING(( "NULL shaccdb" ));
740 0 : return NULL;
741 0 : }
742 :
743 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shaccdb, fd_accdb_align() ) ) ) {
744 0 : FD_LOG_WARNING(( "misaligned shaccdb" ));
745 0 : return NULL;
746 0 : }
747 :
748 12 : return (fd_accdb_t*)shaccdb;
749 12 : }
750 :
751 : fd_accdb_t *
752 : fd_accdb_join_readonly( void * ljoin,
753 : fd_accdb_shmem_t * shmem,
754 : ulong * my_epoch_slot_rw,
755 0 : int fd_ro ) {
756 0 : if( FD_UNLIKELY( !ljoin ) ) {
757 0 : FD_LOG_WARNING(( "NULL ljoin" ));
758 0 : return NULL;
759 0 : }
760 :
761 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)ljoin, fd_accdb_align() ) ) ) {
762 0 : FD_LOG_WARNING(( "misaligned ljoin" ));
763 0 : return NULL;
764 0 : }
765 :
766 0 : if( FD_UNLIKELY( !my_epoch_slot_rw ) ) {
767 0 : FD_LOG_WARNING(( "NULL my_epoch_slot_rw" ));
768 0 : return NULL;
769 0 : }
770 :
771 0 : ulong max_live_slots = shmem->max_live_slots;
772 0 : ulong max_accounts = shmem->max_accounts;
773 0 : ulong max_account_writes_per_slot = shmem->max_account_writes_per_slot;
774 0 : ulong partition_cnt = shmem->partition_cnt;
775 :
776 0 : ulong chain_cnt = fd_ulong_pow2_up( (max_accounts>>1) + (max_accounts&1UL) );
777 0 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
778 :
779 : /* Recompute the same shmem scratch layout that fd_accdb_shmem_new
780 : used. All FD_SCRATCH_ALLOC_APPEND calls here only compute pointer
781 : offsets — they do not write to shmem. */
782 0 : FD_SCRATCH_ALLOC_INIT( l, shmem );
783 0 : FD_SCRATCH_ALLOC_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
784 0 : void * _fork_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
785 0 : void * _descends_sets = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
786 0 : void * _acc_map = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
787 0 : void * _acc_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
788 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_txn_t), txn_max*sizeof(fd_accdb_txn_t) );
789 0 : FD_SCRATCH_ALLOC_APPEND( l, partition_pool_align(), partition_pool_footprint( partition_cnt ) );
790 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
791 0 : FD_SCRATCH_ALLOC_APPEND( l, compaction_dlist_align(), compaction_dlist_footprint() );
792 0 : }
793 0 : FD_SCRATCH_ALLOC_APPEND( l, deferred_free_dlist_align(), deferred_free_dlist_footprint() );
794 :
795 0 : FD_SCRATCH_ALLOC_INIT( l2, ljoin );
796 0 : fd_accdb_t * accdb = FD_SCRATCH_ALLOC_APPEND( l2, fd_accdb_align(), sizeof(fd_accdb_t) );
797 0 : void * _local_fork_pool = FD_SCRATCH_ALLOC_APPEND( l2, alignof(fd_accdb_fork_t), max_live_slots*sizeof(fd_accdb_fork_t) );
798 :
799 0 : accdb->fd = fd_ro;
800 0 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_IDLE;
801 0 : accdb->shmem = shmem;
802 0 : FD_TEST( acc_pool_join( accdb->acc_pool_join, shmem->acc_pool, _acc_pool_ele, max_accounts ) );
803 0 : accdb->acc_pool = accdb->acc_pool_join->ele;
804 0 : accdb->acc_map = _acc_map;
805 0 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache[ c ] = (uchar *)shmem + shmem->cache_region_off[ c ];
806 :
807 : /* Writer-only structures: leave NULL so any accidental writer-path
808 : call from a readonly joiner crashes loudly rather than corrupting
809 : state. */
810 0 : accdb->partition_pool = NULL;
811 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) accdb->compaction_dlist[ k ] = NULL;
812 0 : accdb->deferred_free_dlist = NULL;
813 0 : accdb->delta.chains = NULL;
814 0 : accdb->delta.pool = NULL;
815 :
816 0 : FD_TEST( fork_pool_join( accdb->fork_shmem_pool, shmem->fork_pool, _fork_pool_ele, max_live_slots ) );
817 0 : accdb->fork_pool = _local_fork_pool;
818 0 : for( ulong i=0UL; i<max_live_slots; i++ ) {
819 0 : fd_accdb_fork_t * fork = &accdb->fork_pool[ i ];
820 0 : fork->shmem = fork_pool_ele( accdb->fork_shmem_pool, i );
821 0 : fork->descends = descends_set_join( (uchar *)_descends_sets + i*descends_set_footprint( max_live_slots ) );
822 0 : FD_TEST( fork->shmem );
823 0 : FD_TEST( fork->descends );
824 0 : }
825 :
826 : /* my_epoch_slot_rw points at memory owned by this joiner (e.g. a
827 : private per-tile fseq) that the joiner can write to. The
828 : accdb tile sees it via its external_epoch_slots[] array (mapped
829 : read-only) and includes it in its compaction epoch scan.
830 : Storing through this pointer is the only side effect a readonly
831 : joiner has on shared state. */
832 0 : accdb->my_epoch_slot = my_epoch_slot_rw;
833 :
834 : /* Readonly joiners do not own external slots themselves; only the
835 : compaction tile / writer joiners do. */
836 0 : accdb->external_epoch_slots = NULL;
837 0 : accdb->external_epoch_cnt = 0UL;
838 0 : accdb->bounce = NULL;
839 0 : accdb->stage = NULL;
840 :
841 0 : accdb->deferred_acc_buf = NULL;
842 0 : accdb->prealloc_failed_at = ULONG_MAX;
843 :
844 0 : memset( accdb->metrics, 0, sizeof(fd_accdb_metrics_t) );
845 0 : memset( &accdb->write_stats, 0, sizeof(accdb->write_stats) );
846 :
847 0 : return accdb;
848 0 : }
849 :
850 : fd_accdb_fork_id_t
851 : fd_accdb_attach_child( fd_accdb_t * accdb,
852 64016 : fd_accdb_fork_id_t parent_fork_id ) {
853 64016 : fd_accdb_shmem_t * shmem = accdb->shmem;
854 :
855 64016 : if( FD_UNLIKELY( shmem->generation==UINT_MAX ) ) FD_LOG_ERR(( "accdb ran out of generation sequence numbers, restart required" ));
856 :
857 : /* The pool can be exhausted by forks that are purged or rooted past
858 : but not yet released. The accdb tile releases them once concurrent
859 : readers drain. */
860 64016 : fd_accdb_fork_shmem_t * acquired = fork_pool_acquire( accdb->fork_shmem_pool );
861 64016 : if( FD_UNLIKELY( !acquired ) ) {
862 0 : wait_idle( accdb );
863 0 : acquired = fork_pool_acquire( accdb->fork_shmem_pool );
864 0 : FD_CHECK_CRIT( acquired, "accdb fork pool exhausted" );
865 0 : }
866 64016 : ulong idx = fork_pool_idx( accdb->fork_shmem_pool, acquired );
867 :
868 64016 : fd_accdb_fork_t * fork = &accdb->fork_pool[ idx ];
869 64016 : fd_accdb_fork_id_t fork_id = { .val = (ushort)idx };
870 :
871 : /* The slot's previous fork may still have its bit set in the
872 : descends_set of a fork that descended from it. That bit was
873 : harmless while the slot was retired (a rooted fork's accounts are
874 : visible by generation, a purged fork had no live descendants), but
875 : it would make the new fork look like an ancestor. Clear it before
876 : the new fork can write any account. */
877 1076800 : for( ulong i=0UL; i<shmem->max_live_slots; i++ ) descends_set_remove( accdb->fork_pool[ i ].descends, idx );
878 :
879 64016 : fork->shmem->child_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
880 64016 : fork->shmem->sibling_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
881 64016 : fork->shmem->generation = shmem->generation++;
882 64016 : fork->shmem->txn_head = UINT_MAX;
883 :
884 64016 : if( FD_UNLIKELY( parent_fork_id.val==USHORT_MAX ) ) {
885 : /* Published as both the applied and the requested root, so the
886 : accdb tile has nothing to root. The request keeps purge_seq
887 : (the accdb tile is idle, so every purge is applied). */
888 12 : FD_CHECK_CRIT( shmem->root_fork_id.val==USHORT_MAX, "accdb root fork already exists" );
889 12 : fork->shmem->parent_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
890 12 : descends_set_null( fork->descends );
891 12 : FD_VOLATILE( shmem->root_fork_id ) = fork_id;
892 12 : fd_accdb_req_t req = shmem->req;
893 12 : req.root_gen = fork->shmem->generation;
894 12 : publish_req( accdb, req );
895 64004 : } else {
896 : /* Forks below the requested root have a single child that the
897 : accdb tile may be rooting. Every other live fork descends from
898 : the requested root and so is newer. */
899 64004 : fd_accdb_fork_t * parent = &accdb->fork_pool[ parent_fork_id.val ];
900 64004 : FD_CHECK_CRIT( shmem->root_fork_id.val!=USHORT_MAX, "attach_child before the root fork exists" );
901 64004 : FD_CHECK_CRIT( parent->shmem->generation>=shmem->req.root_gen, "attach_child parent is an ancestor of the root" );
902 64004 : fork->shmem->parent_id = parent_fork_id;
903 64004 : fork->shmem->sibling_id = parent->shmem->child_id;
904 64004 : descends_set_copy( fork->descends, parent->descends );
905 64004 : descends_set_insert( fork->descends, parent_fork_id.val );
906 64004 : parent->shmem->child_id = fork_id;
907 64004 : }
908 :
909 64016 : FD_TEST( !descends_set_test( fork->descends, fork_id.val ) );
910 :
911 64016 : return fork_id;
912 64016 : }
913 :
914 : /* evict_clear_acc_cache_ref atomically tears down acc->cache_idx and
915 : acc->executable_size.CACHE_VALID for an acc that is being evicted
916 : from cache line (size_class, line_idx). The caller must already
917 : hold a pin or an exclusive eviction claim on the line to prevent
918 : its reuse.
919 :
920 : The naive sequence (clear cache_idx, clear VALID) lets a reader in
921 : cold_load_acc see VALID=1 and read a stale INVAL cache_idx, which
922 : decodes to an OOB cache_line pointer. The reverse sequence (clear
923 : VALID, clear cache_idx) lets a concurrent cold_load_acc observe
924 : VALID=0/CLAIM=0 and start publishing a *new* cache_idx + VALID=1
925 : between our two stores; our later cache_idx=INVAL would then
926 : stomp on the cold-loader's published idx.
927 :
928 : We close both races by acquiring CACHE_CLAIM_BIT before mutating
929 : acc->cache_idx. cold_load_acc spins while CLAIM is held, so it
930 : cannot enter the publish path concurrently. If CLAIM is held with
931 : VALID clear, the binding is being replaced or has already been
932 : cleared, so we return. If CLAIM is held with VALID set, wait for
933 : its holder, then retry acquiring CLAIM and checking the binding.
934 : After mutation we release CLAIM.
935 :
936 : Verifies acc->cache_idx still encodes (size_class, line_idx) before
937 : clobbering, in case the acc was concurrently re-published into a
938 : different line (e.g. by a previous cold_load_acc completing before
939 : we arrived). */
940 :
941 : static inline void
942 : evict_clear_acc_cache_ref( fd_accdb_accmeta_t * accmeta,
943 : ulong size_class,
944 101 : ulong line_idx ) {
945 101 : uint expected_cidx = FD_ACCDB_ACC_CIDX_PACK( (uint)size_class, (uint)line_idx );
946 :
947 : /* CAS-acquire CLAIM. A holder with VALID clear is a cold-loader
948 : publishing a different line, or a clearer that already severed the
949 : binding: bail. A holder with VALID set is another evictor,
950 : acc_unlink or the deferred-free drain, none of which block while
951 : holding CLAIM: wait, so whoever evicts the current line clears the
952 : binding. */
953 101 : for(;;) {
954 101 : uint cur = FD_VOLATILE_CONST( accmeta->executable_size );
955 101 : if( FD_UNLIKELY( cur & FD_ACCDB_SIZE_CACHE_CLAIM_BIT ) ) {
956 0 : if( !FD_ACCDB_SIZE_CACHE_VALID( cur ) ) return;
957 0 : fd_racesan_hook( "accdb_evict_clear:claim_held" );
958 0 : FD_SPIN_PAUSE();
959 0 : continue;
960 0 : }
961 101 : uint nxt = cur | FD_ACCDB_SIZE_CACHE_CLAIM_BIT;
962 101 : if( FD_LIKELY( FD_ATOMIC_CAS( &accmeta->executable_size, cur, nxt )==cur ) ) break;
963 0 : fd_racesan_hook( "accdb_evict_clear:claim_wait" );
964 0 : FD_SPIN_PAUSE();
965 0 : }
966 :
967 101 : fd_racesan_hook( "accdb_evict_clear:post_claim" );
968 :
969 : /* CLAIM held. If accmeta->cache_idx still points at our line, clear
970 : VALID and INVAL the cache_idx. Otherwise the accmeta was already
971 : re-published into a different line; leave it alone. */
972 101 : uint es = FD_VOLATILE_CONST( accmeta->executable_size );
973 101 : if( FD_LIKELY( FD_ACCDB_SIZE_CACHE_VALID( es ) &&
974 101 : FD_VOLATILE_CONST( accmeta->cache_idx )==expected_cidx ) ) {
975 101 : FD_ATOMIC_FETCH_AND_AND( &accmeta->executable_size, ~FD_ACCDB_SIZE_CACHE_VALID_BIT );
976 101 : fd_racesan_hook( "accdb_evict_clear:pre_inval" );
977 101 : FD_VOLATILE( accmeta->cache_idx ) = FD_ACCDB_ACC_CIDX_INVAL;
978 101 : }
979 :
980 : /* Release CLAIM. */
981 101 : FD_ATOMIC_FETCH_AND_AND( &accmeta->executable_size, ~FD_ACCDB_SIZE_CACHE_CLAIM_BIT );
982 101 : }
983 :
984 : /* Per-join cache free lists.
985 :
986 : Each join pushes freed lines onto its own list per class and pops
987 : from it first, so the common case touches no other join's lines. A
988 : join whose list is empty takes a never-used line, then steals from
989 : another join's list, then evicts (acquire_cache_line). Every free
990 : line is on some list any join can steal from, so reservations still
991 : guarantee a line.
992 :
993 : Pre-eviction (the accdb tile) pushes the lines it frees onto the
994 : lists of the joins that recently ran short (cache_free_hungry), so in
995 : steady state an exec tile pops its own list instead of every exec
996 : tile stealing from the accdb tile's. A list thus has one owner who
997 : pops and pushes, plus the accdb tile pushing and stealers popping.
998 :
999 : cache_free_have lets a stealer skip empty lists. The invariant is
1000 : "list non-empty => bit set": a push into an empty list sets the bit;
1001 : a stealer that finds a list empty clears the bit and then re-reads
1002 : the list, setting the bit back if a push landed in between. */
1003 :
1004 : /* cache_free_push_to pushes a free line onto join j's class list. The
1005 : caller has already invalidated the line (generation and acc_idx
1006 : UINT_MAX, persisted=1) and released refcnt to 0, in that order. */
1007 :
1008 : static inline void
1009 : cache_free_push_to( fd_accdb_t * accdb,
1010 : ulong j,
1011 : ulong size_class,
1012 770110 : fd_accdb_cache_line_t * line ) {
1013 770110 : ulong * ver_top = &accdb->shmem->cache_free[ j ].ver_top[ size_class ];
1014 770110 : ulong line_idx = cache_line_idx( accdb, size_class, line );
1015 770110 : ulong old_vt = FD_VOLATILE_CONST( *ver_top );
1016 770110 : for(;;) {
1017 769409 : uint old_top = (uint)old_vt;
1018 769409 : uint old_ver = (uint)( old_vt>>32 );
1019 769409 : line->next = old_top;
1020 769409 : FD_COMPILER_MFENCE();
1021 769409 : ulong new_vt = ( (ulong)( old_ver+1U )<<32 ) | line_idx;
1022 769409 : ulong seen_vt = FD_ATOMIC_CAS( ver_top, old_vt, new_vt );
1023 769409 : if( FD_LIKELY( seen_vt==old_vt ) ) {
1024 768104 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->cache_free_cnt[ j ].val[ size_class ], 1UL );
1025 768104 : if( FD_UNLIKELY( old_top==UINT_MAX ) ) {
1026 : /* Into an empty list: keep the hint invariant. The CAS above
1027 : is a full barrier, so this load sees any stealer's clear
1028 : that the stealer's re-read would miss. */
1029 12250 : FD_COMPILER_MFENCE();
1030 12250 : fd_racesan_hook( "accdb_push:pre_hint" );
1031 12250 : ulong * have = &accdb->shmem->cache_free_have[ size_class ].bits[ j/64UL ];
1032 12250 : ulong bit = 1UL<<( j%64UL );
1033 12250 : if( !( FD_VOLATILE_CONST( *have ) & bit ) ) FD_ATOMIC_FETCH_AND_OR( have, bit );
1034 12250 : }
1035 768104 : return;
1036 768104 : }
1037 1305 : old_vt = seen_vt; /* retry at once with the head the CAS returned */
1038 1305 : }
1039 770110 : }
1040 :
1041 : /* cache_free_push pushes a free line onto this join's class list. */
1042 :
1043 : static inline void
1044 : cache_free_push( fd_accdb_t * accdb,
1045 : ulong size_class,
1046 400592 : fd_accdb_cache_line_t * line ) {
1047 400592 : cache_free_push_to( accdb, accdb->join_idx, size_class, line );
1048 400592 : }
1049 :
1050 : /* cache_line_free invalidates a claimed line and pushes it onto join
1051 : j's class free list. */
1052 :
1053 : static inline void
1054 : cache_line_free( fd_accdb_t * accdb,
1055 : ulong j,
1056 : ulong size_class,
1057 368680 : fd_accdb_cache_line_t * line ) {
1058 368680 : line->persisted = 1;
1059 368680 : line->acc_idx = UINT_MAX;
1060 368680 : line->key.generation = UINT_MAX;
1061 368680 : FD_COMPILER_MFENCE();
1062 368680 : FD_VOLATILE( line->refcnt ) = 0;
1063 368680 : cache_free_push_to( accdb, j, size_class, line );
1064 368680 : }
1065 :
1066 : /* cache_free_pop_from pops a line from join j's class list, or returns
1067 : NULL if it is empty. When stealing (j is another join), an empty
1068 : list gets its hint bit cleared. */
1069 :
1070 : static inline fd_accdb_cache_line_t *
1071 : cache_free_pop_from( fd_accdb_t * accdb,
1072 : ulong j,
1073 760551 : ulong size_class ) {
1074 760551 : ulong * ver_top = &accdb->shmem->cache_free[ j ].ver_top[ size_class ];
1075 760551 : ulong old_vt = FD_VOLATILE_CONST( *ver_top );
1076 760551 : for(;;) {
1077 759055 : uint old_top = (uint)old_vt;
1078 759055 : if( FD_UNLIKELY( old_top==UINT_MAX ) ) {
1079 4116 : if( j==accdb->join_idx ) return NULL;
1080 : /* Clear the bit, then re-read the list: a push that saw the bit
1081 : still set is visible to the re-read (the clear is a locked
1082 : RMW, a full barrier on x86). */
1083 6 : fd_racesan_hook( "accdb_steal:pre_clear" );
1084 6 : ulong * have = &accdb->shmem->cache_free_have[ size_class ].bits[ j/64UL ];
1085 6 : ulong bit = 1UL<<( j%64UL );
1086 6 : FD_ATOMIC_FETCH_AND_AND( have, ~bit );
1087 6 : FD_COMPILER_MFENCE();
1088 6 : old_vt = FD_VOLATILE_CONST( *ver_top );
1089 6 : if( FD_LIKELY( (uint)old_vt==UINT_MAX ) ) return NULL;
1090 6 : FD_ATOMIC_FETCH_AND_OR( have, bit );
1091 6 : continue;
1092 6 : }
1093 754939 : uint old_ver = (uint)( old_vt>>32 );
1094 754939 : fd_accdb_cache_line_t * top = cache_line( accdb, size_class, old_top );
1095 754939 : uint next = FD_VOLATILE_CONST( top->next );
1096 754939 : fd_racesan_hook( "accdb_freelist:pre_pop_cas" );
1097 754939 : ulong new_vt = ( (ulong)( old_ver+1U )<<32 ) | next;
1098 754939 : ulong seen_vt = FD_ATOMIC_CAS( ver_top, old_vt, new_vt );
1099 769447 : if( FD_LIKELY( seen_vt==old_vt ) ) {
1100 769447 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->cache_free_cnt[ j ].val[ size_class ], 1UL );
1101 769447 : return top;
1102 769447 : }
1103 >1844*10^16 : old_vt = seen_vt;
1104 >1844*10^16 : }
1105 760551 : }
1106 :
1107 : /* cache_free_steal pops a line from another join's class list, or
1108 : returns NULL if every other list is empty. */
1109 :
1110 : static fd_accdb_cache_line_t *
1111 : cache_free_steal( fd_accdb_t * accdb,
1112 0 : ulong size_class ) {
1113 0 : ulong own = accdb->join_idx;
1114 0 : ulong joiner_cnt = fd_ulong_min( FD_VOLATILE_CONST( accdb->shmem->joiner_cnt ), FD_ACCDB_MAX_JOINERS );
1115 0 : for( ulong w=0UL; w*64UL<joiner_cnt; w++ ) {
1116 0 : ulong bits = FD_VOLATILE_CONST( accdb->shmem->cache_free_have[ size_class ].bits[ w ] );
1117 0 : while( bits ) {
1118 0 : ulong j = w*64UL + (ulong)fd_ulong_find_lsb( bits );
1119 0 : bits &= bits-1UL;
1120 0 : if( FD_UNLIKELY( j==own ) ) continue;
1121 0 : fd_accdb_cache_line_t * line = cache_free_pop_from( accdb, j, size_class );
1122 0 : if( FD_LIKELY( line ) ) return line;
1123 0 : }
1124 0 : }
1125 0 : return NULL;
1126 0 : }
1127 :
1128 : /* cache_free_total is the approximate free line count of the class,
1129 : over all joins' lists. */
1130 :
1131 : static inline ulong
1132 : cache_free_total( fd_accdb_shmem_t const * shmem,
1133 0 : ulong size_class ) {
1134 0 : ulong joiner_cnt = fd_ulong_min( FD_VOLATILE_CONST( shmem->joiner_cnt ), FD_ACCDB_MAX_JOINERS );
1135 0 : ulong total = 0UL;
1136 0 : for( ulong j=0UL; j<joiner_cnt; j++ ) total += FD_VOLATILE_CONST( shmem->cache_free_cnt[ j ].val[ size_class ] );
1137 0 : return total;
1138 0 : }
1139 :
1140 : /* cache_free_ring wakes the accdb tile, which parks when idle in
1141 : efficient mode, so pre-eviction refills the free lists. There is
1142 : no cheap global free count on the pop path, so a join rings when
1143 : its own list is empty and the class has no never-used line left,
1144 : rate-limited per join. */
1145 :
1146 : static inline void
1147 0 : cache_free_ring( fd_accdb_t * accdb ) {
1148 0 : if( FD_LIKELY( !accdb->sleep ) ) return;
1149 0 : long now = fd_tickcount();
1150 0 : if( FD_LIKELY( now-accdb->free_ring_ticks<FD_ACCDB_BG_RATE_TICKS ) ) return;
1151 0 : accdb->free_ring_ticks = now;
1152 0 : fd_sleep_ring( accdb->sleep, accdb->sleep_tile_id );
1153 0 : }
1154 :
1155 : /* Per-join reservation credits.
1156 :
1157 : Each join keeps up to cred_max[c] idle credits per tracked class:
1158 : lines reserved on the shared cache_class_used counter but not yet
1159 : promised to an acquire. Acquire spends its own credits first and
1160 : takes a chunk (cred_max/2) from the shared counter when short;
1161 : release returns credits locally and the surplus to the counter.
1162 :
1163 : A join whose reservation fails steals every join's idle credits in
1164 : that class back to the counter before retrying, so idle credits
1165 : never starve anyone and a reservation still only waits for lines
1166 : promised to in-flight acquires. */
1167 :
1168 : static inline int
1169 : res_tracked( fd_accdb_t const * accdb,
1170 7221202 : ulong c ) {
1171 7221202 : return accdb->res_limit[ c ]!=ULONG_MAX;
1172 7221202 : }
1173 :
1174 : /* res_take reserves n lines of tracked class c. Returns 1 on success, 0
1175 : if the class is full (the caller backs out and steals). */
1176 :
1177 : static inline int
1178 : res_take( fd_accdb_t * accdb,
1179 : ulong c,
1180 0 : ulong n ) {
1181 0 : ulong * cred = &accdb->cred[ c ];
1182 0 : for(;;) {
1183 0 : ulong v = FD_VOLATILE_CONST( *cred );
1184 0 : if( v<n ) break;
1185 0 : if( FD_LIKELY( FD_ATOMIC_CAS( cred, v, v-n )==v ) ) return 1;
1186 0 : }
1187 0 : ulong * used = &accdb->shmem->cache_class_used[ c ].val;
1188 0 : ulong limit = accdb->res_limit[ c ];
1189 0 : ulong refill = accdb->cred_max[ c ]/2UL;
1190 0 : if( FD_LIKELY( refill ) ) {
1191 0 : if( FD_LIKELY( FD_ATOMIC_ADD_AND_FETCH( used, n+refill )<=limit ) ) {
1192 0 : fd_racesan_hook( "accdb_res:post_refill" );
1193 0 : FD_ATOMIC_FETCH_AND_ADD( cred, refill );
1194 0 : return 1;
1195 0 : }
1196 0 : FD_ATOMIC_FETCH_AND_SUB( used, n+refill );
1197 0 : }
1198 0 : if( FD_LIKELY( FD_ATOMIC_ADD_AND_FETCH( used, n )<=limit ) ) return 1;
1199 0 : FD_ATOMIC_FETCH_AND_SUB( used, n );
1200 0 : return 0;
1201 0 : }
1202 :
1203 : /* res_give returns n reserved lines of tracked class c: to the join's
1204 : idle credits, and anything over cred_max[c] (down to half of it) to
1205 : the shared counter. */
1206 :
1207 : static inline void
1208 : res_give( fd_accdb_t * accdb,
1209 : ulong c,
1210 0 : ulong n ) {
1211 0 : ulong * used = &accdb->shmem->cache_class_used[ c ].val;
1212 0 : ulong max = accdb->cred_max[ c ];
1213 0 : if( FD_UNLIKELY( !max ) ) { FD_ATOMIC_FETCH_AND_SUB( used, n ); return; }
1214 0 : ulong * cred = &accdb->cred[ c ];
1215 0 : if( FD_LIKELY( FD_ATOMIC_ADD_AND_FETCH( cred, n )<=max ) ) return;
1216 0 : ulong keep = max/2UL;
1217 0 : for(;;) {
1218 0 : ulong v = FD_VOLATILE_CONST( *cred );
1219 0 : fd_racesan_hook( "accdb_res:pre_trim" );
1220 0 : if( v<=keep ) return; /* stolen meanwhile */
1221 0 : if( FD_LIKELY( FD_ATOMIC_CAS( cred, v, keep )==v ) ) { FD_ATOMIC_FETCH_AND_SUB( used, v-keep ); return; }
1222 0 : }
1223 0 : }
1224 :
1225 : /* res_steal returns every join's idle class-c credits to the shared
1226 : counter. Called by a join whose class-c reservation failed. */
1227 :
1228 : static void
1229 : res_steal( fd_accdb_t * accdb,
1230 0 : ulong c ) {
1231 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
1232 0 : ulong joiner_cnt = fd_ulong_min( FD_VOLATILE_CONST( shmem->joiner_cnt ), FD_ACCDB_MAX_JOINERS );
1233 0 : for( ulong j=0UL; j<joiner_cnt; j++ ) {
1234 0 : ulong * cred = &shmem->join_cred[ j ].val[ c ];
1235 0 : if( FD_LIKELY( !FD_VOLATILE_CONST( *cred ) ) ) continue;
1236 0 : ulong s = FD_ATOMIC_XCHG( cred, 0UL );
1237 0 : if( s ) FD_ATOMIC_FETCH_AND_SUB( &shmem->cache_class_used[ c ].val, s );
1238 0 : }
1239 0 : }
1240 :
1241 : /* cache_try_pin attempts a lock-free pin of a cache-hit line. Returns
1242 : the line if successfully pinned, or NULL if the line is being evicted
1243 : or was recycled (ABA). */
1244 :
1245 : static inline fd_accdb_cache_line_t *
1246 : cache_try_pin( fd_accdb_cache_line_t * line,
1247 : uchar const pubkey[ 32 ],
1248 173272 : uint generation ) {
1249 173272 : for(;;) {
1250 173246 : uint old_rc = FD_VOLATILE_CONST( line->refcnt );
1251 173246 : if( FD_UNLIKELY( old_rc==FD_ACCDB_EVICT_SENTINEL ) ) return NULL;
1252 : /* No saturation guard needed: refcnt is a uint and at most
1253 : FD_ACCDB_MAX_JOINERS (256) threads can pin concurrently,
1254 : so old_rc+1 can never reach FD_ACCDB_EVICT_SENTINEL
1255 : (UINT_MAX) or wrap. */
1256 173246 : if( FD_LIKELY( FD_ATOMIC_CAS( &line->refcnt, old_rc, old_rc+1U )==old_rc ) ) {
1257 : /* Pinned. ABA check: verify the key hasn't changed under us. */
1258 173203 : fd_racesan_hook( "accdb_try_pin:post_cas" );
1259 173203 : FD_COMPILER_MFENCE();
1260 173203 : if( FD_UNLIKELY( line->key.generation!=generation ||
1261 173203 : memcmp( line->key.pubkey, pubkey, 32UL ) ) ) {
1262 0 : FD_ATOMIC_FETCH_AND_SUB( &line->refcnt, 1U );
1263 0 : return NULL;
1264 0 : }
1265 173203 : line->referenced = 1;
1266 173203 : fd_racesan_hook( "cache_try_pin:pinned" );
1267 173203 : return line;
1268 173203 : }
1269 43 : fd_racesan_hook( "accdb_try_pin:cas_retry" );
1270 43 : FD_SPIN_PAUSE();
1271 43 : }
1272 173272 : }
1273 :
1274 : /* epoch_drained returns 1 if every joiner's published epoch exceeds
1275 : tag, meaning all readers that were active at epoch=tag have since
1276 : exited their critical sections. */
1277 :
1278 : static int
1279 : epoch_drained( fd_accdb_t * accdb,
1280 64087 : ulong tag ) {
1281 64087 : ulong joiner_cnt = FD_VOLATILE_CONST( accdb->shmem->joiner_cnt );
1282 128169 : for( ulong t=0UL; t<joiner_cnt; t++ ) {
1283 64082 : if( FD_VOLATILE_CONST( accdb->shmem->joiner_epochs[ t ].val )<=tag ) return 0;
1284 64082 : }
1285 64087 : for( ulong t=0UL; t<accdb->external_epoch_cnt; t++ ) {
1286 0 : if( FD_VOLATILE_CONST( *accdb->external_epoch_slots[ t ] )<=tag ) return 0;
1287 0 : }
1288 64087 : return 1;
1289 64087 : }
1290 :
1291 : /* stray_pin_drained waits out every in-flight acquire and returns 1 if
1292 : only the caller's pin remains. */
1293 :
1294 : static int
1295 : stray_pin_drained( fd_accdb_t * accdb,
1296 0 : fd_accdb_cache_line_t * line ) {
1297 0 : ulong tag = FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->epoch, 1UL );
1298 0 : while( !epoch_drained( accdb, tag ) ) {
1299 0 : fd_racesan_hook( "accdb_epoch_drain:wait" );
1300 0 : FD_SPIN_PAUSE();
1301 0 : }
1302 0 : return FD_VOLATILE_CONST( line->refcnt )==1U;
1303 0 : }
1304 :
1305 : /* drain_deferred_frees releases back to their respective pools the
1306 : accs and fork slots retired by a prior request pass. The resources
1307 : cannot be released immediately because concurrent readers may still
1308 : reference them. The caller has checked that every joiner's
1309 : published epoch exceeds deferred_epoch, stamped after they were
1310 : unlinked.
1311 :
1312 : Must be called before creating a new batch (there is at most one
1313 : outstanding at a time). */
1314 :
1315 : static void
1316 64072 : drain_deferred_frees( fd_accdb_t * accdb ) {
1317 64072 : fd_accdb_shmem_t * shmem = accdb->shmem;
1318 64072 : if( FD_UNLIKELY( shmem->deferred_fork_head.val!=USHORT_MAX ) ) {
1319 64070 : fork_pool_release_chain( accdb->fork_shmem_pool,
1320 64070 : fork_pool_ele( accdb->fork_shmem_pool, shmem->deferred_fork_head.val ),
1321 64070 : fork_pool_ele( accdb->fork_shmem_pool, shmem->deferred_fork_tail.val ) );
1322 64070 : shmem->deferred_fork_head = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
1323 64070 : shmem->deferred_fork_tail = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
1324 64070 : }
1325 :
1326 64072 : ulong n = accdb->shmem->deferred_acc_buf_cnt;
1327 64072 : if( FD_LIKELY( !n ) ) return;
1328 :
1329 : /* All readers that could have been holding a captured pointer to any
1330 : of these accs at unlink time have now exited their epoch sections.
1331 : It is safe to materialize pool.next links and hand the chain to
1332 : acc_pool_release_chain. */
1333 8313 : uint * buf = accdb->deferred_acc_buf;
1334 8313 : fd_accdb_accmeta_t * acc_pool = accdb->acc_pool;
1335 :
1336 : /* Late-publish sweep: a concurrent acquire evictor may have published
1337 : a new offset into one of these accmetas after acc_unlink's
1338 : xchg-to-INVAL but before exiting its epoch. Now that the epoch has
1339 : drained, any such publish is complete and visible. Free the
1340 : orphaned disk bytes here, before the accmeta is released to the
1341 : pool and its fields recycled. */
1342 8313 : ulong acc_pool_cap = acc_pool_ele_max( accdb->acc_pool_join );
1343 377137 : for( ulong i=0UL; i<n; i++ ) {
1344 368824 : FD_TEST( (ulong)buf[ i ]<acc_pool_cap );
1345 368824 : fd_accdb_accmeta_t * accmeta = &acc_pool[ buf[ i ] ];
1346 368824 : ulong off = fd_accdb_acc_offset( accmeta );
1347 368824 : if( FD_UNLIKELY( off!=FD_ACCDB_OFF_INVAL ) ) {
1348 0 : ulong entry_sz = (ulong)FD_ACCDB_SIZE_DATA(accmeta->executable_size)+sizeof(fd_accdb_disk_meta_t);
1349 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, off, entry_sz );
1350 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
1351 0 : }
1352 : /* A reader that captured this accmeta before the unlink may have
1353 : cold-loaded it since (tombstones skip the offset wait), publishing
1354 : a line and VALID, and an evictor of that line may be clearing the
1355 : binding right now under CLAIM. Take CLAIM so that evictor has
1356 : finished its cache_idx store before pool.next overwrites it, then
1357 : leave the slot with VALID clear. */
1358 368824 : for(;;) {
1359 368614 : uint cur = FD_VOLATILE_CONST( accmeta->executable_size );
1360 368614 : if( FD_UNLIKELY( cur & FD_ACCDB_SIZE_CACHE_CLAIM_BIT ) ) {
1361 0 : fd_racesan_hook( "accdb_drain:claim_wait" );
1362 0 : FD_SPIN_PAUSE();
1363 0 : continue;
1364 0 : }
1365 368614 : if( FD_LIKELY( FD_ATOMIC_CAS( &accmeta->executable_size, cur, cur | FD_ACCDB_SIZE_CACHE_CLAIM_BIT )==cur ) ) break;
1366 137 : FD_SPIN_PAUSE();
1367 137 : }
1368 368824 : FD_ATOMIC_FETCH_AND_AND( &accmeta->executable_size, ~(FD_ACCDB_SIZE_CACHE_VALID_BIT|FD_ACCDB_SIZE_CACHE_CLAIM_BIT) );
1369 368824 : }
1370 :
1371 368383 : for( ulong i=0UL; i+1UL<n; i++ ) {
1372 360070 : acc_pool[ buf[ i ] ].pool.next = acc_pool_private_cidx( (ulong)buf[ i+1UL ] );
1373 360070 : }
1374 8313 : fd_accdb_accmeta_t * head = &acc_pool[ buf[ 0UL ] ];
1375 8313 : fd_accdb_accmeta_t * tail = &acc_pool[ buf[ n-1UL ] ];
1376 8313 : acc_pool_release_chain( accdb->acc_pool_join, head, tail );
1377 8313 : accdb->shmem->deferred_acc_buf_cnt = 0UL;
1378 8313 : }
1379 :
1380 : /* deferred_acc_append records an unlinked acc index in the side buffer
1381 : for later release once its epoch drains. T2 is the sole writer.
1382 : The chain link from acc->pool.next is NOT laid down here: pool.next
1383 : is union-aliased to cache_idx, and a concurrent cold_load_acc may
1384 : still publish through a captured pointer until the epoch drains.
1385 : Materialization of the chain happens in drain_deferred_frees. */
1386 :
1387 : static inline void
1388 : deferred_acc_append( fd_accdb_t * accdb,
1389 368950 : uint acc_idx ) {
1390 368950 : fd_accdb_shmem_t * shmem = accdb->shmem;
1391 368950 : FD_TEST( shmem->deferred_acc_buf_cnt<shmem->deferred_acc_buf_max );
1392 368950 : accdb->deferred_acc_buf[ shmem->deferred_acc_buf_cnt++ ] = acc_idx;
1393 368950 : }
1394 :
1395 : /* acc_unlink unlinks an account from its hash map chain, frees any
1396 : associated disk bytes, and invalidates a stale cache reference. Does
1397 : NOT release the acc pool slot — the caller is responsible for that
1398 : (or for batching releases).
1399 :
1400 : prev is the previous element in the map chain (UINT_MAX if acc_idx is
1401 : the head).
1402 :
1403 : CONCURRENCY: The chain link being removed is swapped out with a CAS
1404 : so that a concurrent fd_accdb_release prepending to the same chain
1405 : cannot lose its update. If a head-removal CAS fails (a new node was
1406 : prepended since we loaded the head), we re-walk from the new head to
1407 : find the target as an interior node. Interior CAS cannot fail from
1408 : inserts (inserts only touch the head) and only one remover exists at
1409 : a time (advance_root / purge are serialized). */
1410 :
1411 : static inline void
1412 : acc_unlink( fd_accdb_t * accdb,
1413 : uint map_idx,
1414 : uint prev,
1415 368765 : uint acc_idx ) {
1416 368765 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ acc_idx ];
1417 :
1418 : /* Atomically capture and clear the offset. Two races to defuse:
1419 :
1420 : (1) A concurrent fd_accdb_acquire_inner that is CLOCK-evicting the
1421 : cache line currently holding this acc's data may have already
1422 : xchg'd the offset to INVAL during eviction and freed the old disk
1423 : bytes. Without atomicity we would re-read the old offset and
1424 : free those same bytes a second time. The xchg here serializes:
1425 : whoever wins sees the real offset and frees; the loser sees
1426 : INVAL and skips.
1427 :
1428 : (2) That same evictor may also be mid-flight to publish a NEW
1429 : offset after freeing the old disk bytes. That late publish
1430 : lands on an accmeta that is about to be chain-unlinked and
1431 : deferred-released. drain_deferred_
1432 : frees sweeps the deferred buffer after epoch drain to catch
1433 : the late publish and free the orphaned bytes. */
1434 368765 : uint es = FD_VOLATILE_CONST( accmeta->executable_size );
1435 368765 : ulong entry_sz = (ulong)FD_ACCDB_SIZE_DATA(es)+sizeof(fd_accdb_disk_meta_t);
1436 :
1437 : /* Prefetch the cache line the reclaim below will CAS */
1438 368765 : if( FD_UNLIKELY( FD_ACCDB_SIZE_CACHE_VALID( es ) ) ) {
1439 368725 : uint cidx = FD_VOLATILE_CONST( accmeta->cache_idx );
1440 368725 : ulong cls = FD_ACCDB_ACC_CIDX_CLASS( cidx ) & (FD_ACCDB_CACHE_CLASS_CNT-1UL);
1441 368725 : ulong idx = FD_ACCDB_ACC_CIDX_IDX( cidx );
1442 368725 : if( FD_LIKELY( cidx!=FD_ACCDB_ACC_CIDX_INVAL && idx<accdb->shmem->cache_class_max[ cls ] ) ) __builtin_prefetch( cache_line( accdb, cls, idx ), 1, 3 );
1443 368725 : }
1444 :
1445 368765 : ulong old_offset = fd_accdb_acc_xchg_offset( accmeta, FD_ACCDB_OFF_INVAL );
1446 368765 : if( FD_LIKELY( old_offset!=FD_ACCDB_OFF_INVAL ) ) {
1447 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, old_offset, entry_sz );
1448 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
1449 0 : }
1450 368765 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->accounts_total, 1UL );
1451 368765 : accdb->metrics->accounts_deleted++;
1452 :
1453 368765 : if( FD_LIKELY( prev==UINT_MAX ) ) {
1454 : /* Head removal — CAS may fail if a concurrent insert prepended a
1455 : new node. On failure the target is now interior. */
1456 368362 : for(;;) {
1457 368362 : uint old_head = FD_VOLATILE_CONST( accdb->acc_map[ map_idx ] );
1458 368388 : if( FD_LIKELY( old_head==acc_idx ) ) {
1459 368648 : if( FD_LIKELY( FD_ATOMIC_CAS( &accdb->acc_map[ map_idx ], acc_idx, accmeta->map.next )==acc_idx ) ) break;
1460 >1844*10^16 : FD_SPIN_PAUSE();
1461 >1844*10^16 : continue;
1462 368388 : }
1463 : /* Head changed — walk from new head to find prev for interior
1464 : removal. The target must still be in the chain because only
1465 : this thread removes elements. */
1466 >1844*10^16 : prev = old_head;
1467 >1844*10^16 : while( FD_VOLATILE_CONST( accdb->acc_pool[ prev ].map.next )!=acc_idx ) prev = FD_VOLATILE_CONST( accdb->acc_pool[ prev ].map.next );
1468 >1844*10^16 : FD_ATOMIC_CAS( &accdb->acc_pool[ prev ].map.next, acc_idx, accmeta->map.next );
1469 >1844*10^16 : break;
1470 368362 : }
1471 368362 : } else {
1472 403 : FD_ATOMIC_CAS( &accdb->acc_pool[ prev ].map.next, acc_idx, accmeta->map.next );
1473 403 : }
1474 :
1475 368765 : fd_racesan_hook( "accdb_acc_unlink:post_splice" );
1476 :
1477 : /* If the freed acc still has a cached location, invalidate it and
1478 : try to reclaim the cache line so the eviction path does not try
1479 : to write back stale data from a recycled pool slot. Lock-free:
1480 : CAS the refcnt 0 -> EVICT_SENTINEL to claim it exclusively, then
1481 : push to this join's free list. If the line is pinned (refcnt>0),
1482 : skip, the pinner's release will handle it.
1483 :
1484 : Acquire CACHE_CLAIM_BIT before touching acc->cache_idx /
1485 : CACHE_VALID — see evict_clear_acc_cache_ref for the protocol.
1486 : Without CLAIM, a concurrent cold_load_acc can publish a fresh
1487 : (cache_idx, VALID=1) pair into this acc between our two stores,
1488 : and our subsequent cache_idx=INVAL stomps onto the freelist
1489 : pool.next field (the union sibling of cache_idx), corrupting the
1490 : pool. Unlike evict_clear_acc_cache_ref, we cannot bail when CLAIM
1491 : is held: this acc is being permanently unlinked, so we must
1492 : spin-wait for the cold-loader to release CLAIM and then invalidate
1493 : whatever cache_idx is current. */
1494 368765 : uint cur_es;
1495 368782 : for(;;) {
1496 368782 : cur_es = FD_VOLATILE_CONST( accmeta->executable_size );
1497 368782 : if( FD_UNLIKELY( cur_es & FD_ACCDB_SIZE_CACHE_CLAIM_BIT ) ) {
1498 0 : fd_racesan_hook( "accdb_acc_unlink:claim_wait" );
1499 0 : FD_SPIN_PAUSE();
1500 0 : continue;
1501 0 : }
1502 368782 : uint nxt_es = cur_es | FD_ACCDB_SIZE_CACHE_CLAIM_BIT;
1503 368782 : if( FD_LIKELY( FD_ATOMIC_CAS( &accmeta->executable_size, cur_es, nxt_es )==cur_es ) ) break;
1504 196 : FD_SPIN_PAUSE();
1505 196 : }
1506 :
1507 368765 : uint cidx = FD_ACCDB_ACC_CIDX_INVAL;
1508 368765 : int had_valid = FD_ACCDB_SIZE_CACHE_VALID( cur_es );
1509 368765 : if( FD_UNLIKELY( had_valid ) ) {
1510 368623 : cidx = FD_VOLATILE_CONST( accmeta->cache_idx );
1511 : /* Clear VALID before INVAL'ing cache_idx — matches the order in
1512 : evict_clear_acc_cache_ref so cold_load_acc's "VALID=1 +
1513 : cidx=INVAL" spin path resolves on the next iteration when it
1514 : observes VALID=0. */
1515 368623 : FD_ATOMIC_FETCH_AND_AND( &accmeta->executable_size, ~FD_ACCDB_SIZE_CACHE_VALID_BIT );
1516 368623 : FD_VOLATILE( accmeta->cache_idx ) = FD_ACCDB_ACC_CIDX_INVAL;
1517 368623 : }
1518 :
1519 : /* Release CLAIM. */
1520 368765 : FD_ATOMIC_FETCH_AND_AND( &accmeta->executable_size, ~FD_ACCDB_SIZE_CACHE_CLAIM_BIT );
1521 :
1522 369015 : if( FD_UNLIKELY( had_valid ) ) {
1523 369015 : fd_accdb_cache_line_t * stale = cache_line( accdb, FD_ACCDB_ACC_CIDX_CLASS( cidx ), FD_ACCDB_ACC_CIDX_IDX( cidx ) );
1524 369015 : fd_racesan_hook( "acc_unlink:pre_reclaim_cas" );
1525 369015 : uint old_rc = FD_ATOMIC_CAS( &stale->refcnt, 0U, FD_ACCDB_EVICT_SENTINEL );
1526 369015 : fd_racesan_hook( "acc_unlink:post_reclaim_cas" );
1527 369015 : if( FD_LIKELY( !old_rc ) ) {
1528 : /* Claimed. Validate key (ABA, slot could have been recycled
1529 : between our read of cache_idx and the CAS). */
1530 368935 : if( FD_LIKELY( stale->key.generation==accmeta->key.generation &&
1531 368935 : !memcmp( stale->key.pubkey, accmeta->key.pubkey, 32UL ) ) ) {
1532 368672 : cache_line_free( accdb, accdb->join_idx, FD_ACCDB_ACC_CIDX_CLASS( cidx ), stale );
1533 368672 : } else {
1534 : /* Wrong line (ABA). Release claim. */
1535 263 : FD_VOLATILE( stale->refcnt ) = 0;
1536 263 : }
1537 368935 : }
1538 80 : else if( FD_LIKELY( old_rc!=FD_ACCDB_EVICT_SENTINEL ) ) {
1539 : /* The CAS lost to a non-sentinel refcnt, but that does not prove
1540 : `stale` is still our line. Between capturing cidx and here we
1541 : released the claim, so we could have evicted `stale` and
1542 : recycled it to an unrelated account. */
1543 0 : fd_accdb_cache_line_t * mine = cache_try_pin( stale, accmeta->key.pubkey, accmeta->key.generation );
1544 0 : if( FD_LIKELY( mine ) ) {
1545 : /* Genuinely our line, still pinned by a reader. The accmeta
1546 : slot is about to be deferred-released and recycled; if a
1547 : later writeback of this dirty line fires, it would pair the
1548 : recycled accmeta's pubkey with the old owner/data. Set
1549 : persisted so the writeback gate never fires. */
1550 0 : FD_VOLATILE( mine->persisted ) = 1;
1551 :
1552 : /* Only the tombstone self-unlink may be pinned by a reader.
1553 : We (want to) assert lamports==0UL to catch potential bugs/changes
1554 : in high level logic.
1555 : Unfortunately a stray pin could also occur (even though in very
1556 : edge cases), so if lamport!=0 we detect that a stray pin was there.
1557 : In practice, stray_pin_drained() should never hit, and a bug in
1558 : high level logic would still cause the FD_TEST to fail. */
1559 0 : FD_TEST( accmeta->lamports==0UL || stray_pin_drained( accdb, mine ) );
1560 :
1561 0 : FD_ATOMIC_FETCH_AND_SUB( &mine->refcnt, 1U );
1562 0 : }
1563 : /* Else was recycled to a foreign account. Nothing to neutralize,
1564 : leave the line alone. */
1565 80 : } else {
1566 : /* A foreground evictor already claimed this line. It holds its
1567 : epoch acquire and writeback, so drain_deferred_frees cannot
1568 : recycle the slot before it finishes. Its writeback names the
1569 : old account correctly, no poison. */
1570 80 : }
1571 369015 : }
1572 368765 : }
1573 :
1574 : /* fork_slot_defer chains a retired fork pool slot onto the deferred
1575 : fork chain. The slot is released only after concurrent readers that
1576 : could still hold its fork ID (via root_fork_id or descends_set) have
1577 : exited. Stale descends_set bits naming the slot are cleared by
1578 : fd_accdb_attach_child when the slot is reused. */
1579 :
1580 : static inline void
1581 : fork_slot_defer( fd_accdb_t * accdb,
1582 64006 : fd_accdb_fork_id_t fork_id ) {
1583 64006 : fd_accdb_shmem_t * shmem = accdb->shmem;
1584 64006 : if( shmem->deferred_fork_tail.val!=USHORT_MAX ) fork_pool_ele( accdb->fork_shmem_pool, shmem->deferred_fork_tail.val )->pool.next = fork_pool_private_cidx( (ulong)fork_id.val );
1585 64006 : else shmem->deferred_fork_head = fork_id;
1586 64006 : shmem->deferred_fork_tail = fork_id;
1587 64006 : }
1588 :
1589 : /* chain_prewalk prefetches the hash chain walks of up to CHAIN_PREWALK_W
1590 : txn records from txn in lock-step, over the span the caller will walk.
1591 : Read only. Fills txns[] and map_idxs[] and returns the count. */
1592 :
1593 791750 : #define CHAIN_PREWALK_W (16UL)
1594 :
1595 : static ulong
1596 : chain_prewalk( fd_accdb_t * accdb,
1597 : uint txn,
1598 : int purge,
1599 : fd_accdb_txn_t ** txns,
1600 27687 : uint * map_idxs ) {
1601 27687 : uint acc_idx [ CHAIN_PREWALK_W ];
1602 27687 : uint cur [ CHAIN_PREWALK_W ];
1603 27687 : int from_head[ CHAIN_PREWALK_W ];
1604 :
1605 27687 : ulong cnt = 0UL;
1606 395875 : for( ; cnt<CHAIN_PREWALK_W && txn!=UINT_MAX; cnt++ ) {
1607 368188 : fd_accdb_txn_t * txne = txn_pool_ele( accdb->txn_pool, (ulong)txn );
1608 368188 : txns [ cnt ] = txne;
1609 368188 : acc_idx[ cnt ] = txne->acc_pool_idx;
1610 368188 : __builtin_prefetch( &accdb->acc_pool[ txne->acc_pool_idx ], 0, 3 );
1611 368188 : txn = txne->fork.next;
1612 368188 : }
1613 395731 : for( ulong i=0UL; i<cnt; i++ ) {
1614 368044 : fd_accdb_accmeta_t const * acc = &accdb->acc_pool[ acc_idx[ i ] ];
1615 368044 : map_idxs [ i ] = (uint)(fd_hash32( acc->key.pubkey, accdb->shmem->seed ) & (accdb->shmem->chain_cnt-1UL));
1616 368044 : from_head[ i ] = purge || acc->lamports==0UL;
1617 368044 : if( from_head[ i ] ) __builtin_prefetch( &accdb->acc_map[ map_idxs[ i ] ], 1, 3 );
1618 368044 : }
1619 396437 : for( ulong i=0UL; i<cnt; i++ ) {
1620 368769 : if( from_head[ i ] ) cur[ i ] = FD_VOLATILE_CONST( accdb->acc_map[ map_idxs[ i ] ] );
1621 >1844*10^16 : else cur[ i ] = FD_VOLATILE_CONST( accdb->acc_pool[ acc_idx[ i ] ].map.next );
1622 368750 : }
1623 :
1624 30278 : for(;;) {
1625 30278 : ulong live = 0UL;
1626 438697 : for( ulong i=0UL; i<cnt; i++ ) {
1627 408470 : if( cur[ i ]==UINT_MAX || ( purge && cur[ i ]==acc_idx[ i ] ) ) continue;
1628 2555 : cur[ i ] = FD_VOLATILE_CONST( accdb->acc_pool[ cur[ i ] ].map.next );
1629 2555 : live++;
1630 2555 : }
1631 30278 : if( !live ) break;
1632 30278 : }
1633 27687 : return cnt;
1634 27687 : }
1635 :
1636 : static void
1637 : purge_inner( fd_accdb_t * accdb,
1638 64001 : fd_accdb_fork_id_t fork_id ) {
1639 64001 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
1640 :
1641 64001 : fd_accdb_fork_id_t child = fork->shmem->child_id;
1642 64001 : while( child.val!=USHORT_MAX ) {
1643 0 : fd_accdb_fork_id_t next = accdb->fork_pool[ child.val ].shmem->sibling_id;
1644 0 : purge_inner( accdb, child );
1645 0 : child = next;
1646 0 : }
1647 :
1648 64001 : uint txn = fork->shmem->txn_head;
1649 64001 : if( txn!=UINT_MAX ) {
1650 8337 : fd_accdb_txn_t * txn_head = txn_pool_ele( accdb->txn_pool, (ulong)txn );
1651 8337 : fd_accdb_txn_t * txn_tail = NULL;
1652 36029 : while( txn!=UINT_MAX ) {
1653 27692 : fd_accdb_txn_t * txns [ CHAIN_PREWALK_W ];
1654 27692 : uint map_idxs[ CHAIN_PREWALK_W ];
1655 27692 : ulong cnt = chain_prewalk( accdb, txn, 1, txns, map_idxs );
1656 396524 : for( ulong w=0UL; w<cnt; w++ ) {
1657 368832 : fd_accdb_txn_t * txne = txns[ w ];
1658 :
1659 368832 : uint acc_idx = txne->acc_pool_idx;
1660 368832 : uint acc_map_idx = map_idxs[ w ];
1661 :
1662 368832 : uint prev = UINT_MAX;
1663 368832 : uint cur = FD_VOLATILE_CONST( accdb->acc_map[ acc_map_idx ] );
1664 368832 : while( cur!=acc_idx ) {
1665 0 : prev = cur;
1666 0 : cur = FD_VOLATILE_CONST( accdb->acc_pool[ cur ].map.next );
1667 0 : }
1668 :
1669 368832 : fd_racesan_hook( "accdb_purge:pre_unlink" );
1670 368832 : acc_unlink( accdb, acc_map_idx, prev, acc_idx );
1671 368832 : deferred_acc_append( accdb, acc_idx );
1672 :
1673 368832 : txn_tail = txne;
1674 368832 : txn = txne->fork.next;
1675 368832 : }
1676 27692 : }
1677 8337 : txn_pool_release_chain( accdb->txn_pool, txn_head, txn_tail );
1678 8337 : }
1679 :
1680 64001 : fork_slot_defer( accdb, fork_id );
1681 64001 : }
1682 :
1683 : /* root_level roots fork_id, the only child of the current root,
1684 : on the accdb tile. Siblings were already detached and purged by
1685 : fd_accdb_advance_root. */
1686 :
1687 : static void
1688 : root_level( fd_accdb_t * accdb,
1689 0 : fd_accdb_fork_id_t fork_id ) {
1690 0 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
1691 0 : FD_TEST( fork->shmem->parent_id.val==accdb->shmem->root_fork_id.val );
1692 0 : FD_TEST( fork->shmem->sibling_id.val==USHORT_MAX );
1693 :
1694 0 : fd_accdb_fork_t * parent_fork = &accdb->fork_pool[ fork->shmem->parent_id.val ];
1695 :
1696 : /* And for any accounts which were updated in the newly rooted slot,
1697 : we will now never need to access any older version, so we can
1698 : discard any slots earlier than the one we are rooting. */
1699 0 : uint txn = fork->shmem->txn_head;
1700 0 : if( txn!=UINT_MAX ) {
1701 0 : fd_accdb_txn_t * txn_head = txn_pool_ele( accdb->txn_pool, (ulong)txn );
1702 0 : fd_accdb_txn_t * txn_tail = NULL;
1703 0 : while( txn!=UINT_MAX ) {
1704 0 : fd_accdb_txn_t * txns [ CHAIN_PREWALK_W ];
1705 0 : uint map_idxs[ CHAIN_PREWALK_W ];
1706 0 : ulong cnt = chain_prewalk( accdb, txn, 0, txns, map_idxs );
1707 0 : for( ulong w=0UL; w<cnt; w++ ) {
1708 0 : fd_accdb_txn_t * txne = txns[ w ];
1709 :
1710 0 : fd_accdb_accmeta_t const * new_acc = &accdb->acc_pool[ txne->acc_pool_idx ];
1711 0 : uint acc_map_idx = map_idxs[ w ];
1712 :
1713 0 : delta_insert( accdb, new_acc->key.pubkey );
1714 :
1715 : /* Walk from new_acc, not the head: everything ahead of it was
1716 : prepended by a non-ancestor fork. A tombstone needs its
1717 : predecessor so still walks from the head. */
1718 0 : uint prev = UINT_MAX;
1719 0 : uint new_acc_prev = UINT_MAX; /* prev of new_acc on the chain when we encounter it (UINT_MAX if head or never seen) */
1720 0 : int new_acc_seen = 0;
1721 0 : uint acc;
1722 0 : FD_TEST( new_acc->key.generation==fork->shmem->generation && fd_accdb_acc_fork_id( new_acc )==fork_id.val );
1723 0 : if( FD_LIKELY( new_acc->lamports ) ) {
1724 : #if FD_TMPL_USE_HANDHOLDING
1725 : uint chk = FD_VOLATILE_CONST( accdb->acc_map[ acc_map_idx ] );
1726 : while( chk!=UINT_MAX && chk!=txne->acc_pool_idx ) {
1727 : fd_accdb_accmeta_t const * ahead = &accdb->acc_pool[ chk ];
1728 : FD_TEST( !( (ahead->key.generation<=parent_fork->shmem->generation || descends_set_test( fork->descends, fd_accdb_acc_fork_id(ahead) ) ) && !memcmp( new_acc->key.pubkey, ahead->key.pubkey, 32UL ) ) );
1729 : chk = FD_VOLATILE_CONST( ahead->map.next );
1730 : }
1731 : FD_TEST( chk==txne->acc_pool_idx );
1732 : #endif
1733 0 : new_acc_seen = 1;
1734 0 : prev = txne->acc_pool_idx;
1735 0 : acc = FD_VOLATILE_CONST( new_acc->map.next );
1736 0 : } else {
1737 0 : acc = FD_VOLATILE_CONST( accdb->acc_map[ acc_map_idx ] );
1738 0 : FD_TEST( acc!=UINT_MAX );
1739 0 : }
1740 0 : while( acc!=UINT_MAX ) {
1741 0 : if( FD_UNLIKELY( acc==txne->acc_pool_idx ) ) {
1742 0 : new_acc_prev = prev;
1743 0 : new_acc_seen = 1;
1744 0 : prev = acc;
1745 0 : acc = FD_VOLATILE_CONST( new_acc->map.next );
1746 0 : continue;
1747 0 : }
1748 :
1749 0 : fd_accdb_accmeta_t const * cur_acc = &accdb->acc_pool[ acc ];
1750 0 : uint cur_next = FD_VOLATILE_CONST( cur_acc->map.next );
1751 :
1752 0 : if( FD_LIKELY( (cur_acc->key.generation<=parent_fork->shmem->generation || descends_set_test( fork->descends, fd_accdb_acc_fork_id(cur_acc) ) ) && !memcmp( new_acc->key.pubkey, cur_acc->key.pubkey, 32UL ) ) ) {
1753 0 : uint next = cur_next;
1754 0 : fd_racesan_hook( "accdb_advance:pre_unlink" );
1755 0 : acc_unlink( accdb, acc_map_idx, prev, acc );
1756 0 : deferred_acc_append( accdb, acc );
1757 0 : acc = next;
1758 0 : } else {
1759 0 : prev = acc;
1760 0 : acc = cur_next;
1761 0 : }
1762 0 : }
1763 :
1764 : /* If the newly rooted version is a tombstone (lamports==0, e.g.
1765 : account was closed), drop it from the index too: no fork can
1766 : reach it anymore, and keeping it around just wastes a hash
1767 : slot and the disk bytes it occupies. */
1768 0 : if( FD_UNLIKELY( new_acc_seen && new_acc->lamports==0UL ) ) {
1769 0 : uint new_acc_idx = (uint)txne->acc_pool_idx;
1770 0 : acc_unlink( accdb, acc_map_idx, new_acc_prev, new_acc_idx );
1771 0 : deferred_acc_append( accdb, new_acc_idx );
1772 0 : }
1773 :
1774 0 : txn_tail = txne;
1775 0 : txn = txne->fork.next;
1776 0 : }
1777 0 : }
1778 0 : txn_pool_release_chain( accdb->txn_pool, txn_head, txn_tail );
1779 0 : }
1780 :
1781 0 : uint parent_txn = parent_fork->shmem->txn_head;
1782 0 : if( parent_txn!=UINT_MAX ) {
1783 0 : fd_accdb_txn_t * parent_head = txn_pool_ele( accdb->txn_pool, (ulong)parent_txn );
1784 0 : fd_accdb_txn_t * parent_tail = NULL;
1785 0 : while( parent_txn!=UINT_MAX ) {
1786 0 : fd_accdb_txn_t * t = txn_pool_ele( accdb->txn_pool, (ulong)parent_txn );
1787 0 : parent_tail = t;
1788 0 : parent_txn = t->fork.next;
1789 0 : }
1790 0 : txn_pool_release_chain( accdb->txn_pool, parent_head, parent_tail );
1791 0 : }
1792 :
1793 0 : fork->shmem->txn_head = UINT_MAX;
1794 :
1795 : /* Retire the old root. Its accounts stay visible through the
1796 : generation <= root_generation fast path. A reader that loaded the
1797 : old root_fork_id may still read its slot, so the slot is released
1798 : only after the epoch bump that ends the pass drains. */
1799 0 : fork_slot_defer( accdb, fork->shmem->parent_id );
1800 :
1801 0 : fd_racesan_hook( "accdb_advance:pre_publish_root" );
1802 0 : FD_COMPILER_MFENCE();
1803 0 : FD_VOLATILE( accdb->shmem->root_fork_id ) = fork_id;
1804 0 : FD_COMPILER_MFENCE();
1805 0 : fd_racesan_hook( "accdb_advance:post_publish_root" );
1806 0 : }
1807 :
1808 : int
1809 0 : fd_accdb_requests_pending( fd_accdb_t const * accdb ) {
1810 0 : fd_accdb_req_t req = { .u = FD_VOLATILE_CONST( accdb->shmem->req.u ) };
1811 0 : fd_accdb_done_t done = { .u = FD_VOLATILE_CONST( accdb->shmem->done.u ) };
1812 0 : return done.root_gen!=req.root_gen || done.purge_seq!=req.purge_seq || done.retiring;
1813 0 : }
1814 :
1815 : void
1816 : fd_accdb_advance_root( fd_accdb_t * accdb,
1817 0 : fd_accdb_fork_id_t fork_id ) {
1818 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
1819 0 : FD_CHECK_CRIT( fd_accdb_snapshot_sync_state( &shmem->snapshot_sync )!=FD_ACCDB_SNAPSHOT_SYNC_RUNNING,
1820 0 : "fd_accdb_advance_root called during snapshot production" );
1821 :
1822 : /* The caller roots sequentially: fork_id is a child of the last
1823 : requested root. */
1824 0 : fd_accdb_req_t req = shmem->req;
1825 0 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
1826 0 : fd_accdb_fork_id_t parent_id = fork->shmem->parent_id;
1827 0 : FD_CHECK_CRIT( parent_id.val!=USHORT_MAX && accdb->fork_pool[ parent_id.val ].shmem->generation==req.root_gen, "advance_root target is not a child of the root" );
1828 :
1829 : /* Detach the competing siblings for purging. The parent is left with
1830 : fork_id as its only child, which is how the accdb tile finds the
1831 : next level to root. The purges and the root are published in one
1832 : request, and the accdb tile applies a request's purges first. */
1833 0 : fd_accdb_fork_shmem_t * parent = accdb->fork_pool[ parent_id.val ].shmem;
1834 0 : fd_accdb_fork_id_t child = parent->child_id;
1835 0 : while( child.val!=USHORT_MAX ) {
1836 0 : fd_accdb_fork_id_t next = accdb->fork_pool[ child.val ].shmem->sibling_id;
1837 0 : if( child.val!=fork_id.val ) {
1838 0 : accdb->fork_pool[ child.val ].shmem->parent_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
1839 0 : accdb->fork_pool[ child.val ].shmem->sibling_id = req.purge_fork;
1840 0 : req.purge_fork = child;
1841 0 : req.purge_seq++;
1842 0 : }
1843 0 : child = next;
1844 0 : }
1845 0 : fork->shmem->sibling_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
1846 0 : parent->child_id = fork_id;
1847 :
1848 0 : req.root_gen = fork->shmem->generation;
1849 0 : publish_req( accdb, req );
1850 0 : }
1851 :
1852 : void
1853 : fd_accdb_purge( fd_accdb_t * accdb,
1854 63989 : fd_accdb_fork_id_t fork_id ) {
1855 63989 : fd_accdb_shmem_t * shmem = accdb->shmem;
1856 :
1857 : /* Live forks that do not descend from the requested root are the
1858 : root and its ancestors, which are older. A fork newer than the
1859 : root has a parent (unless it was already purged). */
1860 63989 : fd_accdb_req_t req = shmem->req;
1861 63989 : fd_accdb_fork_shmem_t * fork = accdb->fork_pool[ fork_id.val ].shmem;
1862 63989 : FD_CHECK_CRIT( shmem->root_fork_id.val!=USHORT_MAX, "purge before the root fork exists" );
1863 63989 : FD_CHECK_CRIT( fork->generation>req.root_gen, "purge target is the root or its ancestor" );
1864 63989 : fd_accdb_fork_id_t parent_id = fork->parent_id;
1865 63989 : FD_CHECK_CRIT( parent_id.val!=USHORT_MAX, "purge target was already purged" );
1866 :
1867 63989 : fd_accdb_fork_shmem_t * parent = accdb->fork_pool[ parent_id.val ].shmem;
1868 63989 : fd_accdb_fork_id_t next = fork->sibling_id;
1869 64008 : if( parent->child_id.val==fork_id.val ) {
1870 64008 : parent->child_id = next;
1871 >1844*10^16 : } else {
1872 >1844*10^16 : fd_accdb_fork_shmem_t * prev = accdb->fork_pool[ parent->child_id.val ].shmem;
1873 >1844*10^16 : while( prev->sibling_id.val!=fork_id.val ) prev = accdb->fork_pool[ prev->sibling_id.val ].shmem;
1874 >1844*10^16 : prev->sibling_id = next;
1875 >1844*10^16 : }
1876 :
1877 63989 : fork->parent_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
1878 63989 : fork->sibling_id = req.purge_fork;
1879 63989 : req.purge_fork = fork_id;
1880 63989 : req.purge_seq++;
1881 63989 : publish_req( accdb, req );
1882 63989 : }
1883 :
1884 : /* background_release releases the retired batch once every reader
1885 : that could still reference it has exited. Does not wait. Returns
1886 : 1 if no batch is held afterwards. */
1887 :
1888 : static int
1889 128100 : background_release( fd_accdb_t * accdb ) {
1890 128100 : fd_accdb_shmem_t * shmem = accdb->shmem;
1891 128100 : fd_accdb_done_t done = shmem->done;
1892 128100 : if( FD_LIKELY( !done.retiring ) ) return 1;
1893 64088 : if( FD_UNLIKELY( !epoch_drained( accdb, shmem->deferred_epoch ) ) ) return 0;
1894 64088 : drain_deferred_frees( accdb );
1895 64088 : fd_racesan_hook( "accdb_release:pre_done" );
1896 64088 : done.retiring = 0U;
1897 64088 : FD_COMPILER_MFENCE();
1898 64088 : FD_VOLATILE( shmem->done.u ) = done.u;
1899 64088 : FD_COMPILER_MFENCE();
1900 64088 : return 1;
1901 64088 : }
1902 :
1903 : /* background_requests applies the root and purge requests published
1904 : since the last pass, then retires everything they unlinked as one
1905 : batch. Returns 1 if it did any work.
1906 :
1907 : A request's purges are applied before its root, and fd_accdb_advance_root
1908 : requests a level's sibling purges together with the level, so no
1909 : losing sibling is visible through the generation fast path. */
1910 :
1911 : static int
1912 64013 : background_requests( fd_accdb_t * accdb ) {
1913 64013 : fd_accdb_shmem_t * shmem = accdb->shmem;
1914 :
1915 : /* One batch is held at a time. Until it is released, requests wait
1916 : and the tile keeps doing other work. */
1917 64013 : if( FD_UNLIKELY( !background_release( accdb ) ) ) return 0;
1918 :
1919 64013 : fd_accdb_req_t req = { .u = FD_VOLATILE_CONST( shmem->req.u ) };
1920 64013 : fd_accdb_done_t done = shmem->done;
1921 64013 : if( FD_LIKELY( done.root_gen==req.root_gen && done.purge_seq==req.purge_seq ) ) return 0;
1922 64013 : FD_COMPILER_MFENCE();
1923 :
1924 : /* The newest purge_cnt forks of the chain are the ones not yet
1925 : applied. Each still holds its slot, so the links are intact. */
1926 64013 : ulong purge_cnt = (ushort)( req.purge_seq-done.purge_seq );
1927 64013 : fd_accdb_fork_id_t purge = req.purge_fork;
1928 128023 : for( ulong i=0UL; i<purge_cnt; i++ ) {
1929 64010 : fd_accdb_fork_id_t next = accdb->fork_pool[ purge.val ].shmem->sibling_id;
1930 64010 : purge_inner( accdb, purge );
1931 64010 : purge = next;
1932 64010 : }
1933 64013 : done.purge_seq = req.purge_seq;
1934 :
1935 : /* A level unlinks at most two accounts per txn (the replaced rooted
1936 : version and a rooted tombstone). A level that could overflow the
1937 : deferred buffer is left for the next pass, which starts with an
1938 : empty buffer, so passes always progress. */
1939 64013 : for(;;) {
1940 63998 : fd_accdb_fork_shmem_t const * root = accdb->fork_pool[ shmem->root_fork_id.val ].shmem;
1941 63998 : done.root_gen = root->generation;
1942 64006 : if( done.root_gen==req.root_gen ) break;
1943 >1844*10^16 : FD_TEST( done.root_gen<req.root_gen );
1944 >1844*10^16 : fd_accdb_fork_id_t level = root->child_id;
1945 >1844*10^16 : if( shmem->deferred_acc_buf_cnt ) {
1946 0 : ulong unlink_max = 0UL;
1947 0 : for( uint txn=accdb->fork_pool[ level.val ].shmem->txn_head; txn!=UINT_MAX; txn=txn_pool_ele( accdb->txn_pool, (ulong)txn )->fork.next ) unlink_max += 2UL;
1948 0 : if( FD_UNLIKELY( shmem->deferred_acc_buf_cnt+unlink_max>shmem->deferred_acc_buf_max ) ) break;
1949 0 : }
1950 >1844*10^16 : root_level( accdb, level );
1951 >1844*10^16 : }
1952 :
1953 : /* One grace period for everything unlinked or retired in this pass. */
1954 64023 : if( FD_LIKELY( shmem->deferred_acc_buf_cnt || shmem->deferred_fork_head.val!=USHORT_MAX ) ) {
1955 64023 : shmem->deferred_epoch = FD_ATOMIC_FETCH_AND_ADD( &shmem->epoch, 1UL );
1956 64023 : done.retiring = 1U;
1957 64023 : }
1958 :
1959 64013 : fd_racesan_hook( "accdb_requests:pre_done" );
1960 64013 : FD_COMPILER_MFENCE();
1961 64013 : FD_VOLATILE( shmem->done.u ) = done.u;
1962 64013 : FD_COMPILER_MFENCE();
1963 64013 : return 1;
1964 64013 : }
1965 :
1966 : static inline fd_accdb_cache_line_t *
1967 : acquire_cache_line( fd_accdb_t * accdb,
1968 : ulong size_class,
1969 760241 : uint * out_evicted_acc_idx ) {
1970 : /* Priority 1: this join's free list (lines already invalidated). */
1971 760241 : fd_accdb_cache_line_t * result = cache_free_pop_from( accdb, accdb->join_idx, size_class );
1972 769585 : if( FD_LIKELY( result ) ) {
1973 769585 : popped:
1974 767876 : while( FD_UNLIKELY( FD_ATOMIC_CAS( &result->refcnt, 0U, 1U )!=0U ) ) {
1975 0 : fd_racesan_hook( "accdb_freepop:refcnt_wait" );
1976 0 : FD_SPIN_PAUSE();
1977 0 : }
1978 767876 : result->referenced = 0;
1979 767876 : FD_VOLATILE( result->seq ) = result->seq+1U;
1980 767876 : FD_COMPILER_MFENCE();
1981 767876 : *out_evicted_acc_idx = UINT_MAX;
1982 767876 : return result;
1983 769585 : }
1984 :
1985 : /* Priority 2: a never-used line (FAA with undo on overflow), before
1986 : stealing, so each join comes to own the lines it cycles through.
1987 : Load first: once the class is full the counter stops changing. */
1988 >1844*10^16 : ulong max_c = accdb->shmem->cache_class_max[ size_class ];
1989 >1844*10^16 : ulong old_init = max_c;
1990 >1844*10^16 : if( FD_UNLIKELY( FD_VOLATILE_CONST( accdb->shmem->cache_class_init[ size_class ].val )<max_c ) ) {
1991 4132 : old_init = FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->cache_class_init[ size_class ].val, 1UL );
1992 4132 : if( FD_UNLIKELY( old_init>=max_c ) ) FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->cache_class_init[ size_class ].val, 1UL );
1993 4132 : }
1994 >1844*10^16 : if( FD_LIKELY( old_init<max_c ) ) {
1995 4095 : result = cache_line( accdb, size_class, old_init );
1996 4095 : result->refcnt = 1;
1997 4095 : result->persisted = 1;
1998 4095 : result->referenced = 0;
1999 4095 : result->acc_idx = UINT_MAX;
2000 4095 : result->key.generation = UINT_MAX;
2001 4095 : *out_evicted_acc_idx = UINT_MAX;
2002 4095 : return result;
2003 4095 : }
2004 :
2005 : /* Our list is empty and the class has no never-used line left: from
2006 : here on, free lines only come from pre-eviction or eviction, so ask
2007 : pre-eviction for a line next time and wake the accdb tile now. */
2008 >1844*10^16 : {
2009 >1844*10^16 : ulong * hungry = &accdb->shmem->cache_free_hungry[ size_class ].bits[ accdb->join_idx/64UL ];
2010 >1844*10^16 : ulong bit = 1UL<<( accdb->join_idx%64UL );
2011 >1844*10^16 : if( !( FD_VOLATILE_CONST( *hungry ) & bit ) ) FD_ATOMIC_FETCH_AND_OR( hungry, bit );
2012 >1844*10^16 : }
2013 >1844*10^16 : cache_free_ring( accdb );
2014 :
2015 : /* Priority 3: another join's free list. */
2016 >1844*10^16 : result = cache_free_steal( accdb, size_class );
2017 >1844*10^16 : if( FD_LIKELY( result ) ) goto popped;
2018 :
2019 : /* Priority 4: CLOCK sweep ... scan forward giving second chances.
2020 : CLOCK skips free lines, so a line freed onto some list during the
2021 : sweep is invisible to it: after each fruitless lap, check our own
2022 : list (pre-eviction may have pushed to it) and steal again. */
2023 >1844*10^16 : ulong swept = 0UL;
2024 >1844*10^16 : for(;;) {
2025 0 : if( FD_UNLIKELY( ++swept>max_c ) ) {
2026 0 : fd_racesan_hook( "accdb_clock:lap" );
2027 0 : swept = 0UL;
2028 0 : result = cache_free_pop_from( accdb, accdb->join_idx, size_class );
2029 0 : if( result ) goto popped;
2030 0 : result = cache_free_steal( accdb, size_class );
2031 0 : if( result ) goto popped;
2032 0 : }
2033 0 : ulong hand = FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->clock_hand[ size_class ].val, 1UL ) % max_c;
2034 0 : fd_accdb_cache_line_t * line = cache_line( accdb, size_class, hand );
2035 :
2036 0 : if( FD_UNLIKELY( line->key.generation==UINT_MAX && line->acc_idx==UINT_MAX ) ) continue;
2037 :
2038 0 : fd_racesan_hook( "accdb_clock:pre_refcnt" );
2039 0 : uint rc = FD_VOLATILE_CONST( line->refcnt );
2040 0 : if( FD_UNLIKELY( rc!=0U ) ) continue; /* Pinned or being evicted */
2041 :
2042 0 : if( FD_UNLIKELY( line->referenced ) ) {
2043 0 : line->referenced = 0;
2044 0 : continue; /* Second chance */
2045 0 : }
2046 :
2047 0 : if( FD_UNLIKELY( FD_ATOMIC_CAS( &line->refcnt, 0U, FD_ACCDB_EVICT_SENTINEL )!=0U ) ) continue;
2048 :
2049 0 : if( FD_UNLIKELY( line->acc_idx==UINT_MAX && line->key.generation==UINT_MAX ) ) {
2050 0 : FD_VOLATILE( line->refcnt ) = 0;
2051 0 : continue;
2052 0 : }
2053 :
2054 : /* The line is now claimed for eviction (refcnt==EVICT_SENTINEL). A
2055 : concurrent acc_unlink that targets this same line's accmeta will
2056 : observe the sentinel here and take its do-nothing branch — see the
2057 : test_accdb_racesan SENTINEL case. */
2058 0 : fd_racesan_hook( "clock_evict:post_sentinel" );
2059 :
2060 0 : if( FD_LIKELY( line->acc_idx!=UINT_MAX ) ) {
2061 0 : evict_clear_acc_cache_ref( &accdb->acc_pool[ line->acc_idx ], size_class, hand );
2062 0 : }
2063 0 : *out_evicted_acc_idx = line->persisted ? UINT_MAX : line->acc_idx;
2064 0 : line->key.generation = UINT_MAX;
2065 0 : FD_VOLATILE( line->seq ) = line->seq+1U;
2066 0 : FD_COMPILER_MFENCE();
2067 0 : line->refcnt = 1;
2068 0 : line->referenced = 0;
2069 0 : return line;
2070 0 : }
2071 :
2072 >1844*10^16 : FD_TEST( 0 );
2073 >1844*10^16 : return NULL;
2074 >1844*10^16 : }
2075 :
2076 : static int
2077 : grow_locked( fd_accdb_t * accdb,
2078 : ulong partition_idx,
2079 : uchar layer,
2080 : ulong prior_partition_idx,
2081 0 : int preallocated ) {
2082 0 : for( ulong p=accdb->shmem->partition_fallocated; p<=partition_idx; p++ ) {
2083 0 : FD_LOG_INFO(( "growing accounts database from %lu GiB to %lu GiB", p*accdb->shmem->partition_sz/(1UL<<30UL), (p+1UL)*accdb->shmem->partition_sz/(1UL<<30UL) ));
2084 :
2085 0 : long fallocate_nanos = -fd_log_wallclock();
2086 0 : int result = fallocate( accdb->fd, 0, (long)(p*accdb->shmem->partition_sz), (long)accdb->shmem->partition_sz );
2087 0 : fallocate_nanos += fd_log_wallclock();
2088 0 : if( FD_UNLIKELY( -1==result ) ) {
2089 0 : if( FD_LIKELY( preallocated ) ) return errno;
2090 0 : if( FD_LIKELY( errno==ENOSPC ) ) FD_LOG_ERR(( "fallocate() failed (%d-%s). The accounts database filled "
2091 0 : "the disk it is on, trying to grow from %lu GiB to %lu GiB. Please "
2092 0 : "free up disk space and restart the validator.",
2093 0 : errno, fd_io_strerror( errno ), p*accdb->shmem->partition_sz/(1UL<<30UL), (p+1UL)*accdb->shmem->partition_sz/(1UL<<30UL) ));
2094 0 : else FD_LOG_ERR(( "fallocate() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
2095 0 : }
2096 :
2097 0 : FD_VOLATILE( accdb->shmem->partition_fallocated ) = p+1UL;
2098 0 : accdb->shmem->shmetrics->disk_allocated_bytes = (p+1UL)*accdb->shmem->partition_sz;
2099 :
2100 0 : fd_event_accdb_partition_added_t ev = {
2101 0 : .partition_idx = p,
2102 0 : .prior_partition_idx = p==partition_idx ? prior_partition_idx : ULONG_MAX,
2103 0 : .layer = layer,
2104 0 : .old_partition_max = p,
2105 0 : .new_partition_max = p+1UL,
2106 0 : .partition_sz = accdb->shmem->partition_sz,
2107 0 : .disk_allocated_bytes = (p+1UL)*accdb->shmem->partition_sz,
2108 0 : .duration_fallocate_nanos = (ulong)fd_long_max( fallocate_nanos, 0L ),
2109 0 : .preallocated = preallocated,
2110 0 : };
2111 0 : fd_event_report_accdb_partition_added( &ev );
2112 0 : }
2113 0 : return 0;
2114 0 : }
2115 :
2116 : static inline void
2117 : change_partition( fd_accdb_t * accdb,
2118 : accdb_offset_t const * offset_before,
2119 : accdb_offset_t * out_offset,
2120 : int * has_partition,
2121 0 : uchar layer ) {
2122 : /* New data will not fit in the current partition, so we need to
2123 : move to the next one. */
2124 0 : spin_lock_acquire( &accdb->shmem->partition_lock );
2125 0 : ulong partition_idx_before = packed_partition_idx( offset_before );
2126 0 : ulong partition_offset_before = packed_partition_offset( offset_before );
2127 0 : if( FD_LIKELY( *has_partition ) ) {
2128 0 : fd_accdb_partition_t * before = partition_pool_ele( accdb->partition_pool, partition_idx_before );
2129 0 : before->write_offset = partition_offset_before;
2130 0 : }
2131 :
2132 : /* Single rdtsc per partition lifecycle event: stamp the closing
2133 : partition's filled time and the new partition's created time off
2134 : the same sample. */
2135 0 : long now_ticks = (long)fd_tickcount();
2136 :
2137 0 : ulong free_size = accdb->shmem->partition_sz - partition_offset_before;
2138 0 : if( FD_LIKELY( *has_partition ) ) {
2139 0 : fd_accdb_partition_t * old = partition_pool_ele( accdb->partition_pool, partition_idx_before );
2140 0 : FD_ATOMIC_FETCH_AND_ADD( &old->bytes_freed, free_size );
2141 0 : FD_VOLATILE( old->filled_ticks ) = now_ticks;
2142 : /* The tail slack is now committed dead — count it as current
2143 : (written-through) so fragmentation reflects it. */
2144 0 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_current_bytes, free_size );
2145 0 : }
2146 :
2147 0 : if( FD_UNLIKELY( !partition_pool_free( accdb->partition_pool ) ) ) FD_LOG_ERR(( "accounts database file is at capacity" ));
2148 0 : fd_accdb_partition_t * partition = partition_pool_ele_acquire( accdb->partition_pool );
2149 0 : partition->bytes_freed = 0UL;
2150 0 : partition->marked_compaction = 0;
2151 0 : partition->layer = layer;
2152 0 : partition->read_ops = 0UL;
2153 0 : partition->bytes_read = 0UL;
2154 0 : partition->write_ops = 0UL;
2155 0 : partition->bytes_written = 0UL;
2156 0 : partition->write_offset = 0UL;
2157 0 : partition->compaction_offset = 0UL;
2158 0 : partition->created_ticks = now_ticks;
2159 0 : partition->filled_ticks = 0L;
2160 0 : partition->queued = 0;
2161 0 : partition->compacting_now = 0;
2162 :
2163 0 : ulong new_partition_idx = partition_pool_idx( accdb->partition_pool, partition );
2164 0 : int had_partition = *has_partition;
2165 :
2166 : /* Snapshot-load tiering: accounts loaded from a snapshot never get
2167 : a second write, so compaction-driven promotion never fires and
2168 : they would otherwise live in Hot forever. When snapshot_loading
2169 : is set, tag the new partition as Cold up front. We do not set
2170 : has_partition[Cold] / whead[Cold] — those are owned by the
2171 : compaction tile and represent the live Cold write head, which is
2172 : independent of snapshot-loaded partitions that happen to be
2173 : labeled Cold. */
2174 0 : if( FD_UNLIKELY( FD_VOLATILE_CONST( accdb->shmem->snapshot_loading ) && layer==0 ) ) {
2175 0 : FD_VOLATILE( partition->layer ) = FD_ACCDB_COMPACTION_LAYER_CNT-1UL;
2176 0 : }
2177 :
2178 0 : if( FD_UNLIKELY( new_partition_idx>=accdb->shmem->partition_max ) ) FD_VOLATILE( accdb->shmem->partition_max ) = new_partition_idx+1UL;
2179 0 : spin_lock_release( &accdb->shmem->partition_lock );
2180 :
2181 : /* Back the partition before publishing the head, so no writer
2182 : reserves into it while fallocate is running. Layer-0 writers spin
2183 : on the head until then. */
2184 0 : if( FD_UNLIKELY( new_partition_idx>=FD_VOLATILE_CONST( accdb->shmem->partition_fallocated ) ) ) {
2185 0 : spin_lock_acquire( &accdb->shmem->grow_lock );
2186 0 : grow_locked( accdb, new_partition_idx, layer, had_partition ? partition_idx_before : ULONG_MAX, 0 );
2187 0 : spin_lock_release( &accdb->shmem->grow_lock );
2188 0 : }
2189 :
2190 0 : spin_lock_acquire( &accdb->shmem->partition_lock );
2191 0 : *out_offset = accdb_offset( new_partition_idx, 0UL );
2192 0 : FD_COMPILER_MFENCE();
2193 0 : *has_partition = 1;
2194 0 : if( FD_LIKELY( had_partition && partition_idx_before!=new_partition_idx ) ) {
2195 0 : fd_accdb_shmem_try_enqueue_compaction( accdb->shmem, partition_idx_before );
2196 0 : }
2197 0 : spin_lock_release( &accdb->shmem->partition_lock );
2198 0 : }
2199 :
2200 : /* Reserve sz bytes in the layer-0 write head and set
2201 : out_partition_idx to where they landed. Bumps no counters. */
2202 :
2203 : static inline ulong
2204 : reserve_next_write( fd_accdb_t * accdb,
2205 : ulong sz,
2206 0 : ulong * out_partition_idx ) {
2207 0 : for(;;) {
2208 0 : accdb_offset_t offset = { .val = FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->whead[ 0 ].val, sz ) };
2209 0 : if( FD_LIKELY( packed_partition_offset( &offset )+sz<=accdb->shmem->partition_sz ) ) {
2210 0 : *out_partition_idx = packed_partition_idx( &offset );
2211 0 : return packed_partition_file_offset( &offset, accdb->shmem->partition_sz );
2212 0 : }
2213 :
2214 0 : if( FD_UNLIKELY( packed_partition_offset( &offset )>accdb->shmem->partition_sz ) ) {
2215 : /* This can happen if another thread also raced to allocate the
2216 : next write and won. Wait for the partition switch to finish
2217 : before retrying, so we do not keep doing fetch-and-adds that
2218 : advance the offset further past the boundary.
2219 :
2220 : A switch is detected by the head moving to a different
2221 : partition index OR its offset dropping back to a valid position
2222 : (a switch resets the offset to 0). We must not key the wait
2223 : solely on the index changing: the initial write head is a
2224 : sentinel whose packed index can coincide with a real pool
2225 : index. */
2226 0 : ulong stale_partition = packed_partition_idx( &offset );
2227 0 : for(;;) {
2228 0 : accdb_offset_t cur = { .val = FD_VOLATILE_CONST( accdb->shmem->whead[ 0 ].val ) };
2229 0 : if( packed_partition_idx( &cur )!=stale_partition ) break;
2230 0 : if( packed_partition_offset( &cur )<=accdb->shmem->partition_sz ) break;
2231 0 : fd_racesan_hook( "accdb_reserve:switch_wait" );
2232 0 : FD_SPIN_PAUSE();
2233 0 : }
2234 0 : continue;
2235 0 : }
2236 :
2237 0 : change_partition( accdb, &offset, &accdb->shmem->whead[ 0 ], &accdb->shmem->has_partition[ 0 ], 0 );
2238 0 : }
2239 0 : }
2240 :
2241 : /* Reserve sz bytes. Layer-0 write metrics are deferred until
2242 : explicitly flushed. */
2243 :
2244 : static inline ulong
2245 : allocate_next_write( fd_accdb_t * accdb,
2246 0 : ulong sz ) {
2247 0 : ulong partition_idx;
2248 0 : ulong file_offset = reserve_next_write( accdb, sz, &partition_idx );
2249 :
2250 : /* Very rarely the reservation crosses into a new partition. Since
2251 : stats are aggregated per-partition, any accumulated stats are
2252 : flushed and reset to accommodate the new partition. */
2253 0 : if( FD_LIKELY( accdb->write_stats.num_ops ) &&
2254 0 : FD_UNLIKELY( accdb->write_stats.partition_idx!=partition_idx ) ) {
2255 0 : fd_accdb_flush_metrics( accdb );
2256 0 : }
2257 :
2258 0 : accdb->write_stats.partition_idx = partition_idx;
2259 0 : accdb->write_stats.bytes += sz;
2260 0 : accdb->write_stats.num_ops++;
2261 0 : return file_offset;
2262 0 : }
2263 :
2264 : /* write_entry writes one disk entry (meta, then data_sz bytes of data)
2265 : at file offset off, retrying partial writes. */
2266 :
2267 : static void
2268 : write_entry( fd_accdb_t * accdb,
2269 : ulong off,
2270 : fd_accdb_disk_meta_t const * meta,
2271 : void const * data,
2272 0 : ulong data_sz ) {
2273 0 : struct iovec iovs[ 2UL ] = {
2274 0 : { .iov_base = (void *)meta, .iov_len = sizeof(fd_accdb_disk_meta_t) },
2275 0 : { .iov_base = (void *)data, .iov_len = data_sz }
2276 0 : };
2277 0 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t)+data_sz;
2278 0 : ulong written = 0UL;
2279 0 : while( FD_LIKELY( written<entry_sz ) ) {
2280 0 : long result = pwritev2( accdb->fd, iovs, 2, (long)(off+written), 0 );
2281 0 : if( FD_UNLIKELY( -1==result && (errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ) ) ) continue;
2282 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "pwritev2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
2283 0 : else if( FD_UNLIKELY( !result ) ) FD_LOG_ERR(( "accounts database is corrupt, pwritev2() returned 0 at offset %lu with %lu bytes remaining", off+written, entry_sz-written ));
2284 0 : written += (ulong)result;
2285 0 : accdb->metrics->bytes_written += (ulong)result;
2286 0 : accdb->metrics->write_ops++;
2287 :
2288 0 : for( int v=0; v<2; v++ ) {
2289 0 : if( (ulong)result>=iovs[ v ].iov_len ) {
2290 0 : result -= (long)iovs[ v ].iov_len;
2291 0 : iovs[ v ].iov_len = 0UL;
2292 0 : } else {
2293 0 : iovs[ v ].iov_base = (uchar *)iovs[ v ].iov_base + result;
2294 0 : iovs[ v ].iov_len -= (ulong)result;
2295 0 : break;
2296 0 : }
2297 0 : }
2298 0 : }
2299 0 : }
2300 :
2301 : /* release_take_line takes a line of size_class for a commit at release
2302 : time (writable acquires reserve every class but only materialize the
2303 : staging line; the final class is known only after execution). The
2304 : caller holds a reservation for size_class, so a line exists. If the
2305 : line came from CLOCK with a dirty victim, the victim is written back
2306 : and its offset published before the line is returned, exactly as
2307 : acquire's steps 5-10 do (victim offset xchg'd to INVAL first, so
2308 : concurrent readers wait for the publish). Returns the line pinned
2309 : (refcnt 1), persisted, owned by no account. The caller holds its
2310 : epoch, which keeps the victim's accmeta from being recycled. */
2311 :
2312 : static fd_accdb_cache_line_t *
2313 : release_take_line( fd_accdb_t * accdb,
2314 366338 : ulong size_class ) {
2315 366338 : fd_accdb_shmem_t * shmem = accdb->shmem;
2316 366338 : uint victim = UINT_MAX;
2317 366338 : fd_accdb_cache_line_t * line = acquire_cache_line( accdb, size_class, &victim );
2318 366338 : if( FD_UNLIKELY( victim!=UINT_MAX ) ) {
2319 0 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ victim ];
2320 0 : ulong data_sz = (ulong)FD_ACCDB_SIZE_DATA( accmeta->executable_size );
2321 0 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t)+data_sz;
2322 0 : accdb->metrics->accounts_evicted++;
2323 0 : accdb->metrics->accounts_evicted_per_class[ size_class ]++;
2324 :
2325 0 : ulong old_offset = fd_accdb_acc_xchg_offset( accmeta, FD_ACCDB_OFF_INVAL );
2326 0 : if( FD_LIKELY( old_offset!=FD_ACCDB_OFF_INVAL ) ) {
2327 0 : fd_accdb_shmem_bytes_freed( shmem, old_offset, entry_sz );
2328 0 : FD_ATOMIC_FETCH_AND_SUB( &shmem->shmetrics->disk_used_bytes, entry_sz );
2329 0 : }
2330 :
2331 0 : fd_accdb_disk_meta_t meta;
2332 0 : fd_memcpy( meta.pubkey, accmeta->key.pubkey, 32UL );
2333 0 : meta.size = (uint)data_sz;
2334 0 : meta.generation = accmeta->key.generation;
2335 0 : fd_memcpy( meta.owner, line->owner, 32UL );
2336 :
2337 0 : ulong file_off = allocate_next_write( accdb, entry_sz );
2338 0 : write_entry( accdb, file_off, &meta, line+1UL, data_sz );
2339 0 : FD_ATOMIC_FETCH_AND_ADD( &shmem->shmetrics->disk_used_bytes, entry_sz );
2340 0 : fd_racesan_hook( "accdb_release_take:pre_publish" );
2341 0 : FD_COMPILER_MFENCE();
2342 0 : accmeta->offset_fork = fd_accdb_acc_pack_offset_fork( file_off, fd_accdb_acc_fork_id( accmeta ) );
2343 0 : }
2344 366338 : line->persisted = 1;
2345 366338 : line->acc_idx = UINT_MAX;
2346 366338 : return line;
2347 366338 : }
2348 :
2349 : ulong
2350 : fd_accdb_snapshot_reserve_write( fd_accdb_t * accdb,
2351 : ulong sz,
2352 0 : ulong dead_tail_sz ) {
2353 0 : FD_TEST( sz && sz<=accdb->shmem->partition_sz && dead_tail_sz<=sz );
2354 0 : ulong off = allocate_next_write( accdb, sz );
2355 0 : if( dead_tail_sz ) fd_accdb_shmem_bytes_freed( accdb->shmem, off+sz-dead_tail_sz, dead_tail_sz );
2356 0 : return off;
2357 0 : }
2358 :
2359 : /* Compaction write allocation. Single-threaded: only the compaction
2360 : tile calls these, so the compaction write heads do not need atomic
2361 : fetch-and-add. dest_layer is the target layer (1..N-1). */
2362 :
2363 : static inline ulong
2364 : allocate_next_compaction_write( fd_accdb_t * accdb,
2365 : ulong sz,
2366 0 : ulong dest_layer ) {
2367 0 : accdb_offset_t offset = accdb->shmem->whead[ dest_layer ];
2368 0 : if( FD_UNLIKELY( !accdb->shmem->has_partition[ dest_layer ] ||
2369 0 : packed_partition_offset( &offset )+sz>accdb->shmem->partition_sz ) ) {
2370 0 : change_partition( accdb, &offset, &accdb->shmem->whead[ dest_layer ], &accdb->shmem->has_partition[ dest_layer ], (uchar)dest_layer );
2371 0 : offset = accdb->shmem->whead[ dest_layer ];
2372 0 : }
2373 0 : accdb->shmem->whead[ dest_layer ].val += sz;
2374 0 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_current_bytes, sz );
2375 0 : ulong file_offset = packed_partition_file_offset( &offset, accdb->shmem->partition_sz );
2376 0 : fd_accdb_partition_write_bump( accdb, packed_partition_idx( &offset ), sz, 1UL );
2377 0 : return file_offset;
2378 0 : }
2379 :
2380 : /* background_compact relocates batches of records from src_layer
2381 : into the next colder tier (or the same tier for the deepest layer)
2382 : and reclaims retired partitions. Sets *charge_busy if work is done.
2383 :
2384 : src_layer must be in 0..FD_ACCDB_COMPACTION_LAYER_CNT-1. */
2385 :
2386 : static void
2387 : background_compact( fd_accdb_t * accdb,
2388 : ulong src_layer,
2389 0 : int * charge_busy ) {
2390 0 : FD_TEST( accdb->bounce );
2391 :
2392 0 : FD_COMPILER_MFENCE();
2393 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
2394 0 : FD_HW_MFENCE(); /* StoreLoad: epoch store must be globally visible
2395 : before any subsequent loads so the deferred
2396 : reclamation scan does not miss us. */
2397 :
2398 : /* Reclaim any deferred-free partitions whose epoch has been observed
2399 : by all joiners (i.e. no epoch-publishing joiner could still be
2400 : referencing data in them). Scan writer slots [0, joiner_cnt)
2401 : plus each external (read-only) joiner's private epoch fseq. */
2402 0 : ulong min_epoch = ULONG_MAX;
2403 0 : ulong joiner_cnt = FD_VOLATILE_CONST( accdb->shmem->joiner_cnt );
2404 0 : for( ulong t=0UL; t<joiner_cnt; t++ ) {
2405 0 : ulong e = FD_VOLATILE_CONST( accdb->shmem->joiner_epochs[ t ].val );
2406 0 : if( FD_LIKELY( e<min_epoch ) ) min_epoch = e;
2407 0 : }
2408 0 : for( ulong t=0UL; t<accdb->external_epoch_cnt; t++ ) {
2409 0 : ulong e = FD_VOLATILE_CONST( *accdb->external_epoch_slots[ t ] );
2410 0 : if( FD_LIKELY( e<min_epoch ) ) min_epoch = e;
2411 0 : }
2412 0 : for(;;) {
2413 0 : if( FD_LIKELY( deferred_free_dlist_is_empty( accdb->deferred_free_dlist, accdb->partition_pool ) ) ) break;
2414 0 : fd_accdb_partition_t * p = deferred_free_dlist_ele_peek_head( accdb->deferred_free_dlist, accdb->partition_pool );
2415 0 : if( FD_LIKELY( p->epoch_tag>=min_epoch ) ) break;
2416 :
2417 0 : fd_racesan_hook( "accdb_reclaim:pre_free_partition" );
2418 :
2419 0 : spin_lock_acquire( &accdb->shmem->partition_lock );
2420 0 : deferred_free_dlist_ele_pop_head( accdb->deferred_free_dlist, accdb->partition_pool );
2421 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_current_bytes, accdb->shmem->partition_sz );
2422 0 : FD_VOLATILE( p->write_offset ) = 0UL;
2423 0 : partition_pool_ele_release( accdb->partition_pool, p );
2424 0 : spin_lock_release( &accdb->shmem->partition_lock );
2425 0 : }
2426 :
2427 0 : if( FD_LIKELY( compaction_dlist_is_empty( accdb->compaction_dlist[ src_layer ], accdb->partition_pool ) ) ) {
2428 0 : FD_COMPILER_MFENCE();
2429 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
2430 0 : return;
2431 0 : }
2432 0 : fd_accdb_partition_t * compact = compaction_dlist_ele_peek_head( accdb->compaction_dlist[ src_layer ], accdb->partition_pool );
2433 0 : if( FD_UNLIKELY( !compact ) ) {
2434 0 : FD_COMPILER_MFENCE();
2435 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
2436 0 : return;
2437 0 : }
2438 :
2439 : /* Wait until all epoch-publishing joiners that were active when this
2440 : partition was enqueued for compaction have exited, ensuring any
2441 : in-flight pwritev2 to this partition has completed before we start
2442 : reading from it. */
2443 0 : if( FD_UNLIKELY( compact->compaction_ready_epoch>=min_epoch ) ) {
2444 0 : FD_COMPILER_MFENCE();
2445 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
2446 0 : return;
2447 0 : }
2448 :
2449 0 : *charge_busy = 1;
2450 :
2451 0 : if( FD_UNLIKELY( !compact->compacting_now ) ) {
2452 0 : compact->compaction_start_wallclock = fd_log_wallclock();
2453 0 : compact->compaction_accounts_relocated = 0UL;
2454 0 : compact->compaction_bytes_relocated = 0UL;
2455 0 : compact->compaction_dead_records = 0UL;
2456 0 : }
2457 0 : FD_VOLATILE( compact->queued ) = 0;
2458 0 : FD_VOLATILE( compact->compacting_now ) = 1;
2459 :
2460 0 : ulong compact_base = partition_pool_idx( accdb->partition_pool, compact )*accdb->shmem->partition_sz;
2461 0 : ulong dest_layer = fd_ulong_min( src_layer+1UL, FD_ACCDB_COMPACTION_LAYER_CNT-1UL );
2462 :
2463 0 : ulong want = fd_ulong_min( FD_ACCDB_BOUNCE_SZ, compact->write_offset-compact->compaction_offset );
2464 0 : ulong carry = fd_ulong_if( accdb->carry_off==compact_base+compact->compaction_offset, fd_ulong_min( accdb->carry_sz, want ), 0UL );
2465 0 : accdb->carry_sz = 0UL;
2466 0 : ulong bytes_read = carry;
2467 0 : while( bytes_read<want ) {
2468 0 : long result = pread( accdb->fd, accdb->bounce+bytes_read, want-bytes_read, (long)(compact_base+compact->compaction_offset+bytes_read) );
2469 0 : if( FD_UNLIKELY( -1==result && (errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ) ) ) continue;
2470 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "pread() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
2471 0 : else if( FD_UNLIKELY( !result ) ) FD_LOG_ERR(( "accounts database is corrupt, data expected at offset %lu with size %lu exceeded file extents",
2472 0 : compact_base+compact->compaction_offset+bytes_read, want ));
2473 0 : fd_accdb_partition_read_bump( accdb, compact_base+compact->compaction_offset+bytes_read, (ulong)result );
2474 0 : bytes_read += (ulong)result;
2475 0 : }
2476 :
2477 0 : ulong span = 0UL;
2478 0 : for(;;) {
2479 0 : if( FD_UNLIKELY( span+sizeof(fd_accdb_disk_meta_t)>bytes_read ) ) break;
2480 0 : ulong record_sz = sizeof(fd_accdb_disk_meta_t) + (ulong)((fd_accdb_disk_meta_t const *)(accdb->bounce+span))->size;
2481 0 : if( FD_UNLIKELY( span+record_sz>bytes_read ) ) break;
2482 0 : span += record_sz;
2483 0 : }
2484 0 : if( FD_UNLIKELY( !span ) ) FD_LOG_ERR(( "accounts database is corrupt, record at offset %lu larger than %lu bytes",
2485 0 : compact_base+compact->compaction_offset, FD_ACCDB_BOUNCE_SZ ));
2486 :
2487 0 : fd_accdb_accmeta_t * live_accmeta[ IOV_MAX ];
2488 0 : ulong live_packed [ IOV_MAX ];
2489 0 : ulong live_src [ IOV_MAX ];
2490 0 : ulong live_sz [ IOV_MAX ];
2491 0 : ulong live_dst [ IOV_MAX ];
2492 0 : struct iovec iov [ IOV_MAX ];
2493 :
2494 0 : ulong live_cnt = 0UL;
2495 0 : ulong live_bytes = 0UL;
2496 0 : for( ulong off=0UL; off<span; ) {
2497 0 : fd_accdb_disk_meta_t const * meta = (fd_accdb_disk_meta_t const *)(accdb->bounce+off);
2498 :
2499 0 : fd_accdb_accmeta_t * accmeta = NULL;
2500 0 : ulong source_packed = 0UL;
2501 0 : uint acc_idx = FD_VOLATILE_CONST( accdb->acc_map[ fd_hash32( meta->pubkey, accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL) ] );
2502 0 : while( acc_idx!=UINT_MAX ) {
2503 0 : fd_accdb_accmeta_t * candidate = &accdb->acc_pool[ acc_idx ];
2504 0 : uint next_idx = FD_VOLATILE_CONST( candidate->map.next );
2505 0 : ulong candidate_packed = FD_VOLATILE_CONST( candidate->offset_fork );
2506 0 : if( FD_LIKELY( (candidate_packed & FD_ACCDB_OFF_MASK)==compact_base+compact->compaction_offset+off ) ) {
2507 0 : accmeta = candidate;
2508 0 : source_packed = candidate_packed;
2509 0 : break;
2510 0 : }
2511 0 : acc_idx = next_idx;
2512 0 : }
2513 :
2514 0 : ulong record_sz = sizeof(fd_accdb_disk_meta_t) + (ulong)meta->size;
2515 0 : if( FD_UNLIKELY( !accmeta ) ) compact->compaction_dead_records++; /* index entry gone, extent is garbage */
2516 0 : else {
2517 0 : live_accmeta[ live_cnt ] = accmeta;
2518 0 : live_packed [ live_cnt ] = source_packed;
2519 0 : live_src [ live_cnt ] = off;
2520 0 : live_sz [ live_cnt ] = record_sz;
2521 0 : live_cnt++;
2522 0 : live_bytes += record_sz;
2523 0 : }
2524 0 : off += record_sz;
2525 :
2526 0 : if( FD_UNLIKELY( live_cnt==IOV_MAX ) ) {
2527 0 : span = off;
2528 0 : break;
2529 0 : }
2530 0 : }
2531 :
2532 0 : if( FD_LIKELY( live_cnt ) ) {
2533 0 : ulong dest_base = allocate_next_compaction_write( accdb, live_bytes, dest_layer );
2534 0 : ulong dst = dest_base;
2535 0 : for( ulong i=0UL; i<live_cnt; i++ ) { live_dst[ i ] = dst; dst += live_sz[ i ]; }
2536 :
2537 0 : for( ulong i=0UL; i<live_cnt; i++ ) iov[ i ] = (struct iovec){ .iov_base = accdb->bounce+live_src[ i ], .iov_len = live_sz[ i ] };
2538 0 : ulong written = 0UL; ulong iov_idx = 0UL;
2539 0 : while( written<live_bytes ) {
2540 0 : long result = pwritev2( accdb->fd, iov+iov_idx, (int)(live_cnt-iov_idx), (long)(dest_base+written), 0 );
2541 0 : if( FD_UNLIKELY( -1==result && (errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ) ) ) continue;
2542 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "pwritev2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
2543 0 : else if( FD_UNLIKELY( !result ) ) FD_LOG_ERR(( "accounts database is corrupt, pwritev2() returned 0 at offset %lu", dest_base+written ));
2544 0 : written += (ulong)result;
2545 0 : accdb->metrics->copy_ops++;
2546 0 : ulong adv = (ulong)result;
2547 0 : while( adv && iov_idx<live_cnt ) {
2548 0 : if( adv>=iov[ iov_idx ].iov_len ) {
2549 0 : adv -= iov[ iov_idx ].iov_len;
2550 0 : iov_idx++;
2551 0 : }
2552 0 : else {
2553 0 : iov[ iov_idx ].iov_base = (uchar *)iov[ iov_idx ].iov_base+adv;
2554 0 : iov[ iov_idx ].iov_len -= adv;
2555 0 : adv = 0UL;
2556 0 : }
2557 0 : }
2558 0 : }
2559 :
2560 0 : accdb->shmem->shmetrics->accounts_relocated += live_cnt;
2561 0 : accdb->shmem->shmetrics->accounts_relocated_bytes += live_bytes;
2562 0 : compact->compaction_accounts_relocated += live_cnt;
2563 0 : compact->compaction_bytes_relocated += live_bytes;
2564 :
2565 : /* Data is written before the offsets are published, so a
2566 : concurrent acquire never preadv2s an unwritten location. */
2567 0 : FD_COMPILER_MFENCE();
2568 :
2569 0 : for( ulong i=0UL; i<live_cnt; i++ ) {
2570 : /* CAS the offset from the exact source record we copied to the
2571 : new destination. If a concurrent release overwrote the offset
2572 : to FD_ACCDB_OFF_INVAL (dirty sentinel for a new commit), or
2573 : later published a newer on-disk location, the CAS fails and
2574 : the relocated copy is dead on arrival: account it as freed so
2575 : compaction reclaims it. The full packed offset_fork is CASed
2576 : so the fork_id is preserved. */
2577 0 : ulong new_packed = ( live_packed[ i ] & ~FD_ACCDB_OFF_MASK ) | ( live_dst[ i ] & FD_ACCDB_OFF_MASK );
2578 : #if FD_HAS_RACESAN
2579 : fd_memcpy( fd_accdb_dbg_reloc_pubkey, live_accmeta[ i ]->key.pubkey, 32UL );
2580 : fd_accdb_dbg_reloc_dest = live_dst[ i ];
2581 : fd_accdb_dbg_reloc_cnt++;
2582 : #endif
2583 0 : fd_racesan_hook( "accdb_compact:pre_offset_cas" );
2584 0 : if( FD_UNLIKELY( FD_ATOMIC_CAS( &live_accmeta[ i ]->offset_fork, live_packed[ i ], new_packed )!=live_packed[ i ] ) ) {
2585 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, live_dst[ i ], live_sz[ i ] );
2586 0 : }
2587 0 : }
2588 0 : }
2589 :
2590 0 : fd_racesan_hook( "accdb_compact:post_relocate" );
2591 :
2592 0 : compact->compaction_offset += span;
2593 : #if FD_HAS_RACESAN
2594 : fd_accdb_dbg_compact_scanned += span;
2595 : #endif
2596 :
2597 0 : if( FD_LIKELY( span<bytes_read ) ) {
2598 0 : memmove( accdb->bounce, accdb->bounce+span, bytes_read-span );
2599 0 : accdb->carry_off = compact_base+compact->compaction_offset;
2600 0 : accdb->carry_sz = bytes_read-span;
2601 0 : accdb->carry_layer = src_layer;
2602 0 : }
2603 :
2604 0 : if( FD_UNLIKELY( compact->compaction_offset>=compact->write_offset ) ) {
2605 0 : FD_LOG_INFO(( "compaction of partition %lu completed", partition_pool_idx( accdb->partition_pool, compact ) ));
2606 0 : accdb->carry_sz = 0UL; /* nothing is carried past a finished partition */
2607 :
2608 0 : fd_event_accdb_compaction_completed_t ev = {
2609 0 : .partition_idx = partition_pool_idx( accdb->partition_pool, compact ),
2610 0 : .src_layer = (uchar)src_layer,
2611 0 : .dest_layer = (uchar)fd_ulong_min( src_layer+1UL, FD_ACCDB_COMPACTION_LAYER_CNT-1UL ),
2612 0 : .bytes_scanned = compact->write_offset,
2613 0 : .bytes_freed = compact->bytes_freed,
2614 0 : .accounts_relocated = compact->compaction_accounts_relocated,
2615 0 : .bytes_relocated = compact->compaction_bytes_relocated,
2616 0 : .dead_records = compact->compaction_dead_records,
2617 0 : .start_time = (ulong)compact->compaction_start_wallclock,
2618 0 : .end_time = (ulong)fd_log_wallclock(),
2619 0 : };
2620 0 : fd_event_report_accdb_compaction_completed( &ev );
2621 :
2622 : /* Ensure the new acc->offset_fork stores above are visible to other
2623 : cores before the source partition is moved to the deferred-free
2624 : list. On x86 (TSO) hardware store ordering already guarantees
2625 : this, but the compiler fence prevents the compiler from sinking
2626 : the offset store past the inlined pool/dlist mutations below. */
2627 0 : FD_COMPILER_MFENCE();
2628 :
2629 : /* Bump the global epoch and tag this partition so the reclamation
2630 : scan knows when all epoch-publishing joiners that could reference
2631 : data in this partition have exited. */
2632 0 : ulong tag = FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->epoch, 1UL );
2633 0 : compact->epoch_tag = tag;
2634 :
2635 : /* partition_lock serializes these dlist/pool mutations with
2636 : concurrent push_tail in fd_accdb_shmem_bytes_freed and
2637 : partition_pool_ele_acquire in change_partition. Neither fd_dlist
2638 : nor fd_pool are thread-safe, so all mutations must be under the
2639 : same lock. */
2640 0 : spin_lock_acquire( &accdb->shmem->partition_lock );
2641 :
2642 0 : accdb->shmem->shmetrics->partitions_freed++;
2643 0 : compaction_dlist_ele_pop_head( accdb->compaction_dlist[ src_layer ], accdb->partition_pool );
2644 0 : FD_VOLATILE( compact->compacting_now ) = 0;
2645 0 : FD_VOLATILE( compact->queued ) = 0;
2646 0 : deferred_free_dlist_ele_push_tail( accdb->deferred_free_dlist, compact, accdb->partition_pool );
2647 :
2648 0 : accdb->shmem->shmetrics->compactions_completed++;
2649 0 : if( FD_LIKELY( compaction_dlist_is_empty( accdb->compaction_dlist[ src_layer ], accdb->partition_pool ) ) ) {
2650 0 : accdb->shmem->shmetrics->in_compaction = 0;
2651 0 : } else {
2652 0 : fd_accdb_partition_t * next = compaction_dlist_ele_peek_head( accdb->compaction_dlist[ src_layer ], accdb->partition_pool );
2653 0 : FD_LOG_INFO(( "compaction of layer %lu partition %lu started", src_layer, partition_pool_idx( accdb->partition_pool, next ) ));
2654 0 : }
2655 :
2656 0 : spin_lock_release( &accdb->shmem->partition_lock );
2657 0 : }
2658 :
2659 0 : accdb->metrics->bytes_read += bytes_read-carry;
2660 0 : accdb->metrics->bytes_written += live_bytes;
2661 :
2662 0 : FD_COMPILER_MFENCE();
2663 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
2664 0 : }
2665 :
2666 : /* cold_load_acc resolves the cache slot for `acc` when cache_try_pin
2667 : failed. It uses bit 29 of executable_size as a
2668 : single-claimer lock so that two concurrent acquirers cannot each
2669 : install their own cache slot for the same acc (which would orphan
2670 : one slot with a dangling line->acc_idx and eventually corrupt
2671 : acc->cache_valid via CLOCK).
2672 :
2673 : Protocol per acc:
2674 : - If cache_valid is set, retry cache_try_pin (another thread
2675 : finished the cold-load while we were here). On success, mark
2676 : exists_in_cache so the eviction-write phase skips the slot.
2677 : - If claim is set, spin (another thread is mid-cold-load).
2678 : - Otherwise CAS-set the claim bit. Winner allocates a cache
2679 : line, populates the placeholder (acc_idx=UINT_MAX), publishes
2680 : cache_idx, then atomically (CAS-loop) sets cache_valid and
2681 : clears claim.
2682 :
2683 : Eviction sites must acquire the claim bit before clearing
2684 : cache_valid and release it after updating the cache binding. */
2685 :
2686 : static fd_accdb_cache_line_t *
2687 : cold_load_acc( fd_accdb_t * accdb,
2688 : fd_accdb_accmeta_t * accmeta,
2689 : uchar const * pubkey,
2690 : int * out_exists_in_cache,
2691 0 : uint * out_evicted_acc_idx ) {
2692 0 : for(;;) {
2693 0 : uint old_es = FD_VOLATILE_CONST( accmeta->executable_size );
2694 0 : int valid = FD_ACCDB_SIZE_CACHE_VALID( old_es );
2695 0 : int claimed = FD_ACCDB_SIZE_CACHE_CLAIM( old_es );
2696 :
2697 0 : if( FD_UNLIKELY( valid ) ) {
2698 : /* old_es snapshot saw VALID=1 but a concurrent
2699 : evict_clear_acc_cache_ref may have cleared VALID and stored
2700 : cache_idx=INVAL between our snapshot and this load. Decoding
2701 : INVAL would yield a wild cache_line pointer; retry the loop
2702 : instead (next iteration will see VALID=0). */
2703 0 : uint cidx = FD_VOLATILE_CONST( accmeta->cache_idx );
2704 0 : if( FD_UNLIKELY( cidx==FD_ACCDB_ACC_CIDX_INVAL ) ) {
2705 0 : fd_racesan_hook( "accdb_cold_load:inval_wait" );
2706 0 : FD_SPIN_PAUSE();
2707 0 : continue;
2708 0 : }
2709 0 : fd_accdb_cache_line_t * hit = cache_line( accdb, FD_ACCDB_ACC_CIDX_CLASS( cidx ), FD_ACCDB_ACC_CIDX_IDX( cidx ) );
2710 0 : fd_racesan_hook( "accdb_cold_load:pre_try_pin" );
2711 0 : fd_accdb_cache_line_t * pinned = cache_try_pin( hit, pubkey, accmeta->key.generation );
2712 0 : if( FD_LIKELY( pinned ) ) {
2713 0 : *out_exists_in_cache = 1;
2714 0 : *out_evicted_acc_idx = UINT_MAX;
2715 0 : return pinned;
2716 0 : }
2717 0 : fd_racesan_hook( "accdb_cold_load:pin_retry" );
2718 0 : FD_SPIN_PAUSE();
2719 0 : continue;
2720 0 : }
2721 :
2722 0 : if( FD_UNLIKELY( claimed ) ) {
2723 0 : fd_racesan_hook( "accdb_cold_load:claim_wait" );
2724 0 : FD_SPIN_PAUSE();
2725 0 : continue;
2726 0 : }
2727 :
2728 0 : if( FD_UNLIKELY( FD_ATOMIC_CAS( &accmeta->executable_size, old_es, old_es | FD_ACCDB_SIZE_CACHE_CLAIM_BIT )!=old_es ) ) {
2729 0 : fd_racesan_hook( "accdb_cold_load:claim_cas_retry" );
2730 0 : FD_SPIN_PAUSE();
2731 0 : continue;
2732 0 : }
2733 :
2734 : /* We hold the claim. Allocate a cache line and publish. */
2735 0 : ulong size_class = fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( old_es ) );
2736 0 : fd_accdb_cache_line_t * line = acquire_cache_line( accdb, size_class, out_evicted_acc_idx );
2737 0 : fd_memcpy( line->key.pubkey, accmeta->key.pubkey, 32UL );
2738 0 : line->key.generation = accmeta->key.generation;
2739 : /* Leave acc_idx at UINT_MAX (the "loading" sentinel) until the disk-read
2740 : completion fence has passed. Concurrent threads that
2741 : pin via cache_idx will spin on this before copying the owner. */
2742 0 : line->acc_idx = UINT_MAX;
2743 0 : FD_COMPILER_MFENCE();
2744 0 : FD_VOLATILE( accmeta->cache_idx ) = FD_ACCDB_ACC_CIDX_PACK( (uint)size_class, (uint)cache_line_idx( accdb, size_class, line ) );
2745 0 : FD_COMPILER_MFENCE();
2746 :
2747 0 : fd_racesan_hook( "accdb_cold_load:pre_valid" );
2748 :
2749 : /* Atomically set CACHE_VALID_BIT and clear CACHE_CLAIM_BIT.
2750 : Evictors acquire CLAIM before clearing VALID, so they cannot
2751 : clear it while this cold-loader holds CLAIM. Preserve the
2752 : data length, executable bit and other unrelated flags. */
2753 0 : for(;;) {
2754 0 : uint cur = FD_VOLATILE_CONST( accmeta->executable_size );
2755 0 : uint nxt = (cur & ~FD_ACCDB_SIZE_CACHE_CLAIM_BIT) | FD_ACCDB_SIZE_CACHE_VALID_BIT;
2756 0 : if( FD_LIKELY( FD_ATOMIC_CAS( &accmeta->executable_size, cur, nxt )==cur ) ) break;
2757 0 : FD_SPIN_PAUSE();
2758 0 : }
2759 :
2760 0 : *out_exists_in_cache = 0;
2761 0 : return line;
2762 0 : }
2763 0 : }
2764 :
2765 533139 : #define RESERVATION_TYPE_SIMPLE (0)
2766 7291 : #define RESERVATION_TYPE_MAYBE_PROGRAMDATA (1)
2767 7283 : #define RESERVATION_TYPE_ALREADY_RESERVED (2)
2768 :
2769 : static void
2770 : fd_accdb_acquire_inner( fd_accdb_t * accdb,
2771 : fd_accdb_fork_id_t fork_id,
2772 : int reservation_type,
2773 : ulong reserved_cnt,
2774 : ulong pubkeys_cnt,
2775 : uchar const * const * pubkeys,
2776 : int const * writable,
2777 549044 : fd_acc_t * out_accs ) {
2778 549044 : accdb->metrics->acquire_calls++;
2779 :
2780 549044 : ulong max_acquire_cnt = accdb->shmem->bundle_enabled ? FD_ACCDB_MAX_ACQUIRE_CNT : FD_ACCDB_MAX_TX_ACCOUNT_LOCKS;
2781 549044 : FD_TEST( pubkeys_cnt<=max_acquire_cnt );
2782 :
2783 549044 : FD_TEST( FD_VOLATILE_CONST( *accdb->my_epoch_slot )==ULONG_MAX );
2784 :
2785 549044 : FD_COMPILER_MFENCE();
2786 549044 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
2787 549044 : FD_HW_MFENCE(); /* StoreLoad: epoch store must be globally visible
2788 : before any subsequent loads so the deferred
2789 : reclamation scan does not miss us */
2790 :
2791 : // STEP 1.
2792 : // Locate each account in the fork and index structure, to determine
2793 : // if it already exists, its size and other metadata, and which
2794 : // specific slot (generation) it was last written in.
2795 :
2796 549044 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
2797 549044 : uint root_generation = accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
2798 :
2799 549044 : fd_racesan_hook( "accdb_acquire:post_root_gen" );
2800 :
2801 549044 : fd_accdb_accmeta_t * accmetas[ FD_ACCDB_MAX_ACQUIRE_CNT ];
2802 549044 : ulong acc_map_idxs[ FD_ACCDB_MAX_ACQUIRE_CNT ];
2803 :
2804 : /* Walk the hash chain for each pubkey and take the first visible
2805 : match. Correctness relies on newer entries always being prepended
2806 : to the chain head, which is guaranteed because replay processes
2807 : writes in slot order and release always inserts at the head.
2808 :
2809 : CONCURRENCY: This chain walk runs epoch-protected. A concurrent
2810 : fd_accdb_release may prepend a new node to the same chain while
2811 : we walk it. This is safe on x86-64 (TSO): the releasing thread
2812 : stores all acc fields (pubkey, generation, map.next, ...) before
2813 : publishing the new head via a CAS on acc_map[idx], and TSO
2814 : guarantees a reading core that observes the new head also observes
2815 : all prior stores to the node. A reader that does not yet see the
2816 : new head simply sees an older (still valid) version of the chain.
2817 : On weakly-ordered architectures an explicit acquire fence would be
2818 : needed before the chain walk and a release fence in
2819 : fd_accdb_release before the head-pointer store. Multiple
2820 : concurrent releases serialize on the CAS of the chain head. */
2821 1165941 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
2822 616897 : acc_map_idxs[ i ] = fd_hash32( pubkeys[ i ], accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL);
2823 616897 : uint acc = FD_VOLATILE_CONST( accdb->acc_map[ acc_map_idxs[ i ] ] );
2824 665526 : while( acc!=UINT_MAX ) {
2825 221785 : fd_accdb_accmeta_t const * candidate_acc = &accdb->acc_pool[ acc ];
2826 221785 : uint next_acc = FD_VOLATILE_CONST( candidate_acc->map.next );
2827 :
2828 221785 : fd_racesan_hook( "accdb_acquire:post_next" );
2829 :
2830 221785 : if( FD_UNLIKELY( (candidate_acc->key.generation>root_generation &&
2831 221785 : fd_accdb_acc_fork_id(candidate_acc)!=fork_id.val &&
2832 221785 : !descends_set_test( fork->descends, fd_accdb_acc_fork_id(candidate_acc) )) ) ||
2833 221785 : memcmp( pubkeys[ i ], candidate_acc->key.pubkey, 32UL ) ) {
2834 48629 : acc = next_acc;
2835 48629 : continue;
2836 48629 : }
2837 :
2838 173156 : break;
2839 221785 : }
2840 616897 : if( FD_UNLIKELY( acc==UINT_MAX ) ) accmetas[ i ] = NULL;
2841 175376 : else accmetas[ i ] = &accdb->acc_pool[ acc ];
2842 :
2843 : #if FD_TMPL_USE_HANDHOLDING
2844 : if( FD_UNLIKELY( accmetas[ i ] ) ) {
2845 : fd_accdb_accmeta_t const * sel = accmetas[ i ];
2846 : FD_TEST( !memcmp( sel->key.pubkey, pubkeys[ i ], 32UL ) );
2847 : FD_TEST( sel->key.generation<=root_generation ||
2848 : fd_accdb_acc_fork_id( sel )==fork_id.val ||
2849 : descends_set_test( fork->descends, fd_accdb_acc_fork_id( sel ) ) );
2850 : FD_TEST( sel->key.generation<=FD_VOLATILE_CONST( accdb->shmem->generation ) );
2851 : }
2852 : #endif
2853 :
2854 616897 : if( FD_UNLIKELY( accmetas[ i ] && !writable[ i ] && !accmetas[ i ]->lamports ) ) accmetas[ i ] = NULL;
2855 :
2856 : /* Attribute this acquired account to a size class for per-class
2857 : rate metrics. Use the account's current size class when known;
2858 : otherwise (new account) bucket as class 0. */
2859 616897 : ulong acq_class = 0UL;
2860 616897 : if( FD_LIKELY( accmetas[ i ] ) ) acq_class = fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) );
2861 616897 : if( FD_LIKELY( writable[ i ] ) ) accdb->metrics->writable_accounts_acquired_per_class[ acq_class ]++;
2862 213998 : else accdb->metrics->accounts_acquired_per_class[ acq_class ]++;
2863 616897 : }
2864 :
2865 : // STEP 2.
2866 : // The two-phase programdata acquire (acquire_a then acquire_b)
2867 : // works as follows: acquire_a (RESERVATION_TYPE_MAYBE_PROGRAMDATA)
2868 : // over-reserves one slot in every live size class per candidate
2869 : // account (reserved_cnt total per class), because it does not yet
2870 : // know which accounts have programdata or what size class it lands
2871 : // in. acquire_b then resolves the actual programdata pubkeys and
2872 : // re-enters here with RESERVATION_TYPE_ALREADY_RESERVED to refund
2873 : // the surplus. Keep one reservation per found programdata account
2874 : // in its own size class (consumed later by release) and give the
2875 : // rest back.
2876 549044 : if( FD_UNLIKELY( reservation_type==RESERVATION_TYPE_ALREADY_RESERVED ) ) {
2877 7275 : ulong refund[ FD_ACCDB_CACHE_CLASS_CNT ] = {0};
2878 65431 : for( ulong j=0UL; j<FD_ACCDB_CACHE_CLASS_CNT; j++ ) {
2879 58156 : if( FD_LIKELY( res_tracked( accdb, j ) ) ) refund[ j ] = reserved_cnt;
2880 58156 : }
2881 7295 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
2882 20 : if( FD_LIKELY( accmetas[ i ] ) ) {
2883 20 : ulong cls = fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) );
2884 20 : if( FD_LIKELY( res_tracked( accdb, cls ) ) ) {
2885 0 : FD_TEST( refund[ cls ]>0UL );
2886 0 : refund[ cls ]--;
2887 0 : }
2888 20 : }
2889 20 : }
2890 65411 : for( ulong k=0UL; k<FD_ACCDB_CACHE_CLASS_CNT; k++ ) {
2891 58136 : if( FD_UNLIKELY( refund[ k ] ) ) res_give( accdb, k, refund[ k ] );
2892 58136 : }
2893 7275 : }
2894 :
2895 : // STEP 3.
2896 : // We are potentially going to need to read the account data off of
2897 : // disk into the cache, if the account(s) are not in the cache so
2898 : // reserve the necessary cache space. This is done with an "atomic
2899 : // subtract" spin loop on the cache class counters, which is
2900 : // actually faster than doing a real CAS on a packed ulong.
2901 : //
2902 : // For reads, we only need space to copy the account data into a
2903 : // single right-sized cache line, but for writes ... we need to
2904 : // reserve one of every size class. The reason is we are going to
2905 : // need a 10MiB staging buffer for the executor to write to (it may
2906 : // grow the account, so needs the max size class). Even if the
2907 : // account is already in the 10MiB cache class, we need another one
2908 : // because a transaction can fail half way, so we need scratch space
2909 : // to be able to unwind.
2910 : //
2911 : // So we acquire one of each size class. Then when the transaction
2912 : // finishes, if it succeeded, we will copy the data back to the
2913 : // whichever size-class is now right-sized post execution.
2914 549044 : if( FD_LIKELY( reservation_type==RESERVATION_TYPE_SIMPLE || reservation_type==RESERVATION_TYPE_MAYBE_PROGRAMDATA ) ) {
2915 548379 : ulong requested_buckets[ FD_ACCDB_CACHE_CLASS_CNT ] = {0};
2916 1161424 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
2917 613045 : if( FD_LIKELY( accmetas[ i ] || writable[ i ] ) ) {
2918 539037 : if( FD_LIKELY( accmetas[ i ] ) ) {
2919 173183 : if( FD_UNLIKELY( res_tracked( accdb, fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) ) ) ) ) {
2920 0 : requested_buckets[ fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) ) ]++;
2921 0 : }
2922 173183 : }
2923 539037 : if( FD_UNLIKELY( writable[ i ] ) ) {
2924 3543668 : for( ulong j=0UL; j<FD_ACCDB_CACHE_CLASS_CNT; j++ ) {
2925 3143306 : if( FD_UNLIKELY( res_tracked( accdb, j ) ) ) {
2926 0 : requested_buckets[ j ]++;
2927 0 : }
2928 3143306 : }
2929 400362 : }
2930 539037 : }
2931 :
2932 613045 : if( FD_LIKELY( reservation_type==RESERVATION_TYPE_MAYBE_PROGRAMDATA ) ) {
2933 : /* Any account could also have an implied reference to a
2934 : programdata account, which we don't know yet ... so we need to
2935 : reserve worst case space if they all went to the same size
2936 : class. This reservation runs unconditionally per pubkey (not
2937 : gated on accmetas/writable) so that acquire_b can refund based on
2938 : pubkeys_cnt without needing to re-derive the live-account set. */
2939 654565 : for( ulong j=0UL; j<FD_ACCDB_CACHE_CLASS_CNT; j++ ) {
2940 581797 : if( FD_UNLIKELY( res_tracked( accdb, j ) ) ) {
2941 0 : requested_buckets[ j ]++;
2942 0 : }
2943 581797 : }
2944 72768 : }
2945 613045 : }
2946 :
2947 : /* TODO: This over-reserves cache slots for writable accounts that
2948 : already exist. For each such account we reserve one line in the
2949 : account's size class (for the read into cache) AND one line in
2950 : every size class (so release can take the final class's line). But if the
2951 : account is already resident in cache (which is the common case
2952 : for hot accounts), the read-into-cache line is unnecessary — we
2953 : will get a cache hit when acquiring cache lines and never use it.
2954 : The fix is to probe acc->cache_idx here and skip the per-account
2955 : size class reservation when a hit is found. This would reduce
2956 : peak reservation by up to one line per
2957 : writable account per acquire batch, lowering contention on the
2958 : cache class counters and allowing smaller cache provisioning. */
2959 :
2960 : /* Reserve cache slots from the join's credits, else the shared
2961 : counters (res_take). If a class is full, give back the classes
2962 : already taken, return every join's idle credits in the full class
2963 : to the shared counter, and retry. */
2964 548379 : for(;;) {
2965 537272 : ulong failed = ULONG_MAX;
2966 4824849 : for( ulong i=0UL; i<FD_ACCDB_CACHE_CLASS_CNT; i++ ) {
2967 4287577 : if( FD_LIKELY( !requested_buckets[ i ] ) ) continue;
2968 1319 : if( FD_UNLIKELY( !res_take( accdb, i, requested_buckets[ i ] ) ) ) { failed = i; break; }
2969 1319 : }
2970 540443 : if( FD_LIKELY( failed==ULONG_MAX ) ) break;
2971 >1844*10^16 : accdb->metrics->acquire_failed++;
2972 >1844*10^16 : for( ulong j=0UL; j<failed; j++ ) {
2973 0 : if( requested_buckets[ j ] ) res_give( accdb, j, requested_buckets[ j ] );
2974 0 : }
2975 >1844*10^16 : res_steal( accdb, failed );
2976 >1844*10^16 : fd_racesan_hook( "accdb_acquire:reservation_retry" );
2977 >1844*10^16 : FD_SPIN_PAUSE();
2978 >1844*10^16 : }
2979 548379 : }
2980 :
2981 : // STEP 4.
2982 : // For any accounts that are not in cache, we now need to actually
2983 : // retrieve the cache pointers from our structures. Space has been
2984 : // reserved already, so this step is guaranteed to succeed, and is
2985 : // just pulling the cache lines out of the free lists and marking
2986 : // them as in-use.
2987 : //
2988 : // This step is fully lock-free. Cache hits are pinned with an
2989 : // atomic CAS on refcnt (cache_try_pin). Eviction uses the CLOCK
2990 : // algorithm. The per-join free lists provide immediate recycling
2991 : // of fully-freed lines.
2992 :
2993 549044 : int exists_in_cache[ FD_ACCDB_MAX_ACQUIRE_CNT ];
2994 549044 : fd_accdb_cache_line_t * original_cache_line[ FD_ACCDB_MAX_ACQUIRE_CNT ];
2995 549044 : fd_accdb_cache_line_t * staging_line[ FD_ACCDB_MAX_ACQUIRE_CNT ]; /* writable only */
2996 :
2997 : /* Saved acc_pool indices of evicted dirty cache lines. These are
2998 : captured before clearing acc_idx to UINT_MAX on the line struct, so
2999 : that the loading-sentinel protocol works correctly while the
3000 : evicted account metadata is still available for writeback. */
3001 549044 : uint evicted_staging_acc[ FD_ACCDB_MAX_ACQUIRE_CNT ];
3002 549044 : uint evicted_orig_acc[ FD_ACCDB_MAX_ACQUIRE_CNT ];
3003 :
3004 1155153 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3005 606109 : if( FD_UNLIKELY( !accmetas[ i ] && !writable[ i ] ) ) continue;
3006 :
3007 531734 : original_cache_line[ i ] = NULL;
3008 531734 : if( FD_LIKELY( accmetas[ i ] ) ) {
3009 173422 : if( FD_LIKELY( FD_ACCDB_SIZE_CACHE_VALID( FD_VOLATILE_CONST( accmetas[ i ]->executable_size ) ) ) ) {
3010 : /* Concurrent evict_clear_acc_cache_ref clears VALID then stores
3011 : cache_idx=INVAL. We may have observed VALID=1 just before the
3012 : writer cleared it, so cidx can read as INVAL here; decoding it
3013 : would yield a wild cache_line pointer. Skip on INVAL. Any
3014 : other stale cidx is harmless: cache_try_pin's ABA generation
3015 : check rejects a recycled line. */
3016 173399 : uint cidx = FD_VOLATILE_CONST( accmetas[ i ]->cache_idx );
3017 173399 : if( FD_LIKELY( cidx!=FD_ACCDB_ACC_CIDX_INVAL ) ) {
3018 173398 : fd_accdb_cache_line_t * hit = cache_line( accdb, FD_ACCDB_ACC_CIDX_CLASS( cidx ), FD_ACCDB_ACC_CIDX_IDX( cidx ) );
3019 173398 : fd_racesan_hook( "accdb_acquire:pre_try_pin" );
3020 173398 : original_cache_line[ i ] = cache_try_pin( hit, pubkeys[ i ], accmetas[ i ]->key.generation );
3021 : #if FD_TMPL_USE_HANDHOLDING
3022 : if( FD_LIKELY( original_cache_line[ i ] ) ) {
3023 : FD_TEST( original_cache_line[ i ]->key.generation==accmetas[ i ]->key.generation &&
3024 : !memcmp( original_cache_line[ i ]->key.pubkey, pubkeys[ i ], 32UL ) );
3025 : uint rc = FD_VOLATILE_CONST( original_cache_line[ i ]->refcnt );
3026 : FD_TEST( rc>0U && rc!=FD_ACCDB_EVICT_SENTINEL );
3027 : }
3028 : #endif
3029 173398 : }
3030 173399 : }
3031 173422 : }
3032 531734 : exists_in_cache[ i ] = original_cache_line[ i ]!=NULL;
3033 :
3034 531734 : if( FD_UNLIKELY( writable[ i ] ) ) {
3035 : /* Only the 10 MiB staging line is taken now; the final class is
3036 : known after execution, and release takes that line then
3037 : (release_take_line). Every class is still reserved (step 3),
3038 : so release can always get one. */
3039 395913 : staging_line[ i ] = acquire_cache_line( accdb, 7UL, &evicted_staging_acc[ i ] );
3040 395913 : if( FD_UNLIKELY( accmetas[ i ] && !original_cache_line[ i ] ) ) {
3041 0 : original_cache_line[ i ] = cold_load_acc( accdb, accmetas[ i ], pubkeys[ i ], &exists_in_cache[ i ], &evicted_orig_acc[ i ] );
3042 0 : }
3043 395913 : } else {
3044 135821 : if( FD_UNLIKELY( !original_cache_line[ i ] ) ) {
3045 0 : original_cache_line[ i ] = cold_load_acc( accdb, accmetas[ i ], pubkeys[ i ], &exists_in_cache[ i ], &evicted_orig_acc[ i ] );
3046 0 : }
3047 135821 : }
3048 531734 : }
3049 :
3050 : // STEP 5.
3051 : // For any cache lines we have retrieved, which we might potentially
3052 : // be about to trash (by writing stuff in there), we need to write
3053 : // them back to disk first if they are dirty. This is the process of
3054 : // "persisting" (a/k/a evicting) whatever was previously in the
3055 : // cache line we are about to use.
3056 : //
3057 : // This step does not actually persist the data to disk, it just
3058 : // constructs a series of iovecs (write instructions) which will be
3059 : // used later to do the actual write. The reason is that we want to
3060 : // batch all the writes together into a single writev call, to
3061 : // minimize overhead, and also keep the actual writes at the end of
3062 : // the function and independent of the specific control flow, so
3063 : // that they could be offloaded to another thread of made
3064 : // asynchronous (e.g. with io_uring) in the future without needing
3065 : // to change the rest of the logic.
3066 :
3067 549044 : int write_ops_cnt = 0;
3068 549044 : int write_meta_cnt = 0;
3069 549044 : ulong total_write_sz = 0UL;
3070 549044 : fd_accdb_disk_meta_t write_metas[ 2UL*FD_ACCDB_MAX_ACQUIRE_CNT ];
3071 549044 : struct iovec write_ops[ 4UL*FD_ACCDB_MAX_ACQUIRE_CNT ];
3072 :
3073 1164944 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3074 615900 : if( FD_UNLIKELY( !accmetas[ i ] && !writable[ i ] ) ) continue;
3075 :
3076 541529 : if( FD_UNLIKELY( writable[ i ] ) ) {
3077 403041 : if( FD_UNLIKELY( evicted_staging_acc[ i ]!=UINT_MAX ) ) {
3078 0 : accdb->metrics->accounts_evicted++;
3079 0 : accdb->metrics->accounts_evicted_per_class[ 7UL ]++;
3080 :
3081 0 : fd_accdb_accmeta_t const * evicted = &accdb->acc_pool[ evicted_staging_acc[ i ] ];
3082 0 : fd_racesan_hook( "writeback:pre_synth" );
3083 0 : total_write_sz += sizeof(fd_accdb_disk_meta_t) + FD_ACCDB_SIZE_DATA( evicted->executable_size );
3084 0 : FD_TEST( write_meta_cnt<(int)(sizeof(write_metas)/sizeof(write_metas[0])) );
3085 0 : fd_memcpy( write_metas[ write_meta_cnt ].pubkey, evicted->key.pubkey, 32UL );
3086 0 : write_metas[ write_meta_cnt ].size = FD_ACCDB_SIZE_DATA( evicted->executable_size );
3087 0 : write_metas[ write_meta_cnt ].generation = evicted->key.generation;
3088 0 : fd_memcpy( write_metas[ write_meta_cnt ].owner, staging_line[ i ]->owner, 32UL );
3089 0 : write_ops[ write_ops_cnt++ ] = (struct iovec){ .iov_base = &write_metas[ write_meta_cnt ], .iov_len = sizeof(fd_accdb_disk_meta_t) };
3090 0 : write_meta_cnt++;
3091 0 : write_ops[ write_ops_cnt++ ] = (struct iovec){ .iov_base = staging_line[ i ]+1UL, .iov_len = FD_ACCDB_SIZE_DATA( evicted->executable_size ) };
3092 0 : }
3093 403041 : if( FD_UNLIKELY( accmetas[ i ] && !exists_in_cache[ i ] && evicted_orig_acc[ i ]!=UINT_MAX ) ) {
3094 0 : fd_accdb_accmeta_t const * evicted = &accdb->acc_pool[ evicted_orig_acc[ i ] ];
3095 0 : accdb->metrics->accounts_evicted++;
3096 0 : accdb->metrics->accounts_evicted_per_class[ fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( evicted->executable_size ) ) ]++;
3097 :
3098 0 : total_write_sz += sizeof(fd_accdb_disk_meta_t) + FD_ACCDB_SIZE_DATA( evicted->executable_size );
3099 0 : FD_TEST( write_meta_cnt<(int)(sizeof(write_metas)/sizeof(write_metas[0])) );
3100 0 : fd_memcpy( write_metas[ write_meta_cnt ].pubkey, evicted->key.pubkey, 32UL );
3101 0 : write_metas[ write_meta_cnt ].size = FD_ACCDB_SIZE_DATA( evicted->executable_size );
3102 0 : write_metas[ write_meta_cnt ].generation = evicted->key.generation;
3103 0 : fd_memcpy( write_metas[ write_meta_cnt ].owner, original_cache_line[ i ]->owner, 32UL );
3104 0 : write_ops[ write_ops_cnt++ ] = (struct iovec){ .iov_base = &write_metas[ write_meta_cnt ], .iov_len = sizeof(fd_accdb_disk_meta_t) };
3105 0 : write_meta_cnt++;
3106 0 : write_ops[ write_ops_cnt++ ] = (struct iovec){ .iov_base = original_cache_line[ i ]+1UL, .iov_len = FD_ACCDB_SIZE_DATA( evicted->executable_size ) };
3107 0 : }
3108 403041 : } else {
3109 138488 : if( FD_LIKELY( exists_in_cache[ i ] || evicted_orig_acc[ i ]==UINT_MAX ) ) continue;
3110 890 : fd_accdb_accmeta_t const * evicted = &accdb->acc_pool[ evicted_orig_acc[ i ] ];
3111 890 : accdb->metrics->accounts_evicted++;
3112 890 : accdb->metrics->accounts_evicted_per_class[ fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( evicted->executable_size ) ) ]++;
3113 890 : total_write_sz += sizeof(fd_accdb_disk_meta_t) + FD_ACCDB_SIZE_DATA( evicted->executable_size );
3114 890 : FD_TEST( write_meta_cnt<(int)(sizeof(write_metas)/sizeof(write_metas[0])) );
3115 890 : fd_memcpy( write_metas[ write_meta_cnt ].pubkey, evicted->key.pubkey, 32UL );
3116 890 : write_metas[ write_meta_cnt ].size = FD_ACCDB_SIZE_DATA( evicted->executable_size );
3117 890 : write_metas[ write_meta_cnt ].generation = evicted->key.generation;
3118 890 : fd_memcpy( write_metas[ write_meta_cnt ].owner, original_cache_line[ i ]->owner, 32UL );
3119 890 : write_ops[ write_ops_cnt++ ] = (struct iovec){ .iov_base = &write_metas[ write_meta_cnt ], .iov_len = sizeof(fd_accdb_disk_meta_t) };
3120 890 : write_meta_cnt++;
3121 890 : write_ops[ write_ops_cnt++ ] = (struct iovec){ .iov_base = original_cache_line[ i ]+1UL, .iov_len = FD_ACCDB_SIZE_DATA( evicted->executable_size ) };
3122 890 : }
3123 541529 : }
3124 :
3125 : // STEP 6-7.
3126 : // Compute the file offset for the writes we are about to do and
3127 : // build the pending offset table. The common case is a single
3128 : // atomic fetch-add on the write head, reserving a contiguous
3129 : // region. If the total eviction batch is too large to fit in one
3130 : // partition (extremely unlikely — requires many dirty 10MiB
3131 : // evictions), fall back to per-entry allocation so that each
3132 : // individual write fits in a single partition.
3133 : //
3134 : // The actual stores to evicted->offset_fork and line->persisted
3135 : // are deferred until after pwritev2 completes, so
3136 : // a concurrent acquire spinning on offset==FD_ACCDB_OFF_INVAL
3137 : // does not proceed to preadv2 from a location that hasn't been
3138 : // written.
3139 549044 : int pending_cnt = 0;
3140 549044 : fd_accdb_accmeta_t * pending_accs [ 2UL*FD_ACCDB_MAX_ACQUIRE_CNT ];
3141 549044 : ulong pending_offs [ 2UL*FD_ACCDB_MAX_ACQUIRE_CNT ];
3142 549044 : fd_accdb_cache_line_t * pending_lines[ 2UL*FD_ACCDB_MAX_ACQUIRE_CNT ];
3143 :
3144 549044 : ulong file_offset;
3145 549044 : int batch_contiguous;
3146 549044 : if( FD_LIKELY( total_write_sz && total_write_sz<=accdb->shmem->partition_sz ) ) {
3147 0 : file_offset = allocate_next_write( accdb, total_write_sz );
3148 0 : batch_contiguous = 1;
3149 549044 : } else {
3150 549044 : file_offset = 0UL;
3151 549044 : batch_contiguous = 0;
3152 549044 : }
3153 :
3154 549044 : ulong cumulative_offset = 0UL;
3155 1163611 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3156 614567 : if( FD_UNLIKELY( !accmetas[ i ] && !writable[ i ] ) ) continue;
3157 :
3158 540236 : if( FD_UNLIKELY( writable[ i ] ) ) {
3159 402471 : if( FD_UNLIKELY( evicted_staging_acc[ i ]!=UINT_MAX ) ) {
3160 0 : fd_accdb_accmeta_t * evicted = &accdb->acc_pool[ evicted_staging_acc[ i ] ];
3161 0 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t) + (ulong)FD_ACCDB_SIZE_DATA( evicted->executable_size );
3162 : /* xchg-to-INVAL atomically captures the old offset and prevents
3163 : a concurrent acc_unlink from also reading and freeing it (the
3164 : xchg there will see INVAL and skip). After writeback, the new
3165 : offset is published; cold_load_acc tolerates the transient
3166 : INVAL. The overwrite path uses the same protocol. */
3167 0 : ulong old_off = fd_accdb_acc_xchg_offset( evicted, FD_ACCDB_OFF_INVAL );
3168 0 : if( FD_LIKELY( old_off!=FD_ACCDB_OFF_INVAL ) ) {
3169 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, old_off, entry_sz );
3170 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
3171 0 : }
3172 0 : FD_TEST( pending_cnt<(int)(sizeof(pending_accs)/sizeof(pending_accs[0])) );
3173 0 : pending_accs [ pending_cnt ] = evicted;
3174 0 : if( FD_LIKELY( batch_contiguous ) ) pending_offs[ pending_cnt ] = file_offset + cumulative_offset;
3175 0 : else pending_offs[ pending_cnt ] = allocate_next_write( accdb, entry_sz );
3176 0 : pending_lines[ pending_cnt ] = staging_line[ i ];
3177 0 : pending_cnt++;
3178 0 : cumulative_offset += entry_sz;
3179 0 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
3180 0 : }
3181 402471 : if( FD_UNLIKELY( accmetas[ i ] && !exists_in_cache[ i ] && evicted_orig_acc[ i ]!=UINT_MAX ) ) {
3182 0 : fd_accdb_accmeta_t * evicted = &accdb->acc_pool[ evicted_orig_acc[ i ] ];
3183 0 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t) + (ulong)FD_ACCDB_SIZE_DATA( evicted->executable_size );
3184 0 : ulong old_off = fd_accdb_acc_xchg_offset( evicted, FD_ACCDB_OFF_INVAL );
3185 0 : if( FD_LIKELY( old_off!=FD_ACCDB_OFF_INVAL ) ) {
3186 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, old_off, entry_sz );
3187 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
3188 0 : }
3189 0 : FD_TEST( pending_cnt<(int)(sizeof(pending_accs)/sizeof(pending_accs[0])) );
3190 0 : pending_accs [ pending_cnt ] = evicted;
3191 0 : if( FD_LIKELY( batch_contiguous ) ) pending_offs[ pending_cnt ] = file_offset + cumulative_offset;
3192 0 : else pending_offs[ pending_cnt ] = allocate_next_write( accdb, entry_sz );
3193 0 : pending_lines[ pending_cnt ] = original_cache_line[ i ];
3194 0 : pending_cnt++;
3195 0 : cumulative_offset += entry_sz;
3196 0 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
3197 0 : }
3198 402471 : } else {
3199 137765 : if( FD_LIKELY( exists_in_cache[ i ] || evicted_orig_acc[ i ]==UINT_MAX ) ) continue;
3200 :
3201 174 : fd_accdb_accmeta_t * evicted = &accdb->acc_pool[ evicted_orig_acc[ i ] ];
3202 174 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t) + (ulong)FD_ACCDB_SIZE_DATA( evicted->executable_size );
3203 174 : ulong old_off = fd_accdb_acc_xchg_offset( evicted, FD_ACCDB_OFF_INVAL );
3204 174 : if( FD_LIKELY( old_off!=FD_ACCDB_OFF_INVAL ) ) {
3205 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, old_off, entry_sz );
3206 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
3207 0 : }
3208 174 : FD_TEST( pending_cnt<(int)(sizeof(pending_accs)/sizeof(pending_accs[0])) );
3209 174 : pending_accs [ pending_cnt ] = evicted;
3210 174 : if( FD_LIKELY( batch_contiguous ) ) pending_offs[ pending_cnt ] = file_offset + cumulative_offset;
3211 174 : else pending_offs[ pending_cnt ] = allocate_next_write( accdb, entry_sz );
3212 174 : pending_lines[ pending_cnt ] = original_cache_line[ i ];
3213 174 : pending_cnt++;
3214 174 : cumulative_offset += entry_sz;
3215 174 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->disk_used_bytes, entry_sz );
3216 174 : }
3217 540236 : }
3218 :
3219 : // STEP 8.
3220 : // Fill the output entries with cache pointers and metadata based on
3221 : // the accounts we have located and the cache lines we have
3222 : // reserved.
3223 :
3224 1162584 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3225 613540 : if( FD_UNLIKELY( !accmetas[ i ] && !writable[ i ] ) ) {
3226 74342 : out_accs[ i ].data = NULL;
3227 74342 : out_accs[ i ].data_len = 0UL;
3228 74342 : out_accs[ i ].lamports = 0UL;
3229 74342 : out_accs[ i ].executable = 0;
3230 74342 : memset( out_accs[ i ].owner, 0, 32UL );
3231 74342 : fd_memcpy( out_accs[ i ].pubkey, pubkeys[ i ], 32UL );
3232 74342 : out_accs[ i ].prior_lamports = 0UL;
3233 74342 : out_accs[ i ].prior_data_len = 0UL;
3234 74342 : out_accs[ i ].prior_executable = 0;
3235 74342 : memset( out_accs[ i ].prior_owner, 0, 32UL );
3236 74342 : out_accs[ i ].prior_data = NULL;
3237 74342 : out_accs[ i ].commit = 0;
3238 74342 : out_accs[ i ].pd_write = 0;
3239 74342 : out_accs[ i ]._writable = 0;
3240 74342 : out_accs[ i ]._original_size_class = ULONG_MAX;
3241 74342 : out_accs[ i ]._original_cache_idx = ULONG_MAX;
3242 74342 : continue;
3243 74342 : }
3244 :
3245 539198 : if( FD_LIKELY( !writable[ i ] ) ) out_accs[ i ].data = (uchar *)(original_cache_line[ i ]+1UL);
3246 401700 : else out_accs[ i ].data = (uchar *)(staging_line[ i ]+1UL);
3247 : /* Tombstone reset: agave's account loader returns AccountSharedData::default()
3248 : (System owner, empty data, exec=0) for any account with lamports==0.
3249 : https://github.com/anza-xyz/agave/blob/v2.3.1/svm/src/account_loader.rs#L199-L228 */
3250 539198 : fd_racesan_hook( "accdb_acquire:pre_step7_meta" );
3251 539198 : int tombstone = accmetas[ i ] && accmetas[ i ]->lamports==0UL;
3252 539198 : out_accs[ i ].data_len = ( accmetas[ i ] && !tombstone ) ? FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) : 0UL;
3253 539198 : out_accs[ i ].executable = ( accmetas[ i ] && !tombstone ) ? FD_ACCDB_SIZE_EXEC( accmetas[ i ]->executable_size ) : 0;
3254 539198 : fd_racesan_hook( "accdb_acquire:mid_step7_meta" );
3255 539198 : out_accs[ i ].lamports = accmetas[ i ] ? accmetas[ i ]->lamports : 0UL;
3256 539198 : if( FD_UNLIKELY( !accmetas[ i ] ) ) {
3257 366361 : memset( out_accs[ i ].owner, 0, 32UL );
3258 366361 : memset( out_accs[ i ].prior_owner, 0, 32UL );
3259 366361 : }
3260 : /* For accmetas[i] != NULL, both owners are copied from the cache
3261 : line below after disk reads have completed for cold loads. */
3262 :
3263 539198 : out_accs[ i ].prior_lamports = out_accs[ i ].lamports;
3264 539198 : out_accs[ i ].prior_data_len = out_accs[ i ].data_len;
3265 539198 : out_accs[ i ].prior_executable = out_accs[ i ].executable;
3266 539198 : out_accs[ i ].prior_data = (uchar *)(original_cache_line[ i ] ? (original_cache_line[ i ]+1UL) : NULL);
3267 :
3268 539198 : out_accs[ i ].commit = 0;
3269 539198 : out_accs[ i ].pd_write = 0;
3270 539198 : out_accs[ i ]._writable = writable[ i ];
3271 539198 : if( FD_UNLIKELY( writable[ i ] && accmetas[ i ] ) ) out_accs[ i ]._overwrite = accdb->fork_pool[ fork_id.val ].shmem->generation==accmetas[ i ]->key.generation;
3272 503389 : else out_accs[ i ]._overwrite = 0;
3273 :
3274 539198 : FD_TEST( out_accs[ i ].data_len<=(10UL<<20) );
3275 539198 : FD_TEST( !out_accs[ i ]._overwrite || accdb->fork_pool[ fork_id.val ].shmem->generation==accmetas[ i ]->key.generation );
3276 :
3277 : #if FD_TMPL_USE_HANDHOLDING
3278 : if( FD_UNLIKELY( !writable[ i ] && accmetas[ i ] && !tombstone ) ) {
3279 : ulong cls = fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) );
3280 : FD_TEST( fd_accdb_ptr_in_region( accdb, cls, out_accs[ i ].data ) );
3281 : }
3282 : #endif
3283 :
3284 539198 : if( FD_UNLIKELY( writable[ i ] ) ) {
3285 399940 : out_accs[ i ]._fork_id = fork_id.val;
3286 399940 : out_accs[ i ]._generation = fork->shmem->generation;
3287 399940 : out_accs[ i ]._acc_map_idx = acc_map_idxs[ i ];
3288 399940 : }
3289 539198 : fd_memcpy( out_accs[ i ].pubkey, pubkeys[ i ], 32UL );
3290 :
3291 539198 : if( FD_UNLIKELY( !accmetas[ i ] ) ) {
3292 364780 : out_accs[ i ]._original_size_class = ULONG_MAX;
3293 364780 : out_accs[ i ]._original_cache_idx = ULONG_MAX;
3294 364780 : } else {
3295 174418 : out_accs[ i ]._original_size_class = fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) );
3296 174418 : out_accs[ i ]._original_cache_idx = cache_line_idx( accdb, out_accs[ i ]._original_size_class, original_cache_line[ i ] );
3297 174418 : }
3298 :
3299 539198 : if( FD_UNLIKELY( writable[ i ] ) ) {
3300 399091 : out_accs[ i ]._staging_cache_idx = cache_line_idx( accdb, 7UL, staging_line[ i ] );
3301 399091 : out_accs[ i ]._final_cache_idx = ULONG_MAX;
3302 399091 : }
3303 539198 : }
3304 :
3305 : // STEP 9.
3306 : // Write the dirty eviction data to disk and publish the new offsets
3307 : // BEFORE constructing read iovecs. Cache-line allocation may have
3308 : // evicted a dirty cache line belonging to another account in
3309 : // the same batch whose acc->offset is still FD_ACCDB_OFF_INVAL.
3310 : // The read-iovec loop below spin-waits on
3311 : // offset!=FD_ACCDB_OFF_INVAL, so publishing evicted offsets first
3312 : // prevents an intra-batch deadlock where the thread waits on an
3313 : // offset that only it can resolve.
3314 549044 : if( FD_LIKELY( batch_contiguous ) ) {
3315 : /* Fast path: all evictions fit in one contiguous region. Use the
3316 : pre-built iovec array for a single batched pwritev2 call. */
3317 0 : ulong bytes_written = 0UL;
3318 0 : struct iovec * write_ptr = write_ops;
3319 0 : while( FD_LIKELY( bytes_written<total_write_sz ) ) {
3320 0 : long result = pwritev2( accdb->fd, write_ptr, fd_int_min( write_ops_cnt, IOV_MAX ), (long)(file_offset+bytes_written), 0 );
3321 0 : if( FD_UNLIKELY( -1==result && (errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ) ) ) continue;
3322 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "pwritev2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
3323 0 : else if( FD_UNLIKELY( !result ) ) FD_LOG_ERR(( "accounts database is corrupt, pwritev2() returned 0 at offset %lu with %lu bytes remaining",
3324 0 : file_offset+bytes_written, total_write_sz-bytes_written ));
3325 0 : bytes_written += (ulong)result;
3326 0 : accdb->metrics->bytes_written += (ulong)result;
3327 0 : accdb->metrics->write_ops++;
3328 :
3329 0 : while( write_ops_cnt && (ulong)result>=(ulong)write_ptr[ 0 ].iov_len ) {
3330 0 : result -= (long)write_ptr[ 0 ].iov_len;
3331 0 : write_ptr++;
3332 0 : write_ops_cnt--;
3333 0 : }
3334 0 : if( FD_LIKELY( write_ops_cnt ) ) {
3335 0 : write_ptr[ 0 ].iov_base = (uchar *)write_ptr[ 0 ].iov_base + result;
3336 0 : write_ptr[ 0 ].iov_len -= (ulong)result;
3337 0 : }
3338 0 : }
3339 549044 : } else {
3340 : /* Slow path: total eviction batch exceeds a single partition.
3341 : Write each entry individually using its own allocated offset.
3342 : This path is only taken in extreme edge cases (many concurrent
3343 : dirty 10 MiB evictions). */
3344 549044 : for( int k=0; k<pending_cnt; k++ ) {
3345 0 : struct iovec const * wp = write_ops + 2*k;
3346 0 : write_entry( accdb, pending_offs[ k ], wp[ 0 ].iov_base, wp[ 1 ].iov_base, wp[ 1 ].iov_len );
3347 0 : }
3348 549044 : }
3349 :
3350 : // STEP 10.
3351 : // Now that the data is on disk, publish the evicted account offsets
3352 : // so concurrent acquire threads spinning on
3353 : // offset==FD_ACCDB_OFF_INVAL can proceed. The fence ensures
3354 : // pwritev2 data is globally visible before the offset stores.
3355 549044 : FD_COMPILER_MFENCE();
3356 549044 : for( int k=0; k<pending_cnt; k++ ) {
3357 0 : pending_accs[ k ]->offset_fork = fd_accdb_acc_pack_offset_fork( pending_offs[ k ], fd_accdb_acc_fork_id(pending_accs[ k ]) );
3358 0 : pending_lines[ k ]->persisted = 1;
3359 0 : }
3360 :
3361 : // STEP 11.
3362 : // Now construct iovecs for any reads we need to do of accounts into
3363 : // the cache. For reading accounts, we read them directly into the
3364 : // sole cache line we took (and maybe just evicted). For writing
3365 : // accounts, we read them into the right sized cache line, and later
3366 : // it will be copied to the staging buffer. This is to prevent
3367 : // repeatedly reading the same account off disk into cache, if it is
3368 : // being written cold multiple times and every write fails.
3369 :
3370 549044 : ulong read_ops_cnt = 0UL; /* at most one read per account */
3371 549044 : ulong read_offsets[ FD_ACCDB_MAX_ACQUIRE_CNT ];
3372 549044 : uchar * read_bases[ FD_ACCDB_MAX_ACQUIRE_CNT ];
3373 549044 : ulong read_sizes[ FD_ACCDB_MAX_ACQUIRE_CNT ];
3374 549044 : struct iovec read_ops[ FD_ACCDB_MAX_ACQUIRE_CNT ];
3375 :
3376 1160365 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3377 611321 : if( FD_UNLIKELY( !accmetas[ i ] || exists_in_cache[ i ] ) ) continue;
3378 :
3379 615 : accdb->metrics->accounts_not_found_per_class[ fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) ) ]++;
3380 :
3381 : /* Tombstones (lamports==0) have no on-disk payload to read, and
3382 : background_advance_root may unlink the acc and never assign it a
3383 : disk offset, so the offset_fork spin below would hang forever.
3384 : The final tombstone reset zeros the owner for these accounts.
3385 :
3386 : Exception: when re-creating a closed account on the same fork,
3387 : wait anyway, or a concurrent evictor will later store the
3388 : closed account's disk offset into the new account. */
3389 615 : int tombstone = !accmetas[ i ]->lamports;
3390 615 : if( FD_UNLIKELY( tombstone && !out_accs[ i ]._overwrite ) ) continue;
3391 :
3392 : /* We are guaranteed that if an account is in the cache, the bytes
3393 : are available (all cache operations are atomic via refcnt CAS),
3394 : but we are not guaranteed that if something is _not_ in the cache
3395 : that it has been written back to disk yet. In particular, if we
3396 : are trying to read an account that another thread is in the
3397 : process of evicting, we know they removed it from the cache, but
3398 : we don't know exactly when they will have written it back fully
3399 : to disk, so we may need to wait for that here.
3400 :
3401 : Compaction may concurrently relocate this record, but
3402 : epoch-based safe reclamation guarantees the source partition
3403 : is not freed until all epoch-protected operations that could
3404 : have snapshotted the old offset have exited. So the data at the
3405 : snapshotted offset remains stable for the duration of our
3406 : read and no post-read validation is needed. */
3407 615 : ulong off_packed = FD_VOLATILE_CONST( accmetas[ i ]->offset_fork );
3408 615 : while( FD_UNLIKELY( (off_packed & FD_ACCDB_OFF_MASK)==FD_ACCDB_OFF_INVAL ) ) {
3409 0 : fd_racesan_hook( "accdb_acquire:offset_wait" );
3410 0 : FD_SPIN_PAUSE();
3411 0 : off_packed = FD_VOLATILE_CONST( accmetas[ i ]->offset_fork );
3412 0 : }
3413 615 : if( FD_UNLIKELY( tombstone ) ) continue;
3414 615 : fd_racesan_hook( "accdb_coldload:pre_iovec" );
3415 :
3416 615 : read_offsets[ read_ops_cnt ] = fd_accdb_acc_offset(accmetas[ i ]) + offsetof(fd_accdb_disk_meta_t, owner);
3417 615 : read_bases[ read_ops_cnt ] = original_cache_line[ i ]->owner;
3418 615 : read_sizes[ read_ops_cnt ] = 32UL + FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size );
3419 615 : read_ops[ read_ops_cnt++ ] = (struct iovec){ .iov_base = original_cache_line[ i ]->owner, .iov_len = 32UL + FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size ) };
3420 615 : }
3421 :
3422 : // STEP 12.
3423 : // Almost done... now do the actual reads of accounts into cache,
3424 : // using the iovecs we constructed. This is basically the same loop
3425 : // as the writes, but with preadv2 instead of pwritev2, and that the
3426 : // reads are not necessarily all contiguous, but occur at random
3427 : // offsets.
3428 : //
3429 : // CONCURRENCY: The compaction tile may concurrently relocate a
3430 : // record we are about to read (both are epoch-protected). Epoch-
3431 : // based safe reclamation guarantees the source partition is not
3432 : // freed until all epoch-protected operations that could have
3433 : // snapshotted the old offset have exited, so the data at the
3434 : // remains stable for the duration of this read — no post-read
3435 : // validation or retry is needed.
3436 549044 : for( ulong i=0UL; i<read_ops_cnt; i++ ) {
3437 0 : ulong bytes_read = 0UL;
3438 0 : while( FD_LIKELY( bytes_read<read_sizes[ i ] ) ) {
3439 0 : long result = preadv2( accdb->fd, &read_ops[ i ], 1, (long)(read_offsets[ i ]+bytes_read), 0 );
3440 0 : if( FD_UNLIKELY( -1==result && (errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ) ) ) continue;
3441 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "preadv2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
3442 0 : else if( FD_UNLIKELY( !result ) ) FD_LOG_ERR(( "accounts database is corrupt, data expected at offset %lu with size %lu exceeded file extents",
3443 0 : read_offsets[ i ]+bytes_read, read_sizes[ i ] ));
3444 0 : fd_accdb_partition_read_bump( accdb, read_offsets[ i ]+bytes_read, (ulong)result );
3445 0 : bytes_read += (ulong)result;
3446 0 : accdb->metrics->bytes_read += (ulong)result;
3447 0 : accdb->metrics->read_ops++;
3448 :
3449 0 : read_ops[ i ].iov_base = read_bases[ i ] + bytes_read;
3450 0 : read_ops[ i ].iov_len = read_sizes[ i ] - bytes_read;
3451 0 : }
3452 0 : }
3453 :
3454 : // STEP 13.
3455 : // Publish the real acc index for any cache lines we just loaded
3456 : // from disk, so concurrent threads spinning on acc_idx==UINT_MAX
3457 : // can proceed. The fence ensures all preadv2 data is visible
3458 : // before the sentinel is cleared.
3459 549044 : FD_COMPILER_MFENCE();
3460 1157918 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3461 609058 : if( FD_UNLIKELY( !accmetas[ i ] || exists_in_cache[ i ] ) ) continue;
3462 >1844*10^16 : FD_VOLATILE( original_cache_line[ i ]->acc_idx ) = (uint)( accmetas[ i ] - accdb->acc_pool );
3463 >1844*10^16 : FD_TEST( FD_VOLATILE_CONST( original_cache_line[ i ]->acc_idx )==(uint)( accmetas[ i ] - accdb->acc_pool ) );
3464 >1844*10^16 : }
3465 :
3466 : // STEP 14.
3467 : // Spin-wait for any cache lines found via acc->cache_idx that are
3468 : // still being loaded by another thread's preadv2. The loading
3469 : // thread sets acc_idx to UINT_MAX before publishing cache_idx
3470 : // and publishes the real acc index after its read completes.
3471 : // This step is placed as late as possible to give the loading
3472 : // thread maximum time to finish before we need to spin.
3473 1159256 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3474 609070 : if( FD_UNLIKELY( !accmetas[ i ] && !writable[ i ] ) ) continue;
3475 :
3476 534776 : if( FD_UNLIKELY( !original_cache_line[ i ] ) ) continue;
3477 173198 : if( FD_LIKELY( FD_VOLATILE_CONST( original_cache_line[ i ]->acc_idx )!=UINT_MAX ) ) goto step13_check;
3478 >1844*10^16 : accdb->metrics->accounts_waited++;
3479 >1844*10^16 : while( FD_UNLIKELY( FD_VOLATILE_CONST( original_cache_line[ i ]->acc_idx )==UINT_MAX ) ) {
3480 0 : fd_racesan_hook( "accdb_acquire:step14_load_wait" );
3481 0 : FD_SPIN_PAUSE();
3482 0 : }
3483 >1844*10^16 : step13_check:;
3484 : #if FD_TMPL_USE_HANDHOLDING
3485 : FD_TEST( original_cache_line[ i ]->key.generation==accmetas[ i ]->key.generation &&
3486 : !memcmp( original_cache_line[ i ]->key.pubkey, pubkeys[ i ], 32UL ) );
3487 : #endif
3488 173193 : }
3489 :
3490 : // STEP 15.
3491 : // Now that all reads from disk into original_cache_line have
3492 : // completed (and any concurrent loaders have published their
3493 : // acc_idx after disk-read completion), copy the owner into the output
3494 : // entries. The cache line owner is only valid post-read for cold loads.
3495 1158949 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3496 608763 : if( FD_UNLIKELY( !accmetas[ i ] ) ) continue;
3497 171551 : fd_racesan_hook( "accdb_acquire:pre_step14_owner" );
3498 : /* Tombstone reset: clear the owner of a zero-lamport account. */
3499 171551 : if( FD_UNLIKELY( accmetas[ i ]->lamports==0UL ) ) {
3500 0 : memset( out_accs[ i ].owner, 0, 32UL );
3501 0 : memset( out_accs[ i ].prior_owner, 0, 32UL );
3502 171551 : } else {
3503 171551 : fd_memcpy( out_accs[ i ].owner, original_cache_line[ i ]->owner, 32UL );
3504 171551 : fd_memcpy( out_accs[ i ].prior_owner, original_cache_line[ i ]->owner, 32UL );
3505 171551 : }
3506 171551 : }
3507 :
3508 : // STEP 16.
3509 : // Finally, copy any accounts we are writing into the staging
3510 : // buffers, so they occupy a 10MiB cache line for the execution
3511 : // system.
3512 1157847 : for( ulong i=0UL; i<pubkeys_cnt; i++ ) {
3513 607661 : if( FD_UNLIKELY( !accmetas[ i ] || !writable[ i ] ) ) continue;
3514 :
3515 35426 : ulong copy_sz = (ulong)FD_ACCDB_SIZE_DATA( accmetas[ i ]->executable_size );
3516 35426 : fd_memcpy( staging_line[ i ]+1UL, original_cache_line[ i ]+1UL, copy_sz );
3517 35426 : accdb->metrics->bytes_copied += copy_sz;
3518 35426 : }
3519 :
3520 550186 : FD_COMPILER_MFENCE();
3521 550186 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
3522 550186 : }
3523 :
3524 : void
3525 : fd_accdb_acquire( fd_accdb_t * accdb,
3526 : fd_accdb_fork_id_t fork_id,
3527 : ulong pubkeys_cnt,
3528 : uchar const * const * pubkeys,
3529 : int * writable,
3530 533139 : fd_acc_t * out_accs ) {
3531 533139 : FD_TEST( accdb->acquire_state==FD_ACCDB_ACQUIRE_STATE_IDLE );
3532 533139 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_OPEN;
3533 533139 : fd_accdb_acquire_inner( accdb, fork_id, RESERVATION_TYPE_SIMPLE, 0UL, pubkeys_cnt, pubkeys, writable, out_accs );
3534 533139 : }
3535 :
3536 : void
3537 : fd_accdb_acquire_a( fd_accdb_t * accdb,
3538 : fd_accdb_fork_id_t fork_id,
3539 : ulong pubkeys_cnt,
3540 : uchar const * const * pubkeys,
3541 : int * writable,
3542 7291 : fd_acc_t * out_accs ) {
3543 7291 : FD_TEST( accdb->acquire_state==FD_ACCDB_ACQUIRE_STATE_IDLE );
3544 7291 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_PHASE_A;
3545 7291 : fd_accdb_acquire_inner( accdb, fork_id, RESERVATION_TYPE_MAYBE_PROGRAMDATA, 0UL, pubkeys_cnt, pubkeys, writable, out_accs );
3546 7291 : }
3547 :
3548 : void
3549 : fd_accdb_acquire_b( fd_accdb_t * accdb,
3550 : fd_accdb_fork_id_t fork_id,
3551 : ulong reserved_cnt,
3552 : ulong pubkeys_cnt,
3553 : uchar const * const * pubkeys,
3554 7283 : fd_acc_t * out_accs ) {
3555 7283 : static int const readonly[ FD_ACCDB_MAX_ACQUIRE_CNT ] = {0};
3556 7283 : FD_TEST( accdb->acquire_state==FD_ACCDB_ACQUIRE_STATE_PHASE_A );
3557 7283 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_OPEN;
3558 7283 : fd_accdb_acquire_inner( accdb, fork_id, RESERVATION_TYPE_ALREADY_RESERVED, reserved_cnt, pubkeys_cnt, pubkeys, readonly, out_accs );
3559 7283 : }
3560 :
3561 : /* release_free_staging_line returns an uncommitted staging line to the
3562 : free list. acquire_cache_line via CLOCK leaves line->acc_idx pointing
3563 : at the prior owner. cache_free_push consumers (CLOCK,
3564 : background_preevict) skip lines only when acc_idx==UINT_MAX AND
3565 : gen==UINT_MAX; if we left the stale acc_idx, a future CLOCK pick would
3566 : call evict_clear_acc_cache_ref on the wrong acc and corrupt its
3567 : cache_idx/valid. CAS on refcnt: a stray reader pin may still be
3568 : draining. */
3569 :
3570 : static void
3571 : release_free_staging_line( fd_accdb_t * accdb,
3572 400704 : fd_accdb_cache_line_t * line ) {
3573 400704 : line->acc_idx = UINT_MAX;
3574 400704 : line->key.generation = UINT_MAX;
3575 400704 : line->persisted = 1;
3576 400704 : while( FD_UNLIKELY( FD_ATOMIC_CAS( &line->refcnt, 1U, 0U )!=1U ) ) {
3577 0 : fd_racesan_hook( "accdb_release:dest_refcnt_wait" );
3578 0 : FD_SPIN_PAUSE();
3579 0 : }
3580 400704 : cache_free_push( accdb, 7UL, line );
3581 400704 : }
3582 :
3583 : /* release_inner drains one group of acquired accs but does NOT change the
3584 : handle's acquire_state. The public fd_accdb_release / fd_accdb_release_ab
3585 : wrappers below own the state transition (a single-phase release closes
3586 : the bracket; release_ab drains both phase groups then closes). */
3587 : static void
3588 : release_inner( fd_accdb_t * accdb,
3589 : ulong accs_cnt,
3590 544037 : fd_acc_t * accs ) {
3591 544037 : FD_TEST( accdb->acquire_state==FD_ACCDB_ACQUIRE_STATE_OPEN );
3592 :
3593 544037 : {
3594 544037 : ulong prev = FD_VOLATILE_CONST( *accdb->my_epoch_slot );
3595 544037 : FD_TEST( prev==ULONG_MAX || prev<=FD_VOLATILE_CONST( accdb->shmem->epoch ) );
3596 544037 : }
3597 :
3598 544037 : FD_COMPILER_MFENCE();
3599 544037 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
3600 544037 : FD_HW_MFENCE(); /* StoreLoad: epoch store must be globally visible
3601 : before any subsequent loads so the deferred
3602 : reclamation scan does not miss us. */
3603 :
3604 : // STEP 1.
3605 : // For each cache line which was written to in the 10MiB staging
3606 : // buffer, we may need to copy to the data out to a right sized
3607 : // cache line. Figuring out the target cache line is non-obvious,
3608 : // but follows the more complete logic below this, we just pull the
3609 : // memcpy out so they are not done inside the cache lock.
3610 :
3611 1160875 : for( ulong i=0UL; i<accs_cnt; i++ ) {
3612 616838 : if( FD_UNLIKELY( accs[ i ]._original_size_class==ULONG_MAX && !accs[ i ]._writable ) ) continue;
3613 :
3614 : #if FD_TMPL_USE_HANDHOLDING
3615 : if( FD_LIKELY( accs[ i ]._original_size_class!=ULONG_MAX ) ) {
3616 : FD_TEST( accs[ i ]._original_cache_idx<accdb->shmem->cache_class_max[ accs[ i ]._original_size_class ] );
3617 : }
3618 : if( FD_UNLIKELY( accs[ i ].commit ) ) FD_TEST( accs[ i ]._writable );
3619 : #endif
3620 :
3621 542572 : if( FD_LIKELY( !accs[ i ]._writable || !accs[ i ].commit ) ) continue;
3622 : #if FD_TMPL_USE_HANDHOLDING
3623 : if( FD_UNLIKELY( accs[ i ]._overwrite ) ) {
3624 : FD_TEST( accs[ i ]._writable );
3625 : FD_TEST( accs[ i ]._original_cache_idx!=ULONG_MAX );
3626 : FD_TEST( accs[ i ]._original_size_class!=ULONG_MAX );
3627 : }
3628 : #endif
3629 :
3630 371664 : ulong original_size_class = accs[ i ]._original_size_class;
3631 371664 : ulong new_size_class = fd_accdb_cache_class( accs[ i ].data_len );
3632 371664 : if( FD_UNLIKELY( new_size_class==7UL ) ) continue;
3633 :
3634 371637 : fd_accdb_cache_line_t * target_cache_line;
3635 371637 : if( FD_LIKELY( original_size_class==new_size_class && accs[ i ]._overwrite ) ) {
3636 3923 : target_cache_line = cache_line( accdb, original_size_class, accs[ i ]._original_cache_idx );
3637 367714 : } else {
3638 : /* The final class's line is taken now (acquire only reserved
3639 : it). Step 2 finds it in _final_cache_idx. */
3640 367714 : target_cache_line = release_take_line( accdb, new_size_class );
3641 367714 : accs[ i ]._final_cache_idx = cache_line_idx( accdb, new_size_class, target_cache_line );
3642 367714 : }
3643 :
3644 371637 : fd_accdb_cache_line_t * staging_line = cache_line( accdb, 7UL, accs[ i ]._staging_cache_idx );
3645 :
3646 371637 : fd_racesan_hook( "accdb_commit:pre_owner_write" );
3647 :
3648 : #if FD_TMPL_USE_HANDHOLDING
3649 : if( FD_UNLIKELY( original_size_class==new_size_class && accs[ i ]._overwrite ) ) {
3650 : uint rc = FD_VOLATILE_CONST( target_cache_line->refcnt );
3651 : FD_TEST( target_cache_line->key.generation==accs[ i ]._generation &&
3652 : !memcmp( target_cache_line->key.pubkey, accs[ i ].pubkey, 32UL ) &&
3653 : rc>0U &&
3654 : rc!=FD_ACCDB_EVICT_SENTINEL );
3655 : }
3656 : #endif
3657 :
3658 371637 : fd_memcpy( target_cache_line->owner, accs[ i ].owner, 32UL );
3659 371637 : fd_memcpy( target_cache_line+1UL, staging_line+1UL, accs[ i ].data_len );
3660 371637 : accdb->metrics->bytes_copied += accs[ i ].data_len;
3661 371637 : }
3662 :
3663 : // STEP 2.
3664 : // Now update the metadata structures and free lists to reflect the
3665 : // fact that we are done with these cache lines. This is fully
3666 : // atomic with CLOCK.
3667 :
3668 1155385 : for( ulong i=0UL; i<accs_cnt; i++ ) {
3669 611348 : if( FD_UNLIKELY( accs[ i ]._original_size_class==ULONG_MAX && !accs[ i ]._writable ) ) continue;
3670 :
3671 537069 : ulong original_size_class = accs[ i ]._original_size_class;
3672 537069 : fd_accdb_cache_line_t * original_cache_line = accs[ i ]._original_cache_idx==ULONG_MAX ? NULL : cache_line( accdb, original_size_class, accs[ i ]._original_cache_idx );
3673 : /* For overwrite commits, defer the refcnt decrement on
3674 : original_cache_line until after invalidation completes. If
3675 : we dropped refcnt to 0 here, a concurrent CLOCK sweep could
3676 : CAS(refcnt, 0, EVICT_SENTINEL) and steal the line before we
3677 : get to invalidate it, causing data corruption.
3678 : Non-overwrite and non-commit paths unpin
3679 : immediately because they never invalidate the original line. */
3680 537069 : if( FD_LIKELY( original_cache_line ) ) {
3681 : #if FD_TMPL_USE_HANDHOLDING
3682 : FD_TEST( original_cache_line->refcnt>0U );
3683 : #endif
3684 173500 : if( FD_LIKELY( !accs[ i ]._writable || !accs[ i ].commit || !accs[ i ]._overwrite ) ) {
3685 169449 : FD_ATOMIC_FETCH_AND_SUB( &original_cache_line->refcnt, 1U );
3686 169449 : }
3687 173500 : }
3688 :
3689 537069 : if( FD_LIKELY( !accs[ i ]._writable ) ) {
3690 : /* For readonly accounts, mark as recently used so the CLOCK
3691 : algorithm gives it a second chance before eviction. */
3692 : #if FD_TMPL_USE_HANDHOLDING
3693 : FD_TEST( original_cache_line );
3694 : #endif
3695 137762 : original_cache_line->referenced = 1;
3696 137762 : continue;
3697 137762 : }
3698 :
3699 399307 : fd_accdb_cache_line_t * staging_line = cache_line( accdb, 7UL, accs[ i ]._staging_cache_idx );
3700 :
3701 399307 : if( FD_LIKELY( !accs[ i ].commit ) ) {
3702 : /* If it's writable but it didn't commit, the staging buffer is
3703 : trashed and can be pushed to this join's free list for immediate
3704 : reuse. Whatever buffer it was accessing also gets marked as
3705 : recently used. */
3706 33203 : if( FD_LIKELY( original_cache_line ) ) original_cache_line->referenced = 1;
3707 33203 : release_free_staging_line( accdb, staging_line );
3708 33203 : continue;
3709 33203 : }
3710 :
3711 366104 : ulong new_size_class = fd_accdb_cache_class( accs[ i ].data_len );
3712 366104 : uint original_acc_idx = original_cache_line ? original_cache_line->acc_idx : UINT_MAX;
3713 366104 : fd_accdb_cache_line_t * committed_line;
3714 :
3715 : /* For overwrites, invalidate the on-disk offset BEFORE removing
3716 : the cache acc. This ensures a concurrent acquire that misses
3717 : the cache will see offset==FD_ACCDB_OFF_INVAL and spin-wait,
3718 : rather than reading stale on-disk bytes from the old location.
3719 : The CAS-loop exchange also serializes with a concurrent
3720 : compaction CAS (old_offset -> dest_offset). */
3721 366104 : ulong old_offset = FD_ACCDB_OFF_INVAL;
3722 366104 : if( FD_LIKELY( accs[ i ]._overwrite ) ) {
3723 4024 : fd_accdb_accmeta_t * ow_accmeta = &accdb->acc_pool[ original_acc_idx ];
3724 4024 : fd_racesan_hook( "accdb_overwrite:pre_xchg_offset" );
3725 4024 : old_offset = fd_accdb_acc_xchg_offset( ow_accmeta, FD_ACCDB_OFF_INVAL );
3726 4024 : if( FD_LIKELY( old_offset!=FD_ACCDB_OFF_INVAL ) ) {
3727 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, old_offset, (ulong)FD_ACCDB_SIZE_DATA(ow_accmeta->executable_size)+sizeof(fd_accdb_disk_meta_t) );
3728 0 : FD_ATOMIC_FETCH_AND_SUB( &accdb->shmem->shmetrics->disk_used_bytes, (ulong)FD_ACCDB_SIZE_DATA(ow_accmeta->executable_size)+sizeof(fd_accdb_disk_meta_t) );
3729 0 : }
3730 4024 : }
3731 :
3732 366104 : if( FD_UNLIKELY( new_size_class==7UL ) ) {
3733 : /* The account belongs in the largest size class, and we already
3734 : have it resident in a 10MiB buffer anyway, so no need to copy
3735 : back. If we are "overwriting" (same generation as the account
3736 : came from), then the original can be discarded (pushed to
3737 : this join's free list) and removed from the cache. */
3738 27 : staging_line->persisted = 0;
3739 27 : if( FD_LIKELY( accs[ i ]._overwrite ) ) {
3740 : /* Atomically clear acc.VALID and acc.cache_idx BEFORE freeing
3741 : the line, so a reader cannot observe acc.VALID=1 with
3742 : acc.cache_idx pointing at a line that has been recycled to
3743 : another acc. evict_clear_acc_cache_ref uses the CLAIM
3744 : protocol to serialize with cold_load_acc. */
3745 0 : evict_clear_acc_cache_ref( &accdb->acc_pool[ original_acc_idx ], original_size_class, accs[ i ]._original_cache_idx );
3746 :
3747 : /* Convert our own pin directly into the eviction claim
3748 : (CAS refcnt 1 -> EVICT_SENTINEL) so refcnt never passes
3749 : through 0 while acc_idx/key.generation are still valid:
3750 : in such a window background_preevict could claim and free
3751 : the line, and a claim of our own would then succeed on
3752 : the free-listed line and push it a second time. The CAS
3753 : fails if a concurrent reader that pinned the line via
3754 : cache_try_pin BEFORE evict_clear_acc_cache_ref completed
3755 : still holds a reference (its ABA check on
3756 : line->key.generation is not synchronized with our writes
3757 : to that field); then just drop our pin and leave
3758 : acc_idx/key.generation intact so CLOCK reclaims the line
3759 : once the reader unpins. That reclaim's
3760 : evict_clear_acc_cache_ref is a no-op, but its dirty-
3761 : writeback gate keys on persisted/acc_idx and would
3762 : republish the pre-overwrite bytes over the committed
3763 : version, so set persisted while still pinned. */
3764 0 : original_cache_line->persisted = 1;
3765 0 : fd_racesan_hook( "accdb_release:pre_discard_claim" );
3766 0 : if( FD_LIKELY( FD_ATOMIC_CAS( &original_cache_line->refcnt, 1U, FD_ACCDB_EVICT_SENTINEL )==1U ) ) {
3767 0 : original_cache_line->acc_idx = UINT_MAX;
3768 0 : original_cache_line->key.generation = UINT_MAX;
3769 0 : FD_COMPILER_MFENCE();
3770 0 : FD_VOLATILE( original_cache_line->refcnt ) = 0;
3771 0 : cache_free_push( accdb, original_size_class, original_cache_line );
3772 0 : } else {
3773 0 : FD_ATOMIC_FETCH_AND_SUB( &original_cache_line->refcnt, 1U );
3774 0 : }
3775 0 : }
3776 27 : committed_line = staging_line;
3777 366077 : } else {
3778 : /* The account started in some arbitrary size class, transited
3779 : through a 10MiB staging buffer, and is now being written back
3780 : to some arbitrary (non-10MiB) size class, so we need to copy it
3781 : there. The staging buffer is discarded. If we are going to
3782 : a different size class, and we are "overwriting" (same
3783 : generation), then the original can also be discarded, but if
3784 : we are staying in the same size class, we can reuse the cache
3785 : line in place. */
3786 366077 : fd_accdb_cache_line_t * target_cache_line;
3787 366077 : if( FD_LIKELY( original_size_class==new_size_class && accs[ i ]._overwrite ) ) {
3788 : /* a reader holding a stale acc->cache_idx from this line's
3789 : previous life may hold a transient cache_try_pin pin, so
3790 : our own pin only bounds refcnt from below. */
3791 3922 : uint ow_rc = FD_VOLATILE_CONST( original_cache_line->refcnt );
3792 3922 : FD_TEST( ow_rc>0U && ow_rc!=FD_ACCDB_EVICT_SENTINEL );
3793 3922 : original_cache_line->key.generation = UINT_MAX;
3794 : /* Keep refcnt>=1 through the reuse window so CLOCK cannot
3795 : steal the line between invalidation and re-publish. The
3796 : pin is released after acc->cache_idx has been
3797 : republished. */
3798 3922 : original_cache_line->acc_idx = UINT_MAX;
3799 3922 : target_cache_line = original_cache_line;
3800 362155 : } else {
3801 : /* Step 1 took the final class's line (release_take_line). */
3802 362155 : if( FD_LIKELY( accs[ i ]._overwrite ) ) {
3803 : /* Atomically clear acc.VALID and acc.cache_idx BEFORE freeing
3804 : the line, so a reader cannot observe acc.VALID=1 with
3805 : acc.cache_idx pointing at a line that has been recycled to
3806 : another acc. evict_clear_acc_cache_ref uses the CLAIM
3807 : protocol to serialize with cold_load_acc. See the
3808 : size_class==7 path above for the refcnt claim and
3809 : persisted rationale. */
3810 101 : evict_clear_acc_cache_ref( &accdb->acc_pool[ original_acc_idx ], original_size_class, accs[ i ]._original_cache_idx );
3811 101 : original_cache_line->persisted = 1;
3812 101 : fd_racesan_hook( "accdb_release:pre_discard_claim" );
3813 101 : if( FD_LIKELY( FD_ATOMIC_CAS( &original_cache_line->refcnt, 1U, FD_ACCDB_EVICT_SENTINEL )==1U ) ) {
3814 101 : original_cache_line->acc_idx = UINT_MAX;
3815 101 : original_cache_line->key.generation = UINT_MAX;
3816 101 : FD_COMPILER_MFENCE();
3817 101 : FD_VOLATILE( original_cache_line->refcnt ) = 0;
3818 101 : cache_free_push( accdb, original_size_class, original_cache_line );
3819 101 : } else {
3820 0 : FD_ATOMIC_FETCH_AND_SUB( &original_cache_line->refcnt, 1U );
3821 0 : }
3822 101 : }
3823 :
3824 362155 : target_cache_line = cache_line( accdb, new_size_class, accs[ i ]._final_cache_idx );
3825 362155 : }
3826 :
3827 366077 : target_cache_line->persisted = 0;
3828 366077 : target_cache_line->referenced = 1;
3829 366077 : committed_line = target_cache_line;
3830 366077 : }
3831 :
3832 : /* For non-overwrite commits, the original cache line (if any) still
3833 : holds valid ancestor data but is no longer pinned. Mark it as
3834 : recently used so the CLOCK algorithm retains it. */
3835 366104 : if( FD_UNLIKELY( !accs[ i ]._overwrite && original_cache_line ) ) {
3836 0 : original_cache_line->referenced = 1;
3837 0 : }
3838 :
3839 : /* The committed line keeps refcnt>=1 until acc->cache_idx is
3840 : published (the deferred unpin happens after the publish below).
3841 : The staging line, unless it is the committed line, is fully
3842 : freed to this join's free list. */
3843 366104 : if( new_size_class==7UL ) staging_line->referenced = 1;
3844 366077 : else release_free_staging_line( accdb, staging_line );
3845 :
3846 : /* Update the accounts index for this committed write. For an
3847 : overwrite (same fork+generation), update the existing acc
3848 : in place. Otherwise allocate a new acc, prepend it
3849 : to the hash chain, and record the write in a txn linked to
3850 : the fork so advance_root can clean up old versions. */
3851 366104 : if( FD_LIKELY( accs[ i ]._overwrite ) ) {
3852 4025 : accdb->metrics->accounts_committed_overwrite_per_class[ new_size_class ]++;
3853 4025 : committed_line->acc_idx = original_acc_idx;
3854 :
3855 4025 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ original_acc_idx ];
3856 : /* The offset was already atomically swapped to FD_ACCDB_OFF_INVAL
3857 : and bytes freed above, so just update the metadata and
3858 : re-publish the cache location. CAS-loop preserves CLAIM bit
3859 : (a concurrent evict_clear_acc_cache_ref or acc_unlink may
3860 : hold it) and clears VALID; a plain store would clobber CLAIM
3861 : and break those protocols. */
3862 4025 : uint pd = accs[ i ].pd_write ? FD_ACCDB_SIZE_PD_WRITE_BIT : 0U;
3863 4025 : for(;;) {
3864 4025 : uint cur = FD_VOLATILE_CONST( accmeta->executable_size );
3865 4025 : uint nxt = (cur & (FD_ACCDB_SIZE_CACHE_CLAIM_BIT|FD_ACCDB_SIZE_PD_WRITE_BIT))
3866 4025 : | FD_ACCDB_SIZE_PACK( (uint)accs[ i ].data_len, accs[ i ].executable )
3867 4025 : | pd;
3868 4025 : if( FD_LIKELY( FD_ATOMIC_CAS( &accmeta->executable_size, cur, nxt )==cur ) ) break;
3869 2 : FD_SPIN_PAUSE();
3870 2 : }
3871 4025 : accmeta->lamports = accs[ i ].lamports;
3872 4025 : fd_racesan_hook( "accdb_overwrite:mid_inplace" );
3873 :
3874 4025 : fd_memcpy( committed_line->owner, accs[ i ].owner, 32UL );
3875 4025 : fd_memcpy( committed_line->key.pubkey, accmeta->key.pubkey, 32UL );
3876 4025 : committed_line->key.generation = accmeta->key.generation;
3877 4025 : committed_line->acc_idx = original_acc_idx;
3878 4025 : FD_VOLATILE( accmeta->cache_idx ) = FD_ACCDB_ACC_CIDX_PACK( (uint)new_size_class, (uint)cache_line_idx( accdb, new_size_class, committed_line ) );
3879 : /* Atomic OR so a concurrent evict_clear_acc_cache_ref's CLAIM
3880 : clear (FETCH_AND_AND with ~CLAIM) cannot be lost by an RMW
3881 : race with a plain |= store. */
3882 4025 : FD_ATOMIC_FETCH_AND_OR( &accmeta->executable_size, FD_ACCDB_SIZE_CACHE_VALID_BIT );
3883 :
3884 : /* Now that acc->cache_idx is published, unpin so CLOCK can
3885 : eventually evict it. For same-size overwrites, committed_line
3886 : IS the reused original_cache_line. For cross-size overwrites,
3887 : committed_line is the staging line or the line release_take_line
3888 : returned, still holding the pin taken for it. */
3889 4025 : FD_ATOMIC_FETCH_AND_SUB( &committed_line->refcnt, 1U );
3890 4025 : committed_line->referenced = 1;
3891 362079 : } else {
3892 362079 : accdb->metrics->accounts_committed_new_per_class[ new_size_class ]++;
3893 362079 : fd_accdb_accmeta_t * accmeta = acc_pool_acquire( accdb->acc_pool_join );
3894 362079 : if( FD_UNLIKELY( !accmeta ) ) FD_LOG_ERR(( "accounts database ran out of space, increase [accounts.max_accounts], current value is %lu", acc_pool_ele_max( accdb->acc_pool_join ) ));
3895 362079 : ulong acc_idx = acc_pool_idx( accdb->acc_pool_join, accmeta );
3896 362079 : fd_memcpy( accmeta->key.pubkey, accs[ i ].pubkey, 32UL );
3897 362079 : accmeta->lamports = accs[ i ].lamports;
3898 362079 : accmeta->executable_size = FD_ACCDB_SIZE_PACK( (uint)accs[ i ].data_len, accs[ i ].executable )
3899 362079 : | (accs[ i ].pd_write ? FD_ACCDB_SIZE_PD_WRITE_BIT : 0U);
3900 362079 : accmeta->key.generation = accs[ i ]._generation;
3901 362079 : accmeta->offset_fork = fd_accdb_acc_pack_offset_fork( FD_ACCDB_OFF_INVAL, accs[ i ]._fork_id );
3902 :
3903 : /* Publish in the cache BEFORE the acc_map head so that a
3904 : concurrent acquire that finds this acc in the hash chain will
3905 : also find a cache hit, rather than inserting a conflicting
3906 : placeholder cache acc. */
3907 362079 : committed_line->acc_idx = (uint)acc_idx;
3908 362079 : fd_memcpy( committed_line->owner, accs[ i ].owner, 32UL );
3909 362079 : fd_memcpy( committed_line->key.pubkey, accmeta->key.pubkey, 32UL );
3910 362079 : committed_line->key.generation = accmeta->key.generation;
3911 362079 : FD_VOLATILE( accmeta->cache_idx ) = FD_ACCDB_ACC_CIDX_PACK( (uint)new_size_class, (uint)cache_line_idx( accdb, new_size_class, committed_line ) );
3912 : /* Atomic OR so a concurrent evict_clear_acc_cache_ref's CLAIM
3913 : clear (FETCH_AND_AND with ~CLAIM) cannot be lost by an RMW
3914 : race with a plain |= store. */
3915 362079 : FD_ATOMIC_FETCH_AND_OR( &accmeta->executable_size, FD_ACCDB_SIZE_CACHE_VALID_BIT );
3916 :
3917 : /* Now that acc->cache_idx is published, unpin it so
3918 : CLOCK can eventually evict it. */
3919 362079 : FD_ATOMIC_FETCH_AND_SUB( &committed_line->refcnt, 1U );
3920 362079 : committed_line->referenced = 1;
3921 :
3922 : /* CAS loop to prepend to the hash chain. Succeeds on the first
3923 : try in most cases, but a concurrent acc_unlink CAS removing
3924 : the old head can change acc_map[idx] between our load and
3925 : CAS. Multiple concurrent releases may also race on the head
3926 : pointer — the CAS retry handles this. */
3927 367960 : for(;;) {
3928 367960 : uint old_head = FD_VOLATILE_CONST( accdb->acc_map[ accs[ i ]._acc_map_idx ] );
3929 367960 : accmeta->map.next = old_head;
3930 367960 : FD_COMPILER_MFENCE();
3931 367960 : fd_racesan_hook( "accdb_release:pre_chain_cas" );
3932 367960 : if( FD_LIKELY( FD_ATOMIC_CAS( &accdb->acc_map[ accs[ i ]._acc_map_idx ], old_head, (uint)acc_idx )==old_head ) ) break;
3933 727 : FD_SPIN_PAUSE();
3934 727 : }
3935 :
3936 : /* CONCURRENCY: The cache acc is published before the acc_map
3937 : head so that a concurrent fd_accdb_acquire reader that
3938 : observes the new head also finds a cache hit, preventing
3939 : duplicate cache insertion.
3940 :
3941 : (1) The CAS on acc_map[idx] serializes head-pointer mutations
3942 : from concurrent releases onto the same chain without any
3943 : external lock.
3944 :
3945 : (2) The FD_COMPILER_MFENCE above ensures stores to the acc node
3946 : fields (pubkey, lamports, size, generation, fork_id,
3947 : offset, map.next) are ordered before the CAS that publishes
3948 : the new head. On x86-64 (TSO), hardware also guarantees
3949 : this, but the compiler fence is needed to prevent the
3950 : compiler from reordering the stores. A reader that
3951 : observes the new head is guaranteed to see a fully
3952 : initialized node. A reader that has not yet seen the new
3953 : head simply traverses the previous (still valid) chain.
3954 :
3955 : (3) A concurrent acc_unlink (advance_root / purge) may CAS the
3956 : head away between our load and CAS here. The CAS retry
3957 : loop handles this. */
3958 :
3959 362079 : fd_accdb_txn_t * txn = txn_pool_acquire( accdb->txn_pool );
3960 362079 : FD_TEST( txn ); /* Sized so it always succeeds */
3961 362079 : txn->acc_pool_idx = (uint)acc_idx;
3962 362079 : uint txn_idx = (uint)txn_pool_idx( accdb->txn_pool, txn );
3963 365660 : for(;;) {
3964 365660 : uint old_head = FD_VOLATILE_CONST( accdb->fork_pool[ accs[ i ]._fork_id ].shmem->txn_head );
3965 365660 : txn->fork.next = old_head;
3966 366036 : if( FD_LIKELY( FD_ATOMIC_CAS( &accdb->fork_pool[ accs[ i ]._fork_id ].shmem->txn_head, old_head, txn_idx )==old_head ) ) break;
3967 >1844*10^16 : FD_SPIN_PAUSE();
3968 >1844*10^16 : }
3969 :
3970 362079 : FD_ATOMIC_FETCH_AND_ADD( &accdb->shmem->shmetrics->accounts_total, 1UL );
3971 362079 : }
3972 366104 : }
3973 :
3974 : // STEP 3.
3975 : // Finally, we release the cache class reservations we took at the
3976 : // beginning when we acquired these cache lines. They go back to
3977 : // the join's credits (res_give), surplus to the shared counter.
3978 :
3979 544037 : ulong refund[ FD_ACCDB_CACHE_CLASS_CNT ] = {0};
3980 1154019 : for( ulong i=0UL; i<accs_cnt; i++ ) {
3981 609982 : if( FD_LIKELY( accs[ i ]._original_size_class!=ULONG_MAX ) ) {
3982 173315 : if( FD_UNLIKELY( res_tracked( accdb, accs[ i ]._original_size_class ) ) ) {
3983 0 : refund[ accs[ i ]._original_size_class ]++;
3984 0 : }
3985 173315 : }
3986 609982 : if( FD_UNLIKELY( accs[ i ]._writable ) ) {
3987 3532555 : for( ulong j=0UL; j<FD_ACCDB_CACHE_CLASS_CNT; j++ ) {
3988 3134947 : if( FD_UNLIKELY( res_tracked( accdb, j ) ) ) {
3989 0 : refund[ j ]++;
3990 0 : }
3991 3134947 : }
3992 397608 : }
3993 609982 : }
3994 4848174 : for( ulong k=0UL; k<FD_ACCDB_CACHE_CLASS_CNT; k++ ) {
3995 4304137 : if( FD_UNLIKELY( refund[ k ] ) ) res_give( accdb, k, refund[ k ] );
3996 4304137 : }
3997 :
3998 544037 : FD_COMPILER_MFENCE();
3999 544037 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4000 544037 : }
4001 :
4002 : void
4003 : fd_accdb_release( fd_accdb_t * accdb,
4004 : ulong accs_cnt,
4005 536392 : fd_acc_t * accs ) {
4006 536392 : FD_TEST( accdb->acquire_state==FD_ACCDB_ACQUIRE_STATE_OPEN );
4007 536392 : release_inner( accdb, accs_cnt, accs );
4008 536392 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_IDLE;
4009 536392 : }
4010 :
4011 : void
4012 : fd_accdb_release_ab( fd_accdb_t * accdb,
4013 : ulong accs_cnt,
4014 : fd_acc_t * accs,
4015 : ulong execs_cnt,
4016 7298 : fd_acc_t * execs ) {
4017 7298 : FD_TEST( accdb->acquire_state==FD_ACCDB_ACQUIRE_STATE_OPEN );
4018 7298 : release_inner( accdb, accs_cnt, accs );
4019 7298 : if( FD_LIKELY( execs_cnt ) ) release_inner( accdb, execs_cnt, execs );
4020 7298 : accdb->acquire_state = FD_ACCDB_ACQUIRE_STATE_IDLE;
4021 7298 : }
4022 :
4023 : fd_acc_t
4024 : fd_accdb_read_one( fd_accdb_t * accdb,
4025 : fd_accdb_fork_id_t fork_id,
4026 171791 : uchar const * pubkey ) {
4027 171791 : fd_acc_t acc;
4028 171791 : fd_accdb_acquire( accdb, fork_id, 1UL, &pubkey, (int[]){0}, &acc );
4029 171791 : return acc;
4030 171791 : }
4031 :
4032 : void
4033 : fd_accdb_unread_one( fd_accdb_t * accdb,
4034 171809 : fd_acc_t * acc ) {
4035 171809 : fd_accdb_release( accdb, 1UL, acc );
4036 171809 : }
4037 :
4038 : fd_acc_t
4039 : fd_accdb_write_one( fd_accdb_t * accdb,
4040 : fd_accdb_fork_id_t fork_id,
4041 362554 : uchar const * pubkey ) {
4042 362554 : fd_acc_t acc;
4043 362554 : fd_accdb_acquire( accdb, fork_id, 1UL, &pubkey, (int[]){1}, &acc );
4044 362554 : return acc;
4045 362554 : }
4046 :
4047 : void
4048 : fd_accdb_unwrite_one( fd_accdb_t * accdb,
4049 365896 : fd_acc_t * acc ) {
4050 365896 : fd_accdb_release( accdb, 1UL, acc );
4051 365896 : }
4052 :
4053 : int
4054 : fd_accdb_read_one_nocache( fd_accdb_t * accdb,
4055 : fd_accdb_fork_id_t fork_id,
4056 : uchar const * pubkey,
4057 : ulong * out_lamports,
4058 : int * out_executable,
4059 : uchar * out_owner,
4060 : uchar * out_data,
4061 4965 : ulong * out_data_len ) {
4062 : /* Publish epoch — protects against compaction freeing the partition
4063 : under us during the preadv2 path. Readonly joiners publish to a
4064 : separate writable fseq scanned via external_epoch_slots. */
4065 4965 : FD_COMPILER_MFENCE();
4066 4965 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
4067 4965 : FD_HW_MFENCE();
4068 :
4069 : /// STEP 1.
4070 : /// Walk the hash chain at acc_map[hash(pubkey)] using the same
4071 : // visibility test as fd_accdb_acquire_inner. See that function
4072 : // for the detailed safety argument under concurrent prepend.
4073 4965 : uint root_generation = accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
4074 4965 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
4075 4965 : ulong hash = fd_hash32( pubkey, accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL);
4076 4965 : uint acc_idx = FD_VOLATILE_CONST( accdb->acc_map[ hash ] );
4077 4965 : fd_accdb_accmeta_t const * accmeta = NULL;
4078 5082 : while( acc_idx!=UINT_MAX ) {
4079 5042 : fd_accdb_accmeta_t const * candidate = &accdb->acc_pool[ acc_idx ];
4080 5042 : uint next_idx = FD_VOLATILE_CONST( candidate->map.next );
4081 5042 : if( FD_UNLIKELY( (candidate->key.generation>root_generation &&
4082 5042 : fd_accdb_acc_fork_id(candidate)!=fork_id.val &&
4083 5042 : !descends_set_test( fork->descends, fd_accdb_acc_fork_id(candidate) )) ) ||
4084 5042 : memcmp( pubkey, candidate->key.pubkey, 32UL ) ) {
4085 117 : acc_idx = next_idx;
4086 117 : continue;
4087 117 : }
4088 4925 : accmeta = candidate;
4089 4925 : break;
4090 5042 : }
4091 :
4092 4965 : if( FD_UNLIKELY( !accmeta ) ) {
4093 40 : accdb->metrics->accounts_acquired_per_class[ 0 ]++;
4094 40 : *out_lamports = 0UL;
4095 40 : FD_COMPILER_MFENCE();
4096 40 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4097 40 : return FD_ACCDB_READ_ONE_NOCACHE_MISS;
4098 40 : }
4099 :
4100 : /// STEP 2.
4101 : /// Snapshot acc fields. The acc element's metadata is effectively
4102 : /// immutable from the perspective of cross-fork readers (see the
4103 : /// comment block in fd_accdb.h about cross-fork reads). */
4104 4925 : uint snap_es = FD_VOLATILE_CONST( accmeta->executable_size );
4105 4925 : uint snap_gen = accmeta->key.generation;
4106 4925 : ulong snap_lamports = accmeta->lamports;
4107 4925 : uint snap_cidx = FD_VOLATILE_CONST( accmeta->cache_idx );
4108 4925 : ulong data_len = (ulong)FD_ACCDB_SIZE_DATA( snap_es );
4109 4925 : ulong read_data_len = out_data ? data_len : 0UL;
4110 4925 : int executable = FD_ACCDB_SIZE_EXEC( snap_es );
4111 :
4112 4925 : accdb->metrics->accounts_acquired_per_class[ fd_accdb_cache_class( data_len ) ]++;
4113 :
4114 4925 : if( FD_UNLIKELY( !snap_lamports ) ) {
4115 0 : *out_lamports = 0UL;
4116 0 : FD_COMPILER_MFENCE();
4117 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4118 0 : return FD_ACCDB_READ_ONE_NOCACHE_MISS;
4119 0 : }
4120 :
4121 : /// STEP 3.
4122 : /// Cache hit fast path with try-read-test (ABA) loop. Same
4123 : /// primitives as cache_try_pin: re-check key.generation + pubkey
4124 : /// before and after the bulk copy, and bail to the disk path if the
4125 : /// line was claimed for eviction (refcnt ==
4126 : /// FD_ACCDB_EVICT_SENTINEL). No CAS on refcnt, we never pin the
4127 : /// line.
4128 4925 : if( FD_LIKELY( FD_ACCDB_SIZE_CACHE_VALID( snap_es ) && snap_cidx!=FD_ACCDB_ACC_CIDX_INVAL ) ) {
4129 4925 : ulong cls = FD_ACCDB_ACC_CIDX_CLASS( snap_cidx );
4130 4925 : ulong idx = FD_ACCDB_ACC_CIDX_IDX ( snap_cidx );
4131 4925 : fd_accdb_cache_line_t * line = cache_line( accdb, cls, idx );
4132 :
4133 4925 : for(;;) {
4134 4925 : uint seq0 = FD_VOLATILE_CONST( line->seq );
4135 4925 : FD_COMPILER_MFENCE();
4136 4925 : uint gen0 = FD_VOLATILE_CONST( line->key.generation );
4137 4925 : uint rc0 = FD_VOLATILE_CONST( line->refcnt );
4138 4925 : uint ai0 = FD_VOLATILE_CONST( line->acc_idx );
4139 4925 : if( FD_UNLIKELY( rc0==FD_ACCDB_EVICT_SENTINEL ) ) goto miss;
4140 4925 : if( FD_UNLIKELY( gen0!=snap_gen ) ) goto miss;
4141 4925 : if( FD_UNLIKELY( memcmp( line->key.pubkey, pubkey, 32UL ) ) ) goto miss;
4142 : /* acc_idx==UINT_MAX is the "loading" sentinel set by cold_load_acc
4143 : before the preadv2 fills the line. CACHE_VALID can be observed
4144 : set while the bytes are still stale, so fall to the disk path
4145 : (which spins on offset_fork and reads from the file) rather
4146 : than copying garbage. */
4147 4925 : if( FD_UNLIKELY( ai0==UINT_MAX ) ) goto miss;
4148 :
4149 4925 : FD_COMPILER_MFENCE();
4150 4925 : fd_racesan_hook( "accdb_nocache:pre_copy" );
4151 4925 : memcpy( out_owner, line->owner, 32UL );
4152 4925 : if( read_data_len ) memcpy( out_data, (uchar const *)(line+1UL), read_data_len );
4153 4925 : fd_racesan_hook( "accdb_nocache:post_copy" );
4154 4925 : FD_COMPILER_MFENCE();
4155 :
4156 4925 : uint gen1 = FD_VOLATILE_CONST( line->key.generation );
4157 4925 : uint rc1 = FD_VOLATILE_CONST( line->refcnt );
4158 4925 : uint ai1 = FD_VOLATILE_CONST( line->acc_idx );
4159 4925 : uint seq1 = FD_VOLATILE_CONST( line->seq );
4160 4925 : if( FD_UNLIKELY( seq1!=seq0 ) ) goto miss;
4161 4925 : if( FD_UNLIKELY( rc1==FD_ACCDB_EVICT_SENTINEL ) ) goto miss;
4162 4925 : if( FD_UNLIKELY( gen1!=snap_gen ) ) goto miss;
4163 4925 : if( FD_UNLIKELY( memcmp( line->key.pubkey, pubkey, 32UL ) ) ) goto miss;
4164 4925 : if( FD_UNLIKELY( ai1==UINT_MAX ) ) goto miss;
4165 :
4166 4925 : *out_lamports = snap_lamports;
4167 4925 : *out_executable = executable;
4168 4925 : *out_data_len = data_len;
4169 4925 : accdb->metrics->bytes_copied += read_data_len;
4170 4925 : FD_COMPILER_MFENCE();
4171 4925 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4172 4925 : return FD_ACCDB_READ_ONE_NOCACHE_CACHE;
4173 4925 : }
4174 4925 : }
4175 :
4176 0 : miss:;
4177 0 : accdb->metrics->accounts_not_found_per_class[ fd_accdb_cache_class( FD_ACCDB_SIZE_DATA( snap_es ) ) ]++;
4178 :
4179 : /// STEP 4.
4180 : /// Disk path. Spin until the writer publishes a real offset
4181 : /// (matches STEP 10 of fd_accdb_acquire_inner). Compaction may
4182 : /// concurrently relocate the record, but our published epoch
4183 : /// prevents the source partition from being freed until we exit
4184 : /// our critical section, so the bytes at the snapshotted offset
4185 : /// remain stable for the duration of the read.
4186 0 : fd_racesan_hook( "accdb_nocache:pre_offset" );
4187 0 : ulong off_packed = FD_VOLATILE_CONST( accmeta->offset_fork );
4188 0 : if( FD_UNLIKELY( (off_packed & FD_ACCDB_OFF_MASK)==FD_ACCDB_OFF_INVAL ) ) {
4189 0 : accdb->metrics->accounts_waited++;
4190 0 : while( FD_UNLIKELY( ((off_packed=FD_VOLATILE_CONST( accmeta->offset_fork )) & FD_ACCDB_OFF_MASK)==FD_ACCDB_OFF_INVAL ) ) {
4191 0 : fd_racesan_hook( "accdb_nocache:offset_wait" );
4192 0 : FD_SPIN_PAUSE();
4193 0 : }
4194 0 : }
4195 0 : ulong off = off_packed & FD_ACCDB_OFF_MASK;
4196 0 : fd_racesan_hook( "accdb_nocache:pre_preadv2" );
4197 :
4198 0 : struct iovec iovs[ 2 ] = {
4199 0 : { .iov_base = out_owner, .iov_len = 32UL },
4200 0 : { .iov_base = out_data, .iov_len = read_data_len },
4201 0 : };
4202 0 : ulong total = 32UL+read_data_len;
4203 0 : ulong start = off+offsetof( fd_accdb_disk_meta_t, owner );
4204 0 : ulong got = 0UL;
4205 0 : int nio = read_data_len ? 2 : 1;
4206 0 : while( FD_LIKELY( got<total ) ) {
4207 0 : long result = preadv2( accdb->fd, iovs, nio, (long)(start+got), 0 );
4208 0 : if( FD_UNLIKELY( -1==result && (errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK) ) ) continue;
4209 0 : else if( FD_UNLIKELY( -1==result ) ) FD_LOG_ERR(( "preadv2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
4210 0 : else if( FD_UNLIKELY( !result ) ) FD_LOG_ERR(( "accounts database is corrupt, data expected at offset %lu with size %lu exceeded file extents", start+got, total ));
4211 0 : fd_accdb_partition_read_bump( accdb, start+got, (ulong)result );
4212 0 : got += (ulong)result;
4213 0 : accdb->metrics->bytes_read += (ulong)result;
4214 0 : accdb->metrics->read_ops++;
4215 :
4216 0 : long r = result;
4217 0 : for( int v=0; v<nio; v++ ) {
4218 0 : if( (ulong)r>=iovs[ v ].iov_len ) {
4219 0 : r -= (long)iovs[ v ].iov_len;
4220 0 : iovs[ v ].iov_len = 0UL;
4221 0 : } else {
4222 0 : iovs[ v ].iov_base = (uchar *)iovs[ v ].iov_base + r;
4223 0 : iovs[ v ].iov_len -= (ulong)r;
4224 0 : break;
4225 0 : }
4226 0 : }
4227 0 : }
4228 :
4229 0 : *out_lamports = snap_lamports;
4230 0 : *out_executable = executable;
4231 0 : *out_data_len = data_len;
4232 :
4233 0 : FD_COMPILER_MFENCE();
4234 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4235 0 : return FD_ACCDB_READ_ONE_NOCACHE_DISK;
4236 0 : }
4237 :
4238 : int
4239 : fd_accdb_read_one_meta( fd_accdb_t * accdb,
4240 : fd_accdb_fork_id_t fork_id,
4241 : uchar const * pubkey,
4242 : uchar * out_owner,
4243 4965 : ulong * out_data_len ) {
4244 4965 : ulong lamports;
4245 4965 : int executable;
4246 4965 : return fd_accdb_read_one_nocache( accdb, fork_id, pubkey, &lamports, &executable,
4247 4965 : out_owner, NULL, out_data_len );
4248 4965 : }
4249 :
4250 : int
4251 : fd_accdb_exists( fd_accdb_t * accdb,
4252 : fd_accdb_fork_id_t fork_id,
4253 223555 : uchar const * pubkey ) {
4254 223555 : FD_COMPILER_MFENCE();
4255 223555 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
4256 223555 : FD_HW_MFENCE();
4257 :
4258 223555 : uint root_generation = accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
4259 223555 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
4260 223555 : ulong hash = fd_hash32( pubkey, accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL);
4261 223555 : uint acc = FD_VOLATILE_CONST( accdb->acc_map[ hash ] );
4262 234584 : while( acc!=UINT_MAX ) {
4263 11056 : fd_accdb_accmeta_t const * candidate_acc = &accdb->acc_pool[ acc ];
4264 11056 : uint next_acc = FD_VOLATILE_CONST( candidate_acc->map.next );
4265 :
4266 11056 : if( FD_UNLIKELY( (candidate_acc->key.generation>root_generation && fd_accdb_acc_fork_id(candidate_acc)!=fork_id.val && !descends_set_test( fork->descends, fd_accdb_acc_fork_id(candidate_acc) )) ) || memcmp( pubkey, candidate_acc->key.pubkey, 32UL ) ) {
4267 11029 : acc = next_acc;
4268 11029 : continue;
4269 11029 : }
4270 :
4271 27 : break;
4272 11056 : }
4273 :
4274 223555 : int result;
4275 225589 : if( FD_UNLIKELY( acc==UINT_MAX ) ) result = 0;
4276 >1844*10^16 : else result = !!FD_VOLATILE_CONST( accdb->acc_pool[ acc ].lamports );
4277 :
4278 223555 : FD_COMPILER_MFENCE();
4279 223555 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4280 223555 : return result;
4281 223555 : }
4282 :
4283 : int
4284 : fd_accdb_probe_pd_this_fork( fd_accdb_t * accdb,
4285 : fd_accdb_fork_id_t fork_id,
4286 : uchar const * pubkey,
4287 : int * out_pd_write,
4288 : ulong * out_data_len,
4289 5252 : ulong * out_lamports ) {
4290 5252 : FD_COMPILER_MFENCE();
4291 5252 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
4292 5252 : FD_HW_MFENCE();
4293 :
4294 5252 : uint root_generation = accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
4295 5252 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
4296 5252 : ulong hash = fd_hash32( pubkey, accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL);
4297 5252 : uint acc = FD_VOLATILE_CONST( accdb->acc_map[ hash ] );
4298 5649 : while( acc!=UINT_MAX ) {
4299 417 : fd_accdb_accmeta_t const * candidate_acc = &accdb->acc_pool[ acc ];
4300 417 : uint next_acc = FD_VOLATILE_CONST( candidate_acc->map.next );
4301 :
4302 417 : if( FD_UNLIKELY( (candidate_acc->key.generation>root_generation && fd_accdb_acc_fork_id(candidate_acc)!=fork_id.val && !descends_set_test( fork->descends, fd_accdb_acc_fork_id(candidate_acc) )) ) || memcmp( pubkey, candidate_acc->key.pubkey, 32UL ) ) {
4303 397 : acc = next_acc;
4304 397 : continue;
4305 397 : }
4306 :
4307 20 : break;
4308 417 : }
4309 :
4310 5252 : int pd = 0;
4311 5252 : int gen_match = 0;
4312 5252 : ulong len = 0UL;
4313 5252 : ulong lamports = 0UL;
4314 5252 : if( FD_LIKELY( acc!=UINT_MAX ) ) {
4315 19 : fd_accdb_accmeta_t const * m = &accdb->acc_pool[ acc ];
4316 19 : uint es = FD_VOLATILE_CONST( m->executable_size );
4317 19 : gen_match = ( m->key.generation==fork->shmem->generation );
4318 19 : pd = gen_match && FD_ACCDB_SIZE_PD_WRITE( es );
4319 19 : len = FD_ACCDB_SIZE_DATA( es );
4320 19 : lamports = FD_VOLATILE_CONST( m->lamports );
4321 19 : }
4322 :
4323 5252 : FD_COMPILER_MFENCE();
4324 5252 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4325 :
4326 5252 : *out_pd_write = pd;
4327 5252 : if( gen_match ) {
4328 19 : *out_data_len = len;
4329 19 : *out_lamports = lamports;
4330 19 : }
4331 5252 : return gen_match;
4332 5252 : }
4333 :
4334 : ulong
4335 : fd_accdb_lamports( fd_accdb_t * accdb,
4336 : fd_accdb_fork_id_t fork_id,
4337 7290 : uchar const * pubkey ) {
4338 7290 : FD_COMPILER_MFENCE();
4339 7290 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
4340 7290 : FD_HW_MFENCE();
4341 :
4342 7290 : uint root_generation = accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
4343 7290 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
4344 7290 : ulong hash = fd_hash32( pubkey, accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL);
4345 7290 : uint acc = FD_VOLATILE_CONST( accdb->acc_map[ hash ] );
4346 7573 : while( acc!=UINT_MAX ) {
4347 7288 : fd_accdb_accmeta_t const * candidate_acc = &accdb->acc_pool[ acc ];
4348 7288 : uint next_acc = FD_VOLATILE_CONST( candidate_acc->map.next );
4349 :
4350 7288 : if( FD_UNLIKELY( (candidate_acc->key.generation>root_generation && fd_accdb_acc_fork_id(candidate_acc)!=fork_id.val && !descends_set_test( fork->descends, fd_accdb_acc_fork_id(candidate_acc) )) ) || memcmp( pubkey, candidate_acc->key.pubkey, 32UL ) ) {
4351 283 : acc = next_acc;
4352 283 : continue;
4353 283 : }
4354 :
4355 7005 : break;
4356 7288 : }
4357 :
4358 7290 : ulong result;
4359 7290 : if( FD_UNLIKELY( acc==UINT_MAX ) ) result = 0UL;
4360 7004 : else result = FD_VOLATILE_CONST( accdb->acc_pool[ acc ].lamports );
4361 :
4362 7290 : FD_COMPILER_MFENCE();
4363 7290 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4364 7290 : return result;
4365 7290 : }
4366 :
4367 : ulong
4368 : fd_accdb_data_len( fd_accdb_t * accdb,
4369 : fd_accdb_fork_id_t fork_id,
4370 72537 : uchar const * pubkey ) {
4371 72537 : FD_COMPILER_MFENCE();
4372 72537 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( accdb->shmem->epoch );
4373 72537 : FD_HW_MFENCE();
4374 :
4375 72537 : uint root_generation = accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
4376 72537 : fd_accdb_fork_t * fork = &accdb->fork_pool[ fork_id.val ];
4377 72537 : ulong hash = fd_hash32( pubkey, accdb->shmem->seed )&(accdb->shmem->chain_cnt-1UL);
4378 72537 : uint acc = FD_VOLATILE_CONST( accdb->acc_map[ hash ] );
4379 75363 : while( acc!=UINT_MAX ) {
4380 73701 : fd_accdb_accmeta_t const * candidate_acc = &accdb->acc_pool[ acc ];
4381 73701 : uint next_acc = FD_VOLATILE_CONST( candidate_acc->map.next );
4382 :
4383 73701 : if( FD_UNLIKELY( (candidate_acc->key.generation>root_generation && fd_accdb_acc_fork_id(candidate_acc)!=fork_id.val && !descends_set_test( fork->descends, fd_accdb_acc_fork_id(candidate_acc) )) ) || memcmp( pubkey, candidate_acc->key.pubkey, 32UL ) ) {
4384 2826 : acc = next_acc;
4385 2826 : continue;
4386 2826 : }
4387 :
4388 70875 : break;
4389 73701 : }
4390 :
4391 72537 : ulong result;
4392 72537 : if( FD_UNLIKELY( acc==UINT_MAX || !FD_VOLATILE_CONST( accdb->acc_pool[ acc ].lamports ) ) ) result = 0UL;
4393 70870 : else result = FD_ACCDB_SIZE_DATA( FD_VOLATILE_CONST( accdb->acc_pool[ acc ].executable_size ) );
4394 :
4395 72537 : FD_COMPILER_MFENCE();
4396 72537 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4397 72537 : return result;
4398 72537 : }
4399 :
4400 : /* Dirty lines are staged in accdb->stage and written with one
4401 : pwritev2 per batch, so the kernel's inode lock is taken once per
4402 : batch instead of once per account. Evictions are performed
4403 : per-class in batches of up to PREEVICT_TICK_BUDGET records or
4404 : PREEVICT_BATCH_SZ bytes. While a line sits in a batch its account
4405 : can be read neither from the cache nor from disk, so a reader waits
4406 : for the batch write to finish. */
4407 :
4408 0 : #define PREEVICT_TICK_BUDGET (256UL)
4409 :
4410 : /* preevict_hungry_t is pre-eviction's snapshot of a class's
4411 : cache_free_hungry bits (taken and cleared at the start of a class
4412 : pass). preevict_target deals the lines the pass frees round-robin
4413 : to those joins' lists, or to the pre-evicting join's own list when
4414 : none ran short. A join that ran short once gets as many as one that
4415 : ran short often; any surplus is stealable. Joins the pass gave no
4416 : line (it freed too few) are put back by preevict_hungry_restore, so
4417 : a later pass serves them. */
4418 :
4419 : struct preevict_hungry {
4420 : ulong bits [ FD_ACCDB_MAX_JOINERS/64UL ];
4421 : ulong served[ FD_ACCDB_MAX_JOINERS/64UL ]; /* got a line this pass */
4422 : ulong next; /* join to try first */
4423 : };
4424 :
4425 : typedef struct preevict_hungry preevict_hungry_t;
4426 :
4427 : static void
4428 : preevict_hungry_take( preevict_hungry_t * h,
4429 : fd_accdb_shmem_t * shmem,
4430 : ulong size_class,
4431 0 : int force ) {
4432 0 : h->next = 0UL;
4433 0 : for( ulong w=0UL; w<FD_ACCDB_MAX_JOINERS/64UL; w++ ) {
4434 0 : ulong * bits = &shmem->cache_free_hungry[ size_class ].bits[ w ];
4435 0 : h->bits [ w ] = ( !force && FD_VOLATILE_CONST( *bits ) ) ? FD_ATOMIC_XCHG( bits, 0UL ) : 0UL;
4436 0 : h->served[ w ] = 0UL;
4437 0 : }
4438 0 : }
4439 :
4440 : static void
4441 : preevict_hungry_restore( preevict_hungry_t const * h,
4442 : fd_accdb_shmem_t * shmem,
4443 0 : ulong size_class ) {
4444 0 : for( ulong w=0UL; w<FD_ACCDB_MAX_JOINERS/64UL; w++ ) {
4445 0 : ulong unserved = h->bits[ w ] & ~h->served[ w ];
4446 0 : if( FD_UNLIKELY( unserved ) ) FD_ATOMIC_FETCH_AND_OR( &shmem->cache_free_hungry[ size_class ].bits[ w ], unserved );
4447 0 : }
4448 0 : }
4449 :
4450 : static inline ulong
4451 : preevict_hungry_find( preevict_hungry_t const * h,
4452 0 : ulong from ) {
4453 0 : for( ulong w=from/64UL; w<FD_ACCDB_MAX_JOINERS/64UL; w++ ) {
4454 0 : ulong m = h->bits[ w ] & ( w==from/64UL ? ~0UL<<( from%64UL ) : ~0UL );
4455 0 : if( m ) return w*64UL + (ulong)fd_ulong_find_lsb( m );
4456 0 : }
4457 0 : return ULONG_MAX;
4458 0 : }
4459 :
4460 : static inline ulong
4461 : preevict_target( preevict_hungry_t * h,
4462 0 : fd_accdb_t const * accdb ) {
4463 0 : ulong j = preevict_hungry_find( h, h->next );
4464 0 : if( j==ULONG_MAX ) j = preevict_hungry_find( h, 0UL );
4465 0 : if( j==ULONG_MAX ) return accdb->join_idx;
4466 0 : h->next = j+1UL;
4467 0 : h->served[ j/64UL ] |= 1UL<<( j%64UL );
4468 0 : return j;
4469 0 : }
4470 :
4471 : static void
4472 : preevict_flush( fd_accdb_t * accdb,
4473 : ulong size_class,
4474 : preevict_hungry_t * hungry,
4475 : fd_accdb_cache_line_t ** pend_line,
4476 : ulong const * pend_sz,
4477 : ulong pend_cnt,
4478 0 : ulong stage_sz ) {
4479 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
4480 :
4481 : /* advertise to external observers that writes are in progress */
4482 0 : FD_COMPILER_MFENCE();
4483 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = FD_VOLATILE_CONST( shmem->epoch );
4484 0 : FD_HW_MFENCE();
4485 :
4486 0 : fd_racesan_hook( "preevict:pre_synth" );
4487 :
4488 : /* Invalidate the records so compaction does not relocate them while
4489 : we are evicting. */
4490 0 : ulong freed = 0UL;
4491 0 : for( ulong i=0UL; i<pend_cnt; i++ ) {
4492 0 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ pend_line[ i ]->acc_idx ];
4493 0 : ulong old_offset = fd_accdb_acc_xchg_offset( accmeta, FD_ACCDB_OFF_INVAL );
4494 0 : if( FD_UNLIKELY( old_offset!=FD_ACCDB_OFF_INVAL ) ) {
4495 0 : fd_accdb_shmem_bytes_freed( shmem, old_offset, pend_sz[ i ] );
4496 0 : freed += pend_sz[ i ];
4497 0 : }
4498 0 : }
4499 0 : FD_ATOMIC_FETCH_AND_SUB( &shmem->shmetrics->disk_used_bytes, freed );
4500 :
4501 0 : ulong file_off = allocate_next_write( accdb, stage_sz );
4502 0 : ulong written = 0UL;
4503 0 : while( written<stage_sz ) {
4504 0 : struct iovec iov = { .iov_base = accdb->stage+written, .iov_len = stage_sz-written };
4505 0 : long result = pwritev2( accdb->fd, &iov, 1, (long)(file_off+written), 0 );
4506 0 : if( FD_UNLIKELY( result==-1 && errno==EINTR ) ) continue;
4507 0 : else if( FD_UNLIKELY( result<=0 ) ) FD_LOG_ERR(( "pwritev2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
4508 0 : written += (ulong)result;
4509 0 : accdb->metrics->bytes_written += (ulong)result;
4510 0 : accdb->metrics->write_ops++;
4511 0 : }
4512 0 : FD_ATOMIC_FETCH_AND_ADD( &shmem->shmetrics->disk_used_bytes, stage_sz );
4513 :
4514 0 : fd_racesan_hook( "preevict:post_write" );
4515 :
4516 0 : FD_COMPILER_MFENCE();
4517 0 : ulong off = file_off;
4518 0 : for( ulong i=0UL; i<pend_cnt; i++ ) {
4519 0 : fd_accdb_cache_line_t * line = pend_line[ i ];
4520 0 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ line->acc_idx ];
4521 0 : accmeta->offset_fork = fd_accdb_acc_pack_offset_fork( off, fd_accdb_acc_fork_id( accmeta ) );
4522 0 : off += pend_sz[ i ];
4523 0 : FD_COMPILER_MFENCE();
4524 0 : cache_line_free( accdb, preevict_target( hungry, accdb ), size_class, line );
4525 0 : }
4526 0 : accdb->metrics->accounts_preevicted += pend_cnt;
4527 0 : accdb->metrics->accounts_preevicted_per_class[ size_class ] += pend_cnt;
4528 :
4529 0 : FD_COMPILER_MFENCE();
4530 0 : FD_VOLATILE( *accdb->my_epoch_slot ) = ULONG_MAX;
4531 0 : }
4532 :
4533 : /* background_preevict pre-evicts cache lines in the background to keep
4534 : the per-join free lists populated ahead of demand. For each class
4535 : whose immediately available capacity has dropped below low_water,
4536 : a bounded CLOCK sweep claims lines, writes dirty ones to disk, and
4537 : pushes them onto the free list until available capacity reaches
4538 : target. Immediately available capacity includes both the free
4539 : lists and the never-initialized tail of the class, since foreground
4540 : allocators can consume either path without evicting resident data.
4541 :
4542 : Budget: at most PREEVICT_TICK_BUDGET CLOCK ticks per class per call
4543 : to keep the background loop responsive. The function is called every tick of
4544 : fd_accdb_background, so large refills happen across several ticks
4545 : rather than blocking. The low_water / target thresholds are static
4546 : per-class watermarks computed at initialization; pre-eviction only
4547 : converts resident lines into free-list entries and does not consume
4548 : cache-slot reservations.
4549 :
4550 : The watermark test sums every join's free-list count, reading a
4551 : line each join writes on its hot path; so it runs at most once per
4552 : FD_ACCDB_BG_RATE_TICKS (tens of microseconds) instead of every
4553 : tick of an otherwise idle loop (per join, preevict_interval; tests
4554 : set it with fd_accdb_set_preevict_interval).
4555 :
4556 : force: when non-zero, ignore the watermark and sweep every line in
4557 : every class. Always 0 in normal operation; used only by
4558 : test_accdb_racesan to deterministically exercise the writeback path
4559 : without manufacturing real cache pressure. */
4560 :
4561 : static void
4562 : background_preevict( fd_accdb_t * accdb,
4563 : int * charge_busy,
4564 0 : int force ) {
4565 0 : fd_accdb_shmem_t * shmem = accdb->shmem;
4566 :
4567 0 : if( FD_LIKELY( !force ) ) {
4568 0 : long now = fd_tickcount();
4569 0 : if( FD_LIKELY( now-accdb->preevict_ticks<accdb->preevict_interval ) ) {
4570 0 : *charge_busy |= accdb->preevict_pending;
4571 0 : return;
4572 0 : }
4573 0 : accdb->preevict_ticks = now;
4574 0 : accdb->preevict_pending = 0;
4575 0 : }
4576 :
4577 : /* A batch has to fit one disk partition. */
4578 0 : FD_TEST( accdb->stage );
4579 0 : ulong batch_max = fd_ulong_min( PREEVICT_BATCH_SZ, shmem->partition_sz );
4580 :
4581 0 : fd_accdb_cache_line_t * pend_line[ PREEVICT_TICK_BUDGET ];
4582 0 : ulong pend_sz [ PREEVICT_TICK_BUDGET ];
4583 0 : ulong pend_cnt = 0UL;
4584 0 : ulong stage_sz = 0UL;
4585 :
4586 0 : for( ulong size_class=0UL; size_class<FD_ACCDB_CACHE_CLASS_CNT; size_class++ ) {
4587 0 : ulong target = shmem->cache_free_target[ size_class ];
4588 0 : ulong max_c = shmem->cache_class_max[ size_class ];
4589 0 : ulong init = fd_ulong_min( FD_VOLATILE_CONST( shmem->cache_class_init[ size_class ].val ), max_c );
4590 0 : ulong freec = cache_free_total( shmem, size_class );
4591 0 : ulong live = init>freec ? init-freec : 0UL;
4592 0 : ulong avail = max_c-live;
4593 0 : if( FD_LIKELY( !force && avail>=shmem->cache_free_low_water[ size_class ] ) ) continue;
4594 :
4595 0 : *charge_busy = 1;
4596 :
4597 0 : ulong budget = force ? init : PREEVICT_TICK_BUDGET;
4598 0 : ulong evicted = 0UL;
4599 0 : if( FD_UNLIKELY( force ) ) target = max_c; /* sweep everything */
4600 :
4601 0 : preevict_hungry_t hungry[1];
4602 0 : preevict_hungry_take( hungry, shmem, size_class, force );
4603 :
4604 0 : for( ulong tick=0UL; tick<budget && avail+evicted<target; tick++ ) {
4605 : /* Only sweep the lazily initialized prefix. cache_class_init
4606 : may transiently exceed max_c during the acquire_cache_line
4607 : overflow/undo path, so clamp it before using it as the wrap
4608 : bound. */
4609 0 : init = fd_ulong_min( FD_VOLATILE_CONST( shmem->cache_class_init[ size_class ].val ), max_c );
4610 0 : if( FD_UNLIKELY( !init ) ) break;
4611 :
4612 0 : ulong hand = FD_ATOMIC_FETCH_AND_ADD( &shmem->clock_hand[ size_class ].val, 1UL ) % init;
4613 :
4614 0 : fd_accdb_cache_line_t * line = cache_line( accdb, size_class, hand );
4615 :
4616 0 : if( FD_UNLIKELY( line->key.generation==UINT_MAX && line->acc_idx==UINT_MAX ) ) continue;
4617 :
4618 0 : uint rc = FD_VOLATILE_CONST( line->refcnt );
4619 0 : if( FD_UNLIKELY( rc ) ) continue;
4620 :
4621 0 : if( FD_UNLIKELY( line->referenced ) ) {
4622 0 : line->referenced = 0;
4623 0 : continue;
4624 0 : }
4625 :
4626 0 : if( FD_UNLIKELY( FD_ATOMIC_CAS( &line->refcnt, 0U, FD_ACCDB_EVICT_SENTINEL )!=0U ) ) continue;
4627 :
4628 0 : if( FD_UNLIKELY( line->acc_idx==UINT_MAX && line->key.generation==UINT_MAX ) ) {
4629 0 : FD_VOLATILE( line->refcnt ) = 0;
4630 0 : continue;
4631 0 : }
4632 :
4633 0 : uint acc_idx = line->acc_idx;
4634 : #if FD_TMPL_USE_HANDHOLDING
4635 : uint line_gen FD_FN_UNUSED = line->key.generation;
4636 : #endif
4637 0 : if( FD_LIKELY( acc_idx!=UINT_MAX ) ) {
4638 0 : evict_clear_acc_cache_ref( &accdb->acc_pool[ acc_idx ], size_class, hand );
4639 0 : }
4640 0 : line->key.generation = UINT_MAX;
4641 0 : if( FD_UNLIKELY( line->persisted || acc_idx==UINT_MAX ) ) {
4642 0 : cache_line_free( accdb, preevict_target( hungry, accdb ), size_class, line );
4643 0 : evicted++;
4644 0 : continue;
4645 0 : }
4646 :
4647 0 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ acc_idx ];
4648 : #if FD_TMPL_USE_HANDHOLDING
4649 : FD_TEST( line_gen==accmeta->key.generation &&
4650 : !memcmp( line->key.pubkey, accmeta->key.pubkey, 32UL ) );
4651 : #endif
4652 0 : uint data_sz = FD_ACCDB_SIZE_DATA( accmeta->executable_size );
4653 0 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t)+data_sz;
4654 :
4655 0 : if( FD_UNLIKELY( pend_cnt && ( stage_sz+entry_sz>batch_max || pend_cnt==PREEVICT_TICK_BUDGET ) ) ) {
4656 0 : preevict_flush( accdb, size_class, hungry, pend_line, pend_sz, pend_cnt, stage_sz );
4657 0 : pend_cnt = 0UL;
4658 0 : stage_sz = 0UL;
4659 0 : }
4660 :
4661 0 : fd_accdb_disk_meta_t * meta = (fd_accdb_disk_meta_t *)( accdb->stage+stage_sz );
4662 0 : fd_memcpy( meta->pubkey, accmeta->key.pubkey, 32UL );
4663 0 : meta->size = data_sz;
4664 0 : meta->generation = accmeta->key.generation;
4665 0 : fd_memcpy( meta->owner, line->owner, 32UL );
4666 0 : fd_memcpy( meta+1, line+1UL, data_sz );
4667 :
4668 0 : pend_line[ pend_cnt ] = line;
4669 0 : pend_sz [ pend_cnt ] = entry_sz;
4670 0 : pend_cnt++;
4671 0 : stage_sz += entry_sz;
4672 0 : evicted++;
4673 0 : }
4674 :
4675 0 : if( pend_cnt ) {
4676 0 : preevict_flush( accdb, size_class, hungry, pend_line, pend_sz, pend_cnt, stage_sz );
4677 0 : pend_cnt = 0UL;
4678 0 : stage_sz = 0UL;
4679 0 : }
4680 0 : preevict_hungry_restore( hungry, shmem, size_class );
4681 :
4682 : /* Still below low water (budget ran out or too little was
4683 : evictable): the next scan will sweep this class again, so stay
4684 : busy until then rather than letting the tile park. */
4685 0 : if( FD_UNLIKELY( !force && avail+evicted<shmem->cache_free_low_water[ size_class ] ) ) {
4686 0 : accdb->preevict_pending = 1;
4687 0 : }
4688 0 : }
4689 0 : }
4690 :
4691 : int
4692 : fd_accdb_snapshot_write_batch( fd_accdb_t * accdb,
4693 : fd_accdb_fork_id_t fork_id,
4694 : ulong cnt,
4695 : uchar const * const pubkeys[],
4696 : ulong const slots[],
4697 : ulong const lamports[],
4698 : ulong const data_lens[],
4699 : int const executables[],
4700 : ulong const file_offsets[],
4701 : ulong * accounts_ignored,
4702 : ulong * accounts_replaced,
4703 : ulong * accounts_loaded,
4704 : ulong * out_replaced_lamports,
4705 : ulong * out_ignored_lamports,
4706 0 : uchar * results ) {
4707 0 : #define CHAIN_LOCKED (UINT_MAX-1U)
4708 :
4709 0 : FD_TEST( cnt>0UL && cnt<=8UL );
4710 :
4711 0 : int incremental = fork_id.val!=USHORT_MAX;
4712 :
4713 0 : fd_accdb_fork_t * fork = NULL;
4714 0 : uint fork_gen = 0U;
4715 0 : if( FD_UNLIKELY( incremental ) ) {
4716 0 : fork = &accdb->fork_pool[ fork_id.val ];
4717 0 : fork_gen = fork->shmem->generation;
4718 0 : }
4719 :
4720 0 : ulong seed = accdb->shmem->seed;
4721 0 : ulong chain_msk = accdb->shmem->chain_cnt - 1UL;
4722 0 : if( FD_UNLIKELY( !incremental && accdb->shmem->root_fork_id.val==USHORT_MAX ) ) {
4723 0 : FD_LOG_ERR(( "snapshot_write_batch called without a root fork attached" ));
4724 0 : }
4725 0 : uint gen = incremental ? 0U : accdb->fork_pool[ accdb->shmem->root_fork_id.val ].shmem->generation;
4726 :
4727 0 : ulong ignored = 0UL;
4728 0 : ulong replaced = 0UL;
4729 0 : ulong loaded = 0UL;
4730 0 : ulong cross_replaced = 0UL; /* cross-fork overrides (subset of replaced) */
4731 0 : ulong replaced_lamports = 0UL;
4732 0 : ulong ignored_lamports = 0UL;
4733 0 : int result = 0;
4734 :
4735 : /* Snapshot slots are stored in the 32-bit cache_idx scratch field
4736 : during loading. Reject anything that would truncate. */
4737 0 : for( ulong i=0UL; i<cnt; i++ ) {
4738 0 : if( FD_UNLIKELY( slots[ i ]>UINT_MAX ) ) FD_LOG_ERR(( "snapshot slot %lu exceeds 2^32-1, accdb format must be widened", slots[ i ] ));
4739 0 : }
4740 :
4741 : /* Phase 1: compute hashes and prefetch chain heads. */
4742 :
4743 0 : ulong hashes[ 8 ];
4744 0 : for( ulong i=0UL; i<cnt; i++ ) {
4745 0 : hashes[ i ] = fd_hash32( pubkeys[ i ], seed ) & chain_msk;
4746 0 : __builtin_prefetch( &accdb->acc_map[ hashes[ i ] ], 1, 1 );
4747 0 : }
4748 :
4749 : /* Phase 1b: reject same-slot duplicate pubkeys. */
4750 :
4751 0 : for( ulong i=1UL; i<cnt; i++ ) {
4752 0 : for( ulong j=0UL; j<i; j++ ) {
4753 0 : if( FD_UNLIKELY( hashes[ j ]==hashes[ i ] && slots[ j ]==slots[ i ] &&
4754 0 : !memcmp( pubkeys[ j ], pubkeys[ i ], 32UL ) ) ) {
4755 0 : FD_LOG_WARNING(( "corrupt snapshot: duplicate pubkey within a single batch (entries %lu and %lu, slots %lu and %lu)", j, i, slots[ j ], slots[ i ] ));
4756 0 : return -1;
4757 0 : }
4758 0 : }
4759 0 : }
4760 :
4761 0 : fd_accdb_accmeta_t * acquired[ 8 ];
4762 0 : if( FD_UNLIKELY( !acc_pool_acquire_batch( accdb->acc_pool_join, cnt, acquired ) ) ) {
4763 0 : FD_LOG_ERR(( "accounts database ran out of space during snapshot loading, increase [accounts.max_accounts], current value is %lu", acc_pool_ele_max( accdb->acc_pool_join ) ));
4764 0 : }
4765 :
4766 : /* Phase 2: walk and commit under the account chain lock. */
4767 :
4768 0 : ulong used_bytes_added = 0UL;
4769 0 : ulong used_bytes_removed = 0UL;
4770 0 : ulong acquired_used = 0UL;
4771 :
4772 0 : for( ulong i=0UL; i<cnt; i++ ) {
4773 0 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t)+data_lens[ i ];
4774 0 : int skip = 0;
4775 :
4776 0 : uint chain_head;
4777 0 : for(;;) {
4778 0 : chain_head = FD_VOLATILE_CONST( accdb->acc_map[ hashes[ i ] ] );
4779 0 : if( FD_UNLIKELY( chain_head==CHAIN_LOCKED ) ) {
4780 0 : FD_SPIN_PAUSE();
4781 0 : continue;
4782 0 : }
4783 0 : if( FD_LIKELY( FD_ATOMIC_CAS( &accdb->acc_map[ hashes[ i ] ], chain_head, CHAIN_LOCKED )==chain_head ) ) {
4784 0 : break;
4785 0 : }
4786 0 : FD_SPIN_PAUSE();
4787 0 : }
4788 :
4789 0 : fd_accdb_accmeta_t * existing = NULL;
4790 0 : fd_accdb_accmeta_t * cross_existing = NULL; /* cross-fork dup (incremental only) */
4791 0 : uint next_acc = chain_head;
4792 0 : while( next_acc!=UINT_MAX ) {
4793 0 : fd_accdb_accmeta_t * candidate = &accdb->acc_pool[ next_acc ];
4794 :
4795 0 : if( FD_LIKELY( candidate->map.next!=UINT_MAX ) ) {
4796 0 : __builtin_prefetch( &accdb->acc_pool[ candidate->map.next ], 0, 1 );
4797 0 : }
4798 :
4799 0 : if( FD_UNLIKELY( !memcmp( pubkeys[ i ], candidate->key.pubkey, 32UL ) ) ) {
4800 0 : if( FD_LIKELY( (ulong)candidate->cache_idx>slots[ i ] ) ) {
4801 0 : skip = 1;
4802 0 : } else if( FD_UNLIKELY( (ulong)candidate->cache_idx==slots[ i ] ) ) {
4803 0 : FD_LOG_WARNING(( "corrupt snapshot: duplicate account at slot %lu", slots[ i ] ));
4804 0 : FD_COMPILER_MFENCE();
4805 0 : FD_VOLATILE( accdb->acc_map[ hashes[ i ] ] ) = chain_head;
4806 0 : result = -1;
4807 0 : goto fini;
4808 0 : } else if( FD_UNLIKELY( incremental ) && candidate->key.generation!=fork_gen ) {
4809 0 : cross_existing = candidate;
4810 0 : } else {
4811 0 : existing = candidate;
4812 0 : }
4813 0 : break;
4814 0 : }
4815 0 : next_acc = candidate->map.next;
4816 0 : }
4817 :
4818 0 : if( FD_UNLIKELY( skip ) ) {
4819 0 : results[ i ] = FD_ACCDB_SNAPSHOT_WRITE_IGNORED;
4820 0 : FD_COMPILER_MFENCE();
4821 0 : FD_VOLATILE( accdb->acc_map[ hashes[ i ] ] ) = chain_head;
4822 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, file_offsets[ i ], entry_sz );
4823 0 : ignored_lamports += lamports[ i ];
4824 0 : ignored++;
4825 0 : continue;
4826 0 : }
4827 :
4828 0 : fd_accdb_accmeta_t * accmeta;
4829 0 : uint new_head = chain_head;
4830 :
4831 0 : fd_accdb_accmeta_t const * prev = existing ? existing : cross_existing;
4832 0 : if( !prev || !prev->lamports ) results[ i ] = FD_ACCDB_SNAPSHOT_WRITE_LOADED;
4833 0 : else if( cross_existing ) results[ i ] = FD_ACCDB_SNAPSHOT_WRITE_REPLACED_CROSS;
4834 0 : else results[ i ] = FD_ACCDB_SNAPSHOT_WRITE_REPLACED;
4835 :
4836 0 : if( FD_UNLIKELY( existing ) ) {
4837 0 : accmeta = existing;
4838 0 : ulong old_sz = sizeof(fd_accdb_disk_meta_t) + FD_ACCDB_SIZE_DATA( accmeta->executable_size );
4839 0 : fd_accdb_shmem_bytes_freed( accdb->shmem, fd_accdb_acc_offset( accmeta ), old_sz );
4840 0 : used_bytes_removed += old_sz;
4841 0 : replaced_lamports += accmeta->lamports;
4842 0 : replaced++;
4843 0 : } else {
4844 0 : accmeta = acquired[ acquired_used++ ];
4845 :
4846 0 : uint acc_idx = (uint)acc_pool_idx( accdb->acc_pool_join, accmeta );
4847 :
4848 0 : fd_memcpy( accmeta->key.pubkey, pubkeys[ i ], 32UL );
4849 0 : accmeta->key.generation = incremental ? fork_gen : gen;
4850 0 : accmeta->map.next = chain_head;
4851 0 : new_head = acc_idx;
4852 :
4853 0 : if( FD_UNLIKELY( incremental ) ) {
4854 0 : fd_accdb_txn_t * txn = txn_pool_acquire( accdb->txn_pool );
4855 0 : if( FD_UNLIKELY( !txn ) ) FD_LOG_ERR(( "txn pool exhausted during incremental snapshot loading" ));
4856 0 : txn->acc_pool_idx = acc_idx;
4857 0 : uint txn_idx = (uint)txn_pool_idx( accdb->txn_pool, txn );
4858 0 : for(;;) {
4859 0 : uint old_head = FD_VOLATILE_CONST( fork->shmem->txn_head );
4860 0 : txn->fork.next = old_head;
4861 0 : if( FD_LIKELY( FD_ATOMIC_CAS( &fork->shmem->txn_head, old_head, txn_idx )==old_head ) ) break;
4862 0 : FD_SPIN_PAUSE();
4863 0 : }
4864 0 : }
4865 :
4866 0 : if( FD_UNLIKELY( cross_existing ) ) {
4867 0 : replaced_lamports += cross_existing->lamports;
4868 0 : replaced++;
4869 0 : cross_replaced++;
4870 0 : } else {
4871 0 : loaded++;
4872 0 : }
4873 0 : }
4874 :
4875 0 : accmeta->cache_idx = (uint)slots[ i ];
4876 0 : accmeta->lamports = lamports[ i ];
4877 0 : accmeta->executable_size = FD_ACCDB_SIZE_PACK( (uint)data_lens[ i ], executables[ i ] );
4878 0 : ulong file_off = file_offsets[ i ];
4879 0 : accmeta->offset_fork = incremental ? fd_accdb_acc_pack_offset_fork( file_off, fork_id.val ) : file_off;
4880 :
4881 0 : FD_COMPILER_MFENCE();
4882 0 : FD_VOLATILE( accdb->acc_map[ hashes[ i ] ] ) = new_head;
4883 :
4884 0 : used_bytes_added += entry_sz;
4885 0 : }
4886 :
4887 0 : fini:
4888 0 : for( ulong i=acquired_used; i<cnt; i++ ) {
4889 0 : acc_pool_release( accdb->acc_pool_join, acquired[ i ] );
4890 0 : }
4891 :
4892 0 : accdb->write_stats.disk_used_added += used_bytes_added;
4893 0 : accdb->write_stats.disk_used_removed += used_bytes_removed;
4894 0 : accdb->write_stats.accounts_total_added += loaded + cross_replaced;
4895 :
4896 0 : *accounts_ignored = ignored;
4897 0 : *accounts_replaced = replaced;
4898 0 : *accounts_loaded = loaded;
4899 0 : *out_replaced_lamports = replaced_lamports;
4900 0 : *out_ignored_lamports = ignored_lamports;
4901 :
4902 0 : return result;
4903 :
4904 0 : #undef CHAIN_LOCKED
4905 0 : }
4906 :
4907 :
4908 : static void
4909 0 : delta_reset( fd_accdb_t * accdb ) {
4910 0 : if( !accdb->shmem->delta.head ) return; /* clean */
4911 0 : uint * chains = accdb->delta.chains;
4912 0 : ulong chain_cnt = accdb->shmem->delta.chain_cnt;
4913 0 : for( ulong i=0UL; i<chain_cnt; i++ ) chains[ i ] = UINT_MAX;
4914 0 : accdb->shmem->delta.head = 0UL;
4915 0 : }
4916 :
4917 : static int
4918 0 : delta_is_valid( fd_accdb_shmem_t const * accdb ) {
4919 0 : return accdb->delta.head < accdb->delta.ele_max;
4920 0 : }
4921 :
4922 : void
4923 : fd_accdb_background( fd_accdb_t * accdb,
4924 64011 : int * charge_busy ) {
4925 64011 : fd_accdb_shmem_t * shmem = accdb->shmem;
4926 :
4927 64011 : ulong * snap_sync_p = &shmem->snapshot_sync;
4928 64011 : ulong snap_sync = fd_accdb_snapshot_sync_state( snap_sync_p );
4929 :
4930 : /* The batch a pass retires is usually free to release right away
4931 : (no reader mid-epoch). A held batch keeps the tile busy, so it
4932 : does not park with it. A call that applies requests does nothing
4933 : else, as before: single-threaded joins without a compaction buffer
4934 : call it once after a purge. */
4935 64011 : int requests = background_requests( accdb );
4936 64011 : if( FD_UNLIKELY( !background_release( accdb ) ) ) *charge_busy = 1;
4937 64056 : if( FD_UNLIKELY( requests ) ) {
4938 64056 : *charge_busy = 1;
4939 64056 : return;
4940 64056 : }
4941 :
4942 >1844*10^16 : if( FD_UNLIKELY( snap_sync!=FD_ACCDB_SNAPSHOT_SYNC_IDLE ) ) {
4943 0 : switch( snap_sync ) {
4944 0 : case FD_ACCDB_SNAPSHOT_SYNC_RUNNING:
4945 : /* while producing a snapshot, don't do compaction work */
4946 0 : background_preevict( accdb, charge_busy, 0 );
4947 0 : return;
4948 0 : case FD_ACCDB_SNAPSHOT_SYNC_DONE:
4949 0 : fd_accdb_snapshot_sync_advance( snap_sync_p, FD_ACCDB_SNAPSHOT_SYNC_IDLE );
4950 0 : break;
4951 0 : case FD_ACCDB_SNAPSHOT_SYNC_START_FULL:
4952 0 : FD_CHECK_CRIT( !FD_VOLATILE_CONST( shmem->snapshot_loading ),
4953 0 : "snapshot production requested during snapshot load" );
4954 0 : delta_reset( accdb );
4955 0 : fd_accdb_snapshot_sync_advance( snap_sync_p, FD_ACCDB_SNAPSHOT_SYNC_RUNNING );
4956 0 : *charge_busy = 1;
4957 0 : return;
4958 0 : case FD_ACCDB_SNAPSHOT_SYNC_START_INCR:
4959 0 : FD_CHECK_CRIT( !FD_VOLATILE_CONST( shmem->snapshot_loading ),
4960 0 : "snapshot production requested during snapshot load" );
4961 0 : if( delta_is_valid( accdb->shmem ) ) {
4962 0 : fd_accdb_snapshot_sync_advance( snap_sync_p, FD_ACCDB_SNAPSHOT_SYNC_RUNNING );
4963 0 : } else {
4964 : /* cannot produce incrementals because delta ran out of space,
4965 : therefore don't know which accounts changed */
4966 0 : fd_accdb_snapshot_sync_advance( snap_sync_p, FD_ACCDB_SNAPSHOT_SYNC_FAIL );
4967 0 : }
4968 0 : *charge_busy = 1;
4969 0 : return;
4970 0 : case FD_ACCDB_SNAPSHOT_SYNC_FAIL:
4971 : /* wait for client to acknowledge */
4972 0 : break;
4973 0 : default:
4974 0 : FD_LOG_CRIT(( "corrupt snapshot_sync state %lu", snap_sync ));
4975 0 : }
4976 0 : }
4977 :
4978 >1844*10^16 : background_preevict( accdb, charge_busy, 0 );
4979 :
4980 : /* Keep one partition allocated ahead so a write head rotating into a
4981 : new partition never waits on fallocate. */
4982 >1844*10^16 : if( FD_UNLIKELY( FD_VOLATILE_CONST( shmem->partition_max )>=FD_VOLATILE_CONST( shmem->partition_fallocated ) &&
4983 >1844*10^16 : FD_VOLATILE_CONST( shmem->partition_fallocated )!=accdb->prealloc_failed_at ) ) {
4984 0 : spin_lock_acquire( &shmem->grow_lock );
4985 0 : ulong fallocated = shmem->partition_fallocated;
4986 0 : int grow = fallocated<shmem->partition_cnt && FD_VOLATILE_CONST( shmem->partition_max )>=fallocated;
4987 0 : int err = grow ? grow_locked( accdb, fallocated, 0, ULONG_MAX, 1 ) : 0;
4988 0 : spin_lock_release( &shmem->grow_lock );
4989 0 : if( FD_UNLIKELY( err ) ) {
4990 0 : FD_LOG_WARNING(( "fallocate() failed (%d-%s) preallocating the accounts database beyond %lu GiB. "
4991 0 : "The validator will exit soon when the disk fills.",
4992 0 : err, fd_io_strerror( err ), fallocated*shmem->partition_sz/(1UL<<30UL) ));
4993 0 : accdb->prealloc_failed_at = fallocated;
4994 0 : } else if( FD_LIKELY( grow ) ) {
4995 0 : *charge_busy = 1;
4996 0 : return;
4997 0 : }
4998 0 : }
4999 :
5000 >1844*10^16 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
5001 0 : if( FD_UNLIKELY( accdb->carry_sz && k!=accdb->carry_layer ) ) continue; /* would overwrite the carried tail */
5002 0 : background_compact( accdb, k, charge_busy );
5003 0 : }
5004 >1844*10^16 : }
5005 :
5006 : fd_accdb_shmem_metrics_t const *
5007 0 : fd_accdb_shmetrics( fd_accdb_t * accdb ) {
5008 0 : return accdb->shmem->shmetrics;
5009 0 : }
5010 :
5011 : fd_accdb_metrics_t const *
5012 0 : fd_accdb_metrics( fd_accdb_t * accdb ) {
5013 0 : return accdb->metrics;
5014 0 : }
5015 :
5016 : void
5017 : fd_accdb_cache_class_occupancy( fd_accdb_t * accdb,
5018 : ulong * used,
5019 : ulong * max,
5020 0 : ulong * reserved ) {
5021 0 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
5022 0 : ulong cap = accdb->shmem->cache_class_max[ c ];
5023 0 : ulong init = FD_VOLATILE_CONST( accdb->shmem->cache_class_init[ c ].val );
5024 0 : ulong freec = cache_free_total( accdb->shmem, c );
5025 0 : ulong live = init>freec ? init-freec : 0UL;
5026 0 : if( live>cap ) live = cap;
5027 0 : max [ c ] = cap;
5028 0 : used [ c ] = live;
5029 0 : ulong res = FD_VOLATILE_CONST( accdb->shmem->cache_class_used[ c ].val );
5030 0 : if( res!=ULONG_MAX ) { /* in-flight reservations only, not idle credits */
5031 0 : ulong joiner_cnt = fd_ulong_min( FD_VOLATILE_CONST( accdb->shmem->joiner_cnt ), FD_ACCDB_MAX_JOINERS );
5032 0 : ulong idle = 0UL;
5033 0 : for( ulong j=0UL; j<joiner_cnt; j++ ) idle += FD_VOLATILE_CONST( accdb->shmem->join_cred[ j ].val[ c ] );
5034 0 : res = res>idle ? res-idle : 0UL;
5035 0 : }
5036 0 : reserved[ c ] = res;
5037 0 : }
5038 0 : }
5039 :
5040 : void
5041 : fd_accdb_set_preevict_interval( fd_accdb_t * accdb,
5042 0 : long ticks ) {
5043 0 : accdb->preevict_interval = ticks;
5044 0 : }
5045 :
5046 : void
5047 : fd_accdb_cache_class_thresholds( fd_accdb_t * accdb,
5048 : ulong * free_target,
5049 0 : ulong * free_low_water ) {
5050 0 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
5051 0 : free_target [ c ] = accdb->shmem->cache_free_target [ c ];
5052 0 : free_low_water[ c ] = accdb->shmem->cache_free_low_water[ c ];
5053 0 : }
5054 0 : }
5055 :
5056 : #if FD_HAS_RACESAN
5057 :
5058 : /* Force pre-eviction (ignore the watermark) so a deterministic
5059 : single-threaded test can exercise the writeback path without
5060 : manufacturing real cache pressure. Sweeps several times: CLOCK needs
5061 : two visits to evict a recently-touched line (clear the "referenced"
5062 : bit, then evict), and the clock hand position carries across calls, so
5063 : one or two sweeps is not enough to guarantee every eligible line is
5064 : flushed back. */
5065 : void
5066 : fd_accdb_debug_force_preevict( fd_accdb_t * accdb ) {
5067 : for( ulong iter=0UL; iter<8UL; iter++ ) {
5068 : int charge_busy = 0;
5069 : background_preevict( accdb, &charge_busy, 1 );
5070 : }
5071 : }
5072 :
5073 : /* Locate the resident cache line currently holding `pubkey` (most recent
5074 : generation if multiple). Returns 1 and fills out_class/out_idx on a
5075 : hit, 0 if no resident line matches. Test-only helper so the test can
5076 : target a specific line without seeing the opaque fd_accdb struct. */
5077 :
5078 : int
5079 : fd_accdb_debug_find_line( fd_accdb_t * accdb,
5080 : uchar const * pubkey,
5081 : ulong * out_class,
5082 : ulong * out_idx ) {
5083 : int found = 0;
5084 : uint best_gen = 0U;
5085 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
5086 : ulong init = FD_VOLATILE_CONST( accdb->shmem->cache_class_init[ c ].val );
5087 : ulong max_c = accdb->shmem->cache_class_max[ c ];
5088 : if( init>max_c ) init = max_c;
5089 : for( ulong idx=0UL; idx<init; idx++ ) {
5090 : fd_accdb_cache_line_t * line = cache_line( accdb, c, idx );
5091 : if( line->key.generation==UINT_MAX ) continue;
5092 : if( memcmp( line->key.pubkey, pubkey, 32UL ) ) continue;
5093 : if( !found || line->key.generation>=best_gen ) {
5094 : best_gen = line->key.generation;
5095 : *out_class = c;
5096 : *out_idx = idx;
5097 : found = 1;
5098 : }
5099 : }
5100 : }
5101 : return found;
5102 : }
5103 :
5104 : /* Address of one cache line. struct fd_accdb_private is local to this
5105 : translation unit, so a test cannot reach accdb->cache[] itself. */
5106 :
5107 : void *
5108 : fd_accdb_debug_line_addr( fd_accdb_t * accdb,
5109 : ulong size_class,
5110 : ulong line_idx ) {
5111 : return cache_line( accdb, size_class, line_idx );
5112 : }
5113 :
5114 : uint *
5115 : fd_accdb_debug_acc_pool_next( fd_accdb_t * accdb,
5116 : uint acc_idx ) {
5117 : FD_TEST( (ulong)acc_idx<acc_pool_ele_max( accdb->acc_pool_join ) );
5118 : return &accdb->acc_pool[ acc_idx ].pool.next;
5119 : }
5120 :
5121 : /* Deterministically evict a single specified cache line via the
5122 : foreground evictor's claim sequence (CAS refcnt 0->EVICT_SENTINEL),
5123 : then write the dirty line back exactly as fd_accdb_acquire_inner's
5124 : STEP-4 / background_ preevict do (pubkey from accmeta, owner+data
5125 : from the line). Mirrors acquire_cache_line's CLOCK-claim path
5126 : (fd_accdb.c) so a racesan test can reproduce, without a 640+-slot
5127 : cache-pressure rig, the interleaving where acc_unlink observes
5128 : EVICT_SENTINEL on the line it is unlinking.
5129 :
5130 : The fd_racesan_hook("clock_evict:post_sentinel") fires right after
5131 : the sentinel is installed (matching the production foreground path),
5132 : so the test can suspend this fiber holding the sentinel while another
5133 : fiber drives acc_unlink to its reclaim CAS. Returns the captured
5134 : evicted acc_idx (UINT_MAX if the line was clean / unbound). */
5135 :
5136 : uint
5137 : fd_accdb_debug_clock_evict_line( fd_accdb_t * accdb,
5138 : ulong size_class,
5139 : ulong line_idx ) {
5140 : fd_accdb_shmem_t * shmem = accdb->shmem;
5141 : fd_accdb_cache_line_t * line = cache_line( accdb, size_class, line_idx );
5142 :
5143 : /* Claim for eviction, same as acquire_cache_line's CLOCK path. */
5144 : if( FD_UNLIKELY( FD_ATOMIC_CAS( &line->refcnt, 0U, FD_ACCDB_EVICT_SENTINEL )!=0U ) ) return UINT_MAX;
5145 :
5146 : if( FD_UNLIKELY( line->acc_idx==UINT_MAX && line->key.generation==UINT_MAX ) ) {
5147 : FD_VOLATILE( line->refcnt ) = 0;
5148 : return UINT_MAX;
5149 : }
5150 :
5151 : fd_racesan_hook( "clock_evict:post_sentinel" );
5152 :
5153 : uint acc_idx = line->acc_idx;
5154 : if( FD_LIKELY( acc_idx!=UINT_MAX ) ) {
5155 : evict_clear_acc_cache_ref( &accdb->acc_pool[ acc_idx ], size_class, line_idx );
5156 : }
5157 : uint evicted_acc_idx = line->persisted ? UINT_MAX : acc_idx;
5158 : line->key.generation = UINT_MAX;
5159 : fd_racesan_hook( "clock_evict:pre_synth" );
5160 :
5161 : /* Write back the dirty line, exactly like the production writeback
5162 : sites: this is the synthesis that would emit a pubkey=NEW/owner=OLD
5163 : poison record if the accmeta slot had been recycled out from under
5164 : us. In the SENTINEL-vs-acc_unlink race this proves no poison: the
5165 : epoch the evictor holds blocks drain_deferred_frees, so the slot is
5166 : never recycled while we are here. */
5167 : if( FD_UNLIKELY( !line->persisted && acc_idx!=UINT_MAX ) ) {
5168 : fd_accdb_accmeta_t * accmeta = &accdb->acc_pool[ acc_idx ];
5169 : ulong entry_sz = sizeof(fd_accdb_disk_meta_t)+(ulong)FD_ACCDB_SIZE_DATA( accmeta->executable_size );
5170 :
5171 : ulong old_offset = fd_accdb_acc_xchg_offset( accmeta, FD_ACCDB_OFF_INVAL );
5172 : if( FD_LIKELY( old_offset!=FD_ACCDB_OFF_INVAL ) ) {
5173 : fd_accdb_shmem_bytes_freed( shmem, old_offset, entry_sz );
5174 : FD_ATOMIC_FETCH_AND_SUB( &shmem->shmetrics->disk_used_bytes, entry_sz );
5175 : }
5176 :
5177 : fd_accdb_disk_meta_t meta;
5178 : fd_memcpy( meta.pubkey, accmeta->key.pubkey, 32UL );
5179 : meta.size = FD_ACCDB_SIZE_DATA( accmeta->executable_size );
5180 : meta.generation = accmeta->key.generation;
5181 : fd_memcpy( meta.owner, line->owner, 32UL );
5182 :
5183 : struct iovec iovs[ 2UL ] = {
5184 : { .iov_base = &meta, .iov_len = sizeof(fd_accdb_disk_meta_t) },
5185 : { .iov_base = (void *)(line+1UL), .iov_len = FD_ACCDB_SIZE_DATA( accmeta->executable_size ) }
5186 : };
5187 : ulong file_off = allocate_next_write( accdb, entry_sz );
5188 : ulong written = 0UL;
5189 : while( written<entry_sz ) {
5190 : long result = pwritev2( accdb->fd, iovs, 2, (long)(file_off+written), 0 );
5191 : if( FD_UNLIKELY( result==-1 && errno==EINTR ) ) continue;
5192 : else if( FD_UNLIKELY( result<=0 ) ) FD_LOG_ERR(( "pwritev2() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
5193 : written += (ulong)result;
5194 : for( int v=0; v<2; v++ ) {
5195 : if( (ulong)result>=iovs[ v ].iov_len ) { result -= (long)iovs[ v ].iov_len; iovs[ v ].iov_len = 0UL; }
5196 : else { iovs[ v ].iov_base = (uchar *)iovs[ v ].iov_base + result; iovs[ v ].iov_len -= (ulong)result; break; }
5197 : }
5198 : }
5199 : FD_COMPILER_MFENCE();
5200 : accmeta->offset_fork = fd_accdb_acc_pack_offset_fork( file_off, fd_accdb_acc_fork_id(accmeta) );
5201 : FD_ATOMIC_FETCH_AND_ADD( &shmem->shmetrics->disk_used_bytes, entry_sz );
5202 : }
5203 :
5204 : cache_line_free( accdb, accdb->join_idx, size_class, line );
5205 : return evicted_acc_idx;
5206 : }
5207 :
5208 : #endif
|
