Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_accdb_fd_accdb_private_h
2 : #define HEADER_fd_src_flamenco_accdb_fd_accdb_private_h
3 :
4 : #include "fd_accdb_base.h"
5 : #include "fd_accdb_shmem.h"
6 : #include "fd_accdb_cache.h"
7 :
8 : static inline void
9 0 : spin_lock_acquire( int * lock ) {
10 0 : # if FD_HAS_THREADS
11 0 : for(;;) {
12 0 : if( FD_LIKELY( !FD_ATOMIC_CAS( lock, 0, 1 ) ) ) break;
13 0 : FD_SPIN_PAUSE();
14 0 : }
15 : # else
16 : *lock = 1;
17 : # endif
18 0 : FD_COMPILER_MFENCE();
19 0 : }
20 :
21 : static inline void
22 0 : spin_lock_release( int * lock ) {
23 0 : FD_COMPILER_MFENCE();
24 0 : # if FD_HAS_THREADS
25 0 : FD_VOLATILE( *lock ) = 0;
26 : # else
27 : *lock = 0;
28 : # endif
29 0 : }
30 :
31 : struct fd_accdb_txn {
32 : union {
33 : struct { uint next; } pool;
34 : struct { uint next; } fork;
35 : };
36 :
37 : uint acc_pool_idx;
38 : };
39 :
40 : typedef struct fd_accdb_txn fd_accdb_txn_t;
41 :
42 : #define POOL_NAME txn_pool
43 1114814 : #define POOL_ELE_T fd_accdb_txn_t
44 370385 : #define POOL_NEXT pool.next
45 : #define POOL_IDX_T uint
46 2223432 : #define POOL_IDX_WIDTH 32
47 : #define POOL_IMPL_STYLE 0
48 : #define POOL_LAZY 1
49 :
50 : #include "../../util/tmpl/fd_pool_para.c"
51 :
52 : #define SET_NAME descends_set
53 : #define SET_IMPL_STYLE 1
54 : #include "../../util/tmpl/fd_set_dynamic.c"
55 :
56 : /* fd_accdb_req_t is the root / purge request word (T1 -> T2) and
57 : fd_accdb_done_t the matching progress word (T2 -> T1). Each is
58 : written by a single thread with one 8-byte store, so a reader always
59 : sees a consistent set of fields. */
60 :
61 : union __attribute__((aligned(8))) fd_accdb_req {
62 : struct {
63 : uint root_gen; /* generation of the latest requested root */
64 : ushort purge_seq; /* purges requested so far, mod 2^16 */
65 : fd_accdb_fork_id_t purge_fork; /* latest purged fork */
66 : };
67 : ulong u;
68 : };
69 :
70 : typedef union fd_accdb_req fd_accdb_req_t;
71 :
72 : union __attribute__((aligned(8))) fd_accdb_done {
73 : struct {
74 : uint root_gen; /* generation of root_fork_id */
75 : ushort purge_seq; /* purges applied so far, mod 2^16 */
76 : ushort retiring; /* 1 while a retired batch awaits release */
77 : };
78 : ulong u;
79 : };
80 :
81 : typedef union fd_accdb_done fd_accdb_done_t;
82 :
83 : FD_STATIC_ASSERT( sizeof(fd_accdb_req_t )==sizeof(ulong), req );
84 : FD_STATIC_ASSERT( sizeof(fd_accdb_done_t)==sizeof(ulong), done );
85 :
86 : struct fd_accdb_fork_shmem {
87 : uint generation;
88 :
89 : fd_accdb_fork_id_t parent_id;
90 : fd_accdb_fork_id_t child_id;
91 : fd_accdb_fork_id_t sibling_id;
92 :
93 : struct {
94 : ulong next;
95 : } pool;
96 :
97 : uint txn_head;
98 : };
99 :
100 : typedef struct fd_accdb_fork_shmem fd_accdb_fork_shmem_t;
101 :
102 : #define POOL_NAME fork_pool
103 256340 : #define POOL_ELE_T fd_accdb_fork_shmem_t
104 128155 : #define POOL_NEXT pool.next
105 : #define POOL_IDX_T ulong
106 : #define POOL_IMPL_STYLE 0
107 :
108 : #include "../../util/tmpl/fd_pool_para.c"
109 :
110 : struct fd_accdb_partition {
111 : ulong marked_compaction;
112 : ulong write_offset;
113 : ulong compaction_offset;
114 :
115 : ulong bytes_freed;
116 :
117 : uchar layer; /* compaction tier this partition belongs to. */
118 :
119 : ulong read_ops;
120 : ulong bytes_read;
121 : ulong write_ops;
122 : ulong bytes_written;
123 :
124 : /* Tickcount (fd_tickcount) of the partition's lifecycle events. Set
125 : only at partition creation and again when the partition closes
126 : (i.e. the layer's write head rotates off it). */
127 : long created_ticks;
128 : long filled_ticks;
129 :
130 : /* Compaction lifecycle flags. queued is set when the partition is
131 : pushed onto the compaction_dlist and cleared when it is popped.
132 : compacting_now is set by the compaction tile around the actual
133 : compaction work for this partition. */
134 : uchar queued;
135 : uchar compacting_now;
136 :
137 : /* Per-compaction-pass telemetry accumulators for the
138 : accdb_compaction_completed event. Reset when a partition's
139 : compaction begins (compacting_now set in background_compact) and
140 : accumulated as records are scanned/relocated. */
141 : long compaction_start_wallclock; /* timestamp when this pass began */
142 : ulong compaction_accounts_relocated; /* live records moved this pass */
143 : ulong compaction_bytes_relocated; /* bytes moved this pass */
144 : ulong compaction_dead_records; /* records skipped (no live index entry) this pass */
145 :
146 : /* Epoch at which this partition was enqueued for compaction. Set by
147 : fd_accdb_shmem_bytes_freed when the partition crosses the
148 : freed-bytes threshold. The compaction tile will not begin reading
149 : from this partition until all joiners that were in an
150 : epoch-protected critical section at enqueue time have exited,
151 : ensuring any in-flight pwritev2 to this partition has completed. */
152 : ulong compaction_ready_epoch;
153 :
154 : /* Epoch at which this partition was enqueued for deferred freeing.
155 : Set by compaction when the partition finishes compaction, and
156 : checked by the reclamation scan to determine when it is safe to
157 : release the partition back to the pool. */
158 : ulong epoch_tag;
159 :
160 : ulong pool_next;
161 :
162 : ulong dlist_prev;
163 : ulong dlist_next;
164 : };
165 :
166 : typedef struct fd_accdb_partition fd_accdb_partition_t;
167 :
168 : #define POOL_NAME partition_pool
169 : #define POOL_T fd_accdb_partition_t
170 0 : #define POOL_NEXT pool_next
171 : #define POOL_IDX_T ulong
172 : #define POOL_IMPL_STYLE 1
173 :
174 : #include "../../util/tmpl/fd_pool.c"
175 :
176 : #define DLIST_NAME compaction_dlist
177 : #define DLIST_ELE_T fd_accdb_partition_t
178 0 : #define DLIST_PREV dlist_prev
179 0 : #define DLIST_NEXT dlist_next
180 : #define DLIST_IMPL_STYLE 1
181 :
182 : #include "../../util/tmpl/fd_dlist.c"
183 :
184 : /* deferred_free_dlist reuses the same prev/next fields as
185 : compaction_dlist. A partition is in at most one of the two lists at
186 : any time: it is popped from compaction_dlist before being pushed onto
187 : deferred_free_dlist. */
188 :
189 : #define DLIST_NAME deferred_free_dlist
190 : #define DLIST_ELE_T fd_accdb_partition_t
191 0 : #define DLIST_PREV dlist_prev
192 0 : #define DLIST_NEXT dlist_next
193 : #define DLIST_IMPL_STYLE 1
194 :
195 : #include "../../util/tmpl/fd_dlist.c"
196 :
197 : /* Per-join reservation credits (fd_accdb.c res_*). fd_accdb_cred_max
198 : is the most idle credits a join holds per tracked class: at most
199 : FD_ACCDB_CRED_MAX, and all joins together at most
200 : class_max/FD_ACCDB_CRED_DIV. */
201 :
202 0 : #define FD_ACCDB_CRED_MAX (64UL)
203 0 : #define FD_ACCDB_CRED_DIV (2UL)
204 :
205 : static inline ulong
206 : fd_accdb_cred_max( ulong class_max,
207 0 : ulong joiner_cnt ) {
208 0 : return fd_ulong_min( FD_ACCDB_CRED_MAX, class_max/( FD_ACCDB_CRED_DIV*fd_ulong_max( joiner_cnt, 1UL ) ) );
209 0 : }
210 :
211 : struct fd_accdb_cache_key {
212 : uchar pubkey[ 32UL ];
213 : uint generation;
214 : };
215 :
216 : typedef struct fd_accdb_cache_key fd_accdb_cache_key_t;
217 :
218 : struct __attribute__((aligned(64))) fd_accdb_accmeta {
219 : fd_accdb_cache_key_t key;
220 :
221 : struct {
222 : uint next;
223 : } map;
224 :
225 : union {
226 : struct {
227 : uint next;
228 : } pool;
229 : uint cache_idx;
230 : };
231 :
232 : uint executable_size;
233 :
234 : ulong lamports;
235 :
236 : /* Pack offset and fork_id together into a single ulong to pack the
237 : struct into a single 64 byte cache line. This is a performance win
238 : of 2-3%. */
239 : ulong offset_fork;
240 : };
241 :
242 : typedef struct fd_accdb_accmeta fd_accdb_accmeta_t;
243 :
244 : FD_STATIC_ASSERT( alignof(fd_accdb_accmeta_t)==64, layout );
245 : FD_STATIC_ASSERT( sizeof (fd_accdb_accmeta_t)==64, layout );
246 :
247 3635646 : #define FD_ACCDB_OFF_BITS 48UL
248 2951811 : #define FD_ACCDB_OFF_MASK ((1UL<<FD_ACCDB_OFF_BITS)-1UL) /* 0x0000_FFFF_FFFF_FFFF */
249 1101146 : #define FD_ACCDB_OFF_INVAL FD_ACCDB_OFF_MASK /* sentinel: offset bits all-ones */
250 :
251 : /* The in-memory executable_size field in fd_accdb_accmeta_t packs
252 : five things into 32 bits:
253 :
254 : bit 31 executable flag (FD_ACCDB_SIZE_EXEC_BIT)
255 : bit 30 cache_valid flag, in-memory only (FD_ACCDB_SIZE_CACHE_VALID_BIT)
256 : bit 29 cache_claim flag, in-memory only (FD_ACCDB_SIZE_CACHE_CLAIM_BIT)
257 : bit 28 pd_write flag, in-memory only (FD_ACCDB_SIZE_PD_WRITE_BIT)
258 : bits 27..0 data length in bytes (FD_ACCDB_SIZE_MASK)
259 :
260 : The data length is therefore 28 bits, max 256 MiB, still well above
261 : FD_RUNTIME_ACC_SZ_MAX of 10 MiB (enforced by the static assert below).
262 :
263 : The three upper flag bits exist only in the in-memory index, never on
264 : disk:
265 : - cache_valid (bit 30): when set, cache_idx holds a valid
266 : (class, idx) pair; when clear, cache_idx must not be dereferenced
267 : (it may hold a snapshot slot number or garbage).
268 : - cache_claim (bit 29): a short-lived eviction/install lock taken
269 : via CAS while a writer mutates the cache_idx <-> cache-line
270 : binding, so concurrent readers and the evictor do not race.
271 : - pd_write (bit 28): set on a committed version whose write changed
272 : BPF upgradeable-loader deploy status this slot (Deploy/Upgrade/
273 : Extend/Close). Meaningful only while the accmeta's key.generation
274 : equals a live fork's generation (i.e. the version was committed on
275 : that fork this slot); across snapshot/root boundaries the bit is
276 : dead by construction because the generation no longer matches any
277 : live fork. Carried explicitly by the two commit sites in
278 : fd_accdb_release and nowhere else.
279 :
280 : The on-disk fd_accdb_disk_meta_t.size is only the raw data length,
281 : written via FD_ACCDB_SIZE_DATA. It includes neither the executable
282 : bit nor the cache/PD flags. Compaction preserves these record
283 : headers verbatim. */
284 :
285 272184 : #define FD_ACCDB_SIZE_EXEC_BIT (1U<<31)
286 368765 : #define FD_ACCDB_SIZE_CACHE_VALID_BIT (1U<<30)
287 372908 : #define FD_ACCDB_SIZE_CACHE_CLAIM_BIT (1U<<29)
288 4044 : #define FD_ACCDB_SIZE_PD_WRITE_BIT (1U<<28)
289 1006368 : #define FD_ACCDB_SIZE_MASK ((1U<<28)-1U)
290 366104 : #define FD_ACCDB_SIZE_PACK(sz,exec) ((uint)(sz) | ((exec) ? FD_ACCDB_SIZE_EXEC_BIT : 0U))
291 1006368 : #define FD_ACCDB_SIZE_DATA(packed) ((packed) & FD_ACCDB_SIZE_MASK)
292 178049 : #define FD_ACCDB_SIZE_EXEC(packed) (!!((packed) & FD_ACCDB_SIZE_EXEC_BIT))
293 368765 : #define FD_ACCDB_SIZE_CACHE_VALID(p) (!!((p) & FD_ACCDB_SIZE_CACHE_VALID_BIT))
294 0 : #define FD_ACCDB_SIZE_CACHE_CLAIM(p) (!!((p) & FD_ACCDB_SIZE_CACHE_CLAIM_BIT))
295 38 : #define FD_ACCDB_SIZE_PD_WRITE(p) (!!((p) & FD_ACCDB_SIZE_PD_WRITE_BIT))
296 :
297 : FD_STATIC_ASSERT( (10UL<<20) < (1UL<<28), pd_write_bit_collides_with_len );
298 :
299 : static inline ulong
300 368683 : fd_accdb_acc_offset( fd_accdb_accmeta_t const * acc ) {
301 368683 : return acc->offset_fork & FD_ACCDB_OFF_MASK;
302 368683 : }
303 :
304 : static inline ushort
305 318841 : fd_accdb_acc_fork_id( fd_accdb_accmeta_t const * acc ) {
306 318841 : return (ushort)( acc->offset_fork >> FD_ACCDB_OFF_BITS );
307 318841 : }
308 :
309 : static inline ulong
310 : fd_accdb_acc_pack_offset_fork( ulong offset,
311 364994 : ushort fork_id ) {
312 364994 : return ( (ulong)fork_id << FD_ACCDB_OFF_BITS ) | ( offset & FD_ACCDB_OFF_MASK );
313 364994 : }
314 :
315 : /* fd_accdb_acc_xchg_offset atomically replaces the 48-bit offset
316 : portion of acc->offset_fork with new_offset while preserving the
317 : 16-bit fork_id, and returns the previous 48-bit offset. Uses a
318 : CAS loop so that concurrent compaction CAS and release-overwrite
319 : exchanges serialize correctly. */
320 :
321 : static inline ulong
322 : fd_accdb_acc_xchg_offset( fd_accdb_accmeta_t * acc,
323 372410 : ulong new_offset ) {
324 372410 : for(;;) {
325 372410 : ulong old_packed = FD_VOLATILE_CONST( acc->offset_fork );
326 372410 : ulong new_packed = ( old_packed & ~FD_ACCDB_OFF_MASK ) | ( new_offset & FD_ACCDB_OFF_MASK );
327 372410 : if( FD_LIKELY( FD_ATOMIC_CAS( &acc->offset_fork, old_packed, new_packed )==old_packed ) )
328 372168 : return old_packed & FD_ACCDB_OFF_MASK;
329 242 : FD_SPIN_PAUSE();
330 242 : }
331 372410 : }
332 :
333 : /* Packing helpers for the embedded acc cache index. 3 bits class
334 : in bits 31-29, 29 bits line index. INVAL is the sentinel for
335 : "no cached location known". FD_ACCDB_CACHE_LINE_MAX (defined in
336 : fd_accdb_cache.h) is the exclusive upper bound on representable
337 : line indices; per-class slot counts must not exceed it, or cidx
338 : values would alias. */
339 :
340 1282268 : #define FD_ACCDB_ACC_CIDX_IDX_MASK ((uint)(FD_ACCDB_CACHE_LINE_MAX-1UL))
341 737489 : #define FD_ACCDB_ACC_CIDX_INVAL UINT_MAX
342 366205 : #define FD_ACCDB_ACC_CIDX_PACK(c,i) ((uint)( ((uint)(c)<<FD_ACCDB_CACHE_LINE_BITS) | ((uint)(i) & FD_ACCDB_ACC_CIDX_IDX_MASK) ))
343 1284735 : #define FD_ACCDB_ACC_CIDX_CLASS(ci) ((ulong)((uint)(ci) >> FD_ACCDB_CACHE_LINE_BITS))
344 916063 : #define FD_ACCDB_ACC_CIDX_IDX(ci) ((ulong)((uint)(ci) & FD_ACCDB_ACC_CIDX_IDX_MASK))
345 :
346 : #define POOL_NAME acc_pool
347 739932 : #define POOL_ELE_T fd_accdb_accmeta_t
348 370121 : #define POOL_NEXT pool.next
349 : #define POOL_IDX_T uint
350 2225178 : #define POOL_IDX_WIDTH 32
351 : #define POOL_IMPL_STYLE 0
352 : #define POOL_LAZY 1
353 :
354 : #include "../../util/tmpl/fd_pool_para.c"
355 :
356 : struct fd_accdb_cache_line {
357 : fd_accdb_cache_key_t key;
358 :
359 : uint acc_idx;
360 : uint seq; /* incremented each time line is reused */
361 :
362 : uint refcnt;
363 : uchar persisted;
364 : uchar referenced;
365 :
366 : uint next;
367 :
368 : uchar owner[ 32UL ];
369 : };
370 :
371 : typedef struct fd_accdb_cache_line fd_accdb_cache_line_t;
372 :
373 : typedef struct __attribute__((aligned(64))) { ulong val; } accdb_offset_t;
374 :
375 : /* Partition offsets are packed into accdb_offset_t as:
376 : bits 63..51: partition pool index
377 : bits 50..0 : byte offset within the partition */
378 :
379 36 : #define FD_ACCDB_PARTITION_OFF_BITS 51UL
380 :
381 : static FD_FN_CONST inline accdb_offset_t
382 : accdb_offset( ulong partition_idx,
383 36 : ulong partition_offset ) {
384 36 : return (accdb_offset_t){ .val = (partition_idx<<FD_ACCDB_PARTITION_OFF_BITS) | partition_offset };
385 36 : }
386 :
387 : static FD_FN_PURE inline ulong
388 0 : packed_partition_idx( accdb_offset_t const * offset ) {
389 0 : return offset->val>>FD_ACCDB_PARTITION_OFF_BITS;
390 0 : }
391 :
392 : static FD_FN_PURE inline ulong
393 0 : packed_partition_offset( accdb_offset_t const * offset ) {
394 0 : return offset->val & ((1UL<<FD_ACCDB_PARTITION_OFF_BITS)-1UL);
395 0 : }
396 :
397 : static FD_FN_PURE inline ulong
398 : packed_partition_file_offset( accdb_offset_t const * offset,
399 0 : ulong partition_sz ) {
400 0 : return (packed_partition_idx( offset )*partition_sz + packed_partition_offset( offset ));
401 0 : }
402 :
403 : /* Maximum number of concurrent joiners (tiles) that can publish an
404 : epoch in the accdb. Each joiner claims a slot in the shared epoch
405 : array during fd_accdb_new. Must be less than or equal to 256 so
406 : refcnt in cache lines can safely track the number of threads
407 : referencing each cache line without overflow. With a uint refcnt
408 : field, 256 joiners is well within range. */
409 6648 : #define FD_ACCDB_MAX_JOINERS (256UL)
410 : FD_STATIC_ASSERT( FD_ACCDB_MAX_JOINERS%64UL==0UL, joiner_bitmap ); /* cache_free_have words */
411 :
412 : /* EVICT_SENTINEL: stored in refcnt to indicate a cache line is being
413 : claimed by an eviction scan. Any thread seeing this value must treat
414 : the line as unavailable. */
415 : #define FD_ACCDB_EVICT_SENTINEL UINT_MAX
416 :
417 : struct fd_accdb_shmem_private {
418 : int partition_lock __attribute__((aligned(64)));
419 : int grow_lock __attribute__((aligned(64)));
420 :
421 : /* Set non-zero by the snapin tile while a snapshot is being loaded.
422 : Suppresses compaction enqueue so the compaction tile does not race
423 : with bulk snapshot writes; fd_accdb_snapshot_load_end performs a
424 : one-shot sweep to enqueue any partitions that crossed the
425 : fragmentation threshold during the load. */
426 : int snapshot_loading;
427 :
428 : /* Set at construction (fd_accdb_shmem_new) when this validator
429 : supports bundles. A bundle coalesces up to
430 : FD_ACCDB_MAX_TXN_PER_ACQUIRE transactions into one acquire, so when
431 : set, fd_accdb_acquire_inner permits pubkeys_cnt up to
432 : FD_ACCDB_MAX_ACQUIRE_CNT instead of the single-transaction limit
433 : FD_ACCDB_MAX_TX_ACCOUNT_LOCKS. */
434 : int bundle_enabled;
435 :
436 : /* Per-class CLOCK sweep position. Atomically incremented by
437 : eviction scans (modulo cache_class_max[c]). Each element is on
438 : its own cacheline to avoid false sharing between classes. */
439 : struct __attribute__((aligned(64))) { ulong val; } clock_hand[ FD_ACCDB_CACHE_CLASS_CNT ];
440 :
441 : /* Per-join, per-class free lists of cache lines (Treiber stacks, see
442 : fd_accdb.c cache_free_*). ver_top is a 32-bit ABA version (high)
443 : and a line index (low, UINT_MAX = empty). */
444 : struct __attribute__((aligned(64))) { ulong ver_top[ FD_ACCDB_CACHE_CLASS_CNT ]; } cache_free[ FD_ACCDB_MAX_JOINERS ];
445 :
446 : /* Approximate depth of each of those lists. */
447 : struct __attribute__((aligned(64))) { ulong val[ FD_ACCDB_CACHE_CLASS_CNT ]; } cache_free_cnt[ FD_ACCDB_MAX_JOINERS ];
448 :
449 : /* Bit j of cache_free_have[c] is set whenever join j's class-c list
450 : is non-empty (it may also be set when empty). */
451 : struct __attribute__((aligned(64))) { ulong bits[ FD_ACCDB_MAX_JOINERS/64UL ]; } cache_free_have[ FD_ACCDB_CACHE_CLASS_CNT ];
452 :
453 : /* Bit j of cache_free_hungry[c] is set when join j ran out of free
454 : class-c lines (own list empty, no never-used line). Pre-eviction
455 : takes and clears the bits, and hands the lines it frees to those
456 : joins' lists. */
457 : struct __attribute__((aligned(64))) { ulong bits[ FD_ACCDB_MAX_JOINERS/64UL ]; } cache_free_hungry[ FD_ACCDB_CACHE_CLASS_CNT ];
458 :
459 : fd_accdb_fork_id_t root_fork_id;
460 :
461 : ulong seed;
462 :
463 : /* generation is a monotonically increasing counter assigned to each
464 : fork on creation. When the root advances past a fork, its pool
465 : slot (fork_id) is retired and may be recycled by a new fork, making
466 : on-disk fork_id values useless for identifying entries from that
467 : retired fork. But generation persists in disk metadata and is never
468 : recycled.
469 :
470 : Any rooted fork is by definition an ancestor of all live forks, so
471 : entries with generation <= root_fork->generation are
472 : unconditionally visible without consulting descends_set. For
473 : entries with generation > root_fork->generation, the fork_id is
474 : still valid and descends_set is used to check ancestry.
475 :
476 : KEY INVARIANT: descends_set is ONLY consulted when generation >
477 : root_generation, which means the fork_id has NOT been rooted yet
478 : and its pool slot is still live. This is what makes it safe for
479 : fork_slot_defer to eagerly clear descends_set bits for retired
480 : forks: rooted fork bits are dead (bypassed by the generation fast
481 : path), and purged fork bits were already 0 in all live forks'
482 : descends_sets (a purged fork is never an ancestor of a live fork).
483 : */
484 : uint generation;
485 :
486 : /* Lazy initial-allocation counter per size class. Atomically
487 : incremented by acquire_cache_line (with undo on overflow). Each
488 : element is on its own cacheline to avoid false sharing between
489 : classes. */
490 : struct __attribute__((aligned(64))) { ulong val; } cache_class_init[ FD_ACCDB_CACHE_CLASS_CNT ];
491 :
492 : ulong cache_class_max[ FD_ACCDB_CACHE_CLASS_CNT ];
493 :
494 : /* Byte offsets from shmem base to the per-class cache regions. Each
495 : region is cache_class_max[c] * fd_accdb_cache_slot_sz[c] bytes,
496 : with each slot holding an fd_accdb_cache_line_t header followed by
497 : up to (fd_accdb_cache_slot_sz[c] - META_SZ) bytes of account data.
498 : */
499 : ulong cache_region_off[ FD_ACCDB_CACHE_CLASS_CNT ];
500 :
501 : /* Background pre-eviction watermarks (computed once in shmem_new).
502 : cache_free_target[c]: desired free-list depth for class c.
503 : cache_free_low_water[c]: trigger threshold ((target*3)/4). */
504 : ulong cache_free_target [ FD_ACCDB_CACHE_CLASS_CNT ];
505 : ulong cache_free_low_water[ FD_ACCDB_CACHE_CLASS_CNT ];
506 :
507 : /* cache_class_used[i].val holds the number of reserved cache
508 : slots in size class i, taken and returned in chunks by the joins'
509 : reservation credits (fd_accdb.c res_*). Each element is on its
510 : own cacheline to avoid false sharing between classes. The count
511 : includes the joins' idle credits (join_cred), so used[i].val +
512 : available[i] == cache_class_max[i] at all times. */
513 : struct __attribute__((aligned(64))) { ulong val; } cache_class_used[ FD_ACCDB_CACHE_CLASS_CNT ];
514 :
515 : /* Per-layer write heads. whead[0] is the hot (execution) write
516 : head, updated with atomic fetch-and-add by acquire/release
517 : threads. whead[1..N-1] are compaction write heads, each
518 : single-writer (compaction tile only). */
519 : accdb_offset_t whead[ FD_ACCDB_COMPACTION_LAYER_CNT ];
520 : int has_partition[ FD_ACCDB_COMPACTION_LAYER_CNT ];
521 :
522 : ulong partition_cnt;
523 : ulong partition_sz;
524 : ulong partition_max; /* acquired high water, for snapshot recovery */
525 : ulong partition_fallocated; /* partitions backed by the file, can run ahead */
526 :
527 : ulong chain_cnt;
528 : ulong max_live_slots;
529 : ulong max_accounts;
530 : ulong max_account_writes_per_slot;
531 :
532 : /* Hard upper bound on concurrent joiners, set at construction.
533 : Used to determine whether cache_class_used tracking can be
534 : skipped for a given class (when max[c] >= MIN_RESERVED *
535 : joiner_cnt, every reservation succeeds trivially). */
536 : ulong joiner_cnt_max;
537 :
538 : /* Per-joiner worst-case cache reservation, set at construction.
539 : Used by fd_accdb_reset to replicate the cache_class_used sentinel
540 : logic from fd_accdb_shmem_new. */
541 : ulong cache_min_reserved;
542 :
543 : ulong partition_pool_off;
544 :
545 : /* compaction_dlist_off[k] is the byte offset (from shmem base) of
546 : the dlist sentinel for layer k. Partitions at layer k that
547 : reach the freed-bytes threshold are enqueued here for compaction
548 : into layer k+1, or into layer k itself for the deepest layer. */
549 : ulong compaction_dlist_off[ FD_ACCDB_COMPACTION_LAYER_CNT ];
550 :
551 : /* Epoch-based grace periods for compaction and resource reclamation.
552 :
553 : Writers advance epoch when enqueueing compaction or retiring resources.
554 : Compaction and resource reuse wait for the corresponding grace period.
555 :
556 : joiner_epochs[i] holds the epoch observed by joiner i at the start
557 : of its epoch-protected critical section, or ULONG_MAX when idle.
558 : A tagged epoch has passed when all published writer and external
559 : reader epochs are strictly greater than it.
560 :
561 : joiner_cnt is claimed via atomic fetch-and-add in fd_accdb_new
562 : and never decremented. */
563 : ulong epoch __attribute__((aligned(64)));
564 :
565 : /* Synchronization with snapshot producer to inhibit compaction
566 : Holds one of FD_ACCDB_SNAPSHOT_SYNC_* */
567 : ulong snapshot_sync __attribute__((aligned(64)));
568 :
569 : /* Each joiner epoch is padded to a full cache line to prevent
570 : false sharing between joiners writing to adjacent slots. */
571 : struct __attribute__((aligned(64))) { ulong val; } joiner_epochs[ FD_ACCDB_MAX_JOINERS ];
572 : ulong joiner_cnt __attribute__((aligned(64)));
573 : ulong deferred_free_dlist_off;
574 :
575 : /* join_cred[j].val[c] is join j's idle reservation credits in class
576 : c (fd_accdb.c res_*). Only updated atomically: the owner adds and
577 : CAS-takes, a failing reserver exchanges it to 0. */
578 : struct __attribute__((aligned(64))) { ulong val[ FD_ACCDB_CACHE_CLASS_CNT ]; } join_cred[ FD_ACCDB_MAX_JOINERS ];
579 :
580 : fd_accdb_shmem_metrics_t shmetrics[1];
581 :
582 : /* Root / purge requests (T1 -> T2).
583 :
584 : T1 (replay or snapin) owns the fork tree: it alone writes fork
585 : parent / child / sibling links and descends_sets. advance_root and
586 : purge update the tree immediately and leave the index work to T2
587 : (the accdb tile), which never blocks T1.
588 :
589 : req.root_gen is the generation of the latest requested root. Every
590 : fork from root_fork_id up to the requested root's parent has a
591 : single child, and generations increase along that path, so T2
592 : roots root_fork_id's child_id until root_fork_id has root_gen.
593 : Attaching the initial root requests it as the root, so there is
594 : nothing to root. Generations are never reused (reset keeps
595 : counting), so done never matches a request it has not applied.
596 :
597 : req.purge_fork is the latest purged (detached) fork. Each purged
598 : fork links to the one purged before it through its sibling_id (a
599 : detached fork has no siblings). T2 applies the newest
600 : req.purge_seq-done.purge_seq forks of that chain. Each unapplied
601 : purge holds a fork slot, so fewer than 2^16 are ever pending and
602 : their links are never recycled.
603 :
604 : done is T2's progress, stored last in a pass. T2 has work when
605 : done.root_gen or done.purge_seq differ from req. done.retiring is
606 : set while T2 holds a batch of retired accounts / fork slots it
607 : will release to the pools once concurrent readers drain. When
608 : done matches req and retiring is clear, T2 does not touch the
609 : fork, acc or txn pools until the next request. */
610 :
611 : fd_accdb_req_t req __attribute__((aligned(64)));
612 : fd_accdb_done_t done __attribute__((aligned(64)));
613 :
614 : /* T2-only side buffer for deferred acc unlinks. Holds the indices of
615 : accs that have been CAS-unlinked from their map chains in the
616 : current advance_root or purge call but cannot yet have pool.next
617 : written: a concurrent cold_load_acc may stomp it via the cache_idx
618 : union alias. After wait_for_epoch_drain the list is materialized
619 : into pool.next links and released via acc_pool_release_chain.
620 :
621 : T2 is the sole writer (it applies all root / purge requests), so
622 : cnt is plain. Capacity is
623 : txn_max = max_live_slots * max_account_writes_per_slot, the same
624 : bound used to size txn_pool. Stored as a byte offset from shmem
625 : base. */
626 :
627 : ulong deferred_acc_buf_off;
628 : ulong deferred_acc_buf_cnt;
629 : ulong deferred_acc_buf_max;
630 : ulong deferred_epoch; /* epoch tag of the retired batch (accs and fork slots) */
631 :
632 : /* T2-only chain (through pool.next) of fork pool slots retired with
633 : the batch above. Their IDs may still be referenced by concurrent
634 : readers (via descends_set_test or a root_fork_id snapshot).
635 : USHORT_MAX when empty. */
636 : fd_accdb_fork_id_t deferred_fork_head;
637 : fd_accdb_fork_id_t deferred_fork_tail;
638 :
639 : acc_pool_shmem_t acc_pool [1];
640 : fork_pool_shmem_t fork_pool[1];
641 : txn_pool_shmem_t txn_pool [1];
642 :
643 : /* Track accounts modified since full snapshot.
644 :
645 : ------------++++++++....... - pruned blocks
646 : ^ ^ ^ ^ + active blocks
647 : full snap root head incremental (future) . future blocks
648 :
649 : The validator thus needs to track the addresses of all accounts
650 : that have changed since the last full snapshot. accdb forks cannot
651 : be used for this as fork information at the full snapshot slot is
652 : discarded.
653 :
654 : accdb deltas track this missing information. */
655 :
656 : struct {
657 : ulong seed;
658 : ulong chain_off;
659 : uint chain_cnt; /* power of 2 */
660 : uint chain_mask; /* chain_cnt-1, contiguous runs of one bits */
661 : ulong ele_off;
662 : ulong ele_max;
663 : ulong head; /* bump alloc head */
664 : } delta;
665 :
666 : ulong magic; /* ==FD_ACCDB_SHMEM_MAGIC */
667 : };
668 :
669 : /* fd_accdb_shmem_cache_alloc_reset empties the cache line allocator: no
670 : line used yet, every free list empty, no idle reservation credits.
671 : cache_class_max, cache_min_reserved and joiner_cnt_max must be set.
672 : If a class has enough lines for every joiner's worst case at once
673 : (cache_min_reserved per joiner), no reservation can overflow, and
674 : the cache_class_used sentinel ULONG_MAX tells acquire/release to skip
675 : the counter. */
676 :
677 : static inline void
678 12 : fd_accdb_shmem_cache_alloc_reset( fd_accdb_shmem_t * shmem ) {
679 12 : ulong worst = shmem->cache_min_reserved*shmem->joiner_cnt_max;
680 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
681 96 : shmem->clock_hand [ c ].val = 0UL;
682 96 : shmem->cache_class_init[ c ].val = 0UL;
683 96 : shmem->cache_class_used[ c ].val = fd_ulong_if( shmem->cache_class_max[ c ]>=worst, ULONG_MAX, 0UL );
684 480 : for( ulong w=0UL; w<FD_ACCDB_MAX_JOINERS/64UL; w++ ) {
685 384 : shmem->cache_free_have [ c ].bits[ w ] = 0UL;
686 384 : shmem->cache_free_hungry[ c ].bits[ w ] = 0UL;
687 384 : }
688 96 : }
689 3084 : for( ulong j=0UL; j<FD_ACCDB_MAX_JOINERS; j++ ) {
690 27648 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
691 24576 : shmem->cache_free [ j ].ver_top[ c ] = (ulong)UINT_MAX;
692 24576 : shmem->cache_free_cnt[ j ].val [ c ] = 0UL;
693 24576 : shmem->join_cred [ j ].val [ c ] = 0UL;
694 24576 : }
695 3072 : }
696 12 : }
697 :
698 : FD_PROTOTYPES_BEGIN
699 :
700 : /* fd_accdb_set_preevict_interval sets the minimum ticks between this
701 : join's background pre-eviction watermark scans (default ~100000; 0
702 : scans on every fd_accdb_background call). For tests: a
703 : single-threaded test that calls fd_accdb_background once to refill
704 : the free lists sets 0. */
705 :
706 : void
707 : fd_accdb_set_preevict_interval( fd_accdb_t * accdb,
708 : long ticks );
709 :
710 : FD_PROTOTYPES_END
711 :
712 : #endif /* HEADER_fd_src_flamenco_accdb_fd_accdb_private_h */
|
