Line data Source code
1 : #include "fd_accdb_shmem.h"
2 : #include "fd_accdb_private.h"
3 :
4 : #include "../../util/log/fd_log.h"
5 :
6 : #define POOL_NAME partition_pool
7 36 : #define POOL_T fd_accdb_partition_t
8 98304 : #define POOL_NEXT pool_next
9 : #define POOL_IDX_T ulong
10 : #define POOL_IMPL_STYLE 2
11 :
12 : #include "../../util/tmpl/fd_pool.c"
13 :
14 : #define DLIST_NAME compaction_dlist
15 : #define DLIST_ELE_T fd_accdb_partition_t
16 : #define DLIST_PREV dlist_prev
17 0 : #define DLIST_NEXT dlist_next
18 : #define DLIST_IMPL_STYLE 2
19 :
20 : #include "../../util/tmpl/fd_dlist.c"
21 :
22 : #define DLIST_NAME deferred_free_dlist
23 : #define DLIST_ELE_T fd_accdb_partition_t
24 : #define DLIST_PREV dlist_prev
25 0 : #define DLIST_NEXT dlist_next
26 : #define DLIST_IMPL_STYLE 2
27 :
28 : #include "../../util/tmpl/fd_dlist.c"
29 :
30 : FD_FN_CONST ulong
31 36 : fd_accdb_shmem_align( void ) {
32 36 : return FD_ACCDB_SHMEM_ALIGN;
33 36 : }
34 :
35 : fd_accdb_shmem_t *
36 12 : fd_accdb_shmem_join( void * shtc ) {
37 12 : if( FD_UNLIKELY( !shtc ) ) {
38 0 : FD_LOG_WARNING(( "NULL shtc" ));
39 0 : return NULL;
40 0 : }
41 :
42 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shtc, fd_accdb_shmem_align() ) ) ) {
43 0 : FD_LOG_WARNING(( "misaligned shtc" ));
44 0 : return NULL;
45 0 : }
46 :
47 12 : fd_accdb_shmem_t * accdb = (fd_accdb_shmem_t *)shtc;
48 :
49 12 : if( FD_UNLIKELY( accdb->magic!=FD_ACCDB_SHMEM_MAGIC ) ) {
50 0 : FD_LOG_WARNING(( "bad magic" ));
51 0 : return NULL;
52 0 : }
53 12 : return accdb;
54 12 : }
55 :
56 : ulong
57 : fd_accdb_shmem_footprint( ulong max_accounts,
58 : ulong max_live_slots,
59 : ulong max_account_writes_per_slot,
60 : ulong partition_cnt,
61 : ulong cache_footprint,
62 : ulong cache_min_reserved,
63 : ulong joiner_cnt,
64 12 : ulong max_incremental_accounts ) {
65 12 : if( FD_UNLIKELY( !max_accounts ) ) return 0UL;
66 12 : if( FD_UNLIKELY( !max_live_slots ) ) return 0UL;
67 12 : if( FD_UNLIKELY( !max_account_writes_per_slot) ) return 0UL;
68 12 : if( FD_UNLIKELY( !partition_cnt ) ) return 0UL;
69 12 : if( FD_UNLIKELY( !cache_min_reserved ) ) return 0UL;
70 : /* Partition indices are packed into 13 bits of accdb_offset_t
71 : (bits 63..51), so partition_cnt==8192 uses indices 0..8191, the
72 : full 13-bit range. The initial write-head sentinel encodes its
73 : invalidity in the offset bits (partition_offset==partition_sz),
74 : not the index, so it remains distinguishable even when no spare
75 : index value is left. */
76 12 : if( FD_UNLIKELY( partition_cnt>(1UL<<13) ) ) return 0UL;
77 12 : if( FD_UNLIKELY( !joiner_cnt || joiner_cnt>FD_ACCDB_MAX_JOINERS ) ) return 0UL;
78 :
79 12 : if( FD_UNLIKELY( max_accounts>=UINT_MAX ) ) return 0UL;
80 :
81 12 : if( FD_UNLIKELY( max_live_slots>=USHORT_MAX ) ) return 0UL;
82 :
83 12 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
84 12 : if( FD_UNLIKELY( txn_max/max_account_writes_per_slot!=max_live_slots ) ) return 0UL;
85 12 : if( FD_UNLIKELY( txn_max>=UINT_MAX ) ) return 0UL;
86 :
87 12 : ulong descends_fp = descends_set_footprint( max_live_slots );
88 12 : if( FD_UNLIKELY( !descends_fp ) ) return 0UL;
89 12 : if( FD_UNLIKELY( max_live_slots>ULONG_MAX/descends_fp ) ) return 0UL;
90 :
91 12 : ulong chain_cnt = fd_ulong_pow2_up( (max_accounts>>1) + (max_accounts&1UL) );
92 :
93 12 : if( FD_UNLIKELY( chain_cnt>ULONG_MAX/sizeof(uint) ) ) return 0UL;
94 :
95 12 : if( FD_UNLIKELY( !cache_footprint ) ) return 0UL;
96 12 : ulong cache_class_max[ FD_ACCDB_CACHE_CLASS_CNT ];
97 12 : if( FD_UNLIKELY( !fd_accdb_cache_class_cnt( cache_footprint, cache_min_reserved, cache_class_max ) ) ) return 0UL;
98 :
99 12 : if( FD_UNLIKELY( max_incremental_accounts>UINT_MAX ) ) return 0UL;
100 12 : ulong delta_chain_cnt = fd_ulong_pow2_up( (max_incremental_accounts>>1) + (max_incremental_accounts&1UL) );
101 :
102 12 : ulong l;
103 12 : l = FD_LAYOUT_INIT;
104 12 : l = FD_LAYOUT_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
105 12 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
106 12 : l = FD_LAYOUT_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
107 12 : l = FD_LAYOUT_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
108 12 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
109 12 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_txn_t), txn_max*sizeof(fd_accdb_txn_t) );
110 12 : l = FD_LAYOUT_APPEND( l, partition_pool_align(), partition_pool_footprint( partition_cnt ) );
111 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
112 36 : l = FD_LAYOUT_APPEND( l, compaction_dlist_align(), compaction_dlist_footprint() );
113 36 : }
114 12 : l = FD_LAYOUT_APPEND( l, deferred_free_dlist_align(), deferred_free_dlist_footprint() );
115 12 : l = FD_LAYOUT_APPEND( l, alignof(uint), txn_max*sizeof(uint) );
116 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
117 96 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_cache_line_t), cache_class_max[c]*fd_accdb_cache_slot_sz[c] );
118 96 : }
119 12 : l = FD_LAYOUT_APPEND( l, alignof(uint), delta_chain_cnt*sizeof(uint) );
120 12 : l = FD_LAYOUT_APPEND( l, alignof(fd_accdb_delta_t),max_incremental_accounts*sizeof(fd_accdb_delta_t) );
121 12 : return FD_LAYOUT_FINI( l, FD_ACCDB_SHMEM_ALIGN );
122 12 : }
123 :
124 : void *
125 : fd_accdb_shmem_new( void * shmem,
126 : ulong max_accounts,
127 : ulong max_live_slots,
128 : ulong max_account_writes_per_slot,
129 : ulong partition_cnt,
130 : ulong partition_sz,
131 : ulong cache_footprint,
132 : ulong cache_min_reserved,
133 : int bundle_enabled,
134 : ulong seed,
135 : ulong joiner_cnt,
136 12 : ulong max_incremental_accounts ) {
137 12 : if( FD_UNLIKELY( !shmem ) ) {
138 0 : FD_LOG_WARNING(( "NULL shmem" ));
139 0 : return NULL;
140 0 : }
141 :
142 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_accdb_shmem_align() ) ) ) {
143 0 : FD_LOG_WARNING(( "misaligned shmem" ));
144 0 : return NULL;
145 0 : }
146 :
147 12 : if( FD_UNLIKELY( !max_accounts ) ) {
148 0 : FD_LOG_WARNING(( "max_accounts must be non-zero" ));
149 0 : return NULL;
150 0 : }
151 :
152 12 : if( FD_UNLIKELY( !max_live_slots ) ) {
153 0 : FD_LOG_WARNING(( "max_live_slots must be non-zero" ));
154 0 : return NULL;
155 0 : }
156 :
157 12 : if( FD_UNLIKELY( !max_account_writes_per_slot ) ) {
158 0 : FD_LOG_WARNING(( "max_account_writes_per_slot must be non-zero" ));
159 0 : return NULL;
160 0 : }
161 :
162 12 : if( FD_UNLIKELY( !joiner_cnt || joiner_cnt>FD_ACCDB_MAX_JOINERS ) ) {
163 0 : FD_LOG_WARNING(( "joiner_cnt must be in [1, %lu]", FD_ACCDB_MAX_JOINERS ));
164 0 : return NULL;
165 0 : }
166 :
167 12 : if( FD_UNLIKELY( max_live_slots>=USHORT_MAX ) ) {
168 0 : FD_LOG_WARNING(( "max_live_slots must be less than %u", (uint)USHORT_MAX ));
169 0 : return NULL;
170 0 : }
171 :
172 12 : if( FD_UNLIKELY( !partition_cnt ) ) {
173 0 : FD_LOG_WARNING(( "partition_cnt must be non-zero" ));
174 0 : return NULL;
175 0 : }
176 :
177 12 : if( FD_UNLIKELY( partition_cnt>(1UL<<13) ) ) {
178 0 : FD_LOG_WARNING(( "partition_cnt must be at most %lu", 1UL<<13 ));
179 0 : return NULL;
180 0 : }
181 :
182 12 : if( FD_UNLIKELY( !partition_sz ) ) {
183 0 : FD_LOG_WARNING(( "partition_sz must be non-zero" ));
184 0 : return NULL;
185 0 : }
186 :
187 : /* Partition offsets are packed into the low 51 bits of accdb_offset_t
188 : (see FD_ACCDB_PARTITION_OFF_BITS in fd_accdb.c). partition_sz must
189 : be small enough that speculative fetch-and-adds from up to
190 : FD_ACCDB_MAX_JOINERS concurrent threads in allocate_next_write
191 : can never carry the offset field into the partition_idx bits.
192 : Worst case: all joiners each do one FETCH_AND_ADD of partition_sz
193 : before the partition switch completes, starting from an offset of
194 : at most partition_sz-1. */
195 12 : if( FD_UNLIKELY( partition_sz>(1UL<<51)/(FD_ACCDB_MAX_JOINERS+1UL) ) ) {
196 0 : FD_LOG_WARNING(( "partition_sz must be at most %lu", (1UL<<51)/(FD_ACCDB_MAX_JOINERS+1UL) ));
197 0 : return NULL;
198 0 : }
199 :
200 : /* The maximum file offset is (partition_cnt-1)*partition_sz +
201 : partition_sz - 1, which must fit in a signed long (off_t) because
202 : pwritev2, preadv2, and fallocate all take signed offsets. */
203 12 : if( FD_UNLIKELY( partition_cnt>=(ulong)LONG_MAX/partition_sz ) ) {
204 0 : FD_LOG_WARNING(( "partition_cnt*partition_sz must be at most LONG_MAX" ));
205 0 : return NULL;
206 0 : }
207 :
208 : /* The total addressable file space (partition_cnt * partition_sz)
209 : must not exceed 2^FD_ACCDB_OFF_BITS. File offsets are stored in
210 : the 48-bit offset portion of acc->offset_fork, and the all-ones
211 : value FD_ACCDB_OFF_INVAL is reserved as a dirty sentinel. The
212 : allocator guarantees record start offsets are always at least
213 : sizeof(fd_accdb_disk_meta_t) below a partition boundary, so a
214 : total of exactly 2^48 is safe (no valid offset reaches the
215 : sentinel), but exceeding it is not. */
216 12 : if( FD_UNLIKELY( partition_cnt>((1UL<<FD_ACCDB_OFF_BITS)/partition_sz) ) ) {
217 0 : FD_LOG_WARNING(( "partition_cnt*partition_sz must be at most %lu", 1UL<<FD_ACCDB_OFF_BITS ));
218 0 : return NULL;
219 0 : }
220 :
221 : /* partition_sz must be large enough to hold at least one worst-case
222 : account write (disk metadata header + largest cache class payload).
223 : Without this, allocate_next_write can never fit the entry in a
224 : single partition. */
225 12 : ulong min_partition_sz = sizeof(fd_accdb_disk_meta_t) + fd_accdb_cache_slot_sz[ FD_ACCDB_CACHE_CLASS_CNT-1UL ] - FD_ACCDB_CACHE_META_SZ;
226 12 : if( FD_UNLIKELY( partition_sz<min_partition_sz ) ) {
227 0 : FD_LOG_WARNING(( "partition_sz must be at least %lu to fit worst-case account write", min_partition_sz ));
228 0 : return NULL;
229 0 : }
230 :
231 12 : if( FD_UNLIKELY( max_accounts>=UINT_MAX ) ) {
232 0 : FD_LOG_WARNING(( "max_accounts must be less than UINT_MAX" ));
233 0 : return NULL;
234 0 : }
235 :
236 12 : ulong txn_max = max_live_slots * max_account_writes_per_slot;
237 12 : if( FD_UNLIKELY( txn_max/max_account_writes_per_slot!=max_live_slots ) ) {
238 0 : FD_LOG_WARNING(( "max_live_slots*max_account_writes_per_slot overflows" ));
239 0 : return NULL;
240 0 : }
241 12 : if( FD_UNLIKELY( txn_max>=UINT_MAX ) ) {
242 0 : FD_LOG_WARNING(( "max_live_slots*max_account_writes_per_slot must be less than UINT_MAX" ));
243 0 : return NULL;
244 0 : }
245 :
246 12 : ulong descends_fp = descends_set_footprint( max_live_slots );
247 12 : if( FD_UNLIKELY( !descends_fp || max_live_slots>ULONG_MAX/descends_fp ) ) {
248 0 : FD_LOG_WARNING(( "max_live_slots*descends_set_footprint overflows" ));
249 0 : return NULL;
250 0 : }
251 :
252 12 : ulong chain_cnt = fd_ulong_pow2_up( (max_accounts>>1) + (max_accounts&1UL) );
253 :
254 12 : if( FD_UNLIKELY( chain_cnt>ULONG_MAX/sizeof(uint) ) ) {
255 0 : FD_LOG_WARNING(( "chain_cnt*sizeof(uint) overflows" ));
256 0 : return NULL;
257 0 : }
258 :
259 12 : if( FD_UNLIKELY( !cache_min_reserved ) ) {
260 0 : FD_LOG_WARNING(( "cache_min_reserved must be non-zero" ));
261 0 : return NULL;
262 0 : }
263 :
264 12 : ulong cache_class_max[ FD_ACCDB_CACHE_CLASS_CNT ];
265 12 : if( FD_UNLIKELY( !fd_accdb_cache_class_cnt( cache_footprint, cache_min_reserved, cache_class_max ) ) ) {
266 0 : FD_LOG_WARNING(( "invalid cache_footprint" ));
267 0 : return NULL;
268 0 : }
269 : /* cidx packs only FD_ACCDB_CACHE_LINE_BITS bits of line index, so
270 : cache_class_max[c]>FD_ACCDB_CACHE_LINE_MAX would let line indices
271 : alias. fd_accdb_cache_class_cnt clamps this; assert here so any
272 : future regression in the allocator is caught at shmem-new time
273 : rather than as silent cache corruption at runtime. */
274 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) FD_TEST( cache_class_max[ c ]<=FD_ACCDB_CACHE_LINE_MAX );
275 :
276 12 : if( FD_UNLIKELY( max_incremental_accounts>UINT_MAX ) ) {
277 0 : FD_LOG_WARNING(( "max_incremental_accounts must be at most %u", UINT_MAX ));
278 0 : return NULL;
279 0 : }
280 :
281 12 : ulong delta_chain_cnt = fd_ulong_pow2_up( (max_incremental_accounts>>1) + (max_incremental_accounts&1UL) );
282 12 : if( FD_UNLIKELY( delta_chain_cnt>UINT_MAX ) ) {
283 0 : FD_LOG_WARNING(( "max_incremental_accounts must be at most %u", UINT_MAX ));
284 0 : return NULL;
285 0 : }
286 :
287 12 : FD_SCRATCH_ALLOC_INIT( l, shmem );
288 12 : fd_accdb_shmem_t * accdb = FD_SCRATCH_ALLOC_APPEND( l, FD_ACCDB_SHMEM_ALIGN, sizeof(fd_accdb_shmem_t) );
289 12 : void * _fork_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_fork_shmem_t), max_live_slots*sizeof(fd_accdb_fork_shmem_t) );
290 12 : void * _descends_sets = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_live_slots*descends_set_footprint( max_live_slots ) );
291 12 : void * _acc_map = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), chain_cnt*sizeof(uint) );
292 12 : void * _acc_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_accmeta_t), max_accounts*sizeof(fd_accdb_accmeta_t) );
293 12 : void * _txn_pool_ele = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_txn_t), txn_max*sizeof(fd_accdb_txn_t) );
294 12 : void * _partition_pool = FD_SCRATCH_ALLOC_APPEND( l, partition_pool_align(), partition_pool_footprint( partition_cnt ) );
295 0 : void * _compaction_dlists[ FD_ACCDB_COMPACTION_LAYER_CNT ];
296 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
297 36 : _compaction_dlists[ k ] = FD_SCRATCH_ALLOC_APPEND( l, compaction_dlist_align(), compaction_dlist_footprint() );
298 36 : }
299 12 : void * _deferred_free_dlist = FD_SCRATCH_ALLOC_APPEND( l, deferred_free_dlist_align(), deferred_free_dlist_footprint() );
300 12 : void * _deferred_acc_buf = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), txn_max*sizeof(uint) );
301 0 : void * _cache_regions[ FD_ACCDB_CACHE_CLASS_CNT ];
302 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
303 96 : _cache_regions[ c ] = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_cache_line_t), cache_class_max[c]*fd_accdb_cache_slot_sz[c] );
304 96 : }
305 12 : void * _delta_map = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), delta_chain_cnt*sizeof(uint) );
306 12 : void * _delta_pool = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_accdb_delta_t), max_incremental_accounts*sizeof(fd_accdb_delta_t) );
307 :
308 0 : fd_memset( _acc_map, 0xFF, chain_cnt*sizeof(uint) );
309 :
310 12 : FD_TEST( acc_pool_new( accdb->acc_pool ) );
311 12 : acc_pool_t _acc_pool_join[1];
312 12 : FD_TEST( acc_pool_join( _acc_pool_join, accdb->acc_pool, _acc_pool_ele, max_accounts ) );
313 12 : acc_pool_reset( _acc_pool_join );
314 12 : acc_pool_leave( _acc_pool_join );
315 :
316 12 : FD_TEST( fork_pool_new( accdb->fork_pool ) );
317 12 : fork_pool_t _fork_pool_join[1];
318 12 : FD_TEST( fork_pool_join( _fork_pool_join, accdb->fork_pool, _fork_pool_ele, max_live_slots ) );
319 12 : fork_pool_reset( _fork_pool_join );
320 12 : fork_pool_leave( _fork_pool_join );
321 :
322 12 : ulong descends_set_fp = descends_set_footprint( max_live_slots );
323 204 : for( ulong i=0UL; i<max_live_slots; i++ ) {
324 192 : descends_set_t * descends_set = descends_set_join( descends_set_new( (uchar *)_descends_sets + i*descends_set_fp, max_live_slots ) );
325 192 : FD_TEST( descends_set );
326 192 : }
327 :
328 12 : FD_TEST( txn_pool_new( accdb->txn_pool ) );
329 12 : txn_pool_t _txn_pool_join[1];
330 12 : FD_TEST( txn_pool_join( _txn_pool_join, accdb->txn_pool, _txn_pool_ele, txn_max ) );
331 12 : txn_pool_reset( _txn_pool_join );
332 12 : txn_pool_leave( _txn_pool_join );
333 :
334 12 : fd_accdb_partition_t * partition_pool = partition_pool_join( partition_pool_new( _partition_pool, partition_cnt ) );
335 12 : FD_TEST( partition_pool );
336 98316 : for( ulong i=0UL; i<partition_cnt; i++ ) {
337 98304 : partition_pool_ele( partition_pool, i )->write_offset = 0UL;
338 98304 : }
339 :
340 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
341 36 : compaction_dlist_t * dlist = compaction_dlist_join( compaction_dlist_new( _compaction_dlists[ k ] ) );
342 36 : FD_TEST( dlist );
343 36 : }
344 :
345 12 : deferred_free_dlist_t * deferred_free = deferred_free_dlist_join( deferred_free_dlist_new( _deferred_free_dlist ) );
346 12 : FD_TEST( deferred_free );
347 :
348 12 : fd_memset( _delta_map, 0xFF, delta_chain_cnt*sizeof(uint) );
349 :
350 12 : accdb->seed = seed;
351 12 : accdb->root_fork_id = (fd_accdb_fork_id_t){ .val = USHORT_MAX };
352 12 : accdb->generation = 0U;
353 :
354 12 : accdb->partition_lock = 0;
355 12 : accdb->grow_lock = 0;
356 12 : accdb->snapshot_loading = 0;
357 12 : accdb->bundle_enabled = bundle_enabled;
358 :
359 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->clock_hand[ c ].val = 0UL;
360 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_free[ c ].ver_top = (ulong)UINT_MAX;
361 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_free_cnt[ c ].val = 0UL;
362 :
363 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
364 96 : ulong max_c = cache_class_max[ c ];
365 96 : ulong floor_c = fd_ulong_min( cache_min_reserved, max_c );
366 96 : ulong headroom = ( max_c>floor_c ) ? ( max_c - floor_c ) : 0UL;
367 96 : ulong cap = fd_ulong_min( 8192UL, (64UL<<20) / fd_accdb_cache_slot_sz[ c ] );
368 96 : ulong burst_floor = fd_ulong_min( 512UL, headroom/2UL );
369 96 : ulong target = fd_ulong_min( cap, fd_ulong_max( headroom/10UL, burst_floor ) );
370 96 : accdb->cache_free_target [ c ] = target;
371 96 : accdb->cache_free_low_water[ c ] = (target * 3UL) / 4UL;
372 96 : }
373 :
374 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
375 : /* Sentinel: partition_offset == partition_sz forces the first
376 : allocate_next_write to fall into the partition-switch slow path,
377 : which acquires a real partition from the pool.
378 :
379 : The invalidity lives in the offset bits, not the index bits. The
380 : index here (partition_cnt) is only nominally invalid: at the
381 : maximum partition_cnt==8192 it does not fit in the 13-bit index
382 : field and wraps to 0, a perfectly valid pool index. */
383 36 : accdb->whead[ k ] = accdb_offset( partition_cnt, partition_sz );
384 36 : accdb->has_partition[ k ] = 0;
385 36 : }
386 :
387 12 : accdb->chain_cnt = chain_cnt;
388 12 : accdb->max_live_slots = max_live_slots;
389 12 : accdb->max_accounts = max_accounts;
390 12 : accdb->max_account_writes_per_slot = max_account_writes_per_slot;
391 12 : accdb->joiner_cnt_max = joiner_cnt;
392 12 : accdb->cache_min_reserved = cache_min_reserved;
393 12 : accdb->partition_cnt = partition_cnt;
394 12 : accdb->partition_sz = partition_sz;
395 12 : accdb->partition_max = 0UL;
396 12 : accdb->partition_fallocated = 0UL;
397 :
398 12 : accdb->partition_pool_off = (ulong)partition_pool - (ulong)shmem;
399 48 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
400 36 : accdb->compaction_dlist_off[ k ] = (ulong)_compaction_dlists[ k ] - (ulong)shmem;
401 36 : }
402 12 : accdb->deferred_free_dlist_off = (ulong)_deferred_free_dlist - (ulong)shmem;
403 :
404 12 : accdb->deferred_acc_buf_off = (ulong)_deferred_acc_buf - (ulong)shmem;
405 12 : accdb->deferred_acc_buf_cnt = 0UL;
406 12 : accdb->deferred_acc_buf_max = txn_max;
407 12 : accdb->deferred_acc_epoch = 0UL;
408 :
409 12 : accdb->epoch = 1UL;
410 12 : accdb->snapshot_sync = FD_ACCDB_SNAPSHOT_SYNC_IDLE;
411 12 : accdb->joiner_cnt = 0UL;
412 3084 : for( ulong i=0UL; i<FD_ACCDB_MAX_JOINERS; i++ ) accdb->joiner_epochs[ i ].val = ULONG_MAX;
413 :
414 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_class_init[ c ].val = 0UL;
415 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_class_max[ c ] = cache_class_max[ c ];
416 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) accdb->cache_region_off[ c ] = (ulong)_cache_regions[ c ] - (ulong)shmem;
417 :
418 : /* Pre-initialize every cache slot's metadata to the "empty" sentinel
419 : (gen=UINT_MAX, acc_idx=UINT_MAX, refcnt=0). Without this, the
420 : lazy-init path in acquire_cache_line bumps cache_class_init before
421 : writing the sentinels into the freshly-claimed line; a concurrent
422 : background_preevict reading the bumped init counter could then sweep
423 : a slot whose memory still reads as zero, see (gen=0, acc_idx=0)
424 : instead of the skip predicate, CAS refcnt 0->EVICT_SENTINEL, and
425 : "evict" a line the lazy-init owner is about to publish. */
426 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
427 96 : ulong slot_sz = fd_accdb_cache_slot_sz[ c ];
428 39909204 : for( ulong i=0UL; i<cache_class_max[ c ]; i++ ) {
429 39909108 : fd_accdb_cache_line_t * line = (fd_accdb_cache_line_t *)( (uchar *)_cache_regions[ c ] + i*slot_sz );
430 39909108 : line->key.generation = UINT_MAX;
431 39909108 : line->acc_idx = UINT_MAX;
432 39909108 : line->refcnt = 0U;
433 39909108 : line->referenced = 0;
434 39909108 : line->persisted = 1;
435 39909108 : line->seq = 0U;
436 39909108 : }
437 96 : }
438 :
439 : /* If a class has enough slots for every joiner's worst case
440 : simultaneously (cache_min_reserved per joiner), no reservation can
441 : ever overflow. Sentinel ULONG_MAX tells acquire/release to skip
442 : the atomic counters entirely. */
443 108 : for( ulong c=0UL; c<FD_ACCDB_CACHE_CLASS_CNT; c++ ) {
444 96 : if( cache_class_max[ c ]>=cache_min_reserved*joiner_cnt ) accdb->cache_class_used[ c ].val = ULONG_MAX;
445 0 : else accdb->cache_class_used[ c ].val = 0UL;
446 96 : }
447 :
448 12 : accdb->delta.seed = seed+1UL;
449 12 : accdb->delta.chain_off = (ulong)_delta_map - (ulong)shmem;
450 12 : accdb->delta.chain_cnt = (uint)delta_chain_cnt;
451 12 : accdb->delta.chain_mask = (uint)delta_chain_cnt - 1U;
452 12 : accdb->delta.ele_off = (ulong)_delta_pool - (ulong)shmem;
453 12 : accdb->delta.ele_max = max_incremental_accounts;
454 12 : accdb->delta.head = 0UL;
455 :
456 12 : memset( accdb->shmetrics, 0, sizeof( fd_accdb_shmem_metrics_t ) );
457 12 : accdb->shmetrics->accounts_capacity = max_accounts;
458 :
459 12 : accdb->cmd_op = FD_ACCDB_CMD_IDLE;
460 12 : accdb->cmd_fork_id = USHORT_MAX;
461 :
462 12 : FD_COMPILER_MFENCE();
463 12 : FD_VOLATILE( accdb->magic ) = FD_ACCDB_SHMEM_MAGIC;
464 12 : FD_COMPILER_MFENCE();
465 :
466 12 : return (void *)accdb;
467 12 : }
468 :
469 : void
470 : fd_accdb_shmem_try_enqueue_compaction( fd_accdb_shmem_t * accdb,
471 0 : ulong partition_idx ) {
472 : /* Caller must hold partition_lock. */
473 :
474 0 : fd_accdb_partition_t * partition_pool = (fd_accdb_partition_t *)( (uchar *)accdb + accdb->partition_pool_off );
475 0 : fd_accdb_partition_t * partition = partition_pool_ele( partition_pool, partition_idx );
476 :
477 0 : if( FD_UNLIKELY( partition->bytes_freed<(accdb->partition_sz*FD_ACCDB_COMPACTION_THRESHOLD_PCT/100UL) ) ) return;
478 0 : if( FD_UNLIKELY( partition->marked_compaction ) ) return;
479 :
480 : /* While a snapshot load is in flight, defer all compaction so the
481 : compaction tile cannot race with the bulk loader. Anything that
482 : crosses the threshold here will be re-checked by
483 : fd_accdb_snapshot_load_end's sweep when loading completes. */
484 0 : if( FD_UNLIKELY( FD_VOLATILE_CONST( accdb->snapshot_loading ) ) ) return;
485 :
486 : /* Do not enqueue any currently active write-head partition. Its
487 : write_offset is not yet finalized, so compaction cannot determine
488 : the valid data range. The partition_lock serializes this check
489 : with change_partition, so it is not racy. */
490 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
491 0 : if( FD_UNLIKELY( accdb->has_partition[ k ] && packed_partition_idx( &accdb->whead[ k ] )==partition_idx ) ) return;
492 0 : }
493 :
494 0 : uchar layer = partition->layer;
495 0 : compaction_dlist_t * compaction_dlist = (compaction_dlist_t *)( (uchar *)accdb + accdb->compaction_dlist_off[ layer ] );
496 :
497 0 : partition->marked_compaction = 1;
498 0 : partition->compaction_offset = 0UL;
499 0 : partition->compaction_ready_epoch = FD_ATOMIC_FETCH_AND_ADD( &accdb->epoch, 1UL );
500 0 : partition->queued = 1;
501 0 : if( FD_LIKELY( compaction_dlist_is_empty( compaction_dlist, partition_pool ) ) ) {
502 0 : FD_LOG_INFO(( "compaction of layer %u partition %lu started", (uint)layer, partition_pool_idx( partition_pool, partition ) ));
503 0 : }
504 0 : compaction_dlist_ele_push_tail( compaction_dlist, partition, partition_pool );
505 0 : accdb->shmetrics->in_compaction = 1;
506 0 : accdb->shmetrics->compactions_requested++;
507 0 : }
508 :
509 : void
510 : fd_accdb_shmem_bytes_freed( fd_accdb_shmem_t * accdb,
511 : ulong offset,
512 0 : ulong sz ) {
513 0 : fd_accdb_partition_t * partition_pool = (fd_accdb_partition_t *)( (uchar *)accdb + accdb->partition_pool_off );
514 :
515 0 : ulong partition_idx = offset/accdb->partition_sz;
516 0 : fd_accdb_partition_t * partition = partition_pool_ele( partition_pool, partition_idx );
517 : /* Launder the pointer: GCC derives partition from (accdb + off) and so
518 : believes __builtin_object_size( &partition->bytes_freed )==0, which
519 : trips a spurious -Wstringop-overflow on the atomic add below. */
520 0 : FD_COMPILER_FORGET( partition );
521 0 : FD_ATOMIC_FETCH_AND_ADD( &partition->bytes_freed, sz );
522 :
523 : /* Fast-path exit: skip the lock if clearly below threshold or
524 : already enqueued. */
525 0 : if( FD_LIKELY( partition->bytes_freed<(accdb->partition_sz*FD_ACCDB_COMPACTION_THRESHOLD_PCT/100UL) ) ) return;
526 0 : if( FD_UNLIKELY( partition->marked_compaction ) ) return;
527 :
528 0 : spin_lock_acquire( &accdb->partition_lock );
529 0 : fd_accdb_shmem_try_enqueue_compaction( accdb, partition_idx );
530 0 : spin_lock_release( &accdb->partition_lock );
531 0 : }
532 :
533 : ulong
534 0 : fd_accdb_shmem_partition_max( fd_accdb_shmem_t const * accdb ) {
535 0 : return accdb->partition_max;
536 0 : }
537 :
538 : ulong
539 0 : fd_accdb_shmem_partition_sz( fd_accdb_shmem_t const * accdb ) {
540 0 : return accdb->partition_sz;
541 0 : }
542 :
543 : void
544 : fd_accdb_shmem_partition_info( fd_accdb_shmem_t const * accdb,
545 : ulong partition_idx,
546 0 : fd_accdb_shmem_partition_info_t * out ) {
547 0 : fd_accdb_partition_t const * partition_pool = (fd_accdb_partition_t const *)( (uchar const *)accdb + accdb->partition_pool_off );
548 0 : fd_accdb_partition_t const * p = partition_pool_ele_const( partition_pool, partition_idx );
549 :
550 0 : out->file_offset = partition_idx * accdb->partition_sz;
551 0 : out->is_write_head = 0;
552 : /* If this partition is currently the active write head for any
553 : layer, partition->write_offset is stale (it's only updated at
554 : handoff in change_partition). The live tip lives in whead[layer].
555 : Surface the live value so the GUI shows real-time fill, not the
556 : "0 until rolled" snapshot. The tip is a reservation that can
557 : briefly overrun the partition, so clamp rather than show >100%. */
558 0 : ulong head_off = ULONG_MAX;
559 0 : for( ulong k=0UL; k<FD_ACCDB_COMPACTION_LAYER_CNT; k++ ) {
560 0 : if( !FD_VOLATILE_CONST( accdb->has_partition[ k ] ) ) continue;
561 0 : accdb_offset_t whead = { .val = FD_VOLATILE_CONST( accdb->whead[ k ].val ) };
562 0 : if( packed_partition_idx( &whead )==partition_idx ) {
563 0 : head_off = packed_partition_offset( &whead );
564 0 : out->is_write_head = 1;
565 0 : break;
566 0 : }
567 0 : }
568 0 : if( FD_UNLIKELY( out->is_write_head ) ) {
569 0 : out->write_offset_raw = head_off;
570 0 : out->write_offset = fd_ulong_min( head_off, accdb->partition_sz );
571 0 : } else {
572 0 : out->write_offset_raw = FD_VOLATILE_CONST( p->write_offset );
573 0 : out->write_offset = out->write_offset_raw;
574 0 : }
575 0 : out->bytes_freed = FD_VOLATILE_CONST( p->bytes_freed );
576 0 : out->compaction_offset = FD_VOLATILE_CONST( p->compaction_offset );
577 0 : out->read_ops = FD_VOLATILE_CONST( p->read_ops );
578 0 : out->bytes_read = FD_VOLATILE_CONST( p->bytes_read );
579 0 : out->write_ops = FD_VOLATILE_CONST( p->write_ops );
580 0 : out->bytes_written = FD_VOLATILE_CONST( p->bytes_written );
581 0 : out->created_ticks = (long)FD_VOLATILE_CONST( p->created_ticks );
582 0 : out->filled_ticks = (long)FD_VOLATILE_CONST( p->filled_ticks );
583 0 : out->layer = p->layer;
584 0 : uchar compacting = FD_VOLATILE_CONST( p->compacting_now );
585 0 : uchar queued = FD_VOLATILE_CONST( p->queued );
586 0 : out->compaction_state = compacting ? 2 : ( queued ? 1 : 0 );
587 0 : }
588 :
589 : FD_STATIC_ASSERT( sizeof(((fd_accdb_shmem_writer_barrier_t *)0)->bits)*8UL==FD_ACCDB_MAX_JOINERS, barrier_width );
590 :
591 : void
592 : fd_accdb_shmem_writer_barrier_capture( fd_accdb_shmem_t const * accdb,
593 0 : fd_accdb_shmem_writer_barrier_t * barrier ) {
594 0 : memset( barrier->bits, 0, sizeof(barrier->bits) );
595 0 : ulong joiner_cnt = FD_VOLATILE_CONST( accdb->joiner_cnt );
596 0 : for( ulong t=0UL; t<joiner_cnt; t++ ) {
597 0 : if( FD_VOLATILE_CONST( accdb->joiner_epochs[ t ].val )==ULONG_MAX ) continue;
598 0 : barrier->bits[ t/64UL ] |= 1UL<<(t%64UL);
599 0 : }
600 0 : }
601 :
602 : ulong
603 : fd_accdb_shmem_writer_barrier_poll( fd_accdb_shmem_t const * accdb,
604 0 : fd_accdb_shmem_writer_barrier_t * barrier ) {
605 0 : ulong remain = 0UL;
606 0 : for( ulong w=0UL; w<sizeof(barrier->bits)/sizeof(ulong); w++ ) {
607 0 : ulong bits = barrier->bits[ w ];
608 0 : while( bits ) {
609 0 : ulong b = (ulong)fd_ulong_find_lsb( bits );
610 0 : bits &= bits-1UL;
611 0 : if( FD_VOLATILE_CONST( accdb->joiner_epochs[ w*64UL+b ].val )==ULONG_MAX ) {
612 0 : barrier->bits[ w ] &= ~(1UL<<b);
613 0 : }
614 0 : }
615 0 : remain |= barrier->bits[ w ];
616 0 : }
617 0 : return remain;
618 0 : }
619 :
620 : ulong const *
621 0 : fd_accdb_shmem_snapshot_sync( fd_accdb_shmem_t const * accdb ) {
622 0 : return &accdb->snapshot_sync;
623 0 : }
|
