Line data Source code
1 : #ifndef HEADER_fd_src_disco_store_fd_store_h
2 : #define HEADER_fd_src_disco_store_fd_store_h
3 :
4 : #include "../../disco/shred/fd_fec_set.h"
5 : #include "../../ballet/shred/fd_shred.h"
6 : #include "../../flamenco/fd_rwlock.h"
7 : #include "../../flamenco/fd_flamenco_base.h"
8 : #include "../../util/fd_hash32.h"
9 : #include "../../util/hist/fd_histf.h"
10 : #include "../../util/shmem/fd_shmem.h"
11 : #include "../../util/tile/fd_tile.h"
12 :
13 : #define FD_STORE_ALIGN (128UL)
14 :
15 : /* The launcher creates the Store backing file before starting any tile
16 : and passes these descriptors through exec. Keep them adjacent to the
17 : accdb descriptor range, but distinct from it. */
18 0 : #define FD_STORE_FD_RW (123459)
19 0 : #define FD_STORE_FD_RO (123458)
20 :
21 : /* Spill and cache slots are page aligned. */
22 0 : #define FD_STORE_PAYLOAD_PAGE_SZ (FD_SHMEM_NORMAL_PAGE_SZ)
23 :
24 : FD_FN_CONST static inline ulong
25 0 : fd_store_payload_slot_sz( ulong fec_data_max ) {
26 0 : ulong rounded;
27 0 : if( FD_UNLIKELY( __builtin_uaddl_overflow( fec_data_max, FD_STORE_PAYLOAD_PAGE_SZ-1UL, &rounded ) ) ) return 0UL;
28 0 : return rounded & ~(FD_STORE_PAYLOAD_PAGE_SZ-1UL);
29 0 : }
30 0 : #define FD_STORE_MAGIC (0xf17eda2ce75702ebUL) /* firedancer store version 11 */
31 :
32 0 : #define FD_STORE_FEC_DATA_EMPTY (0U)
33 0 : #define FD_STORE_FEC_DATA_RAM_WRITING (1U)
34 0 : #define FD_STORE_FEC_DATA_RAM_READY (2U)
35 0 : #define FD_STORE_FEC_DATA_DISK (3U)
36 0 : #define FD_STORE_FEC_DATA_CONSUMED (4U)
37 0 : #define FD_STORE_FEC_DATA_SPILLING (5U)
38 :
39 : /* Shred ring keys are slot<<32 | shred_idx, so slot<2^32 and shred_idx
40 : <2^32. The per-slot hint reuses the key with bit 31 as the valid
41 : flag, which further bounds shred_idx<2^31 (see FD_SHREDB_HINT_VALID). */
42 :
43 : #define FD_SHREDB_KEY_SLOT_MAX (1UL<<32)
44 0 : #define FD_SHREDB_HINT_VALID (1UL<<31)
45 :
46 : FD_FN_CONST static inline ulong
47 0 : fd_shredb_key_pack( ulong slot, uint shred_idx ) {
48 0 : return (slot << 32) | (ulong)shred_idx;
49 0 : }
50 :
51 : FD_FN_CONST static inline ulong
52 0 : fd_shredb_key_slot( ulong key ) {
53 0 : return fd_ulong_extract( key, 32, 63 );
54 0 : }
55 :
56 : FD_FN_CONST static inline uint
57 0 : fd_shredb_key_shred_idx( ulong key ) {
58 0 : return (uint)fd_ulong_extract( key, 0, 31 );
59 0 : }
60 :
61 : typedef __attribute__((aligned(4))) ulong fd_shredb_map_key_t;
62 :
63 : FD_STATIC_ASSERT( sizeof(fd_shredb_map_key_t)==8UL, shredb_map_key_footprint );
64 : FD_STATIC_ASSERT( alignof(fd_shredb_map_key_t)==4UL, shredb_map_key_align );
65 :
66 : struct fd_shredb_shred_entry {
67 : fd_shredb_map_key_t key;
68 : uint next;
69 : };
70 : typedef struct fd_shredb_shred_entry fd_shredb_shred_entry_t;
71 :
72 : FD_STATIC_ASSERT( sizeof(fd_shredb_shred_entry_t)==12UL, shredb_shred_entry_footprint );
73 :
74 : #define MAP_NAME fd_shredb_shred_map
75 0 : #define MAP_ELE_T fd_shredb_shred_entry_t
76 : #define MAP_KEY_T fd_shredb_map_key_t
77 0 : #define MAP_KEY key
78 0 : #define MAP_IDX_T uint
79 0 : #define MAP_NEXT next
80 : #include "../../util/tmpl/fd_map_chain_para.c"
81 :
82 : struct fd_shredb_root_key {
83 : uchar root[ FD_SHRED_MERKLE_NODE_SZ ];
84 : uchar off; /* 0 <= off < 32 */
85 : };
86 : typedef struct fd_shredb_root_key fd_shredb_root_key_t;
87 :
88 : FD_STATIC_ASSERT( sizeof(fd_shredb_root_key_t)==21UL, shredb_root_key_footprint );
89 :
90 : static inline fd_shredb_root_key_t
91 : fd_shredb_root_key( uchar const * merkle_root,
92 0 : uint shred_idx ) {
93 0 : fd_shredb_root_key_t key;
94 0 : memcpy( key.root, merkle_root, FD_SHRED_MERKLE_NODE_SZ );
95 0 : key.off = (uchar)(shred_idx % FD_FEC_SHRED_CNT);
96 0 : return key;
97 0 : }
98 :
99 : /* The pad keeps next 4-byte aligned, the map takes pointers to it. */
100 : struct fd_shredb_root_entry {
101 : fd_shredb_root_key_t key;
102 : uchar pad[ 3 ];
103 : uint next;
104 : };
105 : typedef struct fd_shredb_root_entry fd_shredb_root_entry_t;
106 :
107 : FD_STATIC_ASSERT( sizeof(fd_shredb_root_entry_t)==28UL, shredb_root_entry_footprint );
108 :
109 : #define MAP_NAME fd_shredb_root_map
110 0 : #define MAP_ELE_T fd_shredb_root_entry_t
111 : #define MAP_KEY_T fd_shredb_root_key_t
112 0 : #define MAP_KEY key
113 0 : #define MAP_KEY_EQ(k0,k1) (!memcmp( (k0), (k1), sizeof(fd_shredb_root_key_t) ))
114 0 : #define MAP_KEY_HASH(key,seed) fd_hash( (seed), (key), sizeof(fd_shredb_root_key_t) )
115 0 : #define MAP_IDX_T uint
116 0 : #define MAP_NEXT next
117 : #include "../../util/tmpl/fd_map_chain_para.c"
118 :
119 0 : #define FD_SHREDB_CELL_INVALID (0UL)
120 0 : #define FD_SHREDB_CELL_WRITING (1UL)
121 0 : #define FD_SHREDB_CELL_READY (2UL)
122 0 : #define FD_SHREDB_CELL_STATE_MASK (3UL)
123 :
124 : struct __attribute__((aligned(64))) fd_shredb_entry {
125 : ulong tag;
126 : ulong key;
127 : uchar root[ FD_SHRED_MERKLE_NODE_SZ ]; /* merkle root prefix of the shred's FEC set */
128 : ushort shred_sz;
129 : uchar shred[ FD_SHRED_MAX_SZ ];
130 : };
131 : typedef struct fd_shredb_entry fd_shredb_entry_t;
132 :
133 : FD_STATIC_ASSERT( sizeof(fd_shredb_entry_t)==1280UL, shred_disk_entry_footprint );
134 :
135 : #define FD_SHREDB_MAX_SIZE_GIB (((ulong)UINT_MAX*sizeof(fd_shredb_entry_t))/(1UL<<30))
136 :
137 : FD_FN_CONST static inline ulong
138 0 : fd_shredb_max_shreds( ulong gib ) {
139 0 : if( FD_UNLIKELY( !gib || gib>(ULONG_MAX>>30) ) ) return 0UL;
140 0 : return (gib<<30) / sizeof(fd_shredb_entry_t);
141 0 : }
142 :
143 : FD_FN_CONST static inline ulong
144 0 : fd_shredb_max_slots( ulong gib ) {
145 0 : return fd_ulong_max( !!gib, fd_shredb_max_shreds( gib )/FD_FEC_SHRED_CNT );
146 0 : }
147 :
148 : struct __attribute__((aligned(FD_STORE_ALIGN))) fd_store_fec {
149 : fd_hash_t key;
150 : uint data_idx; /* RAM cache slot when RAM_*, spill-file slot when DISK */
151 : uint next; /* managed by fd_pool / fd_map_chain_para */
152 : uint data_sz; /* sz of the FEC set payload, <= fec_data_max */
153 : uint cache_prev; /* RAM_READY LRU links, UINT_MAX when unlinked */
154 : uint cache_next;
155 : uint slot; /* slot_to_fecs key, raised to the last published root */
156 : uint slot_next; /* managed by slot_to_fecs */
157 : ushort data_pin_cnt; /* active payload views */
158 : uchar data_state; /* FD_STORE_FEC_DATA_* */
159 : uchar data_consume_pending;
160 : ushort shred_sz[ FD_FEC_SHRED_CNT ]; /* payload size of each data shred */
161 : };
162 : typedef struct fd_store_fec fd_store_fec_t;
163 :
164 : FD_STATIC_ASSERT( sizeof(fd_store_fec_t)==FD_STORE_ALIGN, fd_store_fec_footprint );
165 :
166 : #define POOL_NAME fd_store_pool
167 0 : #define POOL_ELE_T fd_store_fec_t
168 : #define POOL_IDX_T uint
169 : #include "../../util/tmpl/fd_pool_para.c"
170 :
171 : #define MAP_NAME fd_store_map
172 0 : #define MAP_ELE_T fd_store_fec_t
173 : #define MAP_KEY_T fd_hash_t
174 0 : #define MAP_KEY key
175 0 : #define MAP_KEY_EQ(k0,k1) (!memcmp((k0),(k1), sizeof(fd_hash_t)))
176 0 : #define MAP_KEY_HASH(key,seed) fd_hash32( (key)->uc, (seed) )
177 0 : #define MAP_IDX_T uint
178 : #include "../../util/tmpl/fd_map_chain_para.c"
179 :
180 : /* slot_to_fecs exists only under Alpenglow, with one partition per
181 : shred tile. Each inserter appends only to its own partition, so
182 : concurrent inserts under the shared fec_lock never touch the same
183 : chain. Only fd_store_publish removes, under the exclusive
184 : fec_lock. */
185 :
186 : #define MAP_NAME fd_store_slot_to_fecs
187 : #define MAP_ELE_T fd_store_fec_t
188 : #define MAP_KEY_T uint
189 0 : #define MAP_KEY slot
190 0 : #define MAP_NEXT slot_next
191 0 : #define MAP_IDX_T uint
192 : #define MAP_MULTI 1
193 : #include "../../util/tmpl/fd_map_chain.c"
194 :
195 : struct fd_store {
196 : ulong magic;
197 : ulong fec_max;
198 : ulong fec_data_max;
199 : ulong store_gaddr;
200 : ulong map_gaddr;
201 : ulong pool_mem_gaddr;
202 : ulong pool_ele_gaddr;
203 : int alpenglow; /* slot_to_fecs exists only if set */
204 : ulong shred_tile_cnt;
205 : ulong slot_to_fecs_gaddr; /* shred_tile_cnt maps, slot_to_fecs_sz apart */
206 : ulong slot_to_fecs_sz;
207 : ulong root; /* last published root */
208 : atomic_ulong slot_lo; /* no FEC is linked below it, ULONG_MAX before the first insert */
209 :
210 : ulong payload_slot_sz;
211 : ulong payload_sz; /* logical spill region size: payload_slot_sz*fec_max */
212 : ulong wire_off; /* byte offset where the rserve wire region begins */
213 :
214 : /* RAM FEC payload cache. cache_slot_cnt is usually much smaller than
215 : fec_max. cache_free is a stack of slot indices protected by
216 : cache_lock. */
217 : ulong cache_slot_cnt;
218 : ulong cache_data_gaddr;
219 : ulong cache_free_gaddr;
220 : ulong cache_free_cnt;
221 : ulong cache_free_target;
222 : ulong cache_free_low_water;
223 : uint cache_preevict_active;
224 : uint cache_lru_head;
225 : uint cache_lru_tail;
226 : ulong cache_pinned_cnt;
227 : ulong spill_free_gaddr;
228 : ulong spill_free_cnt;
229 : ulong spill_slot_cnt;
230 : atomic_ulong spill_live_cnt;
231 : atomic_ulong fec_spill_cnt;
232 : atomic_ulong fec_spill_bytes;
233 : atomic_ulong fec_spill_read_cnt;
234 : atomic_ulong fec_spill_read_bytes;
235 : fd_rwlock_t cache_lock;
236 :
237 : /* Spilled payloads are read into this buffer under spill_read_lock. */
238 : ulong spill_read_data_gaddr;
239 : fd_rwlock_t spill_read_lock;
240 :
241 : /* Reassembly removal returns metadata synchronously, matching the
242 : original Store capacity contract. */
243 : fd_rwlock_t fec_lock;
244 :
245 : /* Arena used by shred tiles for assembly and recovery. */
246 : ulong fec_set_cnt;
247 : ulong fec_sets_gaddr;
248 :
249 : /* On-disk shred index. Lives in the wire region of the shared file
250 : at byte offset wire_off + ring_idx*sizeof(entry). shred_map keys
251 : cells by (slot,idx), first version wins. root_map keys every cell
252 : by (root,idx). Both pools are indexed by ring_idx. */
253 : ulong shred_map_gaddr;
254 : ulong shred_pool_gaddr;
255 : int has_root_map; /* 0 without alpenglow, root_{map,pool}_gaddr unused */
256 : ulong root_map_gaddr;
257 : ulong root_pool_gaddr;
258 : ulong shred_tag_gaddr;
259 : ulong slot_hint_gaddr;
260 : ulong disk_max_shreds;
261 : ulong disk_max_slots;
262 : ulong max_shreds_per_block; /* bounds shred idxs, <=FD_SHREDB_HINT_VALID */
263 : atomic_ulong disk_reservation_head;
264 : atomic_ulong disk_cnt;
265 : atomic_ulong disk_insert_cnt;
266 : atomic_ulong disk_write_bytes;
267 : };
268 : typedef struct fd_store fd_store_t;
269 :
270 : FD_PROTOTYPES_BEGIN
271 :
272 : /* Store contains a Merkle-root keyed FEC map, a payload cache, and an
273 : on-disk shred ring.
274 :
275 : Shred reserves a payload, fills it, inserts the FEC, and then notifies
276 : Replay. The corresponding reassembly node owns the FEC until
277 : removal.
278 : Removal waits for payload users, then returns the payload and metadata
279 : synchronously. fec_max therefore covers reassembly and complete-FEC
280 : messages in flight.
281 :
282 : Payloads enter the RAM cache and spill by LRU to page-sized file slots.
283 : Freed slots are immediately eligible for reuse and keep their disk
284 : blocks; a new slot is only used when none is free. The file layout
285 : is:
286 :
287 : [ sparse spill slots (payload_slot_sz*fec_max) ][ shred ring ]
288 :
289 : wire_off is the fixed start of the shred ring. Spill slots that
290 : were never used do not consume disk blocks.
291 :
292 : Shred-ring writers reserve cells and mark only the target cell WRITING
293 : while its pwrite is in progress. */
294 :
295 : FD_FN_CONST static inline ulong
296 0 : fd_store_align( void ) {
297 0 : return FD_STORE_PAYLOAD_PAGE_SZ;
298 0 : }
299 :
300 : static inline int
301 : fd_store_layout_append( ulong * l,
302 : ulong align,
303 : ulong cnt,
304 0 : ulong ele_sz ) {
305 0 : ulong bytes;
306 0 : ulong rounded;
307 0 : ulong next;
308 0 : if( FD_UNLIKELY( !align || !fd_ulong_is_pow2( align ) ) ) return -1;
309 0 : if( FD_UNLIKELY( __builtin_umull_overflow( cnt, ele_sz, &bytes ) ) ) return -1;
310 0 : if( FD_UNLIKELY( __builtin_uaddl_overflow( *l, align-1UL, &rounded ) ) ) return -1;
311 0 : rounded &= ~(align-1UL);
312 0 : if( FD_UNLIKELY( __builtin_uaddl_overflow( rounded, bytes, &next ) ) ) return -1;
313 0 : *l = next;
314 0 : return 0;
315 0 : }
316 :
317 : FD_FN_CONST static inline ulong
318 : fd_store_footprint( ulong fec_max,
319 : ulong fec_data_max,
320 : ulong shred_storage_gib,
321 : ulong shred_cache_bytes,
322 : ulong fec_set_cnt,
323 : int alpenglow,
324 0 : ulong shred_tile_cnt ) {
325 0 : if( FD_UNLIKELY( !fec_max || !fec_data_max || fec_max>UINT_MAX || fec_data_max>UINT_MAX ||
326 0 : shred_storage_gib>FD_SHREDB_MAX_SIZE_GIB ) ) return 0UL;
327 0 : ulong chain_cnt = fd_store_map_chain_cnt_est( fec_max );
328 0 : ulong slot_to_fecs_sz = fd_ulong_align_up( fd_store_slot_to_fecs_footprint( fd_store_slot_to_fecs_chain_cnt_est( fec_max ) ), fd_store_slot_to_fecs_align() );
329 0 : ulong payload_slot_sz = fd_store_payload_slot_sz( fec_data_max );
330 0 : if( FD_UNLIKELY( !payload_slot_sz ) ) return 0UL;
331 0 : ulong cache_slot_cnt = shred_cache_bytes
332 0 : ? fd_ulong_min( fec_max, fd_ulong_max( 1UL, shred_cache_bytes / payload_slot_sz ) )
333 0 : : fec_max;
334 0 : ulong l = FD_LAYOUT_INIT;
335 0 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_store_align(), 1UL, sizeof(fd_store_t) ) ||
336 0 : fd_store_layout_append( &l, fd_store_map_align(), 1UL, fd_store_map_footprint( chain_cnt ) ) ||
337 0 : fd_store_layout_append( &l, fd_store_pool_align(), 1UL, fd_store_pool_footprint() ) ||
338 0 : fd_store_layout_append( &l, alignof(fd_store_fec_t), fec_max, sizeof(fd_store_fec_t) ) ||
339 0 : fd_store_layout_append( &l, fd_store_slot_to_fecs_align(), alpenglow ? shred_tile_cnt : 0UL, slot_to_fecs_sz ) ||
340 0 : fd_store_layout_append( &l, FD_STORE_PAYLOAD_PAGE_SZ, cache_slot_cnt, payload_slot_sz ) ||
341 0 : fd_store_layout_append( &l, alignof(ulong), cache_slot_cnt, sizeof(ulong) ) ||
342 0 : fd_store_layout_append( &l, alignof(uint), fec_max, sizeof(uint) ) ||
343 0 : fd_store_layout_append( &l, FD_STORE_PAYLOAD_PAGE_SZ, 1UL, payload_slot_sz ) ) ) return 0UL;
344 0 : if( FD_UNLIKELY( fec_set_cnt && fd_store_layout_append( &l, alignof(fd_fec_set_t), fec_set_cnt, sizeof(fd_fec_set_t) ) ) ) return 0UL;
345 0 : if( shred_storage_gib ) {
346 0 : ulong max_shreds = fd_shredb_max_shreds( shred_storage_gib );
347 0 : ulong max_slots = fd_shredb_max_slots( shred_storage_gib );
348 0 : ulong disk_chain_cnt = fd_shredb_shred_map_chain_cnt_est( max_shreds );
349 0 : if( FD_UNLIKELY( !max_shreds || !max_slots ||
350 0 : fd_store_layout_append( &l, fd_shredb_shred_map_align(), 1UL, fd_shredb_shred_map_footprint( disk_chain_cnt ) ) ||
351 0 : fd_store_layout_append( &l, alignof(fd_shredb_shred_entry_t), max_shreds, sizeof(fd_shredb_shred_entry_t) ) ||
352 0 : fd_store_layout_append( &l, alignof(atomic_ulong), max_shreds, sizeof(atomic_ulong) ) ||
353 0 : fd_store_layout_append( &l, alignof(atomic_ulong), max_slots, sizeof(atomic_ulong) ) ) ) return 0UL;
354 0 : if( alpenglow ) {
355 0 : ulong root_chain_cnt = fd_shredb_root_map_chain_cnt_est( max_shreds );
356 0 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_shredb_root_map_align(), 1UL, fd_shredb_root_map_footprint( root_chain_cnt ) ) ||
357 0 : fd_store_layout_append( &l, alignof(fd_shredb_root_entry_t), max_shreds, sizeof(fd_shredb_root_entry_t) ) ) ) return 0UL;
358 0 : }
359 0 : }
360 0 : if( FD_UNLIKELY( fd_store_layout_append( &l, fd_store_align(), 0UL, 1UL ) ) ) return 0UL;
361 0 : return l;
362 0 : }
363 :
364 : /* Formats a footprint-sized, fd_store_align()-aligned region. fec_max
365 : bounds live FECs; fec_data_max bounds each payload. The remaining size
366 : arguments configure the shred ring, RAM cache, and shred-tile arena.
367 : max_shreds_per_block bounds shred idxs in the shred ring, in
368 : [1,FD_SHREDB_HINT_VALID]. The shred ring's root map and the
369 : shred_tile_cnt slot_to_fecs partitions exist only if alpenglow,
370 : which must match the footprint. Does not create the backing file. */
371 :
372 : void *
373 : fd_store_new( void * shmem,
374 : ulong fec_max,
375 : ulong fec_data_max,
376 : ulong shred_storage_gib,
377 : ulong shred_cache_bytes,
378 : ulong fec_set_cnt,
379 : ulong max_shreds_per_block,
380 : ulong seed,
381 : int alpenglow,
382 : ulong shred_tile_cnt );
383 :
384 : fd_store_t * fd_store_join ( void * shstore );
385 : void * fd_store_leave( fd_store_t const * store );
386 : void * fd_store_delete( void * shstore );
387 :
388 : /* Creates and sizes the Store backing file */
389 :
390 : int
391 : fd_store_file_create( char const * path,
392 : ulong wire_off,
393 : ulong disk_max_shreds );
394 :
395 :
396 : FD_FN_PURE static inline fd_wksp_t *
397 0 : fd_store_wksp( fd_store_t const * store ) {
398 0 : return (fd_wksp_t *)( ( (ulong)store ) - store->store_gaddr );
399 0 : }
400 :
401 : /* Optional FEC-set arena, partitioned among shred tile kind_ids. */
402 :
403 : FD_FN_PURE static inline fd_fec_set_t *
404 0 : fd_store_fec_sets( fd_store_t const * store ) {
405 0 : return store->fec_set_cnt ? fd_wksp_laddr_fast( fd_store_wksp( store ), store->fec_sets_gaddr ) : NULL;
406 0 : }
407 :
408 : /* Joins the FEC map using caller-owned local scratch. */
409 :
410 : static inline fd_store_map_t *
411 : fd_store_map_ljoin( fd_store_t const * store,
412 0 : fd_store_map_t * map ) {
413 0 : fd_wksp_t * wksp = fd_store_wksp( store );
414 0 : return fd_store_map_join( map,
415 0 : fd_wksp_laddr_fast( wksp, store->map_gaddr ),
416 0 : fd_wksp_laddr_fast( wksp, store->pool_ele_gaddr ),
417 0 : store->fec_max );
418 0 : }
419 :
420 :
421 : struct fd_store_fec_data_view {
422 : uchar * data;
423 : fd_store_fec_t * fec;
424 : uint flags;
425 : };
426 : typedef struct fd_store_fec_data_view fd_store_fec_data_view_t;
427 :
428 : struct fd_store_fec_spill_stats {
429 : ulong write_cnt;
430 : ulong write_bytes;
431 : ulong write_ticks;
432 : };
433 : typedef struct fd_store_fec_spill_stats fd_store_fec_spill_stats_t;
434 :
435 : struct fd_store_fec_cache_stats {
436 : ulong free_cnt;
437 : ulong max;
438 : ulong target;
439 : ulong low_water;
440 : };
441 : typedef struct fd_store_fec_cache_stats fd_store_fec_cache_stats_t;
442 :
443 : /* Reserves a RAM payload not yet bound to any FEC and returns it, with
444 : its cache slot in *data_idx, or NULL if nothing can be spilled to
445 : make room. Fill it, then hand data_idx to fd_store_insert, which
446 : takes ownership of the slot. spill is optional and reports
447 : synchronous fallback spills. */
448 :
449 : uchar *
450 : fd_store_fec_data_reserve( fd_store_t * store,
451 : int disk_fd,
452 : uint * data_idx,
453 : fd_store_fec_spill_stats_t * spill );
454 :
455 : /* Spills at most one LRU payload while refilling the free reserve. */
456 :
457 : int
458 : fd_store_fec_data_preevict( fd_store_t * store,
459 : int disk_fd,
460 : fd_store_fec_spill_stats_t * spill );
461 :
462 : void
463 : fd_store_fec_cache_stats_query( fd_store_t * store,
464 : fd_store_fec_cache_stats_t * stats );
465 :
466 : /* Pins a published payload. Returns 0 on success and -1 with an empty
467 : view on failure. The store has one spill-read buffer, so a second
468 : spilled view returns -1 while the first is active. RAM views can
469 : coexist. Release every successful view; remove waits for pins. */
470 :
471 : int
472 : fd_store_fec_data_view( fd_store_t * store,
473 : int disk_fd,
474 : fd_store_fec_t * fec,
475 : fd_store_fec_data_view_t * view );
476 :
477 : void
478 : fd_store_fec_data_view_release( fd_store_t * store,
479 : fd_store_fec_data_view_t * view );
480 :
481 : /* Atomically inserts merkle_root, a FEC set of slot, into the caller's
482 : slot_to_fecs partition part (the shred tile's) under Alpenglow. The new FEC takes the payload
483 : reserved by fd_store_fec_data_reserve at data_idx, data_sz bytes
484 : split by shred_sz, and is RAM_READY when linked. Returns
485 : FD_MAP_SUCCESS with the new FEC, or FD_MAP_ERR_KEY with *fec==NULL if
486 : present, in which case the cache slot is freed. Pool exhaustion is a
487 : topology invariant violation. */
488 :
489 : int
490 : fd_store_insert( fd_store_t * store,
491 : fd_store_map_t * map,
492 : fd_hash_t const * merkle_root,
493 : ulong slot,
494 : ulong part,
495 : uint data_idx,
496 : uint data_sz,
497 : ushort const shred_sz[ FD_FEC_SHRED_CNT ],
498 : fd_store_fec_t ** fec );
499 :
500 : /* Lockless lookup; returns NULL if absent. The returned pointer is
501 : borrowed. Retain the corresponding reassembly node or otherwise
502 : exclude remove while using it. */
503 :
504 : fd_store_fec_t *
505 : fd_store_query( fd_store_map_t * map,
506 : fd_hash_t const * merkle_root );
507 :
508 : /* Removes merkle_root after active views and spill I/O finish. Returns
509 : its payload and metadata before returning. Returns 1 if found and 0
510 : otherwise. Not allowed under Alpenglow, use fd_store_publish. */
511 :
512 : int
513 : fd_store_remove( fd_store_t * store,
514 : fd_store_map_t * map,
515 : fd_hash_t const * merkle_root );
516 :
517 : /* Removes every FEC of a slot below root, like fd_store_remove. Walks
518 : the slots from the lowest inserted one to root. Returns how many it
519 : removed. */
520 :
521 : ulong
522 : fd_store_publish( fd_store_t * store,
523 : fd_store_map_t * map,
524 : ulong root );
525 :
526 :
527 : FD_FN_PURE static inline int
528 0 : fd_store_has_disk( fd_store_t const * store ) {
529 0 : return store->disk_max_shreds > 0UL;
530 0 : }
531 :
532 0 : #define FD_STORE_DISK_INSERT_ERR (-1)
533 0 : #define FD_STORE_DISK_INSERT_DUP ( 0)
534 0 : #define FD_STORE_DISK_INSERT_SUCCESS ( 1)
535 :
536 0 : #define FD_STORE_DISK_QUERY_BUSY (-2)
537 0 : #define FD_STORE_DISK_QUERY_MISS (-1)
538 0 : #define FD_STORE_DISK_QUERY_SCAN_LIMIT (-4)
539 :
540 : struct fd_store_disk_stats {
541 : ulong shred_cnt;
542 : ulong current_bytes;
543 : ulong allocated_bytes;
544 : ulong insert_cnt;
545 : ulong write_bytes;
546 : };
547 : typedef struct fd_store_disk_stats fd_store_disk_stats_t;
548 :
549 : /* Persists one (slot,idx) shred, where idx is below max_shreds_per_block,
550 : whose FEC set has merkle root merkle_root (only the first
551 : FD_SHRED_MERKLE_NODE_SZ bytes are used). With a root map, every
552 : version is indexed by (root,idx) and re-inserting a stored (root,idx)
553 : is a no-op. The first version stored for a (slot,idx) is also
554 : indexed by (slot,idx), and without a root map later versions are
555 : dropped. Returns FD_STORE_DISK_INSERT_SUCCESS if written,
556 : FD_STORE_DISK_INSERT_DUP if already stored or dropped, or
557 : FD_STORE_DISK_INSERT_ERR. */
558 :
559 : int
560 : fd_store_disk_insert( fd_store_t * store,
561 : int disk_fd,
562 : fd_shred_t const * shred,
563 : fd_hash_t const * merkle_root );
564 :
565 : /* Copies (slot,shred_idx) to out, where shred_idx is below
566 : max_shreds_per_block. Returns its positive byte count, MISS, or
567 : retryable BUSY. Only finds the first version stored. */
568 :
569 : int
570 : fd_store_disk_query( fd_store_t const * store,
571 : int disk_fd,
572 : ulong slot,
573 : uint shred_idx,
574 : uchar out[ FD_SHRED_MAX_SZ ] );
575 :
576 : /* fd_store_disk_query_root copies the shred at shred_idx of the FEC set
577 : whose merkle root starts with the FD_SHRED_MERKLE_NODE_SZ bytes at
578 : merkle_root to out. Finds any stored version. Returns its positive
579 : byte count, MISS (always, without a root map), or retryable BUSY. */
580 :
581 : int
582 : fd_store_disk_query_root( fd_store_t const * store,
583 : int disk_fd,
584 : uchar const * merkle_root,
585 : uint shred_idx,
586 : uchar out[ FD_SHRED_MAX_SZ ] );
587 :
588 : /* Returns 0 if a query for (slot,shred_idx), or for the highest shred
589 : of slot when shred_idx is UINT_MAX, would certainly miss. */
590 :
591 : int
592 : fd_store_disk_probe( fd_store_t const * store,
593 : ulong slot,
594 : uint shred_idx );
595 :
596 : /* Copies the cached highest stored shred in slot to out. A shred below
597 : min_shred_idx is returned only if it completes the slot. The compact
598 : hint is conservative and never lowered: a collision can keep returning
599 : BUSY and a stale upper bound can keep returning SCAN_LIMIT instead of
600 : risking a lower, incorrect answer. Exact queries are unaffected.
601 : Otherwise returns a positive byte count or MISS. */
602 : int
603 : fd_store_disk_query_highest( fd_store_t const * store,
604 : int disk_fd,
605 : ulong slot,
606 : uint min_shred_idx,
607 : uchar out[ FD_SHRED_MAX_SZ ] );
608 :
609 : /* Takes an approximate telemetry snapshot. Returns 0 or MISS. */
610 :
611 : int
612 : fd_store_disk_stats_query( fd_store_t const * store,
613 : fd_store_disk_stats_t * stats );
614 :
615 : FD_PROTOTYPES_END
616 :
617 : #endif /* HEADER_fd_src_disco_store_fd_store_h */
|
