Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_gui_store.h"
3 :
4 : #include <errno.h> /* errno */
5 : #include <unistd.h> /* close, fallocate */
6 : #include <fcntl.h> /* open, O_RDWR/O_CREAT/O_TRUNC */
7 : #include <sys/mman.h> /* mmap, munmap, PROT_*, MAP_* */
8 :
9 0 : #define FD_GUI_STORE_MAGIC (0xf17e6d0c0117db04UL)
10 :
11 0 : #define FD_GUI_STORE_PAGE_SZ (4096UL)
12 :
13 : struct fd_gui_store_ring {
14 : ulong stride; /* align_up( hdr + val_sz, val_align ) */
15 : ulong region_capacity; /* whole slots per region (REGION_SZ/stride) */
16 : ulong key_off; /* KV key offset within val (0 for TS) */
17 : ulong key_sz; /* KV key width (0 for TS) */
18 : ulong val_sz; /* stored record size */
19 : ulong val_align; /* slot/record alignment */
20 : ulong ts_off; /* TS timestamp offset within val (0 for KV) */
21 : ulong granularity; /* TS window granularity (0 for KV) */
22 : int kind; /* FD_GUI_STORE_KIND_* */
23 : ulong head_cur; /* next cursor to write */
24 : ulong evict_cur; /* logical low-watermark */
25 : ulong tail_cur; /* oldest physically-present slot */
26 : };
27 : typedef struct fd_gui_store_ring fd_gui_store_ring_t;
28 :
29 0 : #define FD_GUI_STORE_SUPER_MAGIC (0xf17e6d0c0117db06UL) /* fd_gui_store regions, versioned */
30 :
31 : struct fd_gui_store_super {
32 : ulong magic;
33 : ulong size_bytes; /* configured ceiling (max file size) */
34 : ulong data_off; /* byte offset of region 0 within the file */
35 : ulong region_sz; /* bytes per region */
36 : ulong region_cnt; /* total regions in the pool */
37 : ulong ring_cnt;
38 : fd_gui_store_ring_t ring[ FD_GUI_STORE_MAX_RINGS ];
39 : };
40 : typedef struct fd_gui_store_super fd_gui_store_super_t;
41 :
42 : struct fd_gui_store_kv_idx_node {
43 : ulong key; /* MAP_KEY: ring->key_hash of the record's key */
44 : ulong cur; /* generation cursor of the ring slot */
45 : ulong next; /* MAP_NEXT (chain link) */
46 : union {
47 : ulong prev; /* MAP_PREV while acquired */
48 : ulong pool_next; /* POOL_NEXT while free */
49 : };
50 : };
51 : typedef struct fd_gui_store_kv_idx_node fd_gui_store_kv_idx_node_t;
52 : FD_STATIC_ASSERT( sizeof(fd_gui_store_kv_idx_node_t)==32UL, fd_gui_store_kv_idx_node );
53 :
54 : #define MAP_NAME fd_gui_store_kv_idx
55 0 : #define MAP_KEY key
56 : #define MAP_KEY_T ulong
57 0 : #define MAP_ELE_T fd_gui_store_kv_idx_node_t
58 0 : #define MAP_NEXT next
59 0 : #define MAP_PREV prev
60 0 : #define MAP_KEY_EQ(k0,k1) ( *(k0)==*(k1) )
61 0 : #define MAP_KEY_HASH(k,s) fd_ulong_hash( *(k) ^ (s) )
62 : #define MAP_MULTI 1
63 : #define MAP_OPTIMIZE_RANDOM_ACCESS_REMOVAL 1
64 : #include "../../util/tmpl/fd_map_chain.c"
65 :
66 : #define POOL_NAME fd_gui_store_kv_pool
67 0 : #define POOL_T fd_gui_store_kv_idx_node_t
68 0 : #define POOL_NEXT pool_next
69 : #include "../../util/tmpl/fd_pool.c"
70 :
71 : #define STACK_NAME fd_gui_store_freelist
72 : #define STACK_T ulong
73 : #include "../../util/tmpl/fd_stack.c"
74 :
75 : struct fd_gui_store_ring_rt {
76 : ulong reg_base; /* logical ordinal of oldest owned region */
77 : ulong reg_cnt; /* number of regions owned */
78 : ulong max_records; /* private KV index allowance */
79 : };
80 : typedef struct fd_gui_store_ring_rt fd_gui_store_ring_rt_t;
81 :
82 : struct fd_gui_store_private {
83 : ulong magic; /* ==FD_GUI_STORE_MAGIC after fd_gui_store_new */
84 : ulong size; /* configured size ceiling in bytes */
85 : ulong ring_cnt;
86 : int fd; /* backing-file fd, or -1 */
87 : void * mapped; /* mmap base (the file) */
88 : ulong mapped_sz; /* size of `mapped` in bytes */
89 : ulong file_sz; /* current allocated file length */
90 : fd_gui_store_super_t * super; /* == mapped (page 0) */
91 : fd_gui_store_kv_idx_t * kv_idx[ FD_GUI_STORE_MAX_RINGS ]; /* per-KV-ring index (RAM); NULL for TS rings */
92 : fd_gui_store_kv_idx_node_t * kv_pool; /* shared KV index node pool (RAM) */
93 : ulong ( * kv_key_hash[ FD_GUI_STORE_MAX_RINGS ] )( void const * key ); /* RAM: per-KV-ring key hash */
94 : int ( * kv_key_cmp [ FD_GUI_STORE_MAX_RINGS ] )( void const * a, void const * b ); /* RAM: per-KV-ring key compare */
95 : ulong * freelist; /* region free list (RAM) */
96 : fd_gui_store_ring_rt_t * ring_rt; /* per-ring region ownership (RAM): ring_cnt rows */
97 : ulong * region_ids; /* per-ring region-id rings (RAM): ring_cnt rows of region_cnt ulongs */
98 : ulong * region_age; /* allocation order per physical region */
99 : ulong next_age;
100 : ulong reserved; /* sum(max(BASE_REGIONS-reg_cnt,0)) */
101 : ulong region_cnt; /* total regions in the pool (== super->region_cnt; ring divisor) */
102 : fd_gui_store_metrics_t metrics[ 1 ]; /* cumulative per-ring metrics */
103 : };
104 :
105 : /* fd_gui_store_region_ring returns `ring_idx` ring's region_id. */
106 : static inline ulong *
107 0 : fd_gui_store_region_ring( fd_gui_store_t * db, ulong ring_idx ) {
108 0 : return db->region_ids + ring_idx*db->region_cnt;
109 0 : }
110 :
111 : FD_FN_CONST ulong
112 0 : fd_gui_store_align( void ) {
113 0 : return 128UL;
114 0 : }
115 :
116 : /* fd_gui_store_ring_stride returns the align-padded slot stride.
117 : Neither kind has a store-added header. */
118 : static inline ulong
119 : fd_gui_store_ring_stride( int kind,
120 : ulong val_sz,
121 0 : ulong val_align ) {
122 0 : (void)kind;
123 0 : return fd_ulong_align_up( val_sz, fd_ulong_max( val_align, 1UL ) );
124 0 : }
125 :
126 : FD_FN_CONST ulong
127 0 : fd_gui_store_min_size( ulong ring_cnt ) {
128 0 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) return 0UL;
129 0 : return fd_ulong_align_up( sizeof(fd_gui_store_super_t), FD_GUI_STORE_PAGE_SZ )
130 0 : + ring_cnt*FD_GUI_STORE_BASE_REGIONS*FD_GUI_STORE_REGION_SZ;
131 0 : }
132 :
133 : static ulong
134 : fd_gui_store_file_footprint( ulong size_bytes,
135 : ulong ring_cnt,
136 : fd_gui_store_desc_t const * descs,
137 : ulong * region_sz_out,
138 0 : ulong * data_off_out ) {
139 0 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS || !descs ) ) return 0UL;
140 :
141 0 : ulong data_off = fd_ulong_align_up( sizeof(fd_gui_store_super_t), FD_GUI_STORE_PAGE_SZ );
142 0 : ulong region_sz = FD_GUI_STORE_REGION_SZ;
143 0 : if( FD_UNLIKELY( size_bytes<fd_gui_store_min_size( ring_cnt ) ) ) return 0UL;
144 0 : ulong avail = size_bytes - data_off;
145 :
146 : /* Every ring must fit at least one slot in a region. */
147 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
148 0 : ulong stride = fd_gui_store_ring_stride( descs[ i ].kind, descs[ i ].val_sz, descs[ i ].val_align );
149 0 : if( FD_UNLIKELY( !stride || stride>region_sz ) ) return 0UL; /* record does not fit a region */
150 0 : }
151 :
152 0 : ulong region_cnt = avail / region_sz;
153 0 : if( FD_UNLIKELY( !region_cnt ) ) return 0UL;
154 :
155 0 : if( region_sz_out ) *region_sz_out = region_sz;
156 0 : if( data_off_out ) *data_off_out = data_off;
157 0 : return region_cnt;
158 0 : }
159 :
160 : static inline ulong
161 0 : fd_gui_store_kv_idx_max( fd_gui_store_desc_t const * desc ) {
162 0 : if( desc->kind!=FD_GUI_STORE_KIND_KV ) return 0UL;
163 0 : return fd_ulong_max( desc->max_records, 1UL );
164 0 : }
165 :
166 : FD_FN_CONST ulong
167 : fd_gui_store_footprint( ulong size_bytes,
168 : ulong ring_cnt,
169 0 : fd_gui_store_desc_t const * descs ) {
170 0 : if( FD_UNLIKELY( ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) return 0UL;
171 :
172 0 : ulong region_cnt = fd_gui_store_file_footprint( size_bytes, ring_cnt, descs, NULL, NULL );
173 0 : if( FD_UNLIKELY( !region_cnt ) ) return 0UL;
174 :
175 0 : ulong pool_max = 0UL;
176 0 : for( ulong i=0UL; i<ring_cnt; i++ ) pool_max += fd_gui_store_kv_idx_max( &descs[ i ] );
177 0 : pool_max = fd_ulong_max( pool_max, 1UL );
178 :
179 0 : ulong l = FD_LAYOUT_INIT;
180 0 : l = FD_LAYOUT_APPEND( l, fd_gui_store_align(), sizeof(fd_gui_store_t) );
181 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
182 0 : if( descs[ i ].kind!=FD_GUI_STORE_KIND_KV ) continue;
183 0 : ulong chain_cnt = fd_gui_store_kv_idx_chain_cnt_est( fd_gui_store_kv_idx_max( &descs[ i ] ) );
184 0 : l = FD_LAYOUT_APPEND( l, fd_gui_store_kv_idx_align(), fd_gui_store_kv_idx_footprint( chain_cnt ) );
185 0 : }
186 0 : l = FD_LAYOUT_APPEND( l, fd_gui_store_kv_pool_align(), fd_gui_store_kv_pool_footprint( pool_max ) );
187 0 : l = FD_LAYOUT_APPEND( l, fd_gui_store_freelist_align(), fd_gui_store_freelist_footprint( region_cnt ) );
188 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_gui_store_ring_rt_t), fd_ulong_max( ring_cnt, 1UL )*sizeof(fd_gui_store_ring_rt_t) );
189 0 : l = FD_LAYOUT_APPEND( l, alignof(ulong), fd_ulong_max( ring_cnt, 1UL )*region_cnt*sizeof(ulong) );
190 0 : l = FD_LAYOUT_APPEND( l, alignof(ulong), region_cnt*sizeof(ulong) );
191 0 : return FD_LAYOUT_FINI( l, fd_gui_store_align() );
192 0 : }
193 :
194 : void *
195 : fd_gui_store_new( void * mem,
196 : char const * path,
197 : ulong size_bytes,
198 : ulong ring_cnt,
199 : ulong seed,
200 0 : fd_gui_store_desc_t const * descs ) {
201 :
202 0 : if( FD_UNLIKELY( !mem ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null mem" )); return NULL; }
203 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_gui_store_align() ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: misaligned mem" )); return NULL; }
204 0 : if( FD_UNLIKELY( !path || !path[ 0 ] ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null path" )); return NULL; }
205 0 : if( FD_UNLIKELY( !size_bytes ) ) { FD_LOG_WARNING(( "fd_gui_store_new: zero size" )); return NULL; }
206 0 : if( FD_UNLIKELY( !ring_cnt || ring_cnt>FD_GUI_STORE_MAX_RINGS ) ) { FD_LOG_WARNING(( "fd_gui_store_new: bad ring_cnt %lu", ring_cnt )); return NULL; }
207 0 : if( FD_UNLIKELY( !descs ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null descs" )); return NULL; }
208 :
209 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
210 0 : if( FD_UNLIKELY( !descs[ i ].name ) ) { FD_LOG_WARNING(( "fd_gui_store_new: null descs[%lu].name", i )); return NULL; }
211 0 : if( FD_UNLIKELY( descs[ i ].kind!=FD_GUI_STORE_KIND_KV && descs[ i ].kind!=FD_GUI_STORE_KIND_TS ) ) {
212 0 : FD_LOG_WARNING(( "fd_gui_store_new: bad descs[%lu].kind %d", i, descs[ i ].kind )); return NULL;
213 0 : }
214 0 : if( FD_UNLIKELY( !descs[ i ].val_sz ) ) {
215 0 : FD_LOG_WARNING(( "fd_gui_store_new: descs[%lu] has zero val_sz", i )); return NULL;
216 0 : }
217 0 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && !descs[ i ].key_sz ) ) {
218 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] has zero key_sz", i )); return NULL;
219 0 : }
220 0 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && ( !descs[ i ].key_hash || !descs[ i ].key_cmp ) ) ) {
221 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] missing key_hash/key_cmp", i )); return NULL;
222 0 : }
223 0 : if( FD_UNLIKELY( descs[ i ].kind==FD_GUI_STORE_KIND_KV && descs[ i ].key_off+descs[ i ].key_sz>descs[ i ].val_sz ) ) {
224 0 : FD_LOG_WARNING(( "fd_gui_store_new: KV descs[%lu] key (off %lu sz %lu) exceeds val_sz %lu", i, descs[ i ].key_off, descs[ i ].key_sz, descs[ i ].val_sz )); return NULL;
225 0 : }
226 0 : }
227 :
228 0 : ulong region_sz, data_off;
229 0 : ulong region_cnt = fd_gui_store_file_footprint( size_bytes, ring_cnt, descs, ®ion_sz, &data_off );
230 0 : if( FD_UNLIKELY( !region_cnt ) ) {
231 0 : FD_LOG_WARNING(( "fd_gui_store_new: store size %lu too small for %lu rings", size_bytes, ring_cnt ));
232 0 : return NULL;
233 0 : }
234 :
235 : /* Shared KV node pool sized to the sum of per-ring max_records. */
236 0 : ulong pool_max = 0UL;
237 0 : for( ulong i=0UL; i<ring_cnt; i++ ) pool_max += fd_gui_store_kv_idx_max( &descs[ i ] );
238 0 : pool_max = fd_ulong_max( pool_max, 1UL );
239 :
240 0 : fd_memset( mem, 0, sizeof(fd_gui_store_t) );
241 0 : FD_SCRATCH_ALLOC_INIT( l, mem );
242 0 : fd_gui_store_t * db = FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_align(), sizeof(fd_gui_store_t) );
243 0 : void * kv_idx_mem[ FD_GUI_STORE_MAX_RINGS ];
244 0 : ulong kv_idx_chains[ FD_GUI_STORE_MAX_RINGS ];
245 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
246 0 : if( descs[ i ].kind!=FD_GUI_STORE_KIND_KV ) { kv_idx_mem[ i ] = NULL; kv_idx_chains[ i ] = 0UL; continue; }
247 0 : kv_idx_chains[ i ] = fd_gui_store_kv_idx_chain_cnt_est( fd_gui_store_kv_idx_max( &descs[ i ] ) );
248 0 : kv_idx_mem[ i ] = FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_kv_idx_align(), fd_gui_store_kv_idx_footprint( kv_idx_chains[ i ] ) );
249 0 : }
250 0 : void * kv_pool_mem= FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_kv_pool_align(), fd_gui_store_kv_pool_footprint( pool_max ) );
251 0 : void * freelist_mem= FD_SCRATCH_ALLOC_APPEND( l, fd_gui_store_freelist_align(), fd_gui_store_freelist_footprint( region_cnt ) );
252 0 : fd_gui_store_ring_rt_t * ring_rt_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_gui_store_ring_rt_t), ring_cnt*sizeof(fd_gui_store_ring_rt_t) );
253 0 : ulong * region_ids_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), ring_cnt*region_cnt*sizeof(ulong) );
254 0 : ulong * region_age_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), region_cnt*sizeof(ulong) );
255 0 : FD_SCRATCH_ALLOC_FINI( l, fd_gui_store_align() );
256 :
257 0 : db->fd = -1;
258 0 : db->mapped = NULL;
259 0 : db->ring_cnt = ring_cnt;
260 0 : db->size = size_bytes;
261 0 : db->ring_rt = ring_rt_mem;
262 0 : db->region_ids = region_ids_mem;
263 0 : db->region_age = region_age_mem;
264 0 : db->reserved = ring_cnt*FD_GUI_STORE_BASE_REGIONS;
265 0 : db->region_cnt = region_cnt;
266 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
267 0 : ring_rt_mem[ i ].reg_base = 0UL;
268 0 : ring_rt_mem[ i ].reg_cnt = 0UL;
269 0 : ring_rt_mem[ i ].max_records = fd_gui_store_kv_idx_max( &descs[ i ] );
270 0 : }
271 :
272 : /* Shared node pool + one ulong-keyed index per KV ring. */
273 0 : if( FD_UNLIKELY( !fd_gui_store_kv_pool_new( kv_pool_mem, pool_max ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: ent_pool_new failed" )); return NULL; }
274 0 : db->kv_pool = fd_gui_store_kv_pool_join( kv_pool_mem );
275 0 : if( FD_UNLIKELY( !db->kv_pool ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_pool join failed" )); return NULL; }
276 0 : for( ulong i=0UL; i<FD_GUI_STORE_MAX_RINGS; i++ ) db->kv_idx[ i ] = NULL;
277 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
278 0 : if( !kv_idx_mem[ i ] ) continue;
279 0 : if( FD_UNLIKELY( !fd_gui_store_kv_idx_new( kv_idx_mem[ i ], kv_idx_chains[ i ], seed+i ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_idx_new[%lu] failed", i )); return NULL; }
280 0 : db->kv_idx[ i ] = fd_gui_store_kv_idx_join( kv_idx_mem[ i ] );
281 0 : if( FD_UNLIKELY( !db->kv_idx[ i ] ) ) { FD_LOG_WARNING(( "fd_gui_store_new: kv_idx join[%lu] failed", i )); return NULL; }
282 0 : }
283 :
284 0 : if( FD_UNLIKELY( !fd_gui_store_freelist_new( freelist_mem, region_cnt ) ) ) { FD_LOG_WARNING(( "fd_gui_store_new: freelist_new failed" )); return NULL; }
285 0 : db->freelist = fd_gui_store_freelist_join( freelist_mem );
286 0 : if( FD_UNLIKELY( !db->freelist ) ) { FD_LOG_WARNING(( "fd_gui_store_new: freelist join failed" )); return NULL; }
287 0 : for( ulong i=region_cnt; i-->0UL; ) fd_gui_store_freelist_push( db->freelist, i );
288 :
289 : /* Wipe + (re)create the backing file. */
290 0 : ulong map_sz = data_off + region_cnt*region_sz;
291 0 : int fd = open( path, O_RDWR | O_CREAT | O_TRUNC, (mode_t)0600 );
292 0 : if( FD_UNLIKELY( fd<0 ) ) { FD_LOG_WARNING(( "fd_gui_store_new: open(%s) failed (%d-%s)", path, errno, fd_io_strerror( errno ) )); return NULL; }
293 0 : if( FD_UNLIKELY( fallocate( fd, 0, 0, (off_t)data_off ) ) ) {
294 0 : FD_LOG_WARNING(( "fd_gui_store_new: fallocate(%s, %lu) failed (%d-%s)", path, data_off, errno, fd_io_strerror( errno ) ));
295 0 : close( fd ); return NULL;
296 0 : }
297 0 : void * mapped = mmap( NULL, map_sz, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0 );
298 0 : if( FD_UNLIKELY( mapped==MAP_FAILED ) ) {
299 0 : FD_LOG_WARNING(( "fd_gui_store_new: mmap(%s, %lu) failed (%d-%s)", path, map_sz, errno, fd_io_strerror( errno ) ));
300 0 : close( fd ); return NULL;
301 0 : }
302 :
303 0 : db->fd = fd;
304 0 : db->mapped = mapped;
305 0 : db->mapped_sz = map_sz;
306 0 : db->file_sz = data_off;
307 0 : db->super = (fd_gui_store_super_t *)mapped;
308 :
309 0 : fd_gui_store_super_t * super = db->super;
310 0 : fd_memset( super, 0, sizeof(fd_gui_store_super_t) );
311 0 : super->size_bytes = size_bytes;
312 0 : super->data_off = data_off;
313 0 : super->region_sz = region_sz;
314 0 : super->region_cnt = region_cnt;
315 0 : super->ring_cnt = ring_cnt;
316 :
317 0 : for( ulong i=0UL; i<ring_cnt; i++ ) {
318 0 : fd_gui_store_ring_t * p = &super->ring[ i ];
319 0 : p->stride = fd_gui_store_ring_stride( descs[ i ].kind, descs[ i ].val_sz, descs[ i ].val_align );
320 0 : p->region_capacity = region_sz / p->stride; /* >=1 by layout */
321 0 : p->key_off = descs[ i ].key_off;
322 0 : p->key_sz = descs[ i ].key_sz;
323 0 : p->val_sz = descs[ i ].val_sz;
324 0 : p->val_align = descs[ i ].val_align;
325 0 : p->ts_off = descs[ i ].ts_off;
326 0 : p->granularity = descs[ i ].granularity;
327 0 : p->kind = descs[ i ].kind;
328 0 : p->head_cur = 0UL;
329 0 : p->evict_cur = 0UL;
330 0 : p->tail_cur = 0UL;
331 0 : db->kv_key_hash[ i ] = descs[ i ].key_hash; /* NULL for TS rings */
332 0 : db->kv_key_cmp [ i ] = descs[ i ].key_cmp; /* NULL for TS rings */
333 0 : }
334 :
335 0 : FD_COMPILER_MFENCE();
336 0 : super->magic = FD_GUI_STORE_SUPER_MAGIC;
337 0 : db->magic = FD_GUI_STORE_MAGIC;
338 0 : FD_COMPILER_MFENCE();
339 :
340 0 : memset( db->metrics, 0, sizeof(fd_gui_store_metrics_t) );
341 :
342 0 : FD_LOG_INFO(( "fd_gui_store: opened %s (ceiling %lu MiB, %lu regions x %lu MiB, %lu rings)",
343 0 : path, size_bytes>>20, region_cnt, region_sz>>20, ring_cnt ));
344 0 : return mem;
345 0 : }
346 :
347 : fd_gui_store_t *
348 0 : fd_gui_store_join( void * mem ) {
349 0 : if( FD_UNLIKELY( !mem ) ) return NULL;
350 0 : fd_gui_store_t * db = (fd_gui_store_t *)mem;
351 0 : if( FD_UNLIKELY( db->magic!=FD_GUI_STORE_MAGIC ) ) { FD_LOG_WARNING(( "fd_gui_store_join: bad magic" )); return NULL; }
352 0 : return db;
353 0 : }
354 :
355 : void *
356 0 : fd_gui_store_leave( fd_gui_store_t * db ) {
357 0 : return (void *)db;
358 0 : }
359 :
360 : void *
361 0 : fd_gui_store_delete( void * mem ) {
362 0 : if( FD_UNLIKELY( !mem ) ) return NULL;
363 0 : fd_gui_store_t * db = (fd_gui_store_t *)mem;
364 0 : if( db->mapped ) munmap( db->mapped, db->mapped_sz );
365 0 : if( db->fd>=0 ) close( db->fd );
366 0 : db->mapped = NULL;
367 0 : db->fd = -1;
368 0 : db->super = NULL;
369 0 : db->magic = 0UL;
370 0 : return mem;
371 0 : }
372 :
373 : ulong
374 0 : fd_gui_store_cnt( fd_gui_store_t const * db ) {
375 0 : return db->ring_cnt;
376 0 : }
377 :
378 : static inline ulong
379 0 : fd_gui_store_committed_bytes( fd_gui_store_t const * db ) {
380 0 : ulong used = db->super->data_off;
381 0 : for( ulong i=0UL; i<db->ring_cnt; i++ ) used += db->ring_rt[ i ].reg_cnt * db->super->region_sz;
382 0 : return used;
383 0 : }
384 :
385 : ulong
386 0 : fd_gui_store_used_bytes( fd_gui_store_t * db ) {
387 0 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
388 0 : return fd_gui_store_committed_bytes( db );
389 0 : }
390 :
391 : ulong
392 0 : fd_gui_store_live_bytes( fd_gui_store_t * db ) {
393 0 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
394 0 : ulong live = 0UL;
395 0 : for( ulong i=0UL; i<db->ring_cnt; i++ ) {
396 0 : fd_gui_store_ring_t const * p = &db->super->ring[ i ];
397 0 : live += ( p->head_cur - p->evict_cur ) * p->stride;
398 0 : }
399 0 : return live;
400 0 : }
401 :
402 : fd_gui_store_metrics_t const *
403 0 : fd_gui_store_metrics( fd_gui_store_t const * db ) {
404 0 : return db->metrics;
405 0 : }
406 :
407 : void
408 : fd_gui_store_ring_stats( fd_gui_store_t * db,
409 : ulong ring_idx,
410 : ulong * used_bytes,
411 : ulong * cap_bytes,
412 : ulong * free_bytes,
413 : ulong * used_slots,
414 : ulong * cap_slots,
415 0 : ulong * free_slots ) {
416 0 : if( FD_LIKELY( db->super && ring_idx<db->ring_cnt ) ) {
417 0 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
418 0 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ ring_idx ];
419 :
420 0 : ulong free_cnt = fd_gui_store_freelist_cnt( db->freelist ) - db->reserved
421 0 : + ( rt->reg_cnt<FD_GUI_STORE_BASE_REGIONS ? FD_GUI_STORE_BASE_REGIONS-rt->reg_cnt : 0UL );
422 :
423 0 : *used_slots = p->head_cur - p->tail_cur;
424 0 : *cap_slots = rt->reg_cnt * p->region_capacity;
425 0 : *free_slots = free_cnt * p->region_capacity;
426 0 : *used_bytes = *used_slots * p->stride;
427 0 : *cap_bytes = rt->reg_cnt * db->super->region_sz;
428 0 : *free_bytes = free_cnt * db->super->region_sz;
429 0 : }
430 0 : }
431 :
432 : ulong
433 0 : fd_gui_store_size( fd_gui_store_t const * db ) {
434 0 : return db->size;
435 0 : }
436 :
437 : ulong
438 0 : fd_gui_store_file_sz( fd_gui_store_t const * db ) {
439 0 : if( FD_UNLIKELY( !db ) ) return 0UL;
440 0 : return db->file_sz;
441 0 : }
442 :
443 : ulong
444 0 : fd_gui_store_free_region_cnt( fd_gui_store_t const * db ) {
445 0 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
446 0 : return fd_gui_store_freelist_cnt( db->freelist );
447 0 : }
448 :
449 : ulong
450 0 : fd_gui_store_shared_free_region_cnt( fd_gui_store_t const * db ) {
451 0 : if( FD_UNLIKELY( !db->super ) ) return 0UL;
452 0 : return fd_gui_store_freelist_cnt( db->freelist ) - db->reserved;
453 0 : }
454 :
455 : int
456 0 : fd_gui_store_fd( fd_gui_store_t const * db ) {
457 0 : if( FD_UNLIKELY( !db ) ) return -1;
458 0 : return db->fd;
459 0 : }
460 :
461 : static inline ulong
462 : fd_gui_store_ring_head_limit( fd_gui_store_t const * db,
463 : ulong ring_idx,
464 0 : fd_gui_store_ring_t const * p ) {
465 0 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ ring_idx ];
466 0 : return ( rt->reg_base + rt->reg_cnt )*p->region_capacity;
467 0 : }
468 :
469 : static int
470 0 : fd_gui_store_region_grow( fd_gui_store_t * db, ulong ring_idx ) {
471 0 : fd_gui_store_ring_rt_t * rt = &db->ring_rt[ ring_idx ];
472 0 : ulong * region_ring = fd_gui_store_region_ring( db, ring_idx );
473 0 : ulong free_cnt = fd_gui_store_freelist_cnt( db->freelist );
474 0 : if( FD_UNLIKELY( !free_cnt || (rt->reg_cnt>=FD_GUI_STORE_BASE_REGIONS && free_cnt<=db->reserved) ) ) return 0;
475 :
476 0 : ulong region_id = fd_gui_store_freelist_pop( db->freelist );
477 0 : ulong ordinal = rt->reg_base + rt->reg_cnt;
478 0 : region_ring[ ordinal % db->region_cnt ] = region_id;
479 0 : rt->reg_cnt++;
480 :
481 0 : ulong region_end = db->super->data_off + ( region_id + 1UL )*db->super->region_sz;
482 0 : if( region_end>db->file_sz ) {
483 0 : ulong grow_sz = region_end - db->file_sz;
484 0 : if( FD_UNLIKELY( fallocate( db->fd, 0, (off_t)db->file_sz, (off_t)grow_sz ) ) ) {
485 : /* roll back the claim */
486 0 : rt->reg_cnt--;
487 0 : fd_gui_store_freelist_push( db->freelist, region_id );
488 0 : FD_LOG_WARNING(( "fd_gui_store: fallocate grow to %lu failed (%d-%s)", region_end, errno, fd_io_strerror( errno ) ));
489 0 : return 0;
490 0 : }
491 0 : db->file_sz = region_end;
492 0 : }
493 : /* Claiming a base region consumes one outstanding entitlement. */
494 0 : db->reserved -= rt->reg_cnt<=FD_GUI_STORE_BASE_REGIONS;
495 0 : db->region_age[ region_id ] = db->next_age++;
496 0 : db->metrics->region_grows[ ring_idx ]++;
497 0 : return 1;
498 0 : }
499 :
500 : static void
501 : fd_gui_store_region_reclaim( fd_gui_store_t * db, ulong ring_idx,
502 0 : fd_gui_store_ring_t * p ) {
503 0 : fd_gui_store_ring_rt_t * rt = &db->ring_rt[ ring_idx ];
504 0 : ulong * region_ring = fd_gui_store_region_ring( db, ring_idx );
505 0 : ulong cap = p->region_capacity;
506 0 : while( rt->reg_cnt && p->tail_cur >= ( rt->reg_base + 1UL )*cap ) {
507 0 : ulong region_id = region_ring[ rt->reg_base % db->region_cnt ];
508 0 : fd_gui_store_freelist_push( db->freelist, region_id );
509 0 : rt->reg_base++;
510 0 : rt->reg_cnt--;
511 : /* The owner's base entitlement survives returning a region. */
512 0 : db->reserved += rt->reg_cnt<FD_GUI_STORE_BASE_REGIONS;
513 0 : db->metrics->region_reclaims[ ring_idx ]++;
514 0 : }
515 0 : }
516 :
517 : static inline void *
518 : fd_gui_store_slot( fd_gui_store_t * db,
519 : ulong ring_idx,
520 : fd_gui_store_ring_t const * p,
521 0 : ulong cur ) {
522 0 : ulong cap = p->region_capacity;
523 0 : ulong ordinal = cur / cap;
524 0 : ulong region_id = fd_gui_store_region_ring( db, ring_idx )[ ordinal % db->region_cnt ];
525 0 : ulong off = db->super->data_off + region_id*db->super->region_sz + ( cur % cap )*p->stride;
526 0 : return (uchar *)db->mapped + off;
527 0 : }
528 :
529 : static inline ulong
530 : fd_gui_store_ts_window( fd_gui_store_ring_t const * p,
531 0 : void const * val ) {
532 0 : long ts = *(long const *)( (uchar const *)val + p->ts_off );
533 0 : ulong gran = fd_ulong_max( p->granularity, 1UL );
534 0 : return (ulong)fd_long_max( ts, 0L ) / gran;
535 0 : }
536 :
537 : static inline uchar const *
538 : fd_gui_store_kv_slot_key( fd_gui_store_t * db,
539 : ulong ring_idx,
540 : fd_gui_store_ring_t const * p,
541 0 : ulong cur ) {
542 0 : return (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur ) + p->key_off;
543 0 : }
544 :
545 : static fd_gui_store_kv_idx_node_t *
546 : fd_gui_store_kv_find_node( fd_gui_store_t * db,
547 : fd_gui_store_ring_t const * p,
548 : ulong ring_idx,
549 0 : void const * key ) {
550 0 : ulong ik = db->kv_key_hash[ ring_idx ]( key );
551 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_idx_ele_query( db->kv_idx[ ring_idx ], &ik, NULL, db->kv_pool );
552 0 : while( node ) {
553 0 : fd_gui_store_kv_idx_node_t * next = (fd_gui_store_kv_idx_node_t *)fd_gui_store_kv_idx_ele_next_const( node, NULL, db->kv_pool );
554 0 : if( FD_UNLIKELY( node->cur<p->evict_cur ) ) {
555 : /* stale: the slot was evicted; lazily drop the dead index node */
556 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
557 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
558 0 : } else {
559 0 : uchar const * slot_key = fd_gui_store_kv_slot_key( db, ring_idx, p, node->cur );
560 0 : if( FD_LIKELY( 0==db->kv_key_cmp[ ring_idx ]( slot_key, key ) ) ) return node;
561 0 : }
562 0 : node = next;
563 0 : }
564 0 : return NULL;
565 0 : }
566 :
567 : int
568 : fd_gui_store_kv_get_or_create( fd_gui_store_t * db,
569 : ulong ring_idx,
570 : void const * key,
571 0 : void ** val_out ) {
572 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_or_create: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
573 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
574 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_or_create: ring_idx %lu is not a KV ring", ring_idx )); return FD_GUI_STORE_ERR; }
575 :
576 : /* Existing key: hand back the live value region for in-place population. */
577 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, key );
578 0 : if( FD_LIKELY( node ) ) {
579 0 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, node->cur );
580 0 : if( val_out ) *val_out = slot;
581 0 : return FD_GUI_STORE_SUCCESS;
582 0 : }
583 :
584 0 : if( FD_UNLIKELY( p->head_cur-p->evict_cur>=db->ring_rt[ ring_idx ].max_records ) ) return FD_GUI_STORE_RING_FULL;
585 : /* Per-ring limits sum to the pool capacity: one ring cannot consume
586 : another ring's index allowance. */
587 0 : FD_TEST( fd_gui_store_kv_pool_free( db->kv_pool ) );
588 0 : if( FD_UNLIKELY( p->head_cur >= fd_gui_store_ring_head_limit( db, ring_idx, p ) ) ) {
589 0 : if( FD_UNLIKELY( !fd_gui_store_region_grow( db, ring_idx ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
590 0 : }
591 :
592 0 : ulong cur = p->head_cur;
593 0 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, cur );
594 : /* Seed the key inside the value so the stored record always carries a
595 : consistent key; the caller fills the remaining value bytes. */
596 0 : fd_memcpy( slot + p->key_off, key, p->key_sz );
597 :
598 0 : fd_gui_store_kv_idx_node_t * n = fd_gui_store_kv_pool_ele_acquire( db->kv_pool );
599 0 : n->key = db->kv_key_hash[ ring_idx ]( key );
600 0 : n->cur = cur;
601 0 : fd_gui_store_kv_idx_ele_insert( db->kv_idx[ ring_idx ], n, db->kv_pool );
602 :
603 0 : p->head_cur = cur + 1UL;
604 0 : if( val_out ) *val_out = slot;
605 0 : return FD_GUI_STORE_SUCCESS;
606 0 : }
607 :
608 : void *
609 : fd_gui_store_kv_get( fd_gui_store_t * db,
610 : ulong ring_idx,
611 0 : void const * key ) {
612 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get: bad ring_idx %lu", ring_idx )); return NULL; }
613 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
614 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get: ring_idx %lu is not a KV ring", ring_idx )); return NULL; }
615 :
616 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, key );
617 0 : db->metrics->kv_lookups[ ring_idx ]++;
618 0 : if( FD_UNLIKELY( !node ) ) return NULL;
619 0 : return (uchar *)fd_gui_store_slot( db, ring_idx, p, node->cur );
620 0 : }
621 :
622 : static uchar const *
623 : fd_gui_store_kv_lowest_gt( fd_gui_store_t * db,
624 : ulong ring_idx,
625 : fd_gui_store_ring_t const * p,
626 : void const * key,
627 0 : uchar const * gt ) {
628 0 : int ( * key_cmp )( void const *, void const * ) = db->kv_key_cmp[ ring_idx ];
629 0 : uchar const * best = NULL;
630 0 : uchar const * best_key = NULL;
631 :
632 0 : if( FD_LIKELY( key ) ) {
633 : /* Fast path: walk the query key's index bucket (records sharing its
634 : hashed leading field), tracking the lowest full key still > gt that
635 : matches `key`. */
636 0 : ulong ik = db->kv_key_hash[ ring_idx ]( key );
637 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_idx_ele_query( db->kv_idx[ ring_idx ], &ik, NULL, db->kv_pool );
638 0 : while( node ) {
639 0 : fd_gui_store_kv_idx_node_t * next = (fd_gui_store_kv_idx_node_t *)fd_gui_store_kv_idx_ele_next_const( node, NULL, db->kv_pool );
640 0 : if( FD_UNLIKELY( node->cur<p->evict_cur ) ) {
641 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
642 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
643 0 : } else {
644 0 : uchar const * val = (uchar const *)fd_gui_store_slot( db, ring_idx, p, node->cur );
645 0 : uchar const * k = val + p->key_off;
646 0 : if( FD_LIKELY( 0==key_cmp( k, key ) ) ) {
647 0 : if( ( !gt || key_cmp( k, gt )>0 ) &&
648 0 : ( !best || key_cmp( k, best_key )<0 ) ) { best = val; best_key = k; }
649 0 : }
650 0 : }
651 0 : node = next;
652 0 : }
653 0 : } else {
654 : /* Slow path: no query key, find the lowest key across the ring. */
655 0 : for( ulong cur=p->evict_cur; cur<p->head_cur; cur++ ) {
656 0 : uchar const * val = (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur );
657 0 : uchar const * k = val + p->key_off;
658 0 : if( ( !gt || key_cmp( k, gt )>0 ) &&
659 0 : ( !best || key_cmp( k, best_key )<0 ) ) { best = val; best_key = k; }
660 0 : }
661 0 : }
662 0 : return best;
663 0 : }
664 :
665 : void *
666 : fd_gui_store_kv_get_any( fd_gui_store_t * db,
667 : ulong ring_idx,
668 0 : void const * key ) {
669 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_any: bad ring_idx %lu", ring_idx )); return NULL; }
670 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
671 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_get_any: ring_idx %lu is not a KV ring", ring_idx )); return NULL; }
672 :
673 0 : db->metrics->kv_lookups[ ring_idx ]++;
674 0 : return (void *)fd_gui_store_kv_lowest_gt( db, ring_idx, p, key, NULL );
675 0 : }
676 :
677 : fd_gui_store_kv_iter_t *
678 : fd_gui_store_kv_iter_begin( fd_gui_store_t * db,
679 : fd_gui_store_kv_iter_t * iter,
680 : ulong ring_idx,
681 0 : void const * key ) {
682 0 : memset( iter, 0, sizeof(fd_gui_store_kv_iter_t) );
683 0 : iter->_db = db;
684 :
685 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: bad ring_idx %lu", ring_idx )); return iter; }
686 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
687 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: ring_idx %lu is not a KV ring", ring_idx )); return iter; }
688 0 : if( FD_UNLIKELY( !key || p->key_sz>sizeof(iter->_key) ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_iter_begin: null key or key_sz %lu too large", p->key_sz )); return iter; }
689 :
690 0 : iter->_ring_idx = ring_idx;
691 0 : fd_memcpy( iter->_key, key, p->key_sz );
692 :
693 0 : db->metrics->kv_lookups[ ring_idx ]++;
694 0 : uchar const * best = fd_gui_store_kv_lowest_gt( db, ring_idx, p, iter->_key, NULL );
695 0 : if( FD_UNLIKELY( !best ) ) return iter; /* empty: _valid stays 0 */
696 :
697 0 : fd_memcpy( iter->_prev_key, best + p->key_off, p->key_sz );
698 0 : iter->_have_prev = 1;
699 0 : iter->_valid = 1;
700 0 : iter->rec = best;
701 0 : iter->key = best + p->key_off;
702 0 : iter->key_sz = p->key_sz;
703 0 : iter->val_sz = p->val_sz;
704 0 : return iter;
705 0 : }
706 :
707 : int
708 0 : fd_gui_store_kv_iter_next( fd_gui_store_kv_iter_t * iter ) {
709 0 : if( FD_UNLIKELY( !iter->_valid ) ) return 0;
710 0 : fd_gui_store_t * db = (fd_gui_store_t *)iter->_db;
711 0 : fd_gui_store_ring_t * p = &db->super->ring[ iter->_ring_idx ];
712 :
713 0 : uchar const * best = fd_gui_store_kv_lowest_gt( db, iter->_ring_idx, p,
714 0 : iter->_key,
715 0 : iter->_have_prev ? iter->_prev_key : NULL );
716 0 : if( FD_UNLIKELY( !best ) ) {
717 0 : iter->_valid = 0;
718 0 : iter->rec = NULL;
719 0 : iter->key = NULL;
720 0 : return 0;
721 0 : }
722 :
723 0 : fd_memcpy( iter->_prev_key, best + p->key_off, p->key_sz );
724 0 : iter->rec = best;
725 0 : iter->key = best + p->key_off;
726 0 : return 1;
727 0 : }
728 :
729 : void
730 : fd_gui_store_kv_scan_begin( fd_gui_store_t * db,
731 : fd_gui_store_kv_scan_t * iter,
732 0 : ulong ring_idx ) {
733 0 : memset( iter, 0, sizeof(*iter) );
734 0 : if( FD_UNLIKELY( !db || ring_idx>=db->ring_cnt || db->super->ring[ ring_idx ].kind!=FD_GUI_STORE_KIND_KV ) ) return;
735 0 : iter->db = db;
736 0 : iter->ring_idx = ring_idx;
737 0 : iter->cur = db->super->ring[ ring_idx ].evict_cur;
738 0 : iter->end = db->super->ring[ ring_idx ].head_cur;
739 0 : iter->append_capacity = fd_gui_store_ring_head_limit( db, ring_idx, &db->super->ring[ ring_idx ] ) - iter->end;
740 0 : }
741 :
742 : int
743 0 : fd_gui_store_kv_scan_next( fd_gui_store_kv_scan_t * iter ) {
744 0 : iter->rec = NULL;
745 0 : if( FD_UNLIKELY( !iter->db ) ) return 0;
746 0 : fd_gui_store_ring_t const * p = &iter->db->super->ring[ iter->ring_idx ];
747 0 : iter->cur = fd_ulong_max( iter->cur, p->evict_cur );
748 0 : if( iter->cur>=iter->end ) return 0;
749 0 : iter->rec = fd_gui_store_slot( iter->db, iter->ring_idx, p, iter->cur++ );
750 0 : return 1;
751 0 : }
752 :
753 : ulong
754 : fd_gui_store_kv_reclaim_prefix( fd_gui_store_t * db,
755 : ulong ring_idx,
756 : ulong budget,
757 : int (*eligible)( void const *, void * ),
758 0 : void * ctx ) {
759 0 : if( FD_UNLIKELY( !db || ring_idx>=db->ring_cnt || !eligible ) ) return 0UL;
760 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
761 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) return 0UL;
762 0 : ulong first = p->evict_cur;
763 0 : while( budget && p->evict_cur<p->head_cur ) {
764 0 : uchar const * rec = fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
765 0 : if( !eligible( rec, ctx ) ) break;
766 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, rec+p->key_off );
767 0 : FD_TEST( node );
768 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
769 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
770 0 : p->evict_cur++;
771 0 : budget--;
772 0 : }
773 0 : ulong count = p->evict_cur - first;
774 0 : if( count ) db->metrics->evicts[ ring_idx ]++;
775 0 : db->metrics->evict_records[ ring_idx ] += count;
776 0 : p->tail_cur = p->evict_cur;
777 0 : fd_gui_store_region_reclaim( db, ring_idx, p );
778 0 : return count;
779 0 : }
780 :
781 : int
782 : fd_gui_store_kv_evict( fd_gui_store_t * db,
783 : ulong ring_idx,
784 : void const * hi_key,
785 : ulong * budget,
786 0 : int * drained ) {
787 0 : *drained = 1;
788 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_evict: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
789 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
790 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_KV ) ) { FD_LOG_WARNING(( "fd_gui_store_kv_evict: ring_idx %lu is not a KV ring", ring_idx )); return FD_GUI_STORE_ERR; }
791 :
792 0 : ulong evict_cur0 = p->evict_cur;
793 :
794 0 : while( p->evict_cur<p->head_cur ) {
795 0 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
796 0 : uchar const * k = slot + p->key_off;
797 :
798 : /* Reached the boundary key. Inserts are not ordered, so we may miss
799 : some entires after this within the eviction range. They will get
800 : caught in a subsequent eviction. Replay slots are inserted in
801 : roughly sorted order so this is acceptable. */
802 0 : if( db->kv_key_cmp[ ring_idx ]( k, hi_key )>=0 ) break;
803 0 : if( !*budget ) { *drained = 0; break; }
804 :
805 : /* Free the index node backing this record */
806 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, ring_idx, k );
807 0 : if( FD_LIKELY( node ) ) {
808 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ ring_idx ], node, db->kv_pool );
809 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
810 0 : }
811 :
812 0 : p->evict_cur++;
813 0 : (*budget)--;
814 0 : db->metrics->evict_records[ ring_idx ]++;
815 0 : }
816 :
817 0 : if( p->evict_cur!=evict_cur0 ) db->metrics->evicts[ ring_idx ]++;
818 :
819 0 : p->tail_cur = p->evict_cur; /* clean prefix: reclaim chases the watermark */
820 0 : fd_gui_store_region_reclaim( db, ring_idx, p );
821 0 : return FD_GUI_STORE_SUCCESS;
822 0 : }
823 :
824 : int
825 : fd_gui_store_ts_emplace( fd_gui_store_t * db,
826 : ulong ring_idx,
827 : long ts,
828 0 : void ** val_out ) {
829 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_emplace: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
830 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
831 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_emplace: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
832 :
833 : /* Scans binary search for their first record, so live records must
834 : be in nondecreasing timestamp order. */
835 0 : if( FD_LIKELY( p->evict_cur<p->head_cur ) ) {
836 0 : long last_ts = *(long const *)( (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->head_cur-1UL ) + p->ts_off );
837 0 : if( FD_UNLIKELY( ts<last_ts ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_emplace: ring_idx %lu timestamp decreased from %ld to %ld", ring_idx, last_ts, ts )); return FD_GUI_STORE_ERR; }
838 0 : }
839 :
840 0 : if( FD_UNLIKELY( p->head_cur >= fd_gui_store_ring_head_limit( db, ring_idx, p ) ) ) {
841 0 : if( FD_UNLIKELY( !fd_gui_store_region_grow( db, ring_idx ) ) ) { db->metrics->map_full[ ring_idx ]++; return FD_GUI_STORE_MAP_FULL; }
842 0 : }
843 :
844 : /* Window is derived from the timestamp embedded in the value; the
845 : value is stored verbatim with no store-added header. Seed the
846 : timestamp so the stored record is consistent with its window. */
847 0 : ulong cur = p->head_cur;
848 0 : uchar * slot = fd_gui_store_slot( db, ring_idx, p, cur );
849 0 : fd_memcpy( slot + p->ts_off, &ts, sizeof(ts) );
850 0 : p->head_cur = cur + 1UL;
851 0 : db->metrics->ts_appends[ ring_idx ]++;
852 0 : *val_out = slot;
853 0 : return FD_GUI_STORE_SUCCESS;
854 0 : }
855 :
856 : int
857 : fd_gui_store_ts_append( fd_gui_store_t * db,
858 : ulong ring_idx,
859 0 : void const * val ) {
860 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_append: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
861 0 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
862 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_append: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
863 0 : long ts;
864 0 : fd_memcpy( &ts, (uchar const *)val + p->ts_off, sizeof(ts) );
865 0 : void * slot = NULL;
866 0 : int rc = fd_gui_store_ts_emplace( db, ring_idx, ts, &slot );
867 0 : if( FD_LIKELY( rc==FD_GUI_STORE_SUCCESS ) ) fd_memcpy( slot, val, p->val_sz );
868 0 : return rc;
869 0 : }
870 :
871 : int
872 : fd_gui_store_ts_live_timestamp_bounds( fd_gui_store_t * db,
873 : ulong ring_idx,
874 : long * out_first_timestamp,
875 0 : long * out_last_timestamp ) {
876 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) return 0;
877 0 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
878 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) return 0;
879 0 : if( FD_UNLIKELY( p->evict_cur>=p->head_cur ) ) return 0; /* empty */
880 :
881 0 : uchar const * first = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
882 0 : uchar const * last = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->head_cur-1UL );
883 0 : *out_first_timestamp = *(long const *)( first + p->ts_off );
884 0 : *out_last_timestamp = *(long const *)( last + p->ts_off );
885 0 : return 1;
886 0 : }
887 :
888 : /* fd_gui_store_ts_lower_bound returns the lowest live cursor whose
889 : record window is >= window, or head_cur if there is none. */
890 :
891 : static ulong
892 : fd_gui_store_ts_lower_bound( fd_gui_store_t * db,
893 : ulong ring_idx,
894 0 : ulong window ) {
895 0 : fd_gui_store_ring_t const * p = &db->super->ring[ ring_idx ];
896 0 : ulong lo = p->evict_cur;
897 0 : ulong hi = p->head_cur;
898 0 : while( lo<hi ) {
899 0 : ulong mid = lo + (hi-lo)/2UL;
900 0 : if( fd_gui_store_ts_window( p, fd_gui_store_slot( db, ring_idx, p, mid ) )<window ) lo = mid + 1UL;
901 0 : else hi = mid;
902 0 : }
903 0 : return lo;
904 0 : }
905 :
906 : static void
907 0 : fd_gui_store_ts_scan_advance( fd_gui_store_ts_iter_t * iter ) {
908 0 : fd_gui_store_t * db = (fd_gui_store_t *)iter->_db;
909 0 : ulong ring_idx = iter->_rec_sz - 1UL;
910 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
911 :
912 0 : ulong cur = fd_ulong_max( (ulong)iter->_cur, p->evict_cur );
913 0 : for( ; cur<iter->_cur_hi; cur++ ) {
914 0 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, cur );
915 0 : ulong window = fd_gui_store_ts_window( p, slot );
916 0 : if( window>iter->_window_hi ) { cur = iter->_cur_hi; break; } /* sorted: nothing further is in range */
917 0 : if( iter->_filter && !iter->_filter( slot, iter->_filter_ctx ) ) continue;
918 0 : iter->_valid = 1;
919 0 : iter->window = window;
920 0 : iter->rec = (void *)slot;
921 0 : iter->_cur = (void *)( cur + 1UL ); /* resume past this record */
922 0 : db->metrics->ts_read_records[ ring_idx ]++;
923 0 : return;
924 0 : }
925 0 : iter->_valid = 0;
926 0 : iter->rec = NULL;
927 0 : iter->_cur = (void *)cur;
928 0 : }
929 :
930 : fd_gui_store_ts_iter_t *
931 : fd_gui_store_ts_scan_begin( fd_gui_store_t * db,
932 : fd_gui_store_ts_iter_t * iter,
933 : ulong ring_idx,
934 : ulong window_lo,
935 : ulong window_hi,
936 : fd_gui_store_ts_filter_fn filter,
937 0 : void * filter_ctx ) {
938 0 : memset( iter, 0, sizeof(fd_gui_store_ts_iter_t) );
939 0 : iter->_filter = filter;
940 0 : iter->_filter_ctx = filter_ctx;
941 :
942 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_scan_begin: bad ring_idx %lu", ring_idx )); return iter; }
943 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
944 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_scan_begin: ring_idx %lu is not a TS ring", ring_idx )); return iter; }
945 :
946 0 : iter->_db = (void *)db;
947 0 : iter->_rec_sz = ring_idx + 1UL; /* ring_idx, biased so 0 means uninitialised */
948 0 : db->metrics->ts_reads[ ring_idx ]++;
949 :
950 0 : ulong lo_cur = window_lo<=window_hi ? fd_gui_store_ts_lower_bound( db, ring_idx, window_lo ) : p->head_cur;
951 0 : iter->_window_hi = window_hi;
952 0 : iter->_cur = (void *)lo_cur;
953 0 : iter->_cur_hi = p->head_cur;
954 0 : fd_gui_store_ts_scan_advance( iter );
955 0 : return iter;
956 0 : }
957 :
958 : int
959 0 : fd_gui_store_ts_scan_next( fd_gui_store_ts_iter_t * iter ) {
960 0 : if( FD_UNLIKELY( !iter->_valid ) ) return 0;
961 0 : fd_gui_store_ts_scan_advance( iter );
962 0 : return iter->_valid;
963 0 : }
964 :
965 : void
966 0 : fd_gui_store_ts_scan_end( fd_gui_store_ts_iter_t * iter ) {
967 0 : iter->_valid = 0;
968 0 : iter->rec = NULL;
969 0 : iter->_db = NULL;
970 0 : }
971 :
972 : int
973 : fd_gui_store_ts_evict( fd_gui_store_t * db,
974 : ulong ring_idx,
975 : ulong hi_window,
976 : ulong * budget,
977 0 : int * drained ) {
978 0 : *drained = 1;
979 0 : if( FD_UNLIKELY( ring_idx>=db->ring_cnt ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_evict: bad ring_idx %lu", ring_idx )); return FD_GUI_STORE_ERR; }
980 0 : fd_gui_store_ring_t * p = &db->super->ring[ ring_idx ];
981 0 : if( FD_UNLIKELY( p->kind!=FD_GUI_STORE_KIND_TS ) ) { FD_LOG_WARNING(( "fd_gui_store_ts_evict: ring_idx %lu is not a TS ring", ring_idx )); return FD_GUI_STORE_ERR; }
982 :
983 0 : ulong evict_cur0 = p->evict_cur;
984 :
985 0 : while( p->evict_cur<p->head_cur ) {
986 0 : uchar const * slot = (uchar const *)fd_gui_store_slot( db, ring_idx, p, p->evict_cur );
987 0 : ulong window = fd_gui_store_ts_window( p, slot );
988 :
989 : /* reached the watermark window. We may miss some entries in-window
990 : above this, they will get reclaimed in a subsequent eviction. */
991 0 : if( window>=hi_window ) break;
992 0 : if( !*budget ) { *drained = 0; break; }
993 0 : p->evict_cur++;
994 0 : (*budget)--;
995 0 : db->metrics->evict_records[ ring_idx ]++;
996 0 : }
997 :
998 0 : if( p->evict_cur!=evict_cur0 ) db->metrics->evicts[ ring_idx ]++;
999 :
1000 0 : p->tail_cur = p->evict_cur;
1001 0 : fd_gui_store_region_reclaim( db, ring_idx, p );
1002 0 : return FD_GUI_STORE_SUCCESS;
1003 0 : }
1004 :
1005 : ulong
1006 : fd_gui_store_reclaim( fd_gui_store_t * db,
1007 : ulong ring_idx,
1008 : ulong budget,
1009 : int (*eligible)( ulong, void const *, void * ),
1010 0 : void * ctx ) {
1011 0 : if( FD_UNLIKELY( !db || !budget || (ring_idx!=ULONG_MAX && ring_idx>=db->ring_cnt) ) ) return 0UL;
1012 0 : ulong best = ULONG_MAX;
1013 0 : ulong age = ULONG_MAX;
1014 0 : for( ulong i=0UL; i<db->ring_cnt; i++ ) {
1015 0 : fd_gui_store_ring_rt_t const * rt = &db->ring_rt[ i ];
1016 0 : fd_gui_store_ring_t const * p = &db->super->ring[ i ];
1017 0 : if( ring_idx==ULONG_MAX ? rt->reg_cnt<=FD_GUI_STORE_BASE_REGIONS : i!=ring_idx ) continue;
1018 0 : if( p->evict_cur==p->head_cur ) continue;
1019 0 : void const * rec = fd_gui_store_slot( db, i, p, p->evict_cur );
1020 0 : if( eligible && !eligible( i, rec, ctx ) ) continue;
1021 0 : ulong id = fd_gui_store_region_ring( db, i )[ rt->reg_base % db->region_cnt ];
1022 0 : if( db->region_age[ id ]<age ) { best = i; age = db->region_age[ id ]; }
1023 0 : }
1024 0 : if( best==ULONG_MAX ) return 0UL;
1025 :
1026 0 : fd_gui_store_ring_t * p = &db->super->ring[ best ];
1027 : /* Do not cross into the next region: after releasing an excess region
1028 : that next region can become part of the protected base. */
1029 0 : ulong stop = fd_ulong_min( p->head_cur, (db->ring_rt[ best ].reg_base+1UL)*p->region_capacity );
1030 0 : ulong first = p->evict_cur;
1031 0 : while( budget && p->evict_cur<stop ) {
1032 0 : uchar const * rec = fd_gui_store_slot( db, best, p, p->evict_cur );
1033 0 : if( eligible && !eligible( best, rec, ctx ) ) break;
1034 0 : if( p->kind==FD_GUI_STORE_KIND_KV ) {
1035 0 : fd_gui_store_kv_idx_node_t * node = fd_gui_store_kv_find_node( db, p, best, rec+p->key_off );
1036 0 : FD_TEST( node );
1037 0 : fd_gui_store_kv_idx_ele_remove_fast( db->kv_idx[ best ], node, db->kv_pool );
1038 0 : fd_gui_store_kv_pool_ele_release( db->kv_pool, node );
1039 0 : }
1040 0 : p->evict_cur++;
1041 0 : budget--;
1042 0 : }
1043 0 : ulong count = p->evict_cur-first;
1044 0 : db->metrics->evicts[ best ] += !!count;
1045 0 : db->metrics->evict_records[ best ] += count;
1046 0 : p->tail_cur = p->evict_cur;
1047 0 : fd_gui_store_region_reclaim( db, best, p );
1048 0 : return count;
1049 0 : }
|
