Line data Source code
1 : #include "fd_epoch_credits.h"
2 : #include "../runtime/fd_runtime_const.h"
3 :
4 : #include <errno.h>
5 : #include <unistd.h>
6 :
7 12 : #define FD_EPOCH_CREDITS_STORE_MAGIC (0xF17EDA2CEC5E7000UL) /* FIREDANCER EPOCH CREDITS STORE V0 */
8 :
9 : /* set_t is the metadata for each fork. This is always in-memory. */
10 651 : #define SET_SZ (sizeof(fd_epoch_credits_t)*FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS)
11 : struct set {
12 : ulong len;
13 : ulong refcnt;
14 : ushort next;
15 : ushort cache_idx; /* USHORT_MAX if the set is not cached */
16 : };
17 : typedef struct set set_t;
18 :
19 : #define POOL_NAME set_pool
20 24 : #define POOL_T set_t
21 : #define POOL_IDX_T ushort
22 : #include "../../util/tmpl/fd_pool.c"
23 :
24 : /* cache_ent_t is in-memory slot holding the epoch credits of one set */
25 : struct cache_ent {
26 : ulong set_idx; /* ULONG_MAX if the entry is empty */
27 : ulong pin_cnt;
28 : ulong lru;
29 : fd_epoch_credits_t credits[ FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ];
30 : };
31 : typedef struct cache_ent cache_ent_t;
32 :
33 : struct fd_epoch_credits_store {
34 : ulong magic;
35 : ulong cache_cnt;
36 : int disk_fd;
37 : ulong set_off;
38 : ulong cache_off;
39 : ulong lru;
40 : };
41 : typedef struct fd_epoch_credits_store fd_epoch_credits_store_t;
42 :
43 : static inline set_t *
44 198677 : sets( fd_epoch_credits_store_t * store ) {
45 198677 : return fd_type_pun( (uchar *)store + store->set_off );
46 198677 : }
47 :
48 : static inline cache_ent_t *
49 2377 : cache( fd_epoch_credits_store_t * store ) {
50 2377 : return fd_type_pun( (uchar *)store + store->cache_off );
51 2377 : }
52 :
53 : static inline set_t *
54 : set_query( fd_epoch_credits_store_t * store,
55 67685 : ulong set_idx ) {
56 67685 : set_t * pool = sets( store );
57 67685 : FD_CHECK_CRIT( set_idx<set_pool_max( pool ), "invariant violation: invalid epoch credits set index" );
58 67685 : return pool + set_idx;
59 67685 : }
60 :
61 : static void
62 : disk_read( fd_epoch_credits_store_t * store,
63 : void * dst,
64 : ulong set_idx,
65 0 : ulong sz ) {
66 0 : ulong off = set_idx * SET_SZ;
67 0 : ulong got = 0UL;
68 0 : while( got<sz ) {
69 0 : long n = pread( store->disk_fd, (uchar *)dst+got, sz-got, (long)(off+got) );
70 0 : if( FD_UNLIKELY( n<0L ) ) {
71 0 : if( FD_LIKELY( errno==EINTR ) ) continue;
72 0 : FD_LOG_CRIT(( "pread(epoch credits spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
73 0 : }
74 0 : if( FD_UNLIKELY( !n ) ) {
75 0 : FD_LOG_CRIT(( "unexpected EOF in epoch credits spill file (set=%lu offset=%lu size=%lu)", set_idx, off+got, sz-got ));
76 0 : }
77 0 : got += (ulong)n;
78 0 : }
79 0 : }
80 :
81 : static void
82 : disk_write( fd_epoch_credits_store_t * store,
83 : void const * src,
84 : ulong set_idx,
85 0 : ulong sz ) {
86 0 : ulong off = set_idx * SET_SZ;
87 0 : ulong put = 0UL;
88 0 : while( put<sz ) {
89 0 : long n = pwrite( store->disk_fd, (uchar const *)src+put, sz-put, (long)(off+put) );
90 0 : if( FD_LIKELY( n>0L ) ) {
91 0 : put += (ulong)n;
92 0 : continue;
93 0 : }
94 0 : if( FD_UNLIKELY( n<0L && errno==EINTR ) ) continue;
95 0 : if( FD_UNLIKELY( !n ) ) errno = EIO;
96 0 : FD_LOG_CRIT(( "pwrite(epoch credits spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
97 0 : }
98 0 : }
99 :
100 : static cache_ent_t *
101 : cache_get( fd_epoch_credits_store_t * store,
102 859 : ulong set_idx ) {
103 859 : set_t * set = sets( store );
104 859 : cache_ent_t * ent = cache( store );
105 859 : if( set[ set_idx ].cache_idx!=USHORT_MAX ) return ent + set[ set_idx ].cache_idx;
106 :
107 : /* Pick cache eviction victim. Can evict anything that's not pinned
108 : and is the LRU. TODO: consider replacing scan with DLL. */
109 647 : cache_ent_t * victim = NULL;
110 1938 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
111 1292 : if( !ent[i].pin_cnt && (!victim || ent[i].lru<victim->lru) ) victim = ent+i;
112 1291 : }
113 647 : FD_CHECK_CRIT( victim, "every epoch credits cache entry is pinned" );
114 :
115 : /* Write back cache entry to disk */
116 647 : if( victim->set_idx!=ULONG_MAX ) {
117 0 : disk_write( store, victim->credits, victim->set_idx, set[ victim->set_idx ].len*sizeof(fd_epoch_credits_t) );
118 0 : set[ victim->set_idx ].cache_idx = USHORT_MAX;
119 0 : }
120 647 : if( set[ set_idx ].len ) disk_read( store, victim->credits, set_idx, set[ set_idx ].len*sizeof(fd_epoch_credits_t) );
121 647 : victim->set_idx = set_idx;
122 647 : set[ set_idx ].cache_idx = (ushort)( victim - ent );
123 647 : return victim;
124 647 : }
125 :
126 : ulong
127 108 : fd_epoch_credits_store_align( void ) {
128 108 : return FD_EPOCH_CREDITS_STORE_ALIGN;
129 108 : }
130 :
131 : ulong
132 : fd_epoch_credits_store_footprint( ulong max_live_slots,
133 60 : ulong cache_cnt ) {
134 60 : if( FD_UNLIKELY( !max_live_slots || max_live_slots>=USHORT_MAX || !cache_cnt ) ) return 0UL;
135 60 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
136 :
137 60 : ulong l = FD_LAYOUT_INIT;
138 60 : l = FD_LAYOUT_APPEND( l, FD_EPOCH_CREDITS_STORE_ALIGN, sizeof(fd_epoch_credits_store_t) );
139 60 : l = FD_LAYOUT_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
140 60 : l = FD_LAYOUT_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
141 60 : return FD_LAYOUT_FINI( l, FD_EPOCH_CREDITS_STORE_ALIGN );
142 60 : }
143 :
144 : void *
145 : fd_epoch_credits_store_new( void * shmem,
146 : int disk_fd,
147 : ulong max_live_slots,
148 12 : ulong cache_cnt ) {
149 12 : if( FD_UNLIKELY( !shmem ) ) {
150 0 : FD_LOG_WARNING(( "NULL shmem" ));
151 0 : return NULL;
152 0 : }
153 :
154 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_epoch_credits_store_align() ) ) ) {
155 0 : FD_LOG_WARNING(( "misaligned shmem" ));
156 0 : return NULL;
157 0 : }
158 :
159 12 : if( FD_UNLIKELY( disk_fd<0 ) ) {
160 0 : FD_LOG_WARNING(( "invalid disk_fd %d", disk_fd ));
161 0 : return NULL;
162 0 : }
163 :
164 12 : if( FD_UNLIKELY( !fd_epoch_credits_store_footprint( max_live_slots, cache_cnt ) ) ) {
165 0 : FD_LOG_WARNING(( "invalid max_live_slots %lu or cache_cnt %lu", max_live_slots, cache_cnt ));
166 0 : return NULL;
167 0 : }
168 :
169 12 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
170 :
171 12 : FD_SCRATCH_ALLOC_INIT( l, shmem );
172 12 : fd_epoch_credits_store_t * store = FD_SCRATCH_ALLOC_APPEND( l, FD_EPOCH_CREDITS_STORE_ALIGN, sizeof(fd_epoch_credits_store_t) );
173 12 : void * set_mem = FD_SCRATCH_ALLOC_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
174 12 : cache_ent_t * cache_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
175 12 : FD_SCRATCH_ALLOC_FINI( l, FD_EPOCH_CREDITS_STORE_ALIGN );
176 :
177 12 : set_t * set_pool = set_pool_join( set_pool_new( set_mem, max_live_slots ) );
178 12 : if( FD_UNLIKELY( !set_pool ) ) {
179 0 : FD_LOG_WARNING(( "failed to create epoch credits set pool" ));
180 0 : return NULL;
181 0 : }
182 :
183 12 : store->cache_cnt = cache_cnt;
184 12 : store->disk_fd = disk_fd;
185 12 : store->set_off = (ulong)set_pool - (ulong)store;
186 12 : store->cache_off = (ulong)cache_mem - (ulong)store;
187 36 : for( ulong i=0UL; i<cache_cnt; i++ ) cache_mem[ i ].pin_cnt = 0UL;
188 12 : fd_epoch_credits_store_reset( store );
189 :
190 12 : FD_COMPILER_MFENCE();
191 12 : FD_VOLATILE( store->magic ) = FD_EPOCH_CREDITS_STORE_MAGIC;
192 12 : FD_COMPILER_MFENCE();
193 :
194 12 : return shmem;
195 12 : }
196 :
197 : fd_epoch_credits_store_t *
198 : fd_epoch_credits_store_join( void * shmem,
199 24 : int disk_fd ) {
200 24 : fd_epoch_credits_store_t * store = shmem;
201 :
202 24 : if( FD_UNLIKELY( !store ) ) {
203 0 : FD_LOG_WARNING(( "NULL shmem" ));
204 0 : return NULL;
205 0 : }
206 :
207 24 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)store, fd_epoch_credits_store_align() ) ) ) {
208 0 : FD_LOG_WARNING(( "misaligned shmem" ));
209 0 : return NULL;
210 0 : }
211 :
212 24 : if( FD_UNLIKELY( store->magic!=FD_EPOCH_CREDITS_STORE_MAGIC ) ) {
213 0 : FD_LOG_WARNING(( "bad magic" ));
214 0 : return NULL;
215 0 : }
216 :
217 24 : if( FD_UNLIKELY( store->disk_fd!=disk_fd ) ) {
218 0 : FD_LOG_WARNING(( "disk_fd mismatch (store=%d join=%d)", store->disk_fd, disk_fd ));
219 0 : return NULL;
220 0 : }
221 :
222 24 : return store;
223 24 : }
224 :
225 : void
226 12 : fd_epoch_credits_store_reset( fd_epoch_credits_store_t * store ) {
227 12 : set_t * pool = sets( store );
228 12 : ulong max = set_pool_max( pool );
229 36 : for( ulong i=0UL; i<max; i++ ) pool[ i ] = (set_t){ .cache_idx = USHORT_MAX };
230 12 : set_pool_reset( pool );
231 12 : store->lru = 0UL;
232 12 : cache_ent_t * ent = cache( store );
233 36 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
234 24 : FD_CHECK_CRIT( !ent[i].pin_cnt, "invariant violation: resetting pinned epoch credits cache" );
235 24 : ent[i].set_idx = ULONG_MAX;
236 24 : ent[i].lru = 0UL;
237 24 : }
238 12 : }
239 :
240 : ushort
241 : fd_epoch_credits_store_new_fork( fd_epoch_credits_store_t * store,
242 65144 : ushort old_fork_id ) {
243 65144 : if( FD_LIKELY( old_fork_id!=USHORT_MAX ) ) fd_epoch_credits_store_release( store, old_fork_id );
244 :
245 65144 : set_t * pool = sets( store );
246 65144 : FD_CHECK_CRIT( set_pool_free( pool ), "invariant violation: no free epoch credits sets" );
247 65144 : ulong set_idx = set_pool_idx_acquire( pool );
248 65144 : pool[ set_idx ].refcnt = 1UL;
249 65144 : return (ushort)set_idx;
250 65144 : }
251 :
252 : void
253 : fd_epoch_credits_store_acquire( fd_epoch_credits_store_t * store,
254 12 : ushort fork_id ) {
255 12 : set_t * set = set_query( store, (ulong)fork_id );
256 12 : FD_CHECK_CRIT( set->refcnt, "invariant violation: acquiring an unreferenced epoch credits set" );
257 12 : set->refcnt++;
258 12 : }
259 :
260 : void
261 : fd_epoch_credits_store_release( fd_epoch_credits_store_t * store,
262 65964 : ushort fork_id ) {
263 65964 : set_t * set = set_query( store, (ulong)fork_id );
264 65964 : FD_CHECK_CRIT( set->refcnt, "invariant violation: releasing an unreferenced epoch credits set" );
265 65964 : if( --set->refcnt ) return;
266 :
267 65093 : set->len = 0UL;
268 65093 : if( set->cache_idx!=USHORT_MAX ) {
269 647 : cache_ent_t * ent = cache( store ) + set->cache_idx;
270 647 : ent->set_idx = ULONG_MAX;
271 647 : ent->lru = 0UL;
272 647 : set->cache_idx = USHORT_MAX;
273 647 : }
274 65093 : set_pool_idx_release( sets( store ), (ulong)fork_id );
275 65093 : }
276 :
277 : fd_epoch_credits_view_t *
278 : fd_epoch_credits_view_init( fd_epoch_credits_view_t * view,
279 : fd_epoch_credits_store_t * store,
280 859 : ushort fork_id ) {
281 859 : if( FD_UNLIKELY( !view || !store ) ) return NULL;
282 :
283 859 : ulong set_idx = (ulong)fork_id;
284 859 : set_t * set = set_query( store, set_idx );
285 859 : cache_ent_t * ent = cache_get( store, set_idx );
286 :
287 859 : set->refcnt++;
288 859 : ent->pin_cnt++;
289 859 : ent->lru = ++store->lru;
290 859 : if( FD_UNLIKELY( !set->len ) ) fd_memset( ent->credits, 0, SET_SZ );
291 :
292 859 : *view = (fd_epoch_credits_view_t) {
293 859 : .credits = ent->credits,
294 859 : .store = store,
295 859 : .len = set->len,
296 859 : .set_idx = set_idx
297 859 : };
298 859 : return view;
299 859 : }
300 :
301 : void
302 859 : fd_epoch_credits_view_fini( fd_epoch_credits_view_t * view ) {
303 859 : if( FD_UNLIKELY( !view || !view->credits ) ) return;
304 :
305 859 : fd_epoch_credits_store_t * store = view->store;
306 859 : set_t * set = set_query( store, view->set_idx );
307 859 : FD_CHECK_CRIT( set->cache_idx<store->cache_cnt, "invariant violation: invalid epoch credits view" );
308 859 : cache_ent_t * ent = cache( store ) + set->cache_idx;
309 859 : FD_CHECK_CRIT( ent->set_idx==view->set_idx && ent->pin_cnt, "invariant violation: invalid epoch credits view" );
310 :
311 859 : FD_CHECK_CRIT( view->len<=FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS, "invariant violation: invalid epoch credits length" );
312 859 : set->len = view->len;
313 :
314 859 : ent->pin_cnt--;
315 859 : fd_epoch_credits_store_release( store, (ushort)view->set_idx );
316 :
317 859 : view->credits = NULL;
318 859 : view->store = NULL;
319 859 : }
320 :
321 : fd_epoch_credits_t const *
322 : fd_epoch_credits_store_peek( fd_epoch_credits_store_t const * store,
323 : ushort fork_id,
324 0 : ulong * len ) {
325 0 : fd_epoch_credits_store_t * s = (fd_epoch_credits_store_t *)store;
326 0 : set_t const * set = set_query( s, (ulong)fork_id );
327 0 : FD_CHECK_CRIT( set->cache_idx<store->cache_cnt, "invariant violation: epoch credits set is not cached" );
328 0 : cache_ent_t const * ent = cache( s ) + set->cache_idx;
329 0 : FD_CHECK_CRIT( ent->set_idx==(ulong)fork_id && ent->pin_cnt, "invariant violation: epoch credits set is not pinned" );
330 0 : *len = set->len;
331 0 : return ent->credits;
332 0 : }
|
