Line data Source code
1 : #include "fd_cost_tracker_store.h"
2 :
3 : #include <errno.h>
4 : #include <unistd.h>
5 :
6 12 : #define FD_COST_TRACKER_STORE_MAGIC (0xF17EDA2CC057B000UL) /* FIREDANCER COST TRACKER STORE V0 */
7 :
8 : /* set_t is the metadata of one fork's cost tracker. An acquired cost
9 : tracker that is not cached is on disk. */
10 : struct set {
11 : ushort next;
12 : ushort cache_idx; /* USHORT_MAX if the cost tracker is not cached */
13 : };
14 : typedef struct set set_t;
15 :
16 : #define POOL_NAME set_pool
17 24 : #define POOL_T set_t
18 : #define POOL_IDX_T ushort
19 : #include "../../util/tmpl/fd_pool.c"
20 :
21 : /* cache_ent_t is an in-memory slot holding one cost tracker, plus the
22 : bookkeeping needed to pin and evict it. */
23 : struct cache_ent {
24 : ulong set_idx; /* ULONG_MAX if the entry is empty */
25 : ulong pin_cnt;
26 : ulong lru;
27 : uchar data[ FD_COST_TRACKER_FOOTPRINT ] __attribute__((aligned(FD_COST_TRACKER_ALIGN)));
28 : };
29 : typedef struct cache_ent cache_ent_t;
30 :
31 : struct fd_cost_tracker_store {
32 : ulong magic;
33 : ulong cache_cnt;
34 : int disk_fd;
35 : ulong bench_max_cost_per_block;
36 : ulong seed;
37 : ulong set_off;
38 : ulong cache_off;
39 : ulong lru;
40 : };
41 :
42 : static inline set_t *
43 262637 : sets( fd_cost_tracker_store_t const * store ) {
44 262637 : return fd_type_pun( (uchar *)store + store->set_off );
45 262637 : }
46 :
47 : static inline cache_ent_t *
48 132487 : cache( fd_cost_tracker_store_t const * store ) {
49 132487 : return fd_type_pun( (uchar *)store + store->cache_off );
50 132487 : }
51 :
52 : static inline set_t *
53 : set_query( fd_cost_tracker_store_t const * store,
54 67410 : ulong set_idx ) {
55 67410 : set_t * pool = sets( store );
56 67410 : FD_CHECK_CRIT( set_idx<set_pool_max( pool ), "invariant violation: invalid cost tracker set index" );
57 67410 : return pool + set_idx;
58 67410 : }
59 :
60 : static void
61 : disk_read( fd_cost_tracker_store_t * store,
62 : void * dst,
63 0 : ulong set_idx ) {
64 0 : ulong off = set_idx * FD_COST_TRACKER_FOOTPRINT;
65 0 : ulong sz = FD_COST_TRACKER_FOOTPRINT;
66 0 : ulong got = 0UL;
67 0 : while( got<sz ) {
68 0 : long n = pread( store->disk_fd, (uchar *)dst+got, sz-got, (long)(off+got) );
69 0 : if( FD_UNLIKELY( n<0L ) ) {
70 0 : if( FD_LIKELY( errno==EINTR ) ) continue;
71 0 : FD_LOG_CRIT(( "pread(cost tracker spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
72 0 : }
73 0 : if( FD_UNLIKELY( !n ) ) {
74 0 : FD_LOG_CRIT(( "unexpected EOF in cost tracker spill file (set=%lu offset=%lu size=%lu)", set_idx, off+got, sz-got ));
75 0 : }
76 0 : got += (ulong)n;
77 0 : }
78 0 : }
79 :
80 : static void
81 : disk_write( fd_cost_tracker_store_t * store,
82 : void const * src,
83 0 : ulong set_idx ) {
84 0 : ulong off = set_idx * FD_COST_TRACKER_FOOTPRINT;
85 0 : ulong sz = FD_COST_TRACKER_FOOTPRINT;
86 0 : ulong put = 0UL;
87 0 : while( put<sz ) {
88 0 : long n = pwrite( store->disk_fd, (uchar const *)src+put, sz-put, (long)(off+put) );
89 0 : if( FD_LIKELY( n>0L ) ) {
90 0 : put += (ulong)n;
91 0 : continue;
92 0 : }
93 0 : if( FD_UNLIKELY( n<0L && errno==EINTR ) ) continue;
94 0 : if( FD_UNLIKELY( !n ) ) errno = EIO;
95 0 : FD_LOG_CRIT(( "pwrite(cost tracker spill file, fd=%d) failed (%i-%s)", store->disk_fd, errno, fd_io_strerror( errno ) ));
96 0 : }
97 0 : }
98 :
99 : /* cache_evict frees the least recently used unpinned cache entry for
100 : the cost tracker of set_idx, writing its previous cost tracker back
101 : to disk. */
102 :
103 : static cache_ent_t *
104 : cache_evict( fd_cost_tracker_store_t * store,
105 65108 : ulong set_idx ) {
106 65108 : set_t * set = sets( store );
107 65108 : cache_ent_t * ent = cache( store );
108 :
109 65108 : cache_ent_t * victim = NULL;
110 195330 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
111 130222 : if( !ent[i].pin_cnt && (!victim || ent[i].lru<victim->lru) ) victim = ent+i;
112 130222 : }
113 65108 : FD_CHECK_CRIT( victim, "every cost tracker cache entry is pinned" );
114 :
115 65108 : if( victim->set_idx!=ULONG_MAX ) {
116 0 : disk_write( store, victim->data, victim->set_idx );
117 0 : set[ victim->set_idx ].cache_idx = USHORT_MAX;
118 0 : }
119 65108 : victim->set_idx = set_idx;
120 65108 : set[ set_idx ].cache_idx = (ushort)( victim - ent );
121 65108 : return victim;
122 65108 : }
123 :
124 : static cache_ent_t *
125 : cache_get( fd_cost_tracker_store_t * store,
126 615 : ulong set_idx ) {
127 615 : set_t * set = set_query( store, set_idx );
128 615 : if( set->cache_idx!=USHORT_MAX ) return cache( store ) + set->cache_idx;
129 :
130 0 : cache_ent_t * ent = cache_evict( store, set_idx );
131 0 : disk_read( store, ent->data, set_idx );
132 0 : return ent;
133 615 : }
134 :
135 : static cache_ent_t *
136 : cached_ent( fd_cost_tracker_store_t const * store,
137 1686 : ulong set_idx ) {
138 1686 : set_t const * set = set_query( store, set_idx );
139 1686 : FD_CHECK_CRIT( set->cache_idx<store->cache_cnt, "invariant violation: cost tracker is not cached" );
140 1686 : return cache( store ) + set->cache_idx;
141 1686 : }
142 :
143 : ulong
144 108 : fd_cost_tracker_store_align( void ) {
145 108 : return FD_COST_TRACKER_STORE_ALIGN;
146 108 : }
147 :
148 : ulong
149 : fd_cost_tracker_store_footprint( ulong max_live_slots,
150 60 : ulong cache_cnt ) {
151 60 : if( FD_UNLIKELY( !max_live_slots || max_live_slots>=USHORT_MAX || !cache_cnt ) ) return 0UL;
152 60 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
153 :
154 60 : ulong l = FD_LAYOUT_INIT;
155 60 : l = FD_LAYOUT_APPEND( l, FD_COST_TRACKER_STORE_ALIGN, sizeof(fd_cost_tracker_store_t) );
156 60 : l = FD_LAYOUT_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
157 60 : l = FD_LAYOUT_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
158 60 : return FD_LAYOUT_FINI( l, FD_COST_TRACKER_STORE_ALIGN );
159 60 : }
160 :
161 : void *
162 : fd_cost_tracker_store_new( void * shmem,
163 : int disk_fd,
164 : ulong max_live_slots,
165 : ulong cache_cnt,
166 : ulong bench_max_cost_per_block,
167 12 : ulong seed ) {
168 12 : if( FD_UNLIKELY( !shmem ) ) {
169 0 : FD_LOG_WARNING(( "NULL shmem" ));
170 0 : return NULL;
171 0 : }
172 :
173 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_cost_tracker_store_align() ) ) ) {
174 0 : FD_LOG_WARNING(( "misaligned shmem" ));
175 0 : return NULL;
176 0 : }
177 :
178 12 : if( FD_UNLIKELY( disk_fd<0 ) ) {
179 0 : FD_LOG_WARNING(( "invalid disk_fd %d", disk_fd ));
180 0 : return NULL;
181 0 : }
182 :
183 12 : if( FD_UNLIKELY( !fd_cost_tracker_store_footprint( max_live_slots, cache_cnt ) ) ) {
184 0 : FD_LOG_WARNING(( "invalid max_live_slots %lu or cache_cnt %lu", max_live_slots, cache_cnt ));
185 0 : return NULL;
186 0 : }
187 :
188 12 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
189 :
190 12 : FD_SCRATCH_ALLOC_INIT( l, shmem );
191 12 : fd_cost_tracker_store_t * store = FD_SCRATCH_ALLOC_APPEND( l, FD_COST_TRACKER_STORE_ALIGN, sizeof(fd_cost_tracker_store_t) );
192 12 : void * set_mem = FD_SCRATCH_ALLOC_APPEND( l, set_pool_align(), set_pool_footprint( max_live_slots ) );
193 12 : cache_ent_t * cache_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(cache_ent_t), sizeof(cache_ent_t)*cache_cnt );
194 12 : FD_SCRATCH_ALLOC_FINI( l, FD_COST_TRACKER_STORE_ALIGN );
195 :
196 12 : set_t * set_pool = set_pool_join( set_pool_new( set_mem, max_live_slots ) );
197 12 : if( FD_UNLIKELY( !set_pool ) ) {
198 0 : FD_LOG_WARNING(( "failed to create cost tracker set pool" ));
199 0 : return NULL;
200 0 : }
201 :
202 12 : store->cache_cnt = cache_cnt;
203 12 : store->disk_fd = disk_fd;
204 12 : store->bench_max_cost_per_block = bench_max_cost_per_block;
205 12 : store->seed = seed;
206 12 : store->set_off = (ulong)set_pool - (ulong)store;
207 12 : store->cache_off = (ulong)cache_mem - (ulong)store;
208 36 : for( ulong i=0UL; i<cache_cnt; i++ ) cache_mem[ i ].pin_cnt = 0UL;
209 12 : fd_cost_tracker_store_reset( store );
210 :
211 12 : FD_COMPILER_MFENCE();
212 12 : FD_VOLATILE( store->magic ) = FD_COST_TRACKER_STORE_MAGIC;
213 12 : FD_COMPILER_MFENCE();
214 :
215 12 : return shmem;
216 12 : }
217 :
218 : fd_cost_tracker_store_t *
219 : fd_cost_tracker_store_join( void * shmem,
220 24 : int disk_fd ) {
221 24 : fd_cost_tracker_store_t * store = shmem;
222 :
223 24 : if( FD_UNLIKELY( !store ) ) {
224 0 : FD_LOG_WARNING(( "NULL shmem" ));
225 0 : return NULL;
226 0 : }
227 :
228 24 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)store, fd_cost_tracker_store_align() ) ) ) {
229 0 : FD_LOG_WARNING(( "misaligned shmem" ));
230 0 : return NULL;
231 0 : }
232 :
233 24 : if( FD_UNLIKELY( store->magic!=FD_COST_TRACKER_STORE_MAGIC ) ) {
234 0 : FD_LOG_WARNING(( "bad magic" ));
235 0 : return NULL;
236 0 : }
237 :
238 24 : if( FD_UNLIKELY( store->disk_fd!=disk_fd ) ) {
239 0 : FD_LOG_WARNING(( "disk_fd mismatch (store=%d join=%d)", store->disk_fd, disk_fd ));
240 0 : return NULL;
241 0 : }
242 :
243 24 : return store;
244 24 : }
245 :
246 : void
247 12 : fd_cost_tracker_store_reset( fd_cost_tracker_store_t * store ) {
248 12 : set_t * pool = sets( store );
249 12 : ulong max = set_pool_max( pool );
250 36 : for( ulong i=0UL; i<max; i++ ) pool[ i ] = (set_t){ .cache_idx = USHORT_MAX };
251 12 : set_pool_reset( pool );
252 12 : store->lru = 0UL;
253 12 : cache_ent_t * ent = cache( store );
254 36 : for( ulong i=0UL; i<store->cache_cnt; i++ ) {
255 24 : FD_CHECK_CRIT( !ent[i].pin_cnt, "invariant violation: resetting pinned cost tracker cache" );
256 24 : ent[i].set_idx = ULONG_MAX;
257 24 : ent[i].lru = 0UL;
258 24 : }
259 12 : }
260 :
261 : ushort
262 : fd_cost_tracker_store_new_fork( fd_cost_tracker_store_t * store,
263 65132 : ushort old_fork_id ) {
264 65132 : if( FD_LIKELY( old_fork_id!=USHORT_MAX ) ) fd_cost_tracker_store_release( store, old_fork_id );
265 :
266 65132 : set_t * pool = sets( store );
267 65132 : FD_CHECK_CRIT( set_pool_free( pool ), "invariant violation: no free cost tracker sets" );
268 65132 : ulong set_idx = set_pool_idx_acquire( pool );
269 65132 : cache_ent_t * ent = cache_evict( store, set_idx );
270 65132 : FD_TEST( fd_cost_tracker_join( fd_cost_tracker_new( ent->data, store->bench_max_cost_per_block, store->seed ) ) );
271 65132 : ent->lru = ++store->lru;
272 65132 : return (ushort)set_idx;
273 65132 : }
274 :
275 : void
276 : fd_cost_tracker_store_release( fd_cost_tracker_store_t * store,
277 65121 : ushort fork_id ) {
278 65121 : set_t * set = set_query( store, (ulong)fork_id );
279 65121 : if( set->cache_idx!=USHORT_MAX ) {
280 65112 : cache_ent_t * ent = cache( store ) + set->cache_idx;
281 65112 : FD_CHECK_CRIT( !ent->pin_cnt, "invariant violation: releasing a pinned cost tracker" );
282 65112 : ent->set_idx = ULONG_MAX;
283 65112 : ent->lru = 0UL;
284 65112 : set->cache_idx = USHORT_MAX;
285 65112 : }
286 65121 : set_pool_idx_release( sets( store ), (ulong)fork_id );
287 65121 : }
288 :
289 : fd_cost_tracker_t *
290 : fd_cost_tracker_store_pin( fd_cost_tracker_store_t * store,
291 615 : ushort fork_id ) {
292 615 : cache_ent_t * ent = cache_get( store, (ulong)fork_id );
293 615 : ent->pin_cnt++;
294 615 : ent->lru = ++store->lru;
295 615 : return fd_type_pun( ent->data );
296 615 : }
297 :
298 : void
299 : fd_cost_tracker_store_unpin( fd_cost_tracker_store_t * store,
300 614 : ushort fork_id ) {
301 614 : cache_ent_t * ent = cached_ent( store, (ulong)fork_id );
302 614 : FD_CHECK_CRIT( ent->pin_cnt, "invariant violation: unpinning an unpinned cost tracker" );
303 614 : ent->pin_cnt--;
304 614 : }
305 :
306 : fd_cost_tracker_t *
307 : fd_cost_tracker_store_peek( fd_cost_tracker_store_t const * store,
308 1073 : ushort fork_id ) {
309 1073 : return fd_type_pun( cached_ent( store, (ulong)fork_id )->data );
310 1073 : }
|
