Line data Source code
1 : #include "fd_prog_load.h"
2 : #include "fd_progcache_user.h"
3 : #include "fd_progcache_reclaim.h"
4 : #include "fd_progcache_clock.h"
5 : #include "../../util/racesan/fd_racesan_target.h"
6 : #include "../../disco/metrics/generated/fd_metrics_enums.h"
7 :
8 : /* FD_PROGCACHE_METRICS_WRITE copies CLASS_CNT-sized arrays through the metrics
9 : enum, so a class added without regenerating the enum would read past them. */
10 :
11 : FD_STATIC_ASSERT( FD_METRICS_ENUM_PROGCACHE_CLASS_CNT==FD_PROGCACHE_CACHE_CLASS_CNT,
12 : progcache_metrics_class_cnt );
13 :
14 : /* Counts class c in a per-class metrics array; an out-of-range class goes uncounted. */
15 :
16 : static inline void
17 : progcache_metric_per_class( ulong * per_class,
18 4801 : ulong c ) {
19 4801 : if( FD_LIKELY( c<FD_PROGCACHE_CACHE_CLASS_CNT ) ) per_class[ c ]++;
20 4801 : }
21 :
22 : FD_TL fd_progcache_metrics_t fd_progcache_metrics_default;
23 :
24 : fd_progcache_t *
25 : fd_progcache_join( fd_progcache_t * cache,
26 : fd_progcache_shmem_t * shmem,
27 : uchar * scratch,
28 12 : ulong scratch_sz ) {
29 12 : if( FD_UNLIKELY( !cache ) ) {
30 0 : FD_LOG_WARNING(( "NULL cache" ));
31 0 : return NULL;
32 0 : }
33 12 : if( FD_LIKELY( scratch_sz ) ) {
34 12 : if( FD_UNLIKELY( !scratch ) ) {
35 0 : FD_LOG_WARNING(( "NULL scratch" ));
36 0 : return NULL;
37 0 : }
38 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)scratch, FD_PROGCACHE_SCRATCH_ALIGN ) ) ) {
39 0 : FD_LOG_WARNING(( "misaligned scratch" ));
40 0 : return NULL;
41 0 : }
42 12 : }
43 12 : memset( cache, 0, sizeof(fd_progcache_t) );
44 12 : if( FD_UNLIKELY( !fd_progcache_shmem_join( cache->join, shmem ) ) ) return NULL;
45 :
46 12 : cache->metrics = &fd_progcache_metrics_default;
47 12 : cache->scratch = scratch;
48 12 : cache->scratch_sz = scratch_sz;
49 :
50 12 : return cache;
51 12 : }
52 :
53 : void *
54 : fd_progcache_leave( fd_progcache_t * cache,
55 12 : fd_progcache_shmem_t ** opt_shmem ) {
56 12 : if( FD_UNLIKELY( !cache ) ) {
57 0 : FD_LOG_WARNING(( "NULL cache" ));
58 0 : return NULL;
59 0 : }
60 :
61 12 : fd_prog_reclaim_work( cache->join );
62 12 : if( FD_UNLIKELY( !fd_progcache_shmem_leave( cache->join, opt_shmem ) ) ) return NULL;
63 12 : cache->scratch = NULL;
64 12 : cache->scratch_sz = 0UL;
65 12 : return cache;
66 12 : }
67 :
68 : /* fd_progcache_load_fork pivots the progcache object to the selected fork
69 : (identified by tip XID): populates cache->lineage with the fork's XIDs,
70 : newest to oldest. Cache lookups only respect records on that lineage.
71 :
72 : load_fork_slow and fd_progcache_query below are internal but not static:
73 : the test suite composes them into a lookup that does not fill (see
74 : test_progcache_common.c). They are absent from fd_progcache_user.h. */
75 :
76 : void
77 : fd_progcache_load_fork_slow( fd_progcache_t * cache,
78 4284 : fd_progcache_fork_id_t fork_id ) {
79 4284 : fd_progcache_lineage_t * lineage = cache->lineage;
80 4284 : fd_progcache_join_t const * ljoin = cache->join;
81 4284 : fd_rwlock_read( &ljoin->shmem->txn.rwlock );
82 4284 : lineage->fork_depth = 0UL;
83 4284 : lineage->tip_txn_idx = ULONG_MAX;
84 4284 : lineage->root = __atomic_load_n( &ljoin->shmem->txn.root, memory_order_acquire );
85 :
86 4284 : ulong txn_max = fd_prog_txnp_max( ljoin->txn.pool );
87 4284 : ulong i;
88 7313 : for( i=0UL;; i++ ) {
89 7313 : if( FD_UNLIKELY( i>=FD_PROGCACHE_DEPTH_MAX ) ) {
90 0 : FD_LOG_CRIT(( "fd_progcache_load_fork: fork depth exceeded max of %lu", (ulong)FD_PROGCACHE_DEPTH_MAX ));
91 0 : }
92 7313 : uint next_idx = (uint)fd_prog_txnm_idx_query_const( ljoin->txn.map, &fork_id, UINT_MAX, ljoin->txn.pool );
93 7313 : if( FD_UNLIKELY( next_idx==UINT_MAX ) ) break;
94 7313 : if( FD_UNLIKELY( (ulong)next_idx >= txn_max ) )
95 0 : FD_LOG_CRIT(( "progcache: corruption detected (load_fork txn_idx=%u txn_max=%lu)", next_idx, txn_max ));
96 7313 : fd_progcache_txn_t * candidate = &ljoin->txn.pool[ next_idx ];
97 :
98 7313 : uint parent_idx = candidate->parent_idx;
99 7313 : FD_TEST( parent_idx!=next_idx );
100 7313 : lineage->fork[ i ] = fork_id;
101 7313 : if( FD_LIKELY( !i ) ) lineage->tip_txn_idx = next_idx;
102 7313 : if( parent_idx==UINT_MAX ) {
103 4283 : i++;
104 4283 : break;
105 4283 : }
106 3030 : if( FD_UNLIKELY( (ulong)parent_idx >= txn_max ) )
107 0 : FD_LOG_CRIT(( "progcache: corruption detected (load_fork parent_idx=%u txn_max=%lu)", parent_idx, txn_max ));
108 3030 : fork_id = ljoin->txn.pool[ parent_idx ].xid;
109 3030 : }
110 :
111 4284 : lineage->fork_depth = i;
112 :
113 4284 : fd_rwlock_unread( &ljoin->shmem->txn.rwlock );
114 :
115 4284 : lineage->root = __atomic_load_n( &ljoin->shmem->txn.root, memory_order_acquire );
116 4284 : }
117 :
118 : static inline void
119 : fd_progcache_load_fork( fd_progcache_t * cache,
120 4801 : fd_progcache_fork_id_t fork_id ) {
121 : /* Skip if already on the correct fork */
122 4801 : fd_progcache_lineage_t * lineage = cache->lineage;
123 4801 : if( FD_LIKELY( (!!lineage->fork_depth) & (lineage->fork[ 0 ]==fork_id ) ) ) return;
124 4283 : fd_progcache_load_fork_slow( cache, fork_id ); /* switch fork */
125 4283 : }
126 :
127 : /* fd_prog_wait_if_loading waits out a peer's in-flight load of rec, so the caller
128 : only ever sees a record whose program is in. Returns rec. */
129 :
130 : static inline fd_progcache_rec_t *
131 : fd_prog_wait_if_loading( fd_progcache_t * cache,
132 113 : fd_progcache_rec_t * rec ) {
133 113 : if( FD_UNLIKELY( fd_prog_state_is_loading( rec ) ) ) {
134 0 : cache->metrics->hit_loading_cnt++;
135 0 : while( fd_prog_state_is_loading( rec ) ) {
136 0 : fd_racesan_hook( "prog_wait_if_loading:spin" );
137 0 : FD_SPIN_PAUSE();
138 0 : }
139 0 : }
140 113 : return rec;
141 113 : }
142 :
143 : /* fd_progcache_search_chain walks one chain for an exact (key, feature_slot,
144 : deploy_slot) match whose fork is on the current lineage. */
145 :
146 : static int
147 : fd_progcache_search_chain( fd_progcache_t const * cache,
148 : ulong chain_idx,
149 : fd_pubkey_t const * key,
150 : ulong feature_slot,
151 : ulong deploy_slot,
152 4797 : fd_progcache_rec_t ** out_rec ) { /* read locked */
153 4797 : *out_rec = NULL;
154 :
155 4797 : fd_progcache_join_t const * ljoin = cache->join;
156 4797 : fd_progcache_lineage_t const * lineage = cache->lineage;
157 4797 : fd_prog_recm_shmem_t * shmap = ljoin->rec.map->map;
158 4797 : fd_prog_recm_shmem_private_chain_t const * chain_tbl = fd_prog_recm_shmem_private_chain_const( shmap, 0UL );
159 4797 : fd_prog_recm_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
160 4797 : fd_progcache_rec_t * rec_tbl = ljoin->rec.ele;
161 4797 : ulong rec_max = ljoin->rec.max;
162 4797 : ulong ver_cnt = FD_VOLATILE_CONST( chain->ver_cnt );
163 :
164 : /* Start a speculative transaction for the chain containing revisions
165 : of the program cache key we are looking for. */
166 4797 : ulong cnt = fd_prog_recm_private_vcnt_cnt( ver_cnt );
167 4797 : if( FD_UNLIKELY( fd_prog_recm_private_vcnt_ver( ver_cnt )&1 ) ) {
168 0 : return FD_MAP_ERR_AGAIN; /* chain is locked */
169 0 : }
170 4797 : FD_COMPILER_MFENCE();
171 4797 : fd_racesan_hook( "prog_search_chain:post_ver_cnt" );
172 4797 : uint ele_idx = chain->head_cidx;
173 :
174 : /* Walk the map chain, remember the best entry */
175 4797 : fd_progcache_rec_t * best = NULL;
176 4798 : for( ulong i=0UL; i<cnt; i++, ele_idx=FD_VOLATILE_CONST( rec_tbl[ ele_idx ].map_next ) ) {
177 114 : if( FD_UNLIKELY( (ulong)ele_idx >= rec_max ) ) return FD_MAP_ERR_AGAIN;
178 114 : fd_progcache_rec_t * rec = &rec_tbl[ ele_idx ];
179 :
180 114 : if( FD_UNLIKELY( ( !fd_pubkey_eq( &rec->pair.prog, key ) ) |
181 114 : ( rec->feature_slot != feature_slot ) |
182 114 : ( rec->deploy_slot != deploy_slot ) ) ) {
183 1 : continue;
184 1 : }
185 :
186 113 : fd_progcache_fork_id_t rec_fork_id = __atomic_load_n( &rec->pair.xid, memory_order_relaxed );
187 113 : if( FD_UNLIKELY( !fd_progcache_lineage_has_xid( lineage, rec_fork_id ) ) ) continue;
188 :
189 113 : if( FD_UNLIKELY( rec->map_next==ele_idx ) ) return FD_MAP_ERR_AGAIN;
190 113 : if( FD_UNLIKELY( rec->map_next!=UINT_MAX && rec->map_next>=rec_max ) ) return FD_MAP_ERR_AGAIN;
191 113 : best = rec;
192 113 : break;
193 113 : }
194 4797 : fd_racesan_hook( "prog_search_chain:pre_tryread" );
195 4797 : if( best && FD_UNLIKELY( !fd_rwlock_tryread( &best->lock ) ) ) {
196 0 : return FD_MAP_ERR_AGAIN;
197 0 : }
198 4797 : fd_racesan_hook( "prog_search_chain:post_tryread" );
199 :
200 : /* Retry if we were overrun */
201 4797 : if( FD_UNLIKELY( FD_VOLATILE_CONST( chain->ver_cnt )!=ver_cnt ) ) {
202 0 : if( best ) fd_rwlock_unread( &best->lock );
203 0 : return FD_MAP_ERR_AGAIN;
204 0 : }
205 :
206 4797 : *out_rec = best;
207 4797 : return FD_MAP_SUCCESS;
208 4797 : }
209 :
210 : fd_progcache_rec_t * /* read locked */
211 : fd_progcache_query( fd_progcache_t * cache,
212 : fd_pubkey_t const * key,
213 : ulong feature_slot,
214 4797 : ulong deploy_slot ) {
215 : /* Hash key to chain */
216 4797 : fd_prog_recm_t const * rec_map = cache->join->rec.map;
217 4797 : ulong hash = fd_progcache_rec_key_hash( key, rec_map->map->seed );
218 4797 : ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
219 :
220 : /* Traverse chain for candidate */
221 4797 : fd_progcache_rec_t * rec = NULL;
222 4797 : for(;;) {
223 4797 : int err = fd_progcache_search_chain( cache, chain_idx, key, feature_slot, deploy_slot, &rec );
224 4798 : if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
225 >1844*10^16 : fd_racesan_hook( "prog_query:retry" );
226 >1844*10^16 : FD_SPIN_PAUSE();
227 : /* FIXME backoff */
228 >1844*10^16 : }
229 4797 : if( FD_LIKELY( !rec ) ) return NULL; /* Program not found, need an insert */
230 :
231 112 : return fd_prog_wait_if_loading( cache, rec );
232 4797 : }
233 :
234 : static void
235 : fd_progcache_rec_push_tail( fd_progcache_rec_t * rec_pool,
236 : fd_progcache_rec_t * rec,
237 : uint * rec_head_idx, /* write locked (txn) */
238 : uint * rec_tail_idx,
239 4686 : ulong rec_max ) {
240 4686 : uint rec_idx = (uint)( rec - rec_pool );
241 4686 : uint rec_prev_idx = *rec_tail_idx;
242 :
243 4686 : if( FD_UNLIKELY( (ulong)rec_idx >= rec_max ) )
244 0 : FD_LOG_CRIT(( "progcache: corruption detected (push_tail rec_idx=%u rec_max=%lu)", rec_idx, rec_max ));
245 4686 : if( FD_UNLIKELY( rec_prev_idx!=UINT_MAX && (ulong)rec_prev_idx >= rec_max ) )
246 0 : FD_LOG_CRIT(( "progcache: corruption detected (push_tail rec_prev_idx=%u rec_max=%lu)", rec_prev_idx, rec_max ));
247 :
248 4686 : rec->prev_idx = rec_prev_idx;
249 4686 : rec->next_idx = UINT_MAX;
250 :
251 4686 : if( rec_prev_idx==UINT_MAX ) {
252 4281 : *rec_head_idx = rec_idx;
253 4281 : } else {
254 405 : rec_pool[ rec_prev_idx ].next_idx = rec_idx;
255 405 : }
256 4686 : *rec_tail_idx = rec_idx;
257 4686 : }
258 :
259 : /* Publishes rec (complete, read-locked) under the txn. Returns the record now
260 : serving this key: rec, or the winner's read-locked record if another tile won
261 : the race, or NULL if a mapped record has this key with different
262 : feature/deploy slots. */
263 :
264 : __attribute__((warn_unused_result))
265 : static fd_progcache_rec_t *
266 : fd_progcache_push( fd_progcache_join_t * cache,
267 : fd_progcache_txn_t * txn, /* write locked */
268 : fd_progcache_rec_t * rec,
269 4686 : void const * prog_addr ) {
270 4686 : FD_TEST( fd_prog_state_is_loading( rec ) );
271 :
272 : /* Determine record's xid-key pair */
273 :
274 4686 : rec->prev_idx = UINT_MAX;
275 4686 : rec->next_idx = UINT_MAX;
276 4686 : memcpy( &rec->pair.prog, prog_addr, 32UL );
277 4686 : if( FD_UNLIKELY( !txn ) ) FD_LOG_CRIT(( "NULL txn" ));
278 4686 : __atomic_store_n( &rec->pair.xid, txn->xid, memory_order_relaxed );
279 :
280 : /* Lock rec_map chain, entering critical section */
281 :
282 4686 : struct {
283 4686 : fd_prog_recm_txn_t txn[1];
284 4686 : fd_prog_recm_txn_private_info_t info[1];
285 4686 : } _map_txn;
286 4686 : fd_prog_recm_txn_t * map_txn = fd_prog_recm_txn_init( _map_txn.txn, cache->rec.map, 1UL );
287 4686 : fd_prog_recm_txn_add( map_txn, &rec->pair, 1 );
288 4686 : int txn_err = fd_prog_recm_txn_try( map_txn, FD_MAP_FLAG_BLOCKING );
289 4686 : if( FD_UNLIKELY( txn_err!=FD_MAP_SUCCESS ) ) {
290 0 : FD_LOG_CRIT(( "Failed to insert progcache record: cannot lock rec map chain: %i-%s", txn_err, fd_map_strerror( txn_err ) ));
291 0 : }
292 4686 : fd_racesan_hook( "prog_push:post_chain_lock" );
293 :
294 : /* Check if record exists */
295 :
296 4686 : fd_prog_recm_query_t query[1];
297 4686 : int query_err = fd_prog_recm_txn_query( cache->rec.map, &rec->pair, NULL, query, 0 );
298 4686 : if( FD_UNLIKELY( query_err==FD_MAP_SUCCESS ) ) {
299 : /* Duplicate: adopt the winner. Requires that a mapped record is never
300 : write-locked. */
301 0 : fd_progcache_rec_t * winner = query->ele;
302 0 : int match = ( winner->feature_slot==rec->feature_slot ) & ( winner->deploy_slot==rec->deploy_slot );
303 0 : if( FD_LIKELY( match ) ) fd_rwlock_read( &winner->lock );
304 0 : fd_prog_recm_txn_test( map_txn );
305 0 : fd_prog_recm_txn_fini( map_txn );
306 0 : return match ? winner : NULL;
307 4686 : } else if( FD_UNLIKELY( query_err!=FD_MAP_ERR_KEY ) ) {
308 0 : FD_LOG_CRIT(( "fd_prog_recm_txn_query failed: %i-%s", query_err, fd_map_strerror( query_err ) ));
309 0 : }
310 :
311 4686 : ulong rec_max = cache->rec.max;
312 :
313 : /* Insert new record */
314 :
315 : /* Link record into the transaction's record list. Ownership is established
316 : before the record becomes findable, so a mapped record has an owner unless
317 : rooting deliberately detached it. */
318 :
319 4686 : fd_progcache_rec_push_tail( cache->rec.ele,
320 4686 : rec,
321 4686 : &txn->rec_head_idx,
322 4686 : &txn->rec_tail_idx,
323 4686 : rec_max );
324 4686 : uint txn_idx_computed = (uint)( txn - cache->txn.pool );
325 4686 : ulong txn_max = fd_prog_txnp_max( cache->txn.pool );
326 4686 : if( FD_UNLIKELY( (ulong)txn_idx_computed >= txn_max ) )
327 0 : FD_LOG_CRIT(( "progcache: corruption detected (push txn_idx=%u txn_max=%lu)", txn_idx_computed, txn_max ));
328 4686 : atomic_store_explicit( &rec->txn_idx, txn_idx_computed, memory_order_release );
329 :
330 4686 : int insert_err = fd_prog_recm_txn_insert( cache->rec.map, rec );
331 4686 : if( FD_UNLIKELY( insert_err!=FD_MAP_SUCCESS ) ) {
332 0 : FD_LOG_CRIT(( "fd_prog_recm_txn_insert failed: %i-%s", insert_err, fd_map_strerror( insert_err ) ));
333 0 : }
334 4686 : fd_racesan_hook( "prog_push:post_map_insert" );
335 :
336 : /* Finish rec_map transaction */
337 :
338 4686 : int test_err = fd_prog_recm_txn_test( map_txn );
339 4686 : if( FD_UNLIKELY( test_err!=FD_MAP_SUCCESS ) ) FD_LOG_CRIT(( "fd_prog_recm_txn_test failed: %i-%s", test_err, fd_map_strerror( test_err ) ));
340 4686 : fd_prog_recm_txn_fini( map_txn );
341 :
342 4686 : return rec;
343 4686 : }
344 :
345 : /* insert_params captures all environment parameters required to load a
346 : program revision into cache. */
347 :
348 : struct insert_params {
349 : fd_pubkey_t prog_addr;
350 : ulong feature_slot;
351 : ulong deploy_slot;
352 : fd_sbpf_elf_info_t elf_info;
353 : fd_sbpf_loader_config_t config;
354 : fd_features_t const * features;
355 : uchar const * bin;
356 : ulong bin_sz;
357 : int peek_err;
358 : };
359 :
360 : typedef struct insert_params insert_params_t;
361 :
362 : static insert_params_t *
363 : insert_params( insert_params_t * p,
364 : fd_pubkey_t const * prog_addr,
365 : fd_prog_load_env_t const * env,
366 : fd_acc_t const * prog_ro,
367 4687 : fd_prog_info_t const * info ) {
368 4687 : memset( p, 0, sizeof(insert_params_t) );
369 :
370 : /* Derive executable info */
371 4687 : uchar const * bin = (uchar const *)prog_ro->data + info->elf_off;
372 4687 : ulong bin_sz = info->elf_sz;
373 :
374 : /* Pre-flight checks, determine required buffer size */
375 :
376 4687 : fd_features_t const * features = env->features;
377 4687 : fd_prog_versions_t versions = fd_prog_versions( features, env->feature_slot );
378 4687 : fd_sbpf_elf_info_t elf_info = {0};
379 4687 : fd_sbpf_loader_config_t config = {
380 4687 : .sbpf_min_version = versions.min_sbpf_version,
381 4687 : .sbpf_max_version = versions.max_sbpf_version,
382 4687 : };
383 4687 : int peek_err = fd_sbpf_elf_peek( &elf_info, bin, bin_sz, &config );
384 :
385 4687 : *p = (insert_params_t) {
386 4687 : .prog_addr = *prog_addr,
387 4687 : .feature_slot = env->feature_slot,
388 4687 : .deploy_slot = info->deploy_slot,
389 4687 : .features = features,
390 4687 : .bin = !peek_err ? bin : NULL,
391 4687 : .bin_sz = !peek_err ? bin_sz : 0UL,
392 4687 : .peek_err = peek_err,
393 4687 : .elf_info = elf_info,
394 4687 : .config = config
395 4687 : };
396 4687 : return p;
397 4687 : }
398 :
399 : /* fd_progcache_spill_acquire takes the next spill frame: an unloaded record with
400 : its spad slot reserved. Requires the spill write lock. */
401 :
402 : static fd_progcache_rec_t * /* read locked */
403 : fd_progcache_spill_acquire( fd_progcache_t * cache,
404 0 : insert_params_t const * params ) {
405 0 : fd_progcache_join_t * join = cache->join;
406 0 : fd_progcache_shmem_t * shmem = join->shmem;
407 0 : FD_TEST( FD_VOLATILE_CONST( shmem->spill.lock.value )==FD_RWLOCK_WRITE_LOCK );
408 : /* Allocate record */
409 :
410 0 : if( FD_UNLIKELY( shmem->spill.rec_used >= FD_MAX_INSTRUCTION_STACK_DEPTH ) ) {
411 0 : FD_LOG_CRIT(( "spill buffer overflow: rec_used=%u rec_max=%lu", shmem->spill.rec_used, FD_MAX_INSTRUCTION_STACK_DEPTH ));
412 0 : }
413 0 : cache->spill_active++;
414 0 : uint rec_idx = shmem->spill.rec_used++;
415 0 : shmem->spill.spad_off[ rec_idx ] = shmem->spill.spad_used;
416 0 : fd_progcache_rec_t * rec = &shmem->spill.rec[ rec_idx ];
417 0 : memset( rec, 0, sizeof(fd_progcache_rec_t) );
418 0 : rec->lock.value = 1; /* read lock; no concurrency, don't need CAS */
419 0 : rec->exists = 1;
420 0 : rec->feature_slot = params->feature_slot;
421 0 : rec->deploy_slot = params->deploy_slot;
422 0 : rec->calldests_off = UINT_MAX; /* non-executable until a load says otherwise */
423 :
424 0 : if( params->peek_err==FD_SBPF_ELF_SUCCESS ) {
425 0 : ulong off0 = fd_ulong_align_up( shmem->spill.spad_used, fd_progcache_val_align() );
426 0 : ulong off1 = off0 + fd_progcache_val_footprint( ¶ms->elf_info );
427 0 : if( FD_UNLIKELY( off1 > FD_PROGCACHE_SPAD_MAX ) ) {
428 0 : FD_LOG_CRIT(( "spill buffer overflow: spad_used=%u val_sz=%lu spad_max=%lu", shmem->spill.spad_used, off1-off0, FD_PROGCACHE_SPAD_MAX ));
429 0 : }
430 0 : rec->data_gaddr = fd_wksp_gaddr_fast( join->data_base, shmem->spill.spad + off0 );
431 0 : rec->data_max = (uint)( off1 - off0 );
432 0 : shmem->spill.spad_used = (uint)off1;
433 0 : }
434 :
435 0 : return rec;
436 0 : }
437 :
438 : static fd_progcache_rec_t * /* read locked */
439 : fd_progcache_insert( fd_progcache_t * cache,
440 4687 : insert_params_t const * params ) {
441 4687 : fd_progcache_join_t * ljoin = cache->join;
442 4687 : fd_progcache_shmem_t * shmem = ljoin->shmem;
443 :
444 4687 : ulong val_footprint = ( params->peek_err==FD_SBPF_ELF_SUCCESS ) ? fd_progcache_val_footprint( ¶ms->elf_info ) : 0UL;
445 4687 : ulong size_class = fd_progcache_cache_class( val_footprint );
446 :
447 4687 : fd_progcache_rec_t * rec = NULL;
448 4687 : int from_spill = 0;
449 :
450 4687 : if( FD_UNLIKELY( cache->spill_active ) ) {
451 0 : rec = fd_progcache_spill_acquire( cache, params );
452 0 : from_spill = 1;
453 4687 : } else {
454 : /* first acquire attempt (outside loop to increase metrics) */
455 4687 : rec = fd_progcache_rec_acquire( ljoin, val_footprint );
456 4687 : if( FD_UNLIKELY( !rec ) ) cache->metrics->class_full_cnt++;
457 4687 : }
458 :
459 : /* spin loop: evict, try spill, acquire (in case another thread freed a slot) */
460 4687 : while( !rec ) {
461 0 : rec = fd_prog_evict( cache, val_footprint );
462 0 : if( FD_LIKELY( rec ) ) break;
463 :
464 0 : if( fd_rwlock_trywrite( &shmem->spill.lock ) ) {
465 0 : rec = fd_progcache_spill_acquire( cache, params );
466 0 : from_spill = 1;
467 0 : break;
468 0 : }
469 :
470 0 : FD_SPIN_PAUSE();
471 0 : rec = fd_progcache_rec_acquire( ljoin, val_footprint );
472 0 : }
473 :
474 : /* Claim the key before loading it. A peer that wants this program finds the
475 : record LOADING and waits, so only one tile runs fd_sbpf_program_load for a
476 : given revision. A spill record is never published. */
477 :
478 4687 : rec->feature_slot = params->feature_slot;
479 4687 : rec->deploy_slot = params->deploy_slot;
480 :
481 4688 : if( FD_LIKELY( !from_spill ) ) {
482 : /* Under the loading sentinel: not LIVE, so the sweep steps over it, and once
483 : push makes it findable a peer waits rather than loading it again. */
484 4688 : fd_prog_state_load_begin( ljoin->rec.ele, (ulong)( rec - ljoin->rec.ele ) );
485 :
486 4688 : fd_racesan_hook( "prog_insert:pre_push" );
487 4688 : fd_rwlock_read( &shmem->txn.rwlock );
488 4688 : ulong txn_idx = cache->lineage->tip_txn_idx;
489 4688 : if( FD_UNLIKELY( txn_idx==ULONG_MAX ) ) FD_LOG_CRIT(( "progcache insert requires a non-root transaction" ));
490 : /* tip_txn_idx may be stale, so revalidate it against the map under the read
491 : lock. A mismatch means execution was dispatched on a dead fork. */
492 4688 : uint live_idx = (uint)fd_prog_txnm_idx_query_const( ljoin->txn.map, &cache->lineage->fork[ 0 ], UINT_MAX, ljoin->txn.pool );
493 4688 : if( FD_UNLIKELY( (ulong)live_idx!=txn_idx ) )
494 0 : FD_LOG_CRIT(( "progcache insert on a published/canceled fork (fork_id=%lu txn_idx=%lu live_idx=%u)",
495 4688 : (ulong)cache->lineage->fork[ 0 ], txn_idx, live_idx ));
496 4688 : fd_progcache_txn_t * txn = &ljoin->txn.pool[ txn_idx ];
497 4688 : fd_rwlock_write( &txn->lock );
498 4688 : fd_progcache_rec_t * mapped = fd_progcache_push( ljoin, txn, rec, ¶ms->prog_addr );
499 4688 : fd_rwlock_unwrite( &txn->lock );
500 4688 : fd_rwlock_unread( &shmem->txn.rwlock );
501 :
502 : /* fd_progcache_push returns rec, the winner of a same-revision race, or
503 : NULL for another revision of this key, which delayed visibility excludes. */
504 4688 : if( FD_UNLIKELY( !mapped ) ) {
505 0 : FD_BASE58_ENCODE_32_BYTES( params->prog_addr.uc, prog_b58 );
506 0 : FD_LOG_CRIT(( "progcache insert found another revision of this program mapped (prog=%s fork_id=%lu feature_slot=%lu deploy_slot=%lu)",
507 0 : prog_b58, (ulong)cache->lineage->fork[ 0 ], params->feature_slot, params->deploy_slot ));
508 :
509 4688 : } else if( FD_UNLIKELY( mapped!=rec ) ) {
510 : /* Another thread published this revision first and may still be loading.
511 : The lookup was counted as a miss; the winner serves it, a hit. */
512 0 : fd_progcache_rec_abandon( ljoin, rec );
513 0 : cache->metrics->miss_cnt--;
514 0 : cache->metrics->hit_cnt++;
515 0 : progcache_metric_per_class( cache->metrics->hit_per_class, size_class );
516 0 : return fd_prog_wait_if_loading( cache, mapped );
517 :
518 4688 : } else {
519 4688 : fd_racesan_hook( "prog_insert:post_claim" );
520 4688 : }
521 4688 : }
522 :
523 : /* Load program */
524 :
525 4687 : if( FD_LIKELY( params->peek_err==FD_SBPF_ELF_SUCCESS ) ) {
526 4156 : cache->metrics->load_cnt++;
527 4156 : long dt = -fd_tickcount();
528 4156 : if( FD_UNLIKELY( !fd_progcache_rec_load( rec, ljoin->data_base, ¶ms->elf_info, ¶ms->config, params->feature_slot,
529 4156 : params->features, params->bin, params->bin_sz, cache->scratch, cache->scratch_sz ) ) ) {
530 : /* Not a valid program (mark cache entry as non-executable) */
531 46 : fd_progcache_rec_nx( rec );
532 46 : }
533 4156 : dt += fd_tickcount();
534 4156 : cache->metrics->cum_load_ticks += (ulong)dt;
535 4156 : } else {
536 531 : fd_progcache_rec_nx( rec );
537 531 : }
538 :
539 4687 : if( FD_UNLIKELY( from_spill ) ) {
540 0 : cache->metrics->spill_cnt++;
541 0 : cache->metrics->spill_tot_sz += rec->rodata_sz;
542 0 : progcache_metric_per_class( cache->metrics->spill_per_class, size_class );
543 :
544 4687 : } else {
545 : /* LOADING -> LIVE, releasing the program to waiters */
546 4687 : fd_prog_state_touch( ljoin->rec.ele, (ulong)( rec - ljoin->rec.ele ) );
547 :
548 4687 : cache->metrics->fill_cnt++;
549 4687 : cache->metrics->fill_tot_sz += rec->rodata_sz;
550 4687 : progcache_metric_per_class( cache->metrics->fill_per_class, size_class );
551 4687 : }
552 :
553 4687 : FD_TEST( rec->exists );
554 4687 : return rec; /* read locked since acquire */
555 4687 : }
556 :
557 : fd_progcache_rec_t * /* read locked */
558 : fd_progcache_pull( fd_progcache_t * cache,
559 : fd_progcache_fork_id_t fork_id,
560 : fd_pubkey_t const * prog_addr,
561 : fd_prog_load_env_t const * env,
562 4802 : fd_acc_t const * prog_ro ) {
563 4802 : if( FD_UNLIKELY( !cache || !cache->join->shmem ) ) FD_LOG_CRIT(( "NULL progcache" ));
564 4802 : long dt = -fd_tickcount();
565 4802 : fd_progcache_load_fork( cache, fork_id );
566 4802 : cache->metrics->lookup_cnt++;
567 :
568 4802 : fd_prog_info_t info[1];
569 4802 : if( FD_UNLIKELY( !fd_prog_info( info, prog_ro ) ) ) return NULL;
570 :
571 4801 : fd_progcache_rec_t * found_rec =
572 4801 : fd_progcache_query( cache, prog_addr, env->feature_slot, info->deploy_slot );
573 4801 : if( FD_LIKELY( found_rec ) ) {
574 113 : cache->metrics->hit_cnt++;
575 : /* Mark the record as recently accessed for CLOCK replacement */
576 113 : fd_prog_state_touch( cache->join->rec.ele, (ulong)( found_rec - cache->join->rec.ele ) );
577 113 : progcache_metric_per_class( cache->metrics->hit_per_class, found_rec->size_class );
578 4688 : } else {
579 4688 : cache->metrics->miss_cnt++;
580 4688 : insert_params_t insert[1];
581 4688 : found_rec = fd_progcache_insert( cache, insert_params( insert, prog_addr, env, prog_ro, info ) );
582 4688 : }
583 :
584 4801 : dt += fd_tickcount();
585 4801 : cache->metrics->cum_pull_ticks += (ulong)dt;
586 4801 : return found_rec;
587 4802 : }
588 :
589 : static void
590 0 : fd_progcache_spill_close( fd_progcache_t * cache ) {
591 0 : FD_TEST( cache->spill_active );
592 0 : cache->spill_active--;
593 :
594 0 : fd_progcache_shmem_t * shmem = cache->join->shmem;
595 :
596 : /* Cascade: rewind rec_used and spad_used while the top record is
597 : closed. This reclaims spill spad memory in LIFO order. */
598 0 : while( shmem->spill.rec_used > 0 &&
599 0 : !shmem->spill.rec[ shmem->spill.rec_used-1 ].exists ) {
600 0 : shmem->spill.rec_used--;
601 0 : shmem->spill.spad_used = shmem->spill.spad_off[ shmem->spill.rec_used ];
602 0 : }
603 :
604 0 : if( cache->spill_active==0 ) {
605 0 : fd_rwlock_t * spill_lock = &shmem->spill.lock;
606 0 : FD_TEST( spill_lock->value==FD_RWLOCK_WRITE_LOCK );
607 0 : FD_TEST( shmem->spill.rec_used==0 );
608 0 : FD_TEST( shmem->spill.spad_used==0 );
609 0 : fd_rwlock_unwrite( spill_lock );
610 0 : }
611 0 : }
612 :
613 : void
614 : fd_progcache_rec_close( fd_progcache_t * cache,
615 4800 : fd_progcache_rec_t * rec ) {
616 4800 : if( FD_UNLIKELY( !rec ) ) return;
617 4800 : if( FD_UNLIKELY( !rec->exists ) ) FD_LOG_CRIT(( "use-after-free: progcache record %p is dead", (void *)rec ));
618 4800 : FD_TEST( FD_VOLATILE_CONST( rec->lock.value )!=0 );
619 4800 : fd_rwlock_unread( &rec->lock );
620 4800 : fd_progcache_shmem_t * shmem = cache->join->shmem;
621 4800 : if( rec >= shmem->spill.rec &&
622 4801 : rec < shmem->spill.rec + FD_MAX_INSTRUCTION_STACK_DEPTH ) {
623 0 : rec->exists = 0;
624 0 : fd_progcache_spill_close( cache );
625 0 : }}
|
