Line data    Source code

       1             : #include "fd_progcache_user.h"
       2             : #include "fd_progcache_clock.h"
       3             : #include "fd_progcache_reclaim.h"
       4             : #include "../../util/racesan/fd_racesan_target.h"
       5             : 
       6             : /* evict_inner is the eviction sweep on one size class, the shared body of
       7             :    fd_prog_evict and fd_prog_preevict.  metrics==NULL is the admin variant:
       8             :    identical scan and claim, no counters. */
       9             : 
      10             : static fd_progcache_rec_t *
      11             : evict_inner( fd_progcache_join_t *    join,
      12             :              fd_progcache_metrics_t * metrics,
      13           0 :              ulong                    class_idx ) {
      14           0 :   fd_progcache_shmem_t * shmem = join->shmem;
      15           0 :   fd_progcache_rec_t *   recs  = join->rec.ele;
      16             : 
      17           0 :   ulong base      = shmem->cache.rec_base [ class_idx ];
      18           0 :   ulong class_max = shmem->cache.class_max[ class_idx ];
      19           0 :   FD_TEST( class_max );
      20             : 
      21             :   /* Do at most 2x class_max (number of slots in class) iterations.
      22             :      If we can't evict, return NULL and we'll try to spill.  Every skip is a
      23             :      continue; the one way out with a slot is the handover at the bottom. */
      24           0 :   fd_progcache_rec_t * claim    = NULL;
      25           0 :   ulong                iter_rem = 2UL*class_max;
      26           0 :   while( iter_rem ) {
      27           0 :     iter_rem--;
      28             : 
      29             :     /* Get a candidate */
      30           0 :     ulong ticket = __atomic_fetch_add( &shmem->cache.clock_hand[ class_idx ].val, 1UL, __ATOMIC_RELAXED );
      31           0 :     fd_progcache_rec_t * rec = recs + base + ( ticket % class_max );
      32             : 
      33           0 :     uchar st = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
      34           0 :     fd_racesan_hook( "prog_clock_evict:post_load_bits" );
      35             : 
      36             :     /* Candidate is free or in flight: skip. */
      37           0 :     if( !( st & FD_PROGCACHE_REC_LIVE ) ) continue;
      38             : 
      39           0 :     if( FD_UNLIKELY( !( st & FD_PROGCACHE_REC_MAPPED ) ) ) {
      40             : 
      41             :       /* Candidate is a zombie, already unmapped, so we can try to
      42             :          write-lock it directly (skip if still read-locked / in use). */
      43             : 
      44           0 :       if( FD_UNLIKELY( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=UINT_MAX ) ) continue;
      45           0 :       if( FD_UNLIKELY( !fd_rwlock_trywrite( &rec->lock ) ) ) continue;
      46             :       /* The write lock freezes the record, so the re-check under it decides. */
      47           0 :       uchar cur = __atomic_load_n( &rec->state, __ATOMIC_RELAXED );
      48           0 :       if( FD_UNLIKELY( ( ( cur & ( FD_PROGCACHE_REC_LIVE|FD_PROGCACHE_REC_MAPPED ) )!=FD_PROGCACHE_REC_LIVE ) |
      49           0 :                        ( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=UINT_MAX ) ) ) {
      50           0 :         fd_rwlock_unwrite( &rec->lock );
      51           0 :         continue;
      52           0 :       }
      53             :       /* Write locked by the trywrite; fall through to fd_progcache_rec_reinit */
      54             : 
      55           0 :     } else {
      56             : 
      57             :       /* Candidate is a live cache record: run CLOCK */
      58           0 :       fd_progcache_rec_key_t pair = rec->pair;
      59             : 
      60             :       /* If there's any read-lock on the record, it'll fail later, so might as well
      61             :          skipping it now (this is just an optimization) */
      62           0 :       if( FD_UNLIKELY( atomic_load_explicit( &rec->lock.value, memory_order_relaxed )!=0 ) ) continue;
      63             : 
      64           0 :       if( st & FD_PROGCACHE_REC_VISITED ) {
      65             :         /* clear the VISITED bit, i.e. CLOCK second chance */
      66           0 :         __atomic_fetch_and( &rec->state, (uchar)~FD_PROGCACHE_REC_VISITED, __ATOMIC_RELAXED );
      67           0 :         continue;
      68           0 :       }
      69             : 
      70             :       /* An attached record is on its fork's list: claim it under the fork lock,
      71             :          or skip it when eviction is rooted-only. */
      72           0 :       uint owner_idx = atomic_load_explicit( &rec->txn_idx, memory_order_relaxed );
      73           0 :       if( FD_UNLIKELY( owner_idx!=UINT_MAX ) ) {
      74           0 : #if FD_PROGCACHE_EVICT_UNROOTED
      75             :         /* Lock order matches fd_progcache_push: txn.rwlock -> txn->lock -> chain -> rec.lock. */
      76           0 :         if( FD_UNLIKELY( (ulong)owner_idx>=shmem->txn.max ) ) continue;
      77           0 :         fd_racesan_hook( "prog_clock_evict:pre_fork_lock" );
      78           0 :         fd_rwlock_read( &shmem->txn.rwlock );
      79             :         /* Recycling a txn slot needs the write lock, so a matching re-read here
      80             :            proves the slot still owns rec. */
      81           0 :         if( FD_UNLIKELY( atomic_load_explicit( &rec->txn_idx, memory_order_relaxed )!=owner_idx ) ) {
      82           0 :           fd_rwlock_unread( &shmem->txn.rwlock );
      83           0 :           continue;
      84           0 :         }
      85           0 :         fd_progcache_txn_t * owner = &join->txn.pool[ owner_idx ];
      86           0 :         if( FD_UNLIKELY( !fd_rwlock_trywrite( &owner->lock ) ) ) {
      87           0 :           fd_rwlock_unread( &shmem->txn.rwlock );
      88           0 :           continue;
      89           0 :         }
      90           0 :         fd_racesan_hook( "prog_clock_evict:pre_delete" );
      91           0 :         long res = fd_prog_delete_rec_claim_txn( join, rec, &pair, owner );
      92           0 :         fd_racesan_hook( "prog_clock_evict:post_delete" );
      93           0 :         fd_rwlock_unwrite( &owner->lock );
      94           0 :         fd_rwlock_unread( &shmem->txn.rwlock );
      95           0 :         if( FD_UNLIKELY( res<0L ) ) continue;
      96             : #else
      97             :         continue;
      98             : #endif
      99           0 :       } else {
     100           0 :         fd_racesan_hook( "prog_clock_evict:pre_delete" );
     101           0 :         long res = fd_prog_delete_rec_claim( join, rec, &pair );
     102           0 :         fd_racesan_hook( "prog_clock_evict:post_delete" );
     103           0 :         if( FD_UNLIKELY( res<0L ) ) continue;
     104           0 :       }
     105             :       /* Write locked by the claim; fall through to fd_progcache_rec_reinit */
     106           0 :     }
     107             : 
     108           0 :     if( FD_LIKELY( metrics ) ) {
     109           0 :       metrics->evict_cnt++;
     110           0 :       metrics->evict_tot_sz += rec->rodata_sz;
     111           0 :       metrics->evict_per_class[ class_idx ]++;
     112           0 :     }
     113             : 
     114             :     /* Unmapped and write locked, so nobody else can take the slot */
     115           0 :     claim = fd_progcache_rec_reinit( join, rec );
     116           0 :     break;
     117           0 :   }
     118             : 
     119           0 :   return claim;
     120           0 : }
     121             : 
     122             : fd_progcache_rec_t *
     123             : fd_prog_evict( fd_progcache_t * cache,
     124           0 :                ulong            sz ) {
     125           0 :   ulong class_idx = fd_progcache_cache_class( sz );
     126           0 :   FD_TEST( class_idx<FD_PROGCACHE_CACHE_CLASS_CNT );
     127           0 :   return evict_inner( cache->join, cache->metrics, class_idx );
     128           0 : }
     129             : 
     130             : ulong
     131             : fd_prog_preevict( fd_progcache_join_t *    join,
     132             :                   fd_progcache_metrics_t * metrics,
     133             :                   ulong                    class_idx,
     134           0 :                   ulong                    free_target ) {
     135           0 :   FD_TEST( class_idx<FD_PROGCACHE_CACHE_CLASS_CNT );
     136           0 :   if( FD_LIKELY( fd_progcache_class_free_cnt( join->shmem, class_idx )>=free_target ) ) return 0UL;
     137           0 :   fd_progcache_rec_t * rec = evict_inner( join, metrics, class_idx );
     138           0 :   if( FD_UNLIKELY( !rec ) ) return 0UL;
     139           0 :   fd_progcache_rec_abandon( join, rec );
     140           0 :   return 1UL;
     141           0 : }
     142             : 
     143             : ulong
     144             : fd_progcache_housekeeping( fd_progcache_join_t *    join,
     145           0 :                            fd_progcache_metrics_t * metrics ) {
     146           0 :   ulong ticket = __atomic_fetch_add( &join->shmem->cache.housekeep_hand.val, 1UL, __ATOMIC_RELAXED );
     147           0 :   return fd_prog_preevict( join, metrics, ticket % FD_PROGCACHE_CACHE_CLASS_CNT, 2UL );
     148           0 : }