Line data    Source code

       1             : #include "fd_bpf_frames.h"
       2             : #include "../fd_rwlock.h"
       3             : 
       4             : union __attribute__((aligned(FD_BPF_FRAMES_ALIGN))) fd_bpf_frame {
       5             :   ulong next; /* pool link while free */
       6             :   uchar data[ BPF_LOADER_SERIALIZATION_FOOTPRINT ];
       7             : };
       8             : typedef union fd_bpf_frame fd_bpf_frame_t;
       9             : 
      10             : #define POOL_NAME frame_pool
      11          24 : #define POOL_T    fd_bpf_frame_t
      12             : #include "../../util/tmpl/fd_pool.c"
      13             : 
      14          12 : #define FD_BPF_FRAMES_MAGIC (0xf17eda2c3bf2a3e0UL) /* FIREDANCE BPF FRAMES V0 */
      15             : 
      16             : struct __attribute__((aligned(FD_BPF_FRAMES_ALIGN))) fd_bpf_frames_shmem {
      17             :   ulong       magic;
      18             :   fd_rwlock_t lock;
      19             :   ulong       has_owner;      /* 1 while some join is the owner, the one
      20             :                                  tile allowed into the owner's reserve */
      21             :   ulong       joiner_cnt;
      22             :   ulong       joiner_cnt_max;
      23             :   ulong       local_cnt;
      24             : 
      25             :   /* local_cnt local frames for each of joiner_cnt_max joins follow,
      26             :      then frame_pool */
      27             : };
      28             : 
      29             : struct fd_bpf_frames {
      30             :   fd_bpf_frames_shmem_t * shmem;
      31             :   fd_bpf_frame_t *        pool;
      32             :   fd_bpf_frame_t *        local;
      33             :   ulong                   local_cnt;
      34             :   ulong                   local_used;  /* local[ 0, local_used ) are in use */
      35             :   ulong                   reserve_cnt; /* frames held back for each of the owner and
      36             :                                           the priority caller, the most shared
      37             :                                           frames one stack can need */
      38             :   int                     is_owner;
      39             :   ulong                   held_cnt;    /* shared frames */
      40             : };
      41             : 
      42             : FD_FN_CONST ulong
      43          84 : fd_bpf_frames_shmem_align( void ) {
      44          84 :   return fd_ulong_max( alignof(fd_bpf_frames_shmem_t), frame_pool_align() );
      45          84 : }
      46             : 
      47             : FD_FN_CONST ulong
      48             : fd_bpf_frames_shmem_footprint( ulong frame_cnt,
      49             :                                ulong joiner_cnt_max,
      50          24 :                                ulong local_cnt ) {
      51          24 :   if( FD_UNLIKELY( local_cnt>FD_MAX_INSTRUCTION_STACK_DEPTH                 ) ) return 0UL;
      52          24 :   if( FD_UNLIKELY( frame_cnt<FD_BPF_FRAMES_HELD_CNT( local_cnt )            ) ) return 0UL;
      53          24 :   if( FD_UNLIKELY( frame_cnt>FD_BPF_FRAMES_CNT_MAX                          ) ) return 0UL;
      54          24 :   if( FD_UNLIKELY( joiner_cnt_max>FD_BPF_FRAMES_CNT_MAX                     ) ) return 0UL;
      55          24 :   if( FD_UNLIKELY( frame_cnt+joiner_cnt_max*local_cnt>FD_BPF_FRAMES_CNT_MAX ) ) return 0UL;
      56          24 :   ulong l = FD_LAYOUT_INIT;
      57          24 :   l = FD_LAYOUT_APPEND( l, alignof(fd_bpf_frames_shmem_t), sizeof(fd_bpf_frames_shmem_t)                   );
      58          24 :   l = FD_LAYOUT_APPEND( l, alignof(fd_bpf_frame_t),        joiner_cnt_max*local_cnt*sizeof(fd_bpf_frame_t) );
      59          24 :   l = FD_LAYOUT_APPEND( l, frame_pool_align(),             frame_pool_footprint( frame_cnt )               );
      60          24 :   return FD_LAYOUT_FINI( l, fd_bpf_frames_shmem_align() );
      61          24 : }
      62             : 
      63             : static fd_bpf_frame_t *
      64          36 : local_frames( fd_bpf_frames_shmem_t * shmem ) {
      65          36 :   return (fd_bpf_frame_t *)( shmem+1 );
      66          36 : }
      67             : 
      68             : static void *
      69          24 : pool_mem( fd_bpf_frames_shmem_t * shmem ) {
      70          24 :   return (void *)fd_ulong_align_up( (ulong)( local_frames( shmem )+shmem->joiner_cnt_max*shmem->local_cnt ), frame_pool_align() );
      71          24 : }
      72             : 
      73             : void *
      74             : fd_bpf_frames_shmem_new( void * shmem,
      75             :                          ulong  frame_cnt,
      76             :                          ulong  joiner_cnt_max,
      77          12 :                          ulong  local_cnt ) {
      78          12 :   if( FD_UNLIKELY( !shmem ) ) {
      79           0 :     FD_LOG_WARNING(( "NULL shmem" ));
      80           0 :     return NULL;
      81           0 :   }
      82          12 :   if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_bpf_frames_shmem_align() ) ) ) {
      83           0 :     FD_LOG_WARNING(( "misaligned shmem" ));
      84           0 :     return NULL;
      85           0 :   }
      86          12 :   if( FD_UNLIKELY( !fd_bpf_frames_shmem_footprint( frame_cnt, joiner_cnt_max, local_cnt ) ) ) {
      87           0 :     FD_LOG_WARNING(( "invalid frame_cnt %lu, joiner_cnt_max %lu or local_cnt %lu", frame_cnt, joiner_cnt_max, local_cnt ));
      88           0 :     return NULL;
      89           0 :   }
      90             : 
      91          12 :   fd_bpf_frames_shmem_t * frames = (fd_bpf_frames_shmem_t *)shmem;
      92          12 :   fd_rwlock_new( &frames->lock );
      93          12 :   frames->has_owner      = 0UL;
      94          12 :   frames->joiner_cnt     = 0UL;
      95          12 :   frames->joiner_cnt_max = joiner_cnt_max;
      96          12 :   frames->local_cnt      = local_cnt;
      97          12 :   FD_TEST( frame_pool_new( pool_mem( frames ), frame_cnt ) );
      98             : 
      99          12 :   FD_COMPILER_MFENCE();
     100          12 :   FD_VOLATILE( frames->magic ) = FD_BPF_FRAMES_MAGIC;
     101          12 :   FD_COMPILER_MFENCE();
     102          12 :   return shmem;
     103          12 : }
     104             : 
     105             : FD_FN_CONST ulong
     106          24 : fd_bpf_frames_align( void ) {
     107          24 :   return alignof(fd_bpf_frames_t);
     108          24 : }
     109             : 
     110             : FD_FN_CONST ulong
     111          12 : fd_bpf_frames_footprint( void ) {
     112          12 :   return sizeof(fd_bpf_frames_t);
     113          12 : }
     114             : 
     115             : fd_bpf_frames_t *
     116             : fd_bpf_frames_join( void * ljoin,
     117          12 :                     void * shframes ) {
     118          12 :   fd_bpf_frames_t *       join  = (fd_bpf_frames_t *)ljoin;
     119          12 :   fd_bpf_frames_shmem_t * shmem = (fd_bpf_frames_shmem_t *)shframes;
     120          12 :   if( FD_UNLIKELY( !join || !fd_ulong_is_aligned( (ulong)join, fd_bpf_frames_align() ) ) ) {
     121           0 :     FD_LOG_WARNING(( "NULL or misaligned ljoin" ));
     122           0 :     return NULL;
     123           0 :   }
     124          12 :   if( FD_UNLIKELY( !shmem || !fd_ulong_is_aligned( (ulong)shmem, fd_bpf_frames_shmem_align() ) ) ) {
     125           0 :     FD_LOG_WARNING(( "NULL or misaligned shframes" ));
     126           0 :     return NULL;
     127           0 :   }
     128          12 :   if( FD_UNLIKELY( shmem->magic!=FD_BPF_FRAMES_MAGIC ) ) {
     129           0 :     FD_LOG_WARNING(( "bad magic" ));
     130           0 :     return NULL;
     131           0 :   }
     132          12 :   ulong idx = FD_ATOMIC_FETCH_AND_ADD( &shmem->joiner_cnt, 1UL );
     133          12 :   if( FD_UNLIKELY( idx>=shmem->joiner_cnt_max ) ) {
     134           0 :     FD_LOG_WARNING(( "too many joins (max %lu)", shmem->joiner_cnt_max ));
     135           0 :     return NULL;
     136           0 :   }
     137             : 
     138          12 :   join->shmem       = shmem;
     139          12 :   join->pool        = frame_pool_join( pool_mem( shmem ) );
     140          12 :   join->local       = local_frames( shmem )+idx*shmem->local_cnt;
     141          12 :   join->local_cnt   = shmem->local_cnt;
     142          12 :   join->reserve_cnt = FD_MAX_INSTRUCTION_STACK_DEPTH-shmem->local_cnt;
     143          12 :   join->local_used  = 0UL;
     144          12 :   join->is_owner    = 0;
     145          12 :   join->held_cnt    = 0UL;
     146          12 :   FD_TEST( join->pool );
     147          12 :   return join;
     148          12 : }
     149             : 
     150             : static uchar *
     151             : fd_bpf_frames_try_acquire( fd_bpf_frames_t * join,
     152       13032 :                            int               priority ) {
     153       13032 :   if( FD_LIKELY( join->local_used<join->local_cnt ) ) {
     154       12941 :     return (uchar *)( join->local+join->local_used++ );
     155       12941 :   }
     156             : 
     157             :   /* Take the lock, and give up right away if no frames are free. */
     158          91 :   fd_bpf_frames_shmem_t * shmem = join->shmem;
     159          91 :   fd_rwlock_write( &shmem->lock );
     160          91 :   ulong free_cnt = frame_pool_free( join->pool );
     161          91 :   if( FD_UNLIKELY( !free_cnt ) ) {
     162           0 :     fd_rwlock_unwrite( &shmem->lock );
     163           0 :     return NULL;
     164           0 :   }
     165             : 
     166             :   /* Become owner if you can. */
     167          91 :   if( !shmem->has_owner ) {
     168          48 :     shmem->has_owner = 1UL;
     169          48 :     join->is_owner   = 1;
     170          48 :   }
     171             : 
     172             :   /* keep is how many free frames this call must leave for others.  The
     173             :      owner never needs more than reserve_cnt frames, so it cannot run
     174             :      into the priority caller's reserve. */
     175          91 :   ulong keep;
     176          92 :   if( priority || join->is_owner ) {
     177          92 :     keep = 0UL;
     178 >1844*10^16 :   } else {
     179 >1844*10^16 :     keep = 2UL*join->reserve_cnt; /* leave both reserves */
     180 >1844*10^16 :   }
     181             : 
     182          91 :   uchar * frame = NULL;
     183          92 :   if( FD_LIKELY( free_cnt>keep ) ) {
     184          92 :     frame = (uchar *)frame_pool_ele_acquire( join->pool );
     185          92 :     join->held_cnt++;
     186          92 :   }
     187             : 
     188          91 :   fd_rwlock_unwrite( &shmem->lock );
     189          91 :   return frame;
     190          91 : }
     191             : 
     192             : uchar *
     193             : fd_bpf_frames_acquire( fd_bpf_frames_t * join,
     194       13039 :                        int               priority ) {
     195       13039 :   for(;;) {
     196       13033 :     uchar * frame = fd_bpf_frames_try_acquire( join, priority );
     197       13033 :     if( FD_LIKELY( frame ) ) return frame;
     198          82 :     FD_SPIN_PAUSE();
     199          82 :   }
     200       13039 : }
     201             : 
     202             : void
     203             : fd_bpf_frames_release( fd_bpf_frames_t * join,
     204       13041 :                        uchar *           frame ) {
     205       13041 :   if( FD_LIKELY( !join->held_cnt ) ) {
     206       12950 :     FD_TEST( join->local_used && frame==(uchar *)( join->local+join->local_used-1UL ) );
     207       12950 :     join->local_used--;
     208       12950 :     return;
     209       12950 :   }
     210             : 
     211          91 :   fd_bpf_frames_shmem_t * shmem = join->shmem;
     212          91 :   fd_rwlock_write( &shmem->lock );
     213          91 :   frame_pool_ele_release( join->pool, (fd_bpf_frame_t *)frame );
     214          91 :   join->held_cnt--;
     215          91 :   if( !join->held_cnt && join->is_owner ) {
     216          48 :     shmem->has_owner = 0UL;
     217          48 :     join->is_owner   = 0;
     218          48 :   }
     219          91 :   fd_rwlock_unwrite( &shmem->lock );
     220          91 : }