Line data    Source code

       1             : /* The waker tile sleeps in epoll_wait(2) on the outer epoll fd on
       2             :    behalf of all waker client tiles, and raises a client's readiness
       3             :    fseq when that client's inner epoll set has a ready fd.  See
       4             :    fd_waker.h for the full wake protocol. */
       5             : 
       6             : #include "fd_waker.h"
       7             : 
       8             : #include "../metrics/fd_metrics.h"
       9             : #include "../stem/fd_stem.h"
      10             : #include "../topo/fd_topo.h"
      11             : #include "../../tango/fseq/fd_fseq.h"
      12             : 
      13             : #include <errno.h>
      14             : #include <sys/epoll.h>
      15             : 
      16             : #include "generated/fd_waker_tile_seccomp.h"
      17             : 
      18           0 : #define EPOLL_TIMEOUT_MILLIS (10)
      19             : 
      20             : struct fd_waker_tile {
      21             :   ulong   client_cnt;
      22             :   ulong * fseq[ FD_WAKER_CLIENT_MAX ];
      23             :   ulong   client_tile_id[ FD_WAKER_CLIENT_MAX ];
      24             : 
      25             :   fd_sleep_t * sleep;
      26             : };
      27             : 
      28             : typedef struct fd_waker_tile fd_waker_tile_t;
      29             : 
      30             : FD_FN_CONST static inline ulong
      31           0 : scratch_align( void ) {
      32           0 :   return alignof(fd_waker_tile_t);
      33           0 : }
      34             : 
      35             : FD_FN_PURE static inline ulong
      36           0 : scratch_footprint( fd_topo_tile_t const * tile ) {
      37           0 :   (void)tile;
      38           0 :   return sizeof(fd_waker_tile_t);
      39           0 : }
      40             : 
      41             : static void
      42             : before_credit( fd_waker_tile_t *   ctx,
      43             :                fd_stem_context_t * stem,
      44           0 :                int *               charge_busy ) {
      45           0 :   (void)stem; (void)charge_busy; /* never busy, the interval is mostly epoll_pwait */
      46             : 
      47           0 :   struct epoll_event evs[ FD_WAKER_CLIENT_MAX ];
      48           0 :   int n = epoll_pwait( FD_WAKER_OUTER_FD, evs, FD_WAKER_CLIENT_MAX, EPOLL_TIMEOUT_MILLIS, NULL );
      49           0 :   if( FD_UNLIKELY( -1==n ) ) {
      50           0 :     if( FD_LIKELY( errno==EINTR ) ) return;
      51           0 :     FD_LOG_ERR(( "epoll_pwait() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
      52           0 :   }
      53           0 :   FD_MCNT_INC( WAKER, EPOLL_WAIT_RETURNED, 1UL );
      54           0 :   if( FD_LIKELY( !n ) ) return;
      55             : 
      56           0 :   for( int i=0; i<n; i++ ) {
      57           0 :     ulong idx = evs[ i ].data.u64;
      58           0 :     FD_TEST( idx<ctx->client_cnt );
      59           0 :     fd_fseq_update( ctx->fseq[ idx ], 1UL );
      60           0 :     if( FD_UNLIKELY( ctx->sleep ) ) fd_sleep_ring( ctx->sleep, ctx->client_tile_id[ idx ] );
      61           0 :   }
      62           0 :   FD_MCNT_INC( WAKER, WAKE_DELIVERED, (ulong)n );
      63           0 : }
      64             : 
      65             : static void
      66             : unprivileged_init( fd_topo_t const *      topo,
      67           0 :                    fd_topo_tile_t const * tile ) {
      68           0 :   void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
      69             : 
      70           0 :   FD_SCRATCH_ALLOC_INIT( l, scratch );
      71           0 :   fd_waker_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_waker_tile_t), sizeof(fd_waker_tile_t) );
      72             : 
      73           0 :   ctx->client_cnt = 0UL;
      74           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
      75           0 :     fd_topo_tile_t const * client = &topo->tiles[ i ];
      76           0 :     ulong idx = client->waker_client_idx;
      77           0 :     if( FD_LIKELY( idx==ULONG_MAX ) ) continue;
      78           0 :     FD_TEST( idx<FD_WAKER_CLIENT_MAX );
      79           0 :     ctx->fseq[ idx ] = fd_fseq_join( fd_topo_obj_laddr( topo, client->waker_fseq_obj_id ) );
      80           0 :     FD_TEST( ctx->fseq[ idx ] );
      81           0 :     ctx->client_tile_id[ idx ] = client->id;
      82           0 :     ctx->client_cnt = fd_ulong_max( ctx->client_cnt, idx+1UL );
      83           0 :   }
      84             : 
      85           0 :   ctx->sleep = NULL;
      86           0 :   if( FD_UNLIKELY( topo->sleep_obj_id!=ULONG_MAX ) ) {
      87           0 :     ctx->sleep = fd_sleep_join( fd_topo_obj_laddr( topo, topo->sleep_obj_id ) );
      88           0 :     FD_TEST( ctx->sleep );
      89           0 :   }
      90             : 
      91           0 :   ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
      92           0 :   if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
      93           0 :     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
      94           0 : }
      95             : 
      96             : static ulong
      97             : populate_allowed_seccomp( fd_topo_t const *      topo,
      98             :                           fd_topo_tile_t const * tile,
      99             :                           ulong                  out_cnt,
     100           0 :                           struct sock_filter *   out ) {
     101           0 :   (void)topo;
     102           0 :   (void)tile;
     103             : 
     104           0 :   populate_sock_filter_policy_fd_waker_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)FD_WAKER_OUTER_FD );
     105           0 :   return sock_filter_policy_fd_waker_tile_instr_cnt;
     106           0 : }
     107             : 
     108             : static ulong
     109             : populate_allowed_fds( fd_topo_t const *      topo,
     110             :                       fd_topo_tile_t const * tile,
     111             :                       ulong                  out_fds_cnt,
     112           0 :                       int *                  out_fds ) {
     113           0 :   (void)tile;
     114             : 
     115           0 :   ulong client_cnt = 0UL;
     116           0 :   for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
     117           0 :     ulong idx = topo->tiles[ i ].waker_client_idx;
     118           0 :     if( FD_UNLIKELY( idx!=ULONG_MAX ) ) client_cnt = fd_ulong_max( client_cnt, idx+1UL );
     119           0 :   }
     120             : 
     121           0 :   if( FD_UNLIKELY( out_fds_cnt<3UL+client_cnt ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
     122             : 
     123           0 :   ulong out_cnt = 0UL;
     124           0 :   out_fds[ out_cnt++ ] = 2; /* stderr */
     125           0 :   if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
     126           0 :     out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
     127           0 :   out_fds[ out_cnt++ ] = FD_WAKER_OUTER_FD; /* waker outer epoll fd (rearm) */
     128           0 :   for( ulong i=0UL; i<client_cnt; i++ ) out_fds[ out_cnt++ ] = FD_WAKER_INNER_FD( i ); /* waker inner epoll fd */
     129           0 :   return out_cnt;
     130           0 : }
     131             : 
     132             : /* The waker tile receives no frags and does not need to park in stem
     133             :    as it parks itself in epoll_wait(2) on behalf of its clients. */
     134             : #define STEM_NEVER_PARK 1
     135           0 : #define STEM_BURST (1UL)
     136           0 : #define STEM_LAZY  ((long)10e6) /* 10ms */
     137             : 
     138           0 : #define STEM_CALLBACK_CONTEXT_TYPE  fd_waker_tile_t
     139           0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_waker_tile_t)
     140             : 
     141           0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
     142             : 
     143             : #include "../stem/fd_stem.c"
     144             : 
     145             : fd_topo_run_tile_t fd_tile_waker = {
     146             :   .name                     = "waker",
     147             :   .populate_allowed_seccomp = populate_allowed_seccomp,
     148             :   .populate_allowed_fds     = populate_allowed_fds,
     149             :   .scratch_align            = scratch_align,
     150             :   .scratch_footprint        = scratch_footprint,
     151             :   .unprivileged_init        = unprivileged_init,
     152             :   .run                      = stem_run,
     153             : };