Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_sleep.h"
3 : #include "../topo/fd_topo.h"
4 :
5 : #include <errno.h>
6 : #include <float.h>
7 : #include <time.h>
8 : #include <linux/futex.h>
9 : #include <sys/epoll.h>
10 : #include <sys/eventfd.h>
11 : #include <sys/syscall.h>
12 : #include <unistd.h>
13 :
14 0 : FD_FN_CONST ulong fd_sleep_align ( void ) { return FD_SLEEP_ALIGN; }
15 0 : FD_FN_CONST ulong fd_sleep_footprint( void ) { return sizeof(fd_sleep_t); }
16 :
17 : void *
18 : fd_sleep_new( void * shmem,
19 0 : double tick_per_ns ) {
20 0 : if( FD_UNLIKELY( !shmem ) ) {
21 0 : FD_LOG_WARNING(( "NULL shmem" ));
22 0 : return NULL;
23 0 : }
24 :
25 0 : if( FD_UNLIKELY( !(tick_per_ns>0.0 && tick_per_ns<=DBL_MAX) ) ) {
26 0 : FD_LOG_WARNING(( "bad tick_per_ns" ));
27 0 : return NULL;
28 0 : }
29 :
30 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_sleep_align() ) ) ) {
31 0 : FD_LOG_WARNING(( "misaligned shmem" ));
32 0 : return NULL;
33 0 : }
34 :
35 0 : fd_sleep_t * sleep = (fd_sleep_t *)shmem;
36 0 : fd_memset( sleep, 0, fd_sleep_footprint() );
37 0 : for( ulong i=0UL; i<FD_SLEEP_TILE_MAX; i++ ) sleep->tile[ i ].word = 1UL; /* running */
38 0 : sleep->tick_per_ns = tick_per_ns;
39 :
40 0 : FD_COMPILER_MFENCE();
41 0 : FD_VOLATILE( sleep->magic ) = FD_SLEEP_MAGIC;
42 0 : FD_COMPILER_MFENCE();
43 :
44 0 : return shmem;
45 0 : }
46 :
47 : ulong
48 : fd_sleep_wake_table( fd_sleep_wake_t * wake,
49 : struct fd_topo const * topo,
50 0 : ulong link_id ) {
51 0 : if( FD_UNLIKELY( topo->sleep_obj_id==ULONG_MAX ) ) return 0UL;
52 0 : ulong mask[ FD_SLEEP_BITS_CNT ] = {0};
53 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
54 0 : fd_topo_tile_t const * consumer = &topo->tiles[ i ];
55 0 : for( ulong j=0UL; j<consumer->in_cnt; j++ ) {
56 0 : if( FD_UNLIKELY( consumer->in_link_id[ j ]==link_id && consumer->in_link_poll[ j ] ) ) {
57 0 : mask[ consumer->id>>6 ] |= 1UL<<(consumer->id&63UL);
58 0 : }
59 0 : }
60 0 : }
61 :
62 0 : ulong cnt = 0UL;
63 0 : for( ulong w=0UL; w<FD_SLEEP_BITS_CNT; w++ ) {
64 0 : if( FD_UNLIKELY( mask[ w ] ) ) wake[ cnt++ ] = (fd_sleep_wake_t){ .w=w, .mask=mask[ w ] };
65 0 : }
66 0 : return cnt;
67 0 : }
68 :
69 : fd_sleep_t *
70 0 : fd_sleep_join( void * shsleep ) {
71 0 : if( FD_UNLIKELY( !shsleep ) ) {
72 0 : FD_LOG_WARNING(( "NULL shsleep" ));
73 0 : return NULL;
74 0 : }
75 :
76 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shsleep, fd_sleep_align() ) ) ) {
77 0 : FD_LOG_WARNING(( "misaligned shsleep" ));
78 0 : return NULL;
79 0 : }
80 :
81 0 : fd_sleep_t * sleep = (fd_sleep_t *)shsleep;
82 :
83 0 : if( FD_UNLIKELY( sleep->magic!=FD_SLEEP_MAGIC ) ) {
84 0 : FD_LOG_WARNING(( "bad magic" ));
85 0 : return NULL;
86 0 : }
87 :
88 0 : return sleep;
89 0 : }
90 :
91 : static inline int
92 0 : woken_cause( ulong const * word ) {
93 0 : return FD_VOLATILE_CONST( word[0] )==FD_SLEEP_WORD( FD_SLEEP_UNPARK_DEADLINE ) ? FD_SLEEP_UNPARK_DEADLINE : FD_SLEEP_UNPARK_RING;
94 0 : }
95 :
96 : int
97 : fd_sleep_park_wait( ulong * word,
98 : long deadline_ticks,
99 0 : double tick_per_ns ) {
100 0 : struct timespec ts;
101 0 : struct timespec * timeout = NULL; /* untimed: only a wake ends the park */
102 0 : if( FD_LIKELY( deadline_ticks!=LONG_MAX ) ) {
103 : /* Absolute timeout, anchored on a clock read taken before the tick
104 : read: preemption anywhere after can only shorten the sleep, never
105 : stretch it past deadline_ticks */
106 0 : clock_gettime( CLOCK_MONOTONIC, &ts );
107 0 : long remaining = (long)((double)(deadline_ticks-fd_tickcount())/tick_per_ns);
108 0 : if( FD_UNLIKELY( remaining<=0L ) ) return FD_SLEEP_UNPARK_DEADLINE;
109 0 : long abs_ns = ts.tv_sec*(long)1e9 + ts.tv_nsec + remaining;
110 0 : ts.tv_sec = abs_ns/(long)1e9;
111 0 : ts.tv_nsec = abs_ns%(long)1e9;
112 0 : timeout = &ts;
113 0 : }
114 :
115 0 : for(;;) {
116 0 : long res = syscall( SYS_futex, (uint *)word, FUTEX_WAIT_BITSET, 0U, timeout, NULL, FUTEX_BITSET_MATCH_ANY );
117 0 : if( FD_LIKELY( !res ) ) return woken_cause( word );
118 :
119 0 : switch( errno ) {
120 0 : case ETIMEDOUT: return FD_SLEEP_UNPARK_DEADLINE;
121 0 : case EAGAIN: return woken_cause( word ); /* word already set */
122 0 : case EINTR: continue; /* signal: same absolute timeout */
123 0 : default: FD_LOG_ERR(( "futex(FUTEX_WAIT_BITSET) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
124 0 : }
125 0 : }
126 0 : }
127 :
128 : void
129 : fd_sleep_wake_one( ulong * word,
130 0 : int cause ) {
131 0 : FD_VOLATILE( word[0] ) = FD_SLEEP_WORD( cause );
132 0 : long res = syscall( SYS_futex, (uint *)word, FUTEX_WAKE, 1, NULL, NULL, 0 );
133 0 : if( FD_UNLIKELY( -1L==res ) ) FD_LOG_ERR(( "futex(FUTEX_WAKE) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
134 0 : }
135 :
136 : void
137 : fd_sleep_wake_eventfd( ulong * word,
138 : int eventfd,
139 0 : int cause ) {
140 0 : FD_VOLATILE( word[0] ) = FD_SLEEP_WORD( cause );
141 0 : ulong one = 1UL;
142 0 : long res = write( eventfd, &one, sizeof(ulong) );
143 0 : if( FD_UNLIKELY( res!=(long)sizeof(ulong) ) ) FD_LOG_ERR(( "write(eventfd %d) failed (%i-%s)", eventfd, errno, fd_io_strerror( errno ) ));
144 0 : }
145 :
146 : int
147 : fd_sleep_park_wait_epoll( int epfd,
148 : ulong const * word,
149 : struct epoll_event * evs,
150 : int ev_max,
151 : int * ev_cnt,
152 : long deadline_ticks,
153 0 : double tick_per_ns ) {
154 0 : *ev_cnt = 0;
155 0 : for(;;) {
156 : /* ms timeout: round up, the sweep delivers the exact deadline.
157 : LONG_MAX: no timeout, the sweep enforces the cap. */
158 0 : int timeout_ms = -1;
159 0 : if( FD_LIKELY( deadline_ticks!=LONG_MAX ) ) {
160 0 : long remaining = (long)((double)(deadline_ticks-fd_tickcount())/tick_per_ns);
161 0 : if( FD_UNLIKELY( remaining<=0L ) ) return FD_SLEEP_UNPARK_DEADLINE;
162 0 : timeout_ms = (int)( (remaining+999999L)/1000000L );
163 0 : }
164 :
165 0 : int n = epoll_pwait( epfd, evs, ev_max, timeout_ms, NULL );
166 0 : if( FD_UNLIKELY( n<0 ) ) {
167 0 : if( FD_LIKELY( errno==EINTR ) ) continue;
168 0 : FD_LOG_ERR(( "epoll_pwait() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
169 0 : }
170 0 : if( FD_UNLIKELY( !n ) ) continue; /* timed out, recheck the deadline */
171 :
172 0 : int doorbell_only = 1;
173 0 : for( int i=0; i<n; i++ ) doorbell_only &= evs[ i ].data.u64==FD_SLEEP_EPOLL_DOORBELL;
174 0 : if( FD_UNLIKELY( doorbell_only ) ) {
175 0 : if( FD_UNLIKELY( !FD_VOLATILE_CONST( word[0] ) ) ) continue; /* stale */
176 0 : if( FD_UNLIKELY( woken_cause( word )==FD_SLEEP_UNPARK_DEADLINE ) ) return FD_SLEEP_UNPARK_DEADLINE;
177 0 : }
178 :
179 0 : *ev_cnt = n;
180 0 : return FD_SLEEP_UNPARK_RING;
181 0 : }
182 0 : }
183 :
184 : void
185 0 : fd_sleep_eventfd_install( fd_topo_t const * topo ) {
186 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
187 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
188 0 : if( FD_LIKELY( !tile->sleep_eventfd ) ) continue;
189 0 : FD_TEST( tile->id<FD_TOPO_MAX_TILES );
190 0 : int fd = eventfd( 0U, EFD_NONBLOCK );
191 0 : if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "eventfd() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
192 0 : if( FD_LIKELY( fd!=FD_SLEEP_EVENTFD( tile->id ) ) ) {
193 0 : if( FD_UNLIKELY( -1==dup2( fd, FD_SLEEP_EVENTFD( tile->id ) ) ) ) FD_LOG_ERR(( "dup2(%d,%d) failed (%i-%s)", fd, FD_SLEEP_EVENTFD( tile->id ), errno, fd_io_strerror( errno ) ));
194 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
195 0 : }
196 0 : }
197 0 : }
|
