Line data Source code
1 : /* The mwaitx tile converts doorbell rings into futex wakes: it naps in
2 : hardware wake-on-store on the doorbell cache line (umonitor/umwait
3 : on Intel, monitorx/mwaitx on AMD, pause spin fallback), and is the
4 : sole issuer of FUTEX_WAKE in the system (a tile parked in epoll on
5 : its own fds is woken through its eventfd instead). It also services
6 : parked tiles' deadlines and runs the verifying sweep (seq_mirror vs
7 : seq_snap) that bounds any lost doorbell to ~one nap. */
8 :
9 : #include "fd_sleep.h"
10 :
11 : #include "../metrics/fd_metrics.h"
12 : #include "../stem/fd_stem.h"
13 : #include "../topo/fd_topo.h"
14 :
15 : #if FD_HAS_X86
16 : #include <cpuid.h>
17 : #include <x86intrin.h>
18 : #endif
19 :
20 : #include <linux/futex.h>
21 :
22 : #include "generated/fd_mwaitx_tile_seccomp.h"
23 :
24 0 : #define MWAITX_NAP_TICKS (100000L) /* ~30us at 3GHz, Intel caps umwait residency anyway */
25 :
26 : struct fd_mwaitx_tile {
27 : fd_sleep_t * sleep;
28 :
29 : ulong tile_cnt;
30 :
31 : int has_waitpkg;
32 : int has_mwaitx;
33 :
34 : /* polled in link ids per tile, for the verifying sweep */
35 : uint in_cnt [ FD_SLEEP_TILE_MAX ];
36 : uint in_link[ FD_SLEEP_TILE_MAX ][ FD_SLEEP_IN_MAX ];
37 :
38 : /* eventfd doorbell per tile, -1 for a futex waiter */
39 : int eventfd[ FD_SLEEP_TILE_MAX ];
40 :
41 : long next_sweep; /* tick of the next deadline/sweep pass */
42 : int napped; /* wakes go out on the stem iteration after a nap */
43 :
44 : ulong metrics_nap;
45 : ulong metrics_wake;
46 : ulong metrics_deadline;
47 : ulong metrics_sweep;
48 : };
49 :
50 : typedef struct fd_mwaitx_tile fd_mwaitx_tile_t;
51 :
52 : FD_FN_CONST static inline ulong
53 0 : scratch_align( void ) {
54 0 : return alignof(fd_mwaitx_tile_t);
55 0 : }
56 :
57 : FD_FN_PURE static inline ulong
58 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
59 0 : (void)tile;
60 0 : return sizeof(fd_mwaitx_tile_t);
61 0 : }
62 :
63 : static void
64 0 : metrics_write( fd_mwaitx_tile_t * ctx ) {
65 0 : FD_MCNT_SET( MWAITX, NAP_TAKEN, ctx->metrics_nap );
66 0 : FD_MCNT_SET( MWAITX, WAKE_ISSUED_DOORBELL, ctx->metrics_wake-ctx->metrics_deadline-ctx->metrics_sweep );
67 0 : FD_MCNT_SET( MWAITX, WAKE_ISSUED_DEADLINE, ctx->metrics_deadline );
68 0 : FD_MCNT_SET( MWAITX, WAKE_ISSUED_SWEEP, ctx->metrics_sweep );
69 0 : }
70 :
71 : static inline int
72 0 : doorbell_rung( ulong const * line ) {
73 0 : ulong any = 0UL;
74 0 : for( ulong w=0UL; w<FD_SLEEP_BITS_CNT; w++ ) any |= FD_VOLATILE_CONST( line[ w ] );
75 0 : return !!any;
76 0 : }
77 :
78 : #if FD_HAS_X86
79 :
80 : __attribute__((target("waitpkg"))) static inline void
81 : idle_umwait( ulong const * line,
82 0 : long deadline_tsc ) {
83 0 : _umonitor( (void *)line );
84 0 : if( FD_UNLIKELY( doorbell_rung( line ) ) ) return;
85 0 : _umwait( 0U /* C0.2 */, (unsigned long long)deadline_tsc );
86 0 : }
87 :
88 : __attribute__((target("mwaitx"))) static inline void
89 : idle_mwaitx( ulong const * line,
90 0 : long nap_ticks ) {
91 0 : _mm_monitorx( (void *)line, 0U, 0U );
92 0 : if( FD_UNLIKELY( doorbell_rung( line ) ) ) return;
93 0 : _mm_mwaitx( 0x2U /* timer */, 0x0U /* C1 */, (unsigned)nap_ticks );
94 0 : }
95 :
96 : #endif
97 :
98 : static inline void
99 : wake( fd_mwaitx_tile_t * ctx,
100 : ulong tid,
101 0 : int cause ) {
102 0 : int eventfd = ctx->eventfd[ tid ];
103 0 : if( FD_UNLIKELY( eventfd>=0 ) ) fd_sleep_wake_eventfd( &ctx->sleep->tile[ tid ].word, eventfd, cause );
104 0 : else fd_sleep_wake_one ( &ctx->sleep->tile[ tid ].word, cause );
105 0 : ctx->metrics_wake++;
106 0 : }
107 :
108 : static void
109 : before_credit( fd_mwaitx_tile_t * ctx,
110 : fd_stem_context_t * stem,
111 0 : int * charge_busy ) {
112 : /* Nap and wake in separate stem iterations, so the nap is billed
113 : idle and only the wakes are billed busy. */
114 0 : if( FD_LIKELY( !ctx->napped ) ) {
115 0 : long deadline = stem->now+MWAITX_NAP_TICKS;
116 0 : #if FD_HAS_X86
117 0 : if( FD_LIKELY( ctx->has_waitpkg ) ) idle_umwait( ctx->sleep->doorbell, deadline );
118 0 : else if( FD_LIKELY( ctx->has_mwaitx ) ) idle_mwaitx( ctx->sleep->doorbell, MWAITX_NAP_TICKS );
119 0 : else
120 0 : #endif
121 0 : while( FD_LIKELY( fd_tickcount()<deadline ) ) {
122 0 : if( FD_UNLIKELY( doorbell_rung( ctx->sleep->doorbell ) ) ) break;
123 0 : FD_SPIN_PAUSE();
124 0 : }
125 :
126 0 : ctx->metrics_nap++;
127 0 : ctx->napped = 1;
128 0 : return;
129 0 : }
130 0 : ctx->napped = 0;
131 :
132 : /* Nap ended ... send wakes if any first for latency. No need to wake
133 : if already woken, and snapshot the parked bits to avoid races. */
134 0 : for( ulong w=0UL; w<FD_SLEEP_BITS_CNT; w++ ) {
135 0 : ulong rung = FD_VOLATILE_CONST( ctx->sleep->doorbell[ w ] );
136 0 : if( FD_LIKELY( !rung ) ) continue;
137 :
138 0 : rung = __atomic_exchange_n( &ctx->sleep->doorbell[ w ], 0UL, __ATOMIC_ACQUIRE );
139 0 : while( rung ) {
140 0 : ulong tid = (w<<6) + (ulong)fd_ulong_find_lsb( rung );
141 0 : rung &= rung-1UL;
142 0 : if( FD_LIKELY( !FD_VOLATILE_CONST( ctx->sleep->tile[ tid ].word ) ) ) {
143 0 : wake( ctx, tid, FD_SLEEP_UNPARK_RING );
144 0 : *charge_busy = 1;
145 0 : }
146 0 : }
147 0 : }
148 :
149 : /* Every nap period, service tile deadlines, and sweep all parked
150 : tiles to catch any lost doorbells due to small unavoidable race
151 : windows in the read-then-park sequence. A tile with nothing due
152 : parks untimed and relies on this pass for its cap, so every parked
153 : tile is visited every pass. */
154 :
155 0 : if( FD_LIKELY( stem->now<ctx->next_sweep ) ) return;
156 0 : ctx->next_sweep = stem->now+MWAITX_NAP_TICKS;
157 :
158 0 : for( ulong w=0UL; w<FD_SLEEP_BITS_CNT; w++ ) {
159 0 : ulong parked = FD_VOLATILE_CONST( ctx->sleep->parked_bits[ w ] );
160 0 : while( parked ) {
161 0 : ulong tid = (w<<6) + (ulong)fd_ulong_find_lsb( parked );
162 0 : parked &= parked-1UL;
163 0 : if( FD_UNLIKELY( FD_VOLATILE_CONST( ctx->sleep->tile[ tid ].word ) ) ) continue; /* already woken */
164 :
165 0 : int cause = -1;
166 0 : if( FD_UNLIKELY( (long)FD_VOLATILE_CONST( ctx->sleep->tile[ tid ].deadline )<=stem->now ) ) {
167 0 : cause = FD_SLEEP_UNPARK_DEADLINE;
168 0 : ctx->metrics_deadline++;
169 0 : } else if( FD_LIKELY( !(FD_VOLATILE_CONST( ctx->sleep->credit_bits[ w ] ) & (1UL<<(tid&63UL))) ) ) {
170 : /* Producer mirror is ahead of the parked tile's snapshot, a
171 : pending frag is available, the doorbell was lost or raced.
172 : A tile parked on backpressure (credit bit) cannot use a
173 : frag; only its deadline or a credit ring wakes it. */
174 0 : for( ulong i=0UL; i<(ulong)ctx->in_cnt[ tid ]; i++ ) {
175 0 : ulong mirror = __atomic_load_n( &ctx->sleep->seq_mirror[ ctx->in_link[ tid ][ i ] ], __ATOMIC_ACQUIRE );
176 0 : if( FD_UNLIKELY( fd_seq_lt( FD_VOLATILE_CONST( ctx->sleep->seq_snap[ tid ][ i ] ), mirror ) ) ) {
177 0 : cause = FD_SLEEP_UNPARK_RING;
178 0 : ctx->metrics_sweep++;
179 0 : break;
180 0 : }
181 0 : }
182 0 : }
183 :
184 0 : if( FD_UNLIKELY( cause>=0 ) ) {
185 0 : wake( ctx, tid, cause );
186 0 : *charge_busy = 1;
187 0 : }
188 0 : }
189 0 : }
190 0 : }
191 :
192 : static void
193 : unprivileged_init( fd_topo_t const * topo,
194 0 : fd_topo_tile_t const * tile ) {
195 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
196 :
197 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
198 0 : fd_mwaitx_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_mwaitx_tile_t), sizeof(fd_mwaitx_tile_t) );
199 :
200 0 : ctx->sleep = fd_sleep_join( fd_topo_obj_laddr( topo, topo->sleep_obj_id ) );
201 0 : FD_TEST( ctx->sleep );
202 0 : ctx->tile_cnt = topo->tile_cnt;
203 :
204 0 : for( ulong i=0UL; i<FD_SLEEP_TILE_MAX; i++ ) ctx->in_cnt[ i ] = 0U;
205 0 : for( ulong i=0UL; i<FD_SLEEP_TILE_MAX; i++ ) ctx->eventfd[ i ] = -1;
206 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
207 0 : fd_topo_tile_t const * t = &topo->tiles[ i ];
208 0 : if( FD_UNLIKELY( t->sleep_eventfd ) ) ctx->eventfd[ t->id ] = FD_SLEEP_EVENTFD( t->id );
209 0 : ulong polled = 0UL;
210 0 : for( ulong j=0UL; j<t->in_cnt; j++ ) {
211 0 : if( FD_UNLIKELY( !t->in_link_poll[ j ] ) ) continue;
212 0 : ctx->in_link[ t->id ][ polled++ ] = (uint)t->in_link_id[ j ];
213 0 : }
214 0 : ctx->in_cnt[ t->id ] = (uint)polled;
215 0 : }
216 :
217 0 : ctx->has_waitpkg = 0;
218 0 : ctx->has_mwaitx = 0;
219 0 : #if FD_HAS_X86
220 0 : uint eax, ebx, ecx, edx;
221 0 : if( FD_LIKELY( __get_cpuid_count( 7U, 0U, &eax, &ebx, &ecx, &edx ) ) ) ctx->has_waitpkg = !!(ecx & (1U<<5U));
222 0 : if( FD_LIKELY( __get_cpuid( 0x80000001U, &eax, &ebx, &ecx, &edx ) ) ) ctx->has_mwaitx = !!(ecx & (1U<<29U));
223 0 : #endif
224 :
225 0 : ctx->next_sweep = 0L;
226 0 : ctx->napped = 0;
227 0 : ctx->metrics_nap = 0UL;
228 0 : ctx->metrics_wake = 0UL;
229 0 : ctx->metrics_deadline = 0UL;
230 0 : ctx->metrics_sweep = 0UL;
231 :
232 0 : if( FD_UNLIKELY( !ctx->has_waitpkg && !ctx->has_mwaitx ) ) FD_LOG_WARNING(( "cpu has neither waitpkg nor mwaitx, mwaitx tile will spin" ));
233 :
234 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
235 0 : if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
236 0 : FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
237 0 : }
238 :
239 : static ulong
240 : populate_allowed_seccomp( fd_topo_t const * topo,
241 : fd_topo_tile_t const * tile,
242 : ulong out_cnt,
243 0 : struct sock_filter * out ) {
244 0 : (void)tile;
245 :
246 0 : ulong id_lo = ULONG_MAX;
247 0 : ulong id_hi = 0UL;
248 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
249 0 : if( FD_LIKELY( !topo->tiles[ i ].sleep_eventfd ) ) continue;
250 0 : id_lo = fd_ulong_min( id_lo, topo->tiles[ i ].id );
251 0 : id_hi = fd_ulong_max( id_hi, topo->tiles[ i ].id+1UL );
252 0 : }
253 0 : if( FD_LIKELY( id_lo==ULONG_MAX ) ) id_lo = id_hi = 0UL;
254 :
255 0 : populate_sock_filter_policy_fd_mwaitx_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(),
256 0 : (uint)FD_SLEEP_EVENTFD( id_lo ), (uint)FD_SLEEP_EVENTFD( id_hi ) );
257 0 : return sock_filter_policy_fd_mwaitx_tile_instr_cnt;
258 0 : }
259 :
260 : static ulong
261 : populate_allowed_fds( fd_topo_t const * topo,
262 : fd_topo_tile_t const * tile,
263 : ulong out_fds_cnt,
264 0 : int * out_fds ) {
265 0 : (void)tile;
266 :
267 0 : if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
268 :
269 0 : ulong out_cnt = 0UL;
270 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
271 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
272 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
273 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
274 0 : if( FD_LIKELY( !topo->tiles[ i ].sleep_eventfd ) ) continue;
275 0 : if( FD_UNLIKELY( out_cnt>=out_fds_cnt ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
276 0 : out_fds[ out_cnt++ ] = FD_SLEEP_EVENTFD( topo->tiles[ i ].id ); /* eventfd doorbell */
277 0 : }
278 0 : return out_cnt;
279 0 : }
280 :
281 : #define STEM_NEVER_PARK 1
282 0 : #define STEM_BURST (1UL)
283 0 : #define STEM_LAZY ((long)10e6) /* 10ms */
284 :
285 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_mwaitx_tile_t
286 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_mwaitx_tile_t)
287 :
288 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
289 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
290 :
291 : #include "../stem/fd_stem.c"
292 :
293 : fd_topo_run_tile_t fd_tile_mwaitx = {
294 : .name = "mwaitx",
295 : .populate_allowed_seccomp = populate_allowed_seccomp,
296 : .populate_allowed_fds = populate_allowed_fds,
297 : .scratch_align = scratch_align,
298 : .scratch_footprint = scratch_footprint,
299 : .unprivileged_init = unprivileged_init,
300 : .run = stem_run,
301 : };
|
