Line data Source code
1 : #ifndef HEADER_fd_src_disco_sleep_fd_sleep_h
2 : #define HEADER_fd_src_disco_sleep_fd_sleep_h
3 :
4 : /* Idle tiles park in futex_wait instead of spinning and are woken
5 : when work arrives. Three roles share one shmem region:
6 :
7 : sleeper a stem tile with no work: flushes its link state, sets
8 : its bit in parked_bits, re-checks its ins once under the
9 : RMW fence, then FUTEX_WAIT_BITSETs on its word, timed
10 : only if a deadline is due before the park cap (the sweep
11 : enforces the cap).
12 :
13 : producer on publish: load parked_bits[w] & a mask precomputed at
14 : boot; if nonzero, locked OR into doorbell[w].
15 :
16 : mwaitx naps in hardware on the doorbell line (umwait/mwaitx),
17 : turns rung bits into FUTEX_WAKEs, and runs the verifying
18 : sweep (seq_mirror vs seq_snap). Lost-doorbell latency
19 : includes producer mirror updates (STEM_LAZY for stem
20 : producers), bounded by the park cap. The only
21 : FUTEX_WAKE issuer.
22 :
23 : word 0 = parked, nonzero = running (the value says why). The
24 : doorbell is a hint: truth is level triggered (seqs, deadline) and
25 : re-checked on every wake, so spurious wakes are absorbed and a lost
26 : hint costs only bounded latency (the sweep, then the tile's own
27 : deadline).
28 :
29 : A tile with its own kernel fds (tile->sleep_eventfd) parks in epoll
30 : instead, with an eventfd doorbell at FD_SLEEP_EVENTFD( id ) that
31 : mwaitx writes in place of FUTEX_WAKE. */
32 :
33 : #include "../../util/fd_util_base.h"
34 :
35 0 : #define FD_SLEEP_ALIGN (128UL)
36 0 : #define FD_SLEEP_MAGIC (0xf17eda2c3751ee90UL) /* firedancer sleep ver 0 */
37 :
38 0 : #define FD_SLEEP_TILE_MAX ( 512UL)
39 0 : #define FD_SLEEP_BITS_CNT (FD_SLEEP_TILE_MAX/64UL)
40 : #define FD_SLEEP_LINK_MAX (1024UL) /* ==FD_TOPO_MAX_LINKS */
41 : #define FD_SLEEP_IN_MAX ( 256UL) /* ==FD_TOPO_MAX_TILE_IN_LINKS */
42 : #define FD_SLEEP_OUT_MAX ( 64UL) /* ==FD_TOPO_MAX_TILE_OUT_LINKS */
43 :
44 0 : #define FD_SLEEP_LINGER_NS (0L) /* park as soon as caught up */
45 0 : #define FD_SLEEP_PARK_CAP_NS (20000000L) /* longest park */
46 0 : #define FD_SLEEP_PARK_MIN_NS (5000L) /* shorter than this and the futex round trip costs more than the spin */
47 :
48 : /* Inherited fd number of a tile's eventfd doorbell */
49 0 : #define FD_SLEEP_EVENTFD_BASE (123600)
50 0 : #define FD_SLEEP_EVENTFD( id ) (FD_SLEEP_EVENTFD_BASE+(int)(id))
51 :
52 : /* epoll data of the eventfd doorbell in a tile's park set */
53 0 : #define FD_SLEEP_EPOLL_DOORBELL (ULONG_MAX)
54 :
55 : struct __attribute__((aligned(FD_SLEEP_ALIGN))) fd_sleep_private {
56 : /* Loaded on every publish, written only at park/unpark */
57 : ulong parked_bits[ FD_SLEEP_BITS_CNT ];
58 : ulong magic; /* ==FD_SLEEP_MAGIC, off the parked_bits line */
59 : double tick_per_ns; /* fd_tickcount ticks per ns, for futex deadlines */
60 : ulong pad0[ 6 ];
61 :
62 : /* Locked OR on wake edges; the line mwaitx monitors. Own line. */
63 : ulong doorbell[ FD_SLEEP_BITS_CNT ];
64 : ulong pad1[ 8 ];
65 :
66 : /* Subset of parked_bits: tiles parked on backpressure, which only a
67 : consumer's credit return can wake. Loaded on every credit return. */
68 : ulong credit_bits[ FD_SLEEP_BITS_CNT ];
69 : ulong pad2[ 8 ];
70 :
71 : /* Indexed by tile->id; written by the owner and mwaitx only */
72 : struct __attribute__((aligned(64UL))) {
73 : ulong word; /* futex word: 0 parked, 1 running */
74 : ulong pad0;
75 : ulong deadline; /* abs fd_tickcount of the next timed obligation */
76 : ulong pad[ 5 ];
77 : } tile[ FD_SLEEP_TILE_MAX ];
78 :
79 : /* Sweep state: producers mirror out seqs at housekeeping, a parking
80 : tile snapshots the next seq it expects per polled in.
81 : mirror[link]>snap means a frag is pending. */
82 : ulong seq_mirror[ FD_SLEEP_LINK_MAX ];
83 : ulong seq_snap[ FD_SLEEP_TILE_MAX ][ FD_SLEEP_IN_MAX ];
84 : };
85 :
86 : typedef struct fd_sleep_private fd_sleep_t;
87 :
88 : /* Wake table entry: the consumers of one out link as a (bitmap word,
89 : mask) pair. One pair per link in practice. */
90 :
91 : struct fd_sleep_wake {
92 : ulong w;
93 : ulong mask;
94 : };
95 :
96 : typedef struct fd_sleep_wake fd_sleep_wake_t;
97 :
98 : struct fd_topo;
99 :
100 : FD_PROTOTYPES_BEGIN
101 :
102 : FD_FN_CONST ulong fd_sleep_align ( void );
103 : FD_FN_CONST ulong fd_sleep_footprint( void );
104 :
105 : /* fd_sleep_new formats shmem (fd_sleep_align aligned, fd_sleep_footprint
106 : bytes) as a sleep region with every tile running. tick_per_ns is
107 : the fd_tickcount rate used to turn park deadlines into futex
108 : timeouts. Returns shmem on success, NULL on failure (logs
109 : details). */
110 :
111 : void * fd_sleep_new( void * shmem,
112 : double tick_per_ns );
113 :
114 : fd_sleep_t * fd_sleep_join( void * shsleep );
115 :
116 : /* fd_sleep_wake_table fills wake (FD_SLEEP_BITS_CNT entries) with the
117 : (word,mask) pairs covering every tile that polls link_id, and
118 : returns the pair count (0 if none, or in performance mode). A
119 : consumer that never parks never has its bit set, so rings to it are
120 : free. */
121 :
122 : ulong
123 : fd_sleep_wake_table( fd_sleep_wake_t * wake,
124 : struct fd_topo const * topo,
125 : ulong link_id );
126 :
127 : /* fd_sleep_ring marks tile_id as having work. Safe from any thread;
128 : ringing a running tile is harmless. */
129 :
130 : static inline void
131 : fd_sleep_ring( fd_sleep_t * sleep,
132 0 : ulong tile_id ) {
133 0 : __atomic_fetch_or( &sleep->doorbell[ tile_id>>6 ], 1UL<<(tile_id&63UL), __ATOMIC_RELEASE );
134 0 : }
135 :
136 : /* fd_sleep_wake_check rings the parked consumers of one out link,
137 : except those parked on backpressure (credit_bits): a frag cannot
138 : help them, only a credit return can. One load per pair; the
139 : second load and the locked OR only on a hit. */
140 :
141 : static inline void
142 : fd_sleep_wake_check( fd_sleep_t * sleep,
143 : fd_sleep_wake_t const * wake,
144 0 : ulong wake_cnt ) {
145 0 : for( ulong k=0UL; k<wake_cnt; k++ ) {
146 0 : ulong rung = FD_VOLATILE_CONST( sleep->parked_bits[ wake[ k ].w ] ) & wake[ k ].mask;
147 0 : if( FD_UNLIKELY( rung ) ) {
148 0 : rung &= ~FD_VOLATILE_CONST( sleep->credit_bits[ wake[ k ].w ] );
149 0 : if( FD_LIKELY( rung ) ) __atomic_fetch_or( &sleep->doorbell[ wake[ k ].w ], rung, __ATOMIC_RELEASE );
150 0 : }
151 0 : }
152 0 : }
153 :
154 : /* Unpark causes, in ParkWake metrics enum order */
155 :
156 0 : #define FD_SLEEP_UNPARK_RING (0)
157 0 : #define FD_SLEEP_UNPARK_DEADLINE (1)
158 : #define FD_SLEEP_UNPARK_PENDING (2)
159 :
160 : /* FD_SLEEP_WORD( cause ) is what a wake stores in the word: nonzero
161 : and distinct per cause */
162 :
163 0 : #define FD_SLEEP_WORD( cause ) (1UL+(ulong)(cause))
164 :
165 : /* fd_sleep_park_wait blocks on word until woken or deadline_ticks
166 : passes (LONG_MAX: no timer, the mwaitx sweep must end the park).
167 : Returns FD_SLEEP_UNPARK_RING or FD_SLEEP_UNPARK_DEADLINE. */
168 :
169 : int
170 : fd_sleep_park_wait( ulong * word,
171 : long deadline_ticks,
172 : double tick_per_ns );
173 :
174 : /* fd_sleep_wake_one wakes the tile parked on word for cause. */
175 :
176 : void
177 : fd_sleep_wake_one( ulong * word,
178 : int cause );
179 :
180 : /* fd_sleep_wake_eventfd is fd_sleep_wake_one for a tile parked in
181 : fd_sleep_park_wait_epoll. */
182 :
183 : void
184 : fd_sleep_wake_eventfd( ulong * word,
185 : int eventfd,
186 : int cause );
187 :
188 : /* fd_sleep_park_wait_epoll is fd_sleep_park_wait for a tile with its
189 : own fds. Blocks in epoll_pwait on epfd until an fd is ready, a wake,
190 : or the deadline. Ready events are returned in evs / *ev_cnt. epfd
191 : must hold the tile's eventfd doorbell registered EPOLLIN|EPOLLET
192 : with data FD_SLEEP_EPOLL_DOORBELL, and the caller must never read
193 : it: a stale doorbell (word still 0) is waited through, which relies
194 : on the edge being consumed by the wait. */
195 :
196 : struct epoll_event;
197 :
198 : int
199 : fd_sleep_park_wait_epoll( int epfd,
200 : ulong const * word,
201 : struct epoll_event * evs,
202 : int ev_max,
203 : int * ev_cnt,
204 : long deadline_ticks,
205 : double tick_per_ns );
206 :
207 : /* fd_sleep_eventfd_install creates the eventfd doorbell of every
208 : sleep_eventfd tile at FD_SLEEP_EVENTFD( tile->id ). */
209 :
210 : void
211 : fd_sleep_eventfd_install( struct fd_topo const * topo );
212 :
213 : FD_PROTOTYPES_END
214 :
215 : #endif /* HEADER_fd_src_disco_sleep_fd_sleep_h */
|
