Line data Source code
1 : /* The failover controller: the state machine the failov tile drives
2 : from its stem callbacks. ctx->action holds the state, one of
3 : FD_FAILOVER_ACTION_*, and set_action is its only writer.
4 :
5 : Two machines share one identity. The holder gives it up through the
6 : demotion flow and the taker adopts it through the handoff request and
7 : promotion flows.
8 :
9 : Holder, giving up the identity Taker, adopting it
10 :
11 : IDLE IDLE
12 : | accept_handoff | run_handoff_cmd
13 : DEMOTE_SWITCH HANDOFF_WAIT_PEER
14 : | the admin tile installs the junk | poll_peer dials and sends
15 : | key, step_demote_switch | HANDOFF_REQUEST, on_demoted_msg
16 : DEMOTE_DRAIN | takes the final state
17 : | the votor's last votes arrive, PROMOTE_WAIT_REPLAY
18 : | step_demote_drain | replay reaches the anchor,
19 : DEMOTE_WAIT_ACK | step_promote_wait_replay
20 : | DEMOTED sent, the peer answers, PROMOTE_WAIT_ADOPT
21 : | on_demoted_answer_msg | the votor adopts the history,
22 : IDLE | step_promote_wait_adopt
23 : PROMOTE_SWITCH
24 : | the admin tile installs the
25 : | identity, step_promote_switch
26 : HANDOFF_WAIT_RESULT
27 : | the peer records our answer,
28 : | on_handoff_result_msg
29 : IDLE
30 :
31 : This is not a translation unit. Keep it free of stem callbacks and
32 : the topology or seccomp, which stay in the tile. */
33 :
34 : /* The state */
35 :
36 : /* FSM state list */
37 : static char const * const action_names[ FD_FAILOVER_ACTION_CNT ] = {
38 : [ FD_FAILOVER_ACTION_IDLE ] = "idle",
39 : [ FD_FAILOVER_ACTION_DEMOTE_SWITCH ] = "demote_switch",
40 : [ FD_FAILOVER_ACTION_DEMOTE_DRAIN ] = "demote_drain",
41 : [ FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ] = "demote_wait_ack",
42 : [ FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ] = "handoff_wait_peer",
43 : [ FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY ] = "promote_wait_replay",
44 : [ FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ] = "promote_wait_adopt",
45 : [ FD_FAILOVER_ACTION_PROMOTE_SWITCH ] = "promote_switch",
46 : [ FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ] = "handoff_wait_result",
47 : };
48 :
49 : static char const *
50 0 : get_action_name( ulong action ) {
51 0 : return action<FD_FAILOVER_ACTION_CNT ? action_names[ action ] : "unknown";
52 0 : }
53 :
54 : /* Phase helpers */
55 : static inline int
56 0 : is_requesting( fd_failover_tile_ctx_t const * ctx ) {
57 0 : return ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT;
58 0 : }
59 :
60 : static inline int
61 0 : is_promoting( fd_failover_tile_ctx_t const * ctx ) {
62 0 : return ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ||
63 0 : ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH;
64 0 : }
65 :
66 : static inline int
67 0 : is_switching_out( fd_failover_tile_ctx_t const * ctx ) {
68 0 : return ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN;
69 0 : }
70 :
71 : static inline int
72 0 : is_switch_pending( fd_failover_tile_ctx_t const * ctx ) {
73 0 : return ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT;
74 0 : }
75 :
76 : /* Change the controller state: the only thing that should ever write to ctx->action! */
77 : static void
78 : set_action( fd_failover_tile_ctx_t * ctx,
79 0 : ulong action ) {
80 0 : FD_LOG_INFO(( "action %s -> %s", get_action_name( ctx->action ), get_action_name( action ) ));
81 0 : if( FD_LIKELY( action!=ctx->action ) ) {
82 0 : ctx->deadline_slot = ctx->replay_slot==FD_FAILOVER_SLOT_NULL ? FD_FAILOVER_SLOT_NULL : fd_ulong_sat_add( ctx->replay_slot, FD_FAILOVER_DEADLINE_SLOTS );
83 0 : ctx->deadline_at = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_DEADLINE_NANOS );
84 0 : }
85 0 : ctx->action = action;
86 0 : }
87 :
88 : /* The current state passed its slot deadline or its clock deadline. */
89 : static int
90 0 : is_overdue( fd_failover_tile_ctx_t const * ctx ) {
91 0 : int slots = ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL && ctx->replay_slot!=FD_FAILOVER_SLOT_NULL && ctx->replay_slot>ctx->deadline_slot;
92 0 : return slots || fd_failover_clock()>=ctx->deadline_at;
93 0 : }
94 :
95 : /* Returns the controller to IDLE with nothing in flight. */
96 : static void
97 0 : reset_controller( fd_failover_tile_ctx_t * ctx ) {
98 0 : fd_memset( &ctx->request, 0, sizeof(ctx->request) );
99 0 : fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) );
100 0 : fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) );
101 0 : fd_memset( &ctx->reply, 0, sizeof(ctx->reply) );
102 0 : fd_memset( &ctx->tx, 0, sizeof(ctx->tx) );
103 0 : ctx->cmd_addr = 0U;
104 0 : ctx->cmd_port = (ushort)0;
105 0 : ctx->adopt_result_fresh = 0;
106 0 : ctx->session.close_after_send = 0;
107 0 : ctx->peer_history.valid = 0;
108 0 : ctx->current_history.valid = 0;
109 0 : ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL;
110 0 : ctx->stuck = 0;
111 :
112 : /* reset to idle */
113 0 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
114 0 : }
115 :
116 : /* Shared helpers: the session, control queue, and log */
117 :
118 : static inline ushort
119 0 : get_dial_port( fd_failover_tile_ctx_t const * ctx ) {
120 0 : return ctx->cmd_port ? ctx->cmd_port : FD_FAILOVER_DEFAULT_PORT;
121 0 : }
122 :
123 : /* The peer on the current session is the bound one. */
124 : static inline int
125 : is_bound_peer( fd_failover_tile_ctx_t const * ctx,
126 0 : fd_failover_peer_t const * bound ) {
127 0 : return is_same_peer( bound, fd_failover_channel_peer_hello( ctx->channel )->boot_id );
128 0 : }
129 :
130 : static void
131 : finish_cmd( fd_failover_tile_ctx_t * ctx,
132 0 : ulong result ) {
133 0 : if( FD_LIKELY( !ctx->cmd_nonce || ctx->cmd_done ) ) return;
134 0 : if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) FD_LOG_NOTICE (( "the failover is done, this machine runs the identity" ));
135 0 : else FD_LOG_WARNING(( "the failover failed (result %#lx)", result ));
136 0 : ctx->cmd_result = result;
137 0 : ctx->cmd_done = 1;
138 0 : }
139 :
140 : /* Updates our session state when the channel's session changes. */
141 : static void
142 0 : sync_session( fd_failover_tile_ctx_t * ctx ) {
143 0 : ulong state = fd_failover_channel_state( ctx->channel );
144 0 : ulong generation = fd_failover_channel_generation( ctx->channel );
145 0 : if( FD_LIKELY( state==ctx->session.state && generation==ctx->session.generation ) ) return;
146 :
147 : /* new generation */
148 0 : ctx->session.state = state;
149 0 : ctx->session.generation = generation;
150 0 : ctx->request.sent = 0;
151 0 : ctx->session.close_after_send = 0;
152 : /* invalidate the previous session's FD_FAILOVER_MSG_HANDOFF_RESULT */
153 0 : if( ctx->tx.valid && ( ctx->tx.type==(ushort)FD_FAILOVER_MSG_HANDOFF_RESULT ||
154 0 : ( ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) ) {
155 0 : ctx->tx.valid = 0;
156 0 : }
157 0 : if( FD_LIKELY( state!=FD_FAILOVER_SESSION_PAIRED ) ) return;
158 0 : fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel );
159 0 : ctx->session.peer_boot_id = peer->boot_id;
160 0 : if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_SENT ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING;
161 0 : if( FD_UNLIKELY( peer->holds ) ) {
162 0 : if( is_promoting( ctx ) ) ctx->promotion.holder_seen = 1;
163 0 : if( ctx->peer_history.valid ) FD_LOG_NOTICE(( "peer boot %016lx runs the identity, discarding the saved final state, keeping the coverage floors", peer->boot_id ));
164 0 : ctx->peer_history.valid = 0;
165 0 : }
166 0 : }
167 :
168 : /* Drops the session and stops dialing. */
169 : static void
170 : hang_up( fd_failover_tile_ctx_t * ctx,
171 0 : long now ) {
172 0 : fd_failover_channel_init_dialer( ctx->channel, 0U, 0 );
173 0 : fd_failover_channel_hangup( ctx->channel, now );
174 0 : sync_session( ctx );
175 0 : }
176 :
177 : /* One-message controller outbox (queue_control is the primitive) */
178 :
179 : static int
180 : queue_control( fd_failover_tile_ctx_t * ctx,
181 : ushort type,
182 : fd_failover_peer_t const * to,
183 : void const * payload,
184 0 : ulong payload_sz ) {
185 0 : if( FD_UNLIKELY( ctx->tx.valid || !payload_sz || payload_sz>sizeof(ctx->tx.payload) ) ) return -1;
186 0 : ctx->tx.type = type;
187 0 : ctx->tx.sz = (ushort)payload_sz;
188 0 : ctx->tx.valid = 1;
189 0 : ctx->tx.to = *to;
190 0 : fd_memcpy( ctx->tx.payload, payload, payload_sz );
191 0 : return 0;
192 0 : }
193 :
194 : /* The holder's hand-over message (consensus-critical payload). */
195 : static int
196 0 : queue_demoted( fd_failover_tile_ctx_t * ctx ) {
197 : /* => DEMOTED */
198 0 : return queue_control( ctx, (ushort)FD_FAILOVER_MSG_DEMOTED, &ctx->handoff.target, ctx->handoff.payload, ctx->handoff.payload_sz );
199 0 : }
200 :
201 : /* Tells the requester how its handoff ended and closes the session once
202 : that has drained. */
203 : static void
204 : queue_handoff_result( fd_failover_tile_ctx_t * ctx,
205 : fd_failover_peer_t const * to,
206 : ulong id,
207 0 : ulong result ) {
208 : /* => HANDOFF_RESULT */
209 0 : fd_failover_peer_t peer;
210 0 : if( !to ) {
211 0 : peer.boot_id = fd_failover_channel_peer_hello( ctx->channel )->boot_id;
212 0 : to = &peer;
213 0 : }
214 0 : fd_failover_handoff_result_t response = { .handoff_id=id, .result=result };
215 0 : if( FD_LIKELY( !queue_control( ctx, FD_FAILOVER_MSG_HANDOFF_RESULT, to, &response, sizeof(response) ) ) ) {
216 0 : ctx->session.close_after_send = 1;
217 0 : }
218 0 : }
219 :
220 : /* Queues our last response to a DEMOTED. */
221 : static void
222 0 : queue_reply( fd_failover_tile_ctx_t * ctx ) {
223 : /* => PROMOTED_ACK or PROMOTE_REJECTED */
224 0 : uchar payload[ sizeof(fd_failover_promote_rejected_t) ];
225 0 : ulong payload_sz = ctx->reply.type==(ushort)FD_FAILOVER_MSG_PROMOTE_ACK
226 0 : ? fd_failover_promote_ack_encode( payload, ctx->reply.handoff_id )
227 0 : : fd_failover_promote_rejected_encode( payload, ctx->reply.handoff_id, ctx->reply.reason );
228 0 : ctx->reply.state = queue_control( ctx, ctx->reply.type, &ctx->reply.peer, payload, payload_sz )
229 0 : ? FD_FAILOVER_REPLY_PENDING
230 0 : : FD_FAILOVER_REPLY_KEPT;
231 0 : }
232 :
233 : /* The taker answers the DEMOTED */
234 : static void
235 : finish_reply( fd_failover_tile_ctx_t * ctx,
236 : fd_failover_peer_t const * peer,
237 : ulong handoff_id,
238 : ushort type,
239 0 : uchar reason ) {
240 0 : ctx->reply.peer = *peer;
241 0 : ctx->reply.handoff_id = handoff_id;
242 0 : ctx->reply.type = type;
243 0 : ctx->reply.reason = reason;
244 0 : queue_reply( ctx );
245 0 : }
246 :
247 : static char const *
248 0 : get_handoff_name( ulong result ) {
249 0 : static char const * const names[ FD_FAILOVER_HANDOFF_CNT ] = {
250 0 : "none", "pending", "taken", "declined", "restarted", "cancelled", "not holder"
251 0 : };
252 0 : return result<FD_FAILOVER_HANDOFF_CNT ? names[ result ] : "unknown";
253 0 : }
254 :
255 : static char const *
256 0 : get_msg_name( ushort type ) {
257 0 : switch( type ) {
258 0 : case (ushort)FD_FAILOVER_MSG_DEMOTED: return "DEMOTED";
259 0 : case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: return "PROMOTE_ACK";
260 0 : case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: return "PROMOTE_REJECTED";
261 0 : case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: return "HANDOFF_RESULT";
262 0 : default: return "unknown";
263 0 : }
264 0 : }
265 :
266 : /* Sends the queued control message once the session can take it. */
267 : static void
268 : flush_pending( fd_failover_tile_ctx_t * ctx,
269 0 : long now ) {
270 0 : if( FD_LIKELY( !ctx->tx.valid ) ) return;
271 0 : if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED || fd_failover_channel_tx_pending( ctx->channel ) ) ) return;
272 0 : if( FD_UNLIKELY( !is_bound_peer( ctx, &ctx->tx.to ) ) ) return;
273 0 : if( FD_UNLIKELY( fd_failover_channel_send( ctx->channel, now, ctx->tx.type, ctx->tx.payload, ctx->tx.sz ) ) ) return;
274 0 : if( ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED && ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT;
275 0 : if( ctx->tx.type==(ushort)FD_FAILOVER_MSG_HANDOFF_RESULT ) {
276 0 : fd_failover_handoff_result_t result;
277 0 : fd_memcpy( &result, ctx->tx.payload, sizeof(result) );
278 0 : FD_LOG_NOTICE(( "handoff %lu: sent HANDOFF_RESULT to peer boot %016lx (result %#lx)", result.handoff_id, ctx->tx.to.boot_id, result.result ));
279 0 : } else {
280 0 : ulong id = ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ? ctx->handoff.id : ctx->reply.handoff_id;
281 0 : FD_LOG_NOTICE(( "handoff %lu: sent %s to peer boot %016lx", id, get_msg_name( ctx->tx.type ), ctx->tx.to.boot_id ));
282 0 : }
283 0 : ctx->tx.valid = 0;
284 0 : }
285 :
286 : static char const *
287 0 : get_reject_name( uint reason ) {
288 0 : static char const * const names[ FD_FAILOVER_REJECT_CNT ] = {
289 0 : "none", "busy", "holds identity", "replay behind", "adoption mismatch", "adoption failed", "switch failed"
290 0 : };
291 0 : return reason<FD_FAILOVER_REJECT_CNT ? names[ reason ] : "unknown";
292 0 : }
293 :
294 : static char const *
295 0 : get_adopt_err_name( ulong result ) {
296 0 : switch( result ) {
297 0 : case FD_VOTOR_ADOPT_ERR_DECODE: return "decode error";
298 0 : case FD_VOTOR_ADOPT_ERR_INVALID: return "invalid";
299 0 : case FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT: return "finality anchor not replayed";
300 0 : case FD_VOTOR_ADOPT_ERR_UNREPLAYED: return "consensus not initialized";
301 0 : case FD_VOTOR_ADOPT_ERR_STALE: return "older than the votes this machine sent as this identity";
302 0 : case FD_VOTOR_ADOPT_ERR_CAPACITY: return "insufficient slot state capacity";
303 0 : default: return "unknown";
304 0 : }
305 0 : }
306 :
307 : static char const *
308 0 : get_source_name( ulong source ) {
309 0 : return source==FD_FAILOVER_SOURCE_PEER ? "the vote history the peer gave us" : "an empty history";
310 0 : }
311 :
312 : /* Identity switching: the admin tile installs the junk key or the identity */
313 :
314 : /* Sets the installed identity this machine currently signs with. */
315 : static void
316 : set_installed( fd_failover_tile_ctx_t * ctx,
317 0 : uchar const * pubkey ) {
318 0 : fd_memcpy( ctx->installed, pubkey, 32UL );
319 0 : ctx->hello.holds = (uchar)holds_identity( ctx );
320 0 : fd_failover_channel_set_holds( ctx->channel, ctx->hello.holds );
321 0 : }
322 :
323 : /* Asks the admin tile to switch to the given key.
324 : Returns the id, or ULONG_MAX if nothing was sent. */
325 : static ulong
326 : request_switch( fd_failover_tile_ctx_t * ctx,
327 : fd_stem_context_t * stem,
328 0 : ulong key ) {
329 0 : if( FD_UNLIKELY( ctx->admin_out_idx==ULONG_MAX || is_switch_pending( ctx ) ) ) return ULONG_MAX;
330 0 : if( FD_UNLIKELY( !++ctx->id_switch.request_id ) ) ctx->id_switch.request_id++;
331 0 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk );
332 0 : fd_memset( out, 0, sizeof(*out) );
333 0 : out->nonce = ctx->id_switch.request_id;
334 0 : fd_failover_switch_req_t req = { .epoch=ctx->id_switch.epoch };
335 0 : fd_memcpy( req.identity, key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->hello.staked_pubkey : ctx->spare_pubkey, 32UL );
336 0 : fd_memcpy( out->payload, &req, sizeof(req) );
337 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
338 0 : fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_SWITCH_REQ, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub );
339 0 : ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark );
340 0 : ctx->id_switch.pending_key = key;
341 0 : ctx->id_switch.fresh = 0;
342 0 : return ctx->id_switch.request_id;
343 0 : }
344 :
345 : /* Ends the switch in flight and returns 1 if the caller should take the result. */
346 : static int
347 : on_switch_resp_msg( fd_failover_tile_ctx_t * ctx,
348 0 : ulong nonce ) {
349 0 : if( FD_UNLIKELY( !is_switch_pending( ctx ) || nonce!=ctx->id_switch.request_id ) ) {
350 0 : FD_LOG_WARNING(( "dropping a stale identity switch answer" ));
351 0 : return 0;
352 0 : }
353 0 : ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT;
354 0 : if( FD_UNLIKELY( ctx->id_switch.discard ) ) {
355 0 : ctx->id_switch.discard = 0;
356 0 : return 0;
357 0 : }
358 0 : ctx->id_switch.fresh = 1;
359 0 : return 1;
360 0 : }
361 :
362 : static ulong
363 0 : take_switch_result( fd_failover_tile_ctx_t * ctx ) {
364 0 : ctx->id_switch.fresh = 0;
365 0 : ctx->id_switch.overdue = 0;
366 0 : return ctx->id_switch.result.result;
367 0 : }
368 :
369 : /* The identity switch in flight passed its deadline. */
370 : static void
371 0 : mark_switch_overdue( fd_failover_tile_ctx_t * ctx ) {
372 0 : ctx->stuck = 1;
373 0 : if( FD_LIKELY( ctx->id_switch.overdue ) ) return;
374 0 : ctx->id_switch.overdue = 1;
375 0 : FD_LOG_WARNING(( "identity switch %lu to %s is overdue, its outcome is unknown, we keep waiting and refuse another switch",
376 0 : ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "the identity" : "the junk key" ));
377 0 : }
378 :
379 : /* Inputs: the vote history, gossip and the operator */
380 :
381 : static void
382 : prepare_history( fd_failover_tile_ctx_t * ctx,
383 0 : fd_votor_hist_msg_t const * hist ) {
384 0 : uchar state[ FD_FAILOVER_ALPENGLOW_STATE_MAX ];
385 0 : ulong state_sz = 0UL;
386 0 : if( FD_UNLIKELY( fd_votor_hist_ser( &hist->hist, state, FD_FAILOVER_ALPENGLOW_STATE_MAX, &state_sz ) || fd_votor_hist_tip( &hist->hist )!=hist->vote_slot ) ) {
387 0 : if( FD_UNLIKELY( !ctx->hist_warned ) ) FD_LOG_WARNING(( "votor produced a vote history that does not match its slot metadata" ));
388 0 : ctx->hist_warned = 1;
389 0 : return;
390 0 : }
391 0 : ctx->hist_warned = 0;
392 0 : fd_memcpy( ctx->current_history.state, state, state_sz );
393 0 : ctx->current_history.valid = 1;
394 0 : ctx->current_history.tip = hist->vote_slot;
395 0 : ctx->current_history.sz = state_sz;
396 0 : }
397 :
398 : static void
399 : on_votor_hist_msg( fd_failover_tile_ctx_t * ctx,
400 0 : fd_votor_hist_msg_t const * hist ) {
401 0 : if( FD_LIKELY( hist->replay_slot!=FD_FAILOVER_SLOT_NULL && ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || hist->replay_slot>ctx->replay_slot ) ) ) ctx->replay_slot = hist->replay_slot;
402 0 : if( FD_LIKELY( hist->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = hist->root_slot;
403 0 : ctx->installed_ranked = hist->ranked;
404 0 : if( FD_UNLIKELY( !fd_memeq( hist->identity.uc, ctx->hello.staked_pubkey, 32UL ) || hist->vote_slot==FD_FAILOVER_SLOT_NULL ) ) return;
405 0 : int producing = holds_identity( ctx ) || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN;
406 0 : int voted = hist->has_vote || ( producing && ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL );
407 0 : int changed = producing && ctx->current_history.valid && hist->vote_slot>=ctx->last_vote_slot;
408 0 : if( FD_LIKELY( !voted && !changed ) ) return;
409 0 : ctx->last_vote_slot = hist->vote_slot;
410 0 : if( voted && ( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot;
411 0 : if( hist->has_vote ) ctx->own_voted = 1;
412 0 : prepare_history( ctx, hist );
413 0 : }
414 :
415 : /* Logs a step of a command at NOTICE. */
416 0 : #define LOG_CMD_STEP( ctx, ... ) do { \
417 0 : FD_LOG_NOTICE(( __VA_ARGS__ )); \
418 0 : fd_cstr_printf( (ctx)->cmd_step[ (ctx)->cmd_step_head%FD_FAILOVER_STEP_CNT ], FD_FAILOVER_BUS_STEP_MAX, NULL, __VA_ARGS__ ); \
419 0 : (ctx)->cmd_step_head++; \
420 0 : if( (ctx)->cmd_step_head-(ctx)->cmd_step_tail>FD_FAILOVER_STEP_CNT ) (ctx)->cmd_step_tail++; \
421 0 : } while(0)
422 :
423 : static void
424 : on_gossip_contact_info_msg( fd_failover_tile_ctx_t * ctx,
425 0 : long now ) {
426 0 : uint addr = ctx->gossip_socket.addr;
427 0 : if( FD_UNLIKELY( !addr || !fd_memeq( ctx->gossip_origin, ctx->hello.staked_pubkey, 32UL ) ) ) return;
428 0 : if( FD_UNLIKELY( addr==ctx->own_gossip.addr && ctx->gossip_socket.port==ctx->own_gossip.port ) ) return;
429 0 : ctx->staked_seen_at = now;
430 0 : if( FD_UNLIKELY( ctx->peer_history.valid ) ) {
431 0 : FD_LOG_NOTICE(( "gossip shows the identity at `" FD_IP4_ADDR_FMT "`, discarding the saved final state, keeping the coverage floors", FD_IP4_ADDR_FMT_ARGS( addr ) ));
432 0 : ctx->peer_history.valid = 0;
433 0 : }
434 0 : if( FD_LIKELY( addr==ctx->staked_addr ) ) return;
435 0 : FD_LOG_NOTICE(( "gossip shows the identity at `" FD_IP4_ADDR_FMT "`, a handoff from this machine dials it", FD_IP4_ADDR_FMT_ARGS( addr ) ));
436 0 : ctx->staked_addr = addr;
437 0 : if( FD_UNLIKELY( ctx->request.id && ctx->request.until && !ctx->request.addr && !ctx->cmd_addr && is_requesting( ctx ) ) ) {
438 0 : ctx->request.addr = addr;
439 0 : ctx->request.until = fd_long_sat_add( now, ctx->request.quick ? FD_FAILOVER_FORCE_WAIT_NANOS : FD_FAILOVER_DEADLINE_NANOS );
440 0 : fd_failover_channel_init_dialer( ctx->channel, addr, get_dial_port( ctx ) );
441 0 : LOG_CMD_STEP( ctx, "handoff %lu: requesting handoff from the machine running the identity at `" FD_IP4_ADDR_FMT ":%hu` using gossip",
442 0 : ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), get_dial_port( ctx ) );
443 0 : }
444 0 : }
445 :
446 : static void
447 : on_admin_operator_action_msg( fd_failover_tile_ctx_t * ctx,
448 0 : fd_failover_operator_t const * operator ) {
449 0 : if( FD_UNLIKELY( operator->epoch<=ctx->id_switch.epoch ) ) return;
450 0 : ctx->id_switch.epoch = operator->epoch;
451 0 : ctx->id_switch.discard = is_switch_pending( ctx );
452 0 : ctx->id_switch.fresh = 0;
453 0 : ctx->id_switch.overdue = 0;
454 0 : if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->request.id || ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE || ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ) {
455 0 : FD_LOG_WARNING(( "the identity changed in %s, dropping request %lu, handoff %lu and reply %lu, the other machine may already run the identity",
456 0 : get_action_name( ctx->action ), ctx->request.id, ctx->handoff.id, ctx->reply.handoff_id ));
457 0 : }
458 0 : finish_cmd( ctx, FD_FAILOVER_CONTROL_RESULT_IDENTITY_CHANGED );
459 0 : hang_up( ctx, fd_failover_clock() );
460 0 : reset_controller( ctx );
461 0 : if( FD_UNLIKELY( !fd_memeq( operator->failover_identity, ctx->hello.staked_pubkey, 32UL ) ) ) {
462 0 : ctx->staked_addr = 0U;
463 0 : ctx->staked_seen_at = 0L;
464 0 : ctx->holder_seen_at = 0L;
465 0 : ctx->peer_floor = FD_FAILOVER_SLOT_NULL;
466 0 : ctx->own_floor = FD_FAILOVER_SLOT_NULL;
467 0 : ctx->own_voted = 0;
468 0 : ctx->target_since = fd_failover_clock();
469 0 : }
470 0 : ulong boot_id = ctx->hello.boot_id;
471 0 : do FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); while( FD_UNLIKELY( !ctx->hello.boot_id || ctx->hello.boot_id==boot_id ) );
472 0 : fd_memcpy( ctx->hello.staked_pubkey, operator->failover_identity, 32UL );
473 0 : set_installed( ctx, operator->identity );
474 0 : fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 };
475 0 : if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) {
476 0 : FD_LOG_ERR(( "failover TLS setup failed after the identity changed" ));
477 0 : }
478 0 : sync_session( ctx );
479 0 : FD_BASE58_ENCODE_32_BYTES( operator->identity, identity );
480 0 : FD_BASE58_ENCODE_32_BYTES( operator->failover_identity, failover_identity );
481 0 : FD_LOG_NOTICE(( "failover now hands off `%s` and this machine runs `%s`, with a new boot %016lx", failover_identity, identity, ctx->hello.boot_id ));
482 0 : }
483 :
484 : /* Demotion: the holder gives up the identity */
485 :
486 : /* Our DEMOTED has been taken to logical completion. */
487 : static void
488 : resolve_handoff( fd_failover_tile_ctx_t * ctx,
489 0 : ulong result ) {
490 0 : if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) ctx->tx.valid = 0;
491 0 : if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) {
492 : /* reset to idle */
493 0 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
494 0 : }
495 0 : ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE;
496 0 : ctx->handoff.result = result;
497 0 : }
498 :
499 : static void
500 : abort_demotion( fd_failover_tile_ctx_t * ctx,
501 0 : char const * why ) {
502 0 : FD_LOG_WARNING(( "handoff %lu: %s, %s", ctx->handoff.id, why, holds_identity( ctx ) ? "this machine keeps the identity" : "nobody runs the identity" ));
503 0 : if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) queue_handoff_result( ctx, &ctx->handoff.target, ctx->handoff.id, FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED );
504 0 : ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE;
505 0 : ctx->stuck = 1;
506 :
507 : /* reset to idle */
508 0 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
509 0 : }
510 :
511 : /* The final history has to be the history of our last vote. */
512 : static int
513 0 : is_final_history_ok( fd_failover_tile_ctx_t const * ctx ) {
514 0 : return ctx->current_history.valid && ctx->current_history.sz && ctx->last_vote_slot!=FD_FAILOVER_SLOT_NULL && ctx->current_history.tip==ctx->last_vote_slot;
515 0 : }
516 :
517 : /* Whether we can hand off now to a machine whose replay is at
518 : replay_slot.
519 : Returns FD_ADMINCTL_RESULT_SUCCESS or an error code. */
520 : static ulong
521 : check_handoff( fd_failover_tile_ctx_t const * ctx,
522 0 : ulong replay_slot ) {
523 0 : if( FD_UNLIKELY( !is_final_history_ok( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY;
524 0 : fd_votor_hist_t hist[1];
525 0 : ulong anchor = fd_votor_hist_de( ctx->current_history.state, ctx->current_history.sz, hist ) ? ctx->last_vote_slot : hist->anchor;
526 0 : if( FD_UNLIKELY( replay_slot==FD_FAILOVER_SLOT_NULL || fd_ulong_sat_add( replay_slot, FD_FAILOVER_DEADLINE_SLOTS )<anchor ) ) return FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND;
527 0 : return FD_ADMINCTL_RESULT_SUCCESS;
528 0 : }
529 :
530 : /* Gives up the identity for the peer that asked (triggers the demotion flow). */
531 : static void
532 : accept_handoff( fd_failover_tile_ctx_t * ctx,
533 0 : ulong handoff_id ) {
534 0 : fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) );
535 0 : ctx->handoff.id = handoff_id;
536 0 : ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING;
537 0 : ctx->handoff.target.boot_id = ctx->session.peer_boot_id;
538 0 : ctx->stuck = 0;
539 0 : FD_LOG_NOTICE(( "handoff %lu: accepted the request from peer boot %016lx, giving up the identity", handoff_id, ctx->handoff.target.boot_id ));
540 :
541 0 : set_action( ctx, FD_FAILOVER_ACTION_DEMOTE_SWITCH );
542 0 : }
543 :
544 : /* Drain vote-history stream up to the key switching watermark, then enqueue the DEMOTED message. */
545 : static void
546 0 : step_demote_drain( fd_failover_tile_ctx_t * ctx ) {
547 0 : if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->hist_seen_seq, 1UL ), ctx->id_switch.result.history_watermark ) ) ) {
548 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) abort_demotion( ctx, "the vote history stream did not report its last votes in time, sending nothing" );
549 0 : return;
550 0 : }
551 0 : if( FD_UNLIKELY( !is_final_history_ok( ctx ) ) ) { abort_demotion( ctx, "no final vote history for the last vote, sending nothing" ); return; }
552 0 : ctx->handoff.payload_sz = fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, ctx->current_history.state, ctx->current_history.sz );
553 0 : if( FD_UNLIKELY( !ctx->handoff.payload_sz ) ) { abort_demotion( ctx, "the final vote history does not encode, sending nothing" ); return; }
554 0 : ctx->handoff.result = FD_FAILOVER_HANDOFF_PENDING;
555 0 : FD_LOG_NOTICE(( "handoff %lu: this machine runs its junk key now, sending the final vote history ending at slot %lu", ctx->handoff.id, ctx->last_vote_slot ));
556 0 : (void)queue_demoted( ctx );
557 :
558 0 : set_action( ctx, FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK );
559 0 : }
560 :
561 : /* Asks the admin tile to switch to the junk key. */
562 : static void
563 : step_demote_switch( fd_failover_tile_ctx_t * ctx,
564 0 : fd_stem_context_t * stem ) {
565 0 : if( FD_UNLIKELY( !is_switch_pending( ctx ) && !ctx->id_switch.fresh ) ) {
566 0 : if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_JUNK )==ULONG_MAX ) ) { abort_demotion( ctx, "an identity switch could not be requested" ); return; }
567 0 : FD_LOG_NOTICE(( "handoff %lu: asked the admin tile to install the junk key", ctx->handoff.id ));
568 0 : return;
569 0 : }
570 0 : if( FD_LIKELY( !ctx->id_switch.fresh ) ) {
571 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) mark_switch_overdue( ctx );
572 0 : return;
573 0 : }
574 0 : ulong result = take_switch_result( ctx );
575 0 : if( FD_UNLIKELY( result!=FD_FAILOVER_SWITCH_OK ) ) {
576 0 : abort_demotion( ctx, result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover is disabled" : result==FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER ? "the identity is an authorized voter" : "the identity switch failed" );
577 0 : return;
578 0 : }
579 :
580 0 : set_installed( ctx, ctx->spare_pubkey );
581 0 : set_action( ctx, FD_FAILOVER_ACTION_DEMOTE_DRAIN );
582 0 : step_demote_drain( ctx );
583 0 : }
584 :
585 : /* DEMOTED is sent out and the peer answers with either PROMOTE_ACK or PROMOTE_REJECTED. */
586 : static void
587 0 : step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) {
588 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) {
589 0 : FD_LOG_WARNING(( "handoff %lu: the peer has not answered, this machine runs the junk key, run `failover promote` to retry", ctx->handoff.id ));
590 0 : ctx->stuck = 1;
591 0 : if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) {
592 0 : ctx->tx.valid = 0;
593 0 : }
594 :
595 : /* reset to idle */
596 0 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
597 0 : return;
598 0 : }
599 0 : if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING && !ctx->tx.valid ) ) {
600 0 : (void)queue_demoted( ctx );
601 0 : }
602 0 : }
603 :
604 : /* Handoff request: the taker asks the holder for the identity */
605 :
606 : /* Our request is paused at its deadline. */
607 : static inline int
608 0 : is_request_paused( fd_failover_tile_ctx_t const * ctx ) {
609 0 : return ctx->request.id && !ctx->request.until && is_requesting( ctx );
610 0 : }
611 :
612 : /* Stops dialing without ending the request. */
613 : static void
614 : pause_request( fd_failover_tile_ctx_t * ctx,
615 : long now,
616 : ulong result,
617 0 : char const * why ) {
618 0 : FD_LOG_WARNING(( "handoff %lu paused in %s, %s, `failover promote` or `failover takeover` resumes it", ctx->request.id, get_action_name( ctx->action ), why ));
619 0 : ctx->request.until = 0L;
620 0 : ctx->stuck = 1;
621 0 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ) finish_cmd( ctx, result );
622 0 : hang_up( ctx, now );
623 0 : }
624 :
625 : /* Ends our request with result, an FD_FAILOVER_HANDOFF_*, and hangs up. */
626 : static void
627 : end_request( fd_failover_tile_ctx_t * ctx,
628 : long now,
629 0 : ulong result ) {
630 0 : FD_TEST( !is_switch_pending( ctx ) && is_requesting( ctx ) );
631 0 : FD_LOG_NOTICE(( "handoff %lu: request ended (%s), %s", ctx->request.id, get_handoff_name( result ), holds_identity( ctx ) ? "this machine runs the identity" : "this machine does not run the identity" ));
632 0 : ctx->request.id = 0UL;
633 0 : ctx->request.until = 0L;
634 0 : ctx->tx.valid = 0;
635 0 : ctx->session.close_after_send = 0;
636 0 : ctx->stuck = result!=FD_FAILOVER_HANDOFF_TAKEN;
637 0 : if( ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ctx->reply.state = FD_FAILOVER_REPLY_KEPT;
638 :
639 : /* reset to idle */
640 0 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
641 0 : hang_up( ctx, now );
642 0 : }
643 :
644 : static void
645 0 : await_request_result( fd_failover_tile_ctx_t * ctx ) {
646 0 : if( FD_UNLIKELY( !ctx->request.until ) ) fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, get_dial_port( ctx ) );
647 0 : ctx->request.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_DEADLINE_NANOS );
648 :
649 0 : set_action( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT );
650 0 : }
651 :
652 : /* Votor adoption: the taker adopts a vote history */
653 :
654 : /* The coverage floor is the higher of the peer's reported last vote and
655 : our highest signed vote. */
656 : static ulong
657 0 : get_coverage_floor( fd_failover_tile_ctx_t const * ctx ) {
658 0 : if( FD_UNLIKELY( ctx->peer_floor==FD_FAILOVER_SLOT_NULL ) ) return ctx->own_floor;
659 0 : if( FD_UNLIKELY( ctx->own_floor ==FD_FAILOVER_SLOT_NULL ) ) return ctx->peer_floor;
660 0 : return fd_ulong_max( ctx->peer_floor, ctx->own_floor );
661 0 : }
662 :
663 : /* Checks whether a forced promotion may go ahead now. Returns
664 : FD_ADMINCTL_RESULT_SUCCESS or the refusal code. */
665 : static ulong
666 0 : check_promotion( fd_failover_tile_ctx_t const * ctx ) {
667 0 : if( FD_UNLIKELY( holds_identity( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS;
668 0 : if( FD_UNLIKELY( ctx->installed_ranked ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDS_OTHER;
669 0 : int cancellable = ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK || ( ctx->request.id && is_requesting( ctx ) );
670 0 : if( FD_UNLIKELY( ( ctx->action!=FD_FAILOVER_ACTION_IDLE && !cancellable ) || is_switch_pending( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
671 0 : return FD_ADMINCTL_RESULT_SUCCESS;
672 0 : }
673 :
674 : /* Sets the finality anchor of the vote history that will be adopted via promotion. */
675 : static void
676 0 : set_adopt_anchor( fd_failover_tile_ctx_t * ctx ) {
677 0 : ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL;
678 0 : ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL;
679 0 : if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ) ) return;
680 0 : fd_votor_hist_t hist[1];
681 0 : if( FD_LIKELY( !fd_votor_hist_de( ctx->promotion.adopt.state, ctx->promotion.adopt.sz, hist ) ) ) ctx->adopt_anchor = hist->anchor;
682 0 : }
683 :
684 : /* Whether replay reached the saved history's finality anchor. */
685 : static int
686 0 : is_replay_ready( fd_failover_tile_ctx_t * ctx ) {
687 0 : if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) {
688 0 : return ctx->adopt_anchor==FD_FAILOVER_SLOT_NULL || ctx->replay_slot>=ctx->adopt_anchor;
689 0 : }
690 0 : ulong floor = ctx->promotion.floor;
691 0 : ctx->empty_vote_after = floor==FD_FAILOVER_SLOT_NULL ? ctx->replay_slot : fd_ulong_max( ctx->replay_slot, floor );
692 0 : FD_STORE( ulong, ctx->promotion.adopt.state, ctx->empty_vote_after );
693 0 : ctx->promotion.adopt.sz = FD_VOTOR_ADOPT_EMPTY_SZ;
694 0 : return 1;
695 0 : }
696 :
697 : /* Sends the promotion's history or empty bound to the votor for the
698 : target identity to the votor tile (which then runs failover_adopt_hist).
699 :
700 : Returns the request id or ULONG_MAX if nothing to send. */
701 : static ulong
702 : publish_adopt_state( fd_failover_tile_ctx_t * ctx,
703 0 : fd_stem_context_t * stem ) {
704 0 : if( FD_UNLIKELY( ctx->adopt_out_idx==ULONG_MAX || !ctx->promotion.adopt.sz ) ) return ULONG_MAX;
705 0 : if( FD_UNLIKELY( !++ctx->adopt_request_id ) ) ctx->adopt_request_id++;
706 0 : uchar * req = fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk );
707 0 : ulong sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+ctx->promotion.adopt.sz;
708 0 : fd_memcpy( req, ctx->hello.staked_pubkey, FD_VOTOR_ADOPT_REQ_HDR_SZ );
709 0 : fd_memcpy( req+FD_VOTOR_ADOPT_REQ_HDR_SZ, ctx->promotion.adopt.state, ctx->promotion.adopt.sz );
710 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
711 0 : fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, sz, 0UL, tspub, tspub );
712 0 : ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark );
713 0 : ctx->adopt_result_fresh = 0;
714 0 : return ctx->adopt_request_id;
715 0 : }
716 :
717 : static ulong
718 0 : get_reject_result( uchar reason ) {
719 0 : switch( reason ) {
720 0 : case (uchar)FD_FAILOVER_REJECT_REPLAY_BEHIND: return FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND;
721 0 : case (uchar)FD_FAILOVER_REJECT_HOLDS_IDENTITY: return FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS;
722 0 : case (uchar)FD_FAILOVER_REJECT_SWITCH_FAILED: return FD_FAILOVER_CONTROL_RESULT_SWITCH_FAILED;
723 0 : default: return FD_FAILOVER_CONTROL_RESULT_ADOPTION_FAILED;
724 0 : }
725 0 : }
726 :
727 : /* Ends an operator promotion at IDLE, or answers the peer and waits for
728 : its result on a cooperative handoff. Returns 1 if it was from a peer. */
729 : static int
730 : end_promotion( fd_failover_tile_ctx_t * ctx,
731 : ushort type,
732 0 : uchar reason ) {
733 0 : if( FD_LIKELY( !ctx->promotion.from_peer ) ) {
734 : /* reset to idle */
735 0 : set_action( ctx, FD_FAILOVER_ACTION_IDLE );
736 0 : return 0;
737 0 : }
738 0 : ctx->peer_history.valid = 0;
739 0 : ctx->promotion.from_peer = 0;
740 0 : await_request_result( ctx );
741 0 : finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, type, reason );
742 0 : return 1;
743 0 : }
744 :
745 : /* Ends the promotion without switching to the identity.. */
746 : static void
747 : reject_promotion( fd_failover_tile_ctx_t * ctx,
748 : uchar reason,
749 0 : int bad_history ) {
750 0 : if( FD_UNLIKELY( bad_history && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_history.valid = 0;
751 0 : ctx->stuck = 1;
752 0 : FD_LOG_WARNING(( "%s failed (%s), this machine does not take the identity", ctx->promotion.label, get_reject_name( reason ) ));
753 0 : finish_cmd( ctx, get_reject_result( reason ) );
754 0 : end_promotion( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason );
755 0 : }
756 :
757 : /* Promotion: the taker installs the identity it adopted */
758 :
759 : static void
760 : start_promotion( fd_failover_tile_ctx_t * ctx,
761 : ulong source,
762 : fd_failover_peer_t const * peer,
763 : ulong handoff_id,
764 0 : ulong flags ) {
765 0 : if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) {
766 0 : resolve_handoff( ctx, FD_FAILOVER_HANDOFF_CANCELLED );
767 0 : }
768 0 : fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) );
769 0 : if( source==FD_FAILOVER_SOURCE_PEER ) ctx->promotion.adopt = ctx->peer_history;
770 0 : else ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL;
771 0 : if( peer ) {
772 0 : ctx->promotion.peer = *peer;
773 0 : ctx->promotion.handoff_id = handoff_id;
774 0 : ctx->promotion.from_peer = 1;
775 0 : fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "handoff %lu", handoff_id );
776 0 : } else {
777 0 : fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "operator promotion" );
778 0 : }
779 0 : ctx->promotion.source = source;
780 0 : ctx->promotion.force = !!( flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER );
781 0 : ctx->promotion.soft = ctx->promotion.force && !( flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE );
782 0 : ctx->promotion.floor = get_coverage_floor( ctx );
783 0 : ctx->current_history = ctx->promotion.adopt;
784 0 : ctx->last_vote_slot = ctx->promotion.adopt.tip;
785 0 : ctx->stuck = 0;
786 0 : set_adopt_anchor( ctx );
787 0 : LOG_CMD_STEP( ctx, "%s: adopting %s%s", ctx->promotion.label, get_source_name( source ),
788 0 : ( flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ) ? ", --force skips the checks on the other machine" :
789 0 : ( flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER ) ? ", the other machine showed no signs of life" : "" );
790 :
791 0 : set_action( ctx, FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY );
792 0 : }
793 :
794 : /* Requests votor to adopt the history. */
795 : static int
796 : publish_adoption( fd_failover_tile_ctx_t * ctx,
797 0 : fd_stem_context_t * stem ) {
798 0 : ulong id = publish_adopt_state( ctx, stem );
799 0 : if( FD_UNLIKELY( id==ULONG_MAX ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return 0; }
800 0 : LOG_CMD_STEP( ctx, "%s: asking the votor to adopt %s", ctx->promotion.label, get_source_name( ctx->promotion.source ) );
801 0 : ctx->promotion.adopt_id = id;
802 0 : ctx->promotion.retry = 0;
803 0 : return 1;
804 0 : }
805 :
806 : /* Waits for replay to reach the history's finality anchor, then asks the votor to adopt it. */
807 : static void
808 : step_promote_wait_replay( fd_failover_tile_ctx_t * ctx,
809 0 : fd_stem_context_t * stem ) {
810 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) {
811 0 : if( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) LOG_CMD_STEP( ctx, "%s: replay reported no slot before the deadline", ctx->promotion.label );
812 0 : else LOG_CMD_STEP( ctx, "%s: replay at slot %lu did not reach the vote history's finality anchor %lu before the deadline", ctx->promotion.label, ctx->replay_slot, ctx->adopt_anchor );
813 0 : reject_promotion( ctx, ctx->promotion.from_peer || ctx->promotion.gave_up ? FD_FAILOVER_REJECT_ADOPTION_FAILED : FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 );
814 0 : return;
815 0 : }
816 0 : if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return;
817 0 : if( FD_UNLIKELY( !is_replay_ready( ctx ) ) ) return; /* empty-history case */
818 0 : if( FD_UNLIKELY( !publish_adoption( ctx, stem ) ) ) return;
819 :
820 0 : set_action( ctx, FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT );
821 0 : }
822 :
823 : /* Waits for the votor's response and then handles it. */
824 : static void
825 : step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx,
826 0 : fd_stem_context_t * stem ) {
827 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; }
828 0 : if( FD_UNLIKELY( ctx->promotion.retry ) ) {
829 0 : if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return;
830 0 : (void)publish_adoption( ctx, stem );
831 0 : return;
832 0 : }
833 0 : if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return;
834 0 : ctx->adopt_result_fresh = 0;
835 0 : if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return;
836 0 : ulong result = ctx->adopt_result.result;
837 0 : char const * err = get_adopt_err_name( result );
838 0 : if( FD_UNLIKELY( result==FD_VOTOR_ADOPT_ERR_UNREPLAYED || result==FD_VOTOR_ADOPT_ERR_CAPACITY ) ) {
839 : /* retry once replay advances */
840 0 : if( FD_UNLIKELY( !ctx->promotion.from_peer && !ctx->promotion.gave_up ) ) {
841 0 : LOG_CMD_STEP( ctx, "%s: the votor is not ready to adopt (%s), retry once consensus is up", ctx->promotion.label, err );
842 0 : reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 );
843 0 : return;
844 0 : }
845 0 : LOG_CMD_STEP( ctx, "%s: the votor is not ready to adopt the vote history (%s), asking again as replay moves", ctx->promotion.label, err );
846 0 : ctx->promotion.retry = 1;
847 0 : ctx->promotion.retry_slot = ctx->replay_slot;
848 0 : return;
849 0 : }
850 0 : if( FD_UNLIKELY( result!=FD_VOTOR_ADOPT_SUCCESS ) ) {
851 0 : reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, result==FD_VOTOR_ADOPT_ERR_DECODE || result==FD_VOTOR_ADOPT_ERR_INVALID || result==FD_VOTOR_ADOPT_ERR_STALE );
852 0 : return;
853 0 : }
854 :
855 : /* check tip and coverage floor */
856 0 : ulong tip = ctx->promotion.adopt.tip;
857 0 : if( FD_UNLIKELY( tip!=FD_FAILOVER_SLOT_NULL && ctx->adopt_result.vote_slot!=tip ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); return; }
858 0 : ulong floor = ctx->promotion.floor;
859 0 : ulong reached = ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ? ctx->adopt_result.vote_bound : ctx->adopt_result.vote_slot;
860 0 : int covered = floor==FD_FAILOVER_SLOT_NULL || ( reached!=FD_FAILOVER_SLOT_NULL && reached>=floor );
861 0 : if( ctx->adopt_result.vote_slot==FD_FAILOVER_SLOT_NULL ) LOG_CMD_STEP( ctx, "%s: the votor adopted %s, no last vote, no vote at or below slot %lu", ctx->promotion.label, get_source_name( ctx->promotion.source ), ctx->adopt_result.vote_bound );
862 0 : else LOG_CMD_STEP( ctx, "%s: the votor adopted %s, last vote at slot %lu, no vote at or below slot %lu", ctx->promotion.label, get_source_name( ctx->promotion.source ), ctx->adopt_result.vote_slot, ctx->adopt_result.vote_bound );
863 0 : if( FD_UNLIKELY( !covered && !ctx->promotion.force ) ) {
864 0 : ctx->promotion.retry = 1;
865 0 : ctx->promotion.retry_slot = FD_FAILOVER_SLOT_NULL;
866 0 : return;
867 0 : }
868 0 : if( FD_UNLIKELY( ctx->promotion.holder_seen && ( !ctx->promotion.force || ctx->promotion.soft ) ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_HOLDS_IDENTITY, 0 ); return; }
869 0 : if( FD_UNLIKELY( tip==FD_FAILOVER_SLOT_NULL ) ) ctx->last_vote_slot = ctx->adopt_result.vote_slot;
870 0 : if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )==ULONG_MAX ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; }
871 0 : LOG_CMD_STEP( ctx, "%s: asked the admin tile to install the identity", ctx->promotion.label );
872 :
873 : /* success, ask the admin tile to install the real identity! */
874 0 : set_action( ctx, FD_FAILOVER_ACTION_PROMOTE_SWITCH );
875 0 : }
876 :
877 : /* Waits for the admin tile to install the identity. */
878 : static void
879 0 : step_promote_switch( fd_failover_tile_ctx_t * ctx ) {
880 0 : if( FD_LIKELY( !ctx->id_switch.fresh ) ) {
881 0 : if( FD_UNLIKELY( is_overdue( ctx ) ) ) mark_switch_overdue( ctx );
882 0 : return;
883 0 : }
884 0 : if( FD_UNLIKELY( take_switch_result( ctx )!=FD_FAILOVER_SWITCH_OK ) ) { reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); return; }
885 0 : if( !ctx->promotion.from_peer ) hang_up( ctx, fd_failover_clock() );
886 :
887 : /* look at me, I'm the holder now */
888 0 : set_installed( ctx, ctx->hello.staked_pubkey );
889 0 : ctx->peer_history.valid = 0;
890 0 : ctx->own_voted = 1;
891 0 : ctx->stuck = 0;
892 0 : FD_LOG_NOTICE(( "%s: this machine runs the identity now, last vote at slot %lu", ctx->promotion.label, ctx->last_vote_slot ));
893 0 : finish_cmd( ctx, FD_ADMINCTL_RESULT_SUCCESS );
894 :
895 : /* send PROMOTE_ACK back to the peer for cooperative handoffs */
896 0 : end_promotion( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, (uchar)FD_FAILOVER_REJECT_NONE );
897 0 : }
898 :
899 : /* The controller step dispatcher */
900 :
901 : /* Advances the controller state (main dispatcher). */
902 : static void
903 : step_controller( fd_failover_tile_ctx_t * ctx,
904 0 : fd_stem_context_t * stem ) {
905 0 : if( FD_UNLIKELY( ctx->reply.state==FD_FAILOVER_REPLY_PENDING && !ctx->tx.valid ) ) queue_reply( ctx );
906 0 : switch( ctx->action ) {
907 0 : case FD_FAILOVER_ACTION_IDLE:
908 0 : case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER:
909 0 : case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: return;
910 0 : case FD_FAILOVER_ACTION_DEMOTE_SWITCH: step_demote_switch( ctx, stem ); return;
911 0 : case FD_FAILOVER_ACTION_DEMOTE_DRAIN: step_demote_drain( ctx ); return;
912 0 : case FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK: step_demote_wait_ack( ctx ); return;
913 0 : case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: step_promote_wait_replay( ctx, stem ); return;
914 0 : case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: step_promote_wait_adopt( ctx, stem ); return;
915 0 : case FD_FAILOVER_ACTION_PROMOTE_SWITCH: step_promote_switch( ctx ); return;
916 0 : default: FD_LOG_ERR(( "unexpected failover action %lu", ctx->action ));
917 0 : }
918 0 : }
919 :
920 : /* Peer messages and polling. Each handler returns 1 when the message
921 : was handled, including one it ignored on purpose, and 0 when it was
922 : malformed, which drops the session. */
923 :
924 : /* HANDOFF_REQUEST, the peer asks us to give up the identity. */
925 : static int
926 : on_handoff_request_msg( fd_failover_tile_ctx_t * ctx,
927 0 : ulong payload_sz ) {
928 0 : fd_failover_handoff_request_t request;
929 0 : if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) return 0;
930 0 : ulong result = FD_ADMINCTL_RESULT_SUCCESS;
931 0 : if( FD_UNLIKELY( request.target_boot_id!=ctx->hello.boot_id ) ) {
932 0 : result = FD_FAILOVER_CONTROL_RESULT_PEER_RESTARTED;
933 0 : } else if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && is_bound_peer( ctx, &ctx->handoff.target ) ) ) {
934 0 : if( is_switching_out( ctx ) ) return 1;
935 0 : if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) {
936 0 : if( !ctx->tx.valid ) (void)queue_demoted( ctx );
937 0 : return 1;
938 0 : }
939 0 : result = ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED;
940 0 : } else if( ctx->action!=FD_FAILOVER_ACTION_IDLE || is_switch_pending( ctx ) || ctx->tx.valid || ctx->session.close_after_send || ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) {
941 0 : result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
942 0 : } else if( !holds_identity( ctx ) ) {
943 0 : result = FD_FAILOVER_CONTROL_RESULT_NOT_HOLDER;
944 0 : } else if( fd_failover_channel_peer_hello( ctx->channel )->holds ) {
945 0 : result = FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS;
946 0 : } else {
947 0 : result = check_handoff( ctx, request.replay_slot );
948 : /* trigger the demotion flow */
949 0 : if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { accept_handoff( ctx, request.handoff_id ); return 1; }
950 0 : }
951 0 : FD_LOG_WARNING(( "handoff %lu refused (result %#lx)", request.handoff_id, result ));
952 0 : queue_handoff_result( ctx, NULL, request.handoff_id, result );
953 0 : return 1;
954 0 : }
955 :
956 : /* HANDOFF_RESULT, the peer recorded how our request ended. */
957 : static int
958 : on_handoff_result_msg( fd_failover_tile_ctx_t * ctx,
959 : ulong payload_sz,
960 0 : long now ) {
961 0 : fd_failover_handoff_result_t result;
962 0 : if( FD_UNLIKELY( !fd_failover_handoff_result_decode( &result, ctx->rx, payload_sz ) ) ) return 0;
963 0 : if( FD_UNLIKELY( !ctx->request.id || result.handoff_id!=ctx->request.id || !is_bound_peer( ctx, &ctx->request.peer ) || !is_requesting( ctx ) ) ) return 1;
964 0 : int failed = result.result!=FD_ADMINCTL_RESULT_SUCCESS || !holds_identity( ctx );
965 0 : if( FD_UNLIKELY( failed && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY &&
966 0 : ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.send_cnt<FD_FAILOVER_REQUEST_RETRIES ) ) {
967 0 : LOG_CMD_STEP( ctx, "handoff %lu: the other machine has no final vote state yet, asking again (send %lu)", result.handoff_id, ctx->request.send_cnt );
968 0 : return 1;
969 0 : }
970 0 : if( FD_UNLIKELY( failed ) ) {
971 0 : FD_LOG_WARNING(( "handoff %lu ended (result %#lx), %s", result.handoff_id, result.result, holds_identity( ctx ) ? "this machine runs the identity and the other may run it too" : "this machine does not run the identity" ));
972 0 : ulong cmd_result = result.result;
973 0 : if( FD_UNLIKELY( cmd_result==FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS ) ) cmd_result = holds_identity( ctx ) ? FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS : FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
974 0 : if( FD_UNLIKELY( cmd_result==FD_ADMINCTL_RESULT_SUCCESS ) ) cmd_result = FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED;
975 0 : finish_cmd( ctx, cmd_result );
976 0 : }
977 0 : end_request( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN );
978 0 : return 1;
979 0 : }
980 :
981 : /* DEMOTED, the peer gave up the identity and hands us its final history.
982 : We call start_promotion to kick off promotion. */
983 : static int
984 : on_demoted_msg( fd_failover_tile_ctx_t * ctx,
985 0 : ulong payload_sz ) {
986 0 : fd_failover_demoted_t demoted;
987 0 : if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) || demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW ) ) return 0;
988 0 : if( FD_UNLIKELY( !ctx->request.id || ctx->request.id!=demoted.handoff_id || !is_bound_peer( ctx, &ctx->request.peer ) ) ) return 1;
989 0 : if( FD_UNLIKELY( demoted.target_boot_id!=ctx->hello.boot_id ) ) return 0;
990 0 : if( FD_UNLIKELY( ctx->promotion.from_peer && ctx->promotion.handoff_id==demoted.handoff_id ) ) return 1;
991 0 : if( FD_UNLIKELY( ctx->reply.state!=FD_FAILOVER_REPLY_NONE && ctx->reply.handoff_id==demoted.handoff_id && is_bound_peer( ctx, &ctx->reply.peer ) ) ) { queue_reply( ctx ); return 1; }
992 0 : uchar reason = holds_identity( ctx ) ? FD_FAILOVER_REJECT_HOLDS_IDENTITY :
993 0 : ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || is_switch_pending( ctx ) ? FD_FAILOVER_REJECT_BUSY : FD_FAILOVER_REJECT_NONE;
994 0 : if( FD_UNLIKELY( reason!=FD_FAILOVER_REJECT_NONE ) ) {
995 0 : FD_LOG_WARNING(( "handoff %lu: refusing the peer's final state (%s)", demoted.handoff_id, get_reject_name( reason ) ));
996 0 : finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason );
997 0 : return 1;
998 0 : }
999 0 : fd_failover_history_t history = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len };
1000 0 : fd_memcpy( history.state, ctx->rx+sizeof(fd_failover_demoted_t), history.sz );
1001 0 : if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || history.tip>ctx->peer_floor ) ctx->peer_floor = history.tip;
1002 0 : ctx->peer_history = history;
1003 0 : start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 );
1004 0 : return 1;
1005 0 : }
1006 :
1007 : /* PROMOTE_ACK or PROMOTE_REJECTED, the peer's response to our DEMOTED. */
1008 : static int
1009 : on_demoted_answer_msg( fd_failover_tile_ctx_t * ctx,
1010 : ushort type,
1011 0 : ulong payload_sz ) {
1012 0 : int taken = type==(ushort)FD_FAILOVER_MSG_PROMOTE_ACK;
1013 0 : fd_failover_promote_ack_t ack;
1014 0 : fd_failover_promote_rejected_t rej;
1015 0 : if( FD_UNLIKELY( taken ? !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) : !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) return 0;
1016 0 : ulong handoff_id = taken ? ack.handoff_id : rej.handoff_id;
1017 0 : if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_NONE || handoff_id!=ctx->handoff.id || !is_bound_peer( ctx, &ctx->handoff.target ) || is_switching_out( ctx ) ) ) return 1;
1018 0 : if( taken ) FD_LOG_NOTICE (( "handoff %lu: the peer took the identity, this machine keeps the junk key", handoff_id ));
1019 0 : else FD_LOG_WARNING(( "handoff %lu: the peer declined the identity (%s), nobody runs it", handoff_id, get_reject_name( rej.reason ) ));
1020 0 : if( taken ) ctx->holder_seen_at = fd_long_max( ctx->holder_seen_at, fd_failover_clock() );
1021 0 : ctx->peer_history.valid = 0;
1022 0 : ctx->stuck = !taken;
1023 0 : resolve_handoff( ctx, taken ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED );
1024 0 : queue_handoff_result( ctx, NULL, handoff_id, taken ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED );
1025 0 : return 1;
1026 0 : }
1027 :
1028 : /* Handles a control message. Anything malformed drops the session. */
1029 : static void
1030 : handle_control( fd_failover_tile_ctx_t * ctx,
1031 : ushort type,
1032 : ulong payload_sz,
1033 0 : long now ) {
1034 0 : int handled;
1035 0 : switch( type ) {
1036 0 : case (ushort)FD_FAILOVER_MSG_HANDOFF_REQUEST: handled = on_handoff_request_msg( ctx, payload_sz ); break;
1037 0 : case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: handled = on_handoff_result_msg ( ctx, payload_sz, now ); break;
1038 0 : case (ushort)FD_FAILOVER_MSG_DEMOTED: handled = on_demoted_msg ( ctx, payload_sz ); break;
1039 0 : case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK:
1040 0 : case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: handled = on_demoted_answer_msg ( ctx, type, payload_sz ); break;
1041 0 : default: handled = 0; break;
1042 0 : }
1043 0 : if( FD_LIKELY( handled ) ) return;
1044 0 : fd_failover_channel_protocol_error( ctx->channel, now );
1045 0 : sync_session( ctx );
1046 0 : }
1047 :
1048 : /* Whether at, our clock or 0 for never, is within FD_FAILOVER_GOSSIP_FRESH_NANOS. */
1049 : static inline int
1050 : is_recent( long at,
1051 0 : long now ) {
1052 0 : return at && now>=at && now-at<FD_FAILOVER_GOSSIP_FRESH_NANOS;
1053 0 : }
1054 :
1055 : /* The machine running the identity refreshed its contact info in gossip
1056 : within FD_FAILOVER_GOSSIP_FRESH_NANOS. */
1057 : static inline int
1058 : is_holder_fresh( fd_failover_tile_ctx_t const * ctx,
1059 0 : long now ) {
1060 0 : return is_recent( ctx->staked_seen_at, now );
1061 0 : }
1062 :
1063 : static void
1064 0 : note_holder_heard( fd_failover_tile_ctx_t * ctx ) {
1065 0 : if( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED || !fd_failover_channel_peer_hello( ctx->channel )->holds ) return;
1066 0 : ctx->holder_seen_at = fd_long_max( ctx->holder_seen_at, fd_failover_channel_peer_heard_at( ctx->channel ) );
1067 0 : }
1068 :
1069 : static int
1070 : is_holder_live( fd_failover_tile_ctx_t const * ctx,
1071 0 : long now ) {
1072 0 : if( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && fd_failover_channel_peer_hello( ctx->channel )->holds ) return 1;
1073 0 : if( now>=ctx->target_since && now-ctx->target_since<FD_FAILOVER_GOSSIP_FRESH_NANOS ) return 1;
1074 0 : return is_holder_fresh( ctx, now ) || is_recent( ctx->holder_seen_at, now );
1075 0 : }
1076 :
1077 : /* Pumps the TCP+TLS peer channel and drives our handoff request on it. */
1078 : static void
1079 : poll_peer( fd_failover_tile_ctx_t * ctx,
1080 : long now,
1081 0 : int * charge_busy ) {
1082 0 : sync_session( ctx );
1083 0 : if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && is_requesting( ctx ) ) ) {
1084 0 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && holds_identity( ctx ) && !is_switch_pending( ctx ) ) {
1085 0 : FD_LOG_WARNING(( "handoff %lu: the peer did not confirm our answer before the deadline, ending the request", ctx->request.id ));
1086 0 : end_request( ctx, now, FD_FAILOVER_HANDOFF_TAKEN );
1087 0 : }
1088 0 : else if( ctx->request.send_cnt ) pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_TIMEOUT, "no answer before the deadline" );
1089 0 : else if( !ctx->request.addr ) pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS, "gossip did not show the identity before the deadline" );
1090 0 : else if( !ctx->request.peer.boot_id ) pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_NO_PEER, "no peer authenticated before the deadline" );
1091 0 : else pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_TIMEOUT, "no handoff request sent before the deadline" );
1092 0 : return;
1093 0 : }
1094 0 : if( FD_UNLIKELY( ctx->session.close_after_send && !ctx->tx.valid && !fd_failover_channel_tx_pending( ctx->channel ) ) ) {
1095 0 : ctx->session.close_after_send = 0;
1096 0 : fd_failover_channel_hangup( ctx->channel, now );
1097 0 : sync_session( ctx );
1098 0 : return;
1099 0 : }
1100 0 : ushort type;
1101 0 : ulong payload_sz;
1102 0 : int delivered = fd_failover_channel_poll( ctx->channel, now, charge_busy, &type, ctx->rx, &payload_sz );
1103 0 : sync_session( ctx );
1104 0 : note_holder_heard( ctx );
1105 0 : if( FD_UNLIKELY( delivered ) ) {
1106 0 : handle_control( ctx, type, payload_sz, now );
1107 0 : sync_session( ctx );
1108 0 : }
1109 0 : if( FD_LIKELY( !ctx->request.id || fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) ) return;
1110 0 : fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel );
1111 0 : if( FD_UNLIKELY( ctx->request.peer.boot_id && !is_bound_peer( ctx, &ctx->request.peer ) ) ) {
1112 0 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && !is_switch_pending( ctx ) ) {
1113 0 : FD_LOG_WARNING(( "handoff %lu: the peer restarted before it confirmed our answer, ending the request", ctx->request.id ));
1114 0 : end_request( ctx, now, holds_identity( ctx ) ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED );
1115 0 : } else if( ctx->request.until ) {
1116 0 : ctx->request.peer_restarted = 1;
1117 0 : pause_request( ctx, now, FD_FAILOVER_CONTROL_RESULT_PEER_RESTARTED, "the peer restarted" );
1118 0 : }
1119 0 : return;
1120 0 : }
1121 0 : if( FD_LIKELY( !ctx->request.until || ctx->request.sent ) ) return;
1122 0 : if( FD_UNLIKELY( !ctx->request.peer.boot_id ) ) {
1123 0 : if( FD_UNLIKELY( !peer->holds ) ) {
1124 0 : if( FD_UNLIKELY( !ctx->request.addr ) ) return;
1125 0 : FD_LOG_WARNING(( "handoff %lu: dialed `" FD_IP4_ADDR_FMT ":%hu`, which does not run the identity", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), get_dial_port( ctx ) ));
1126 0 : finish_cmd( ctx, FD_FAILOVER_CONTROL_RESULT_NOT_HOLDER );
1127 0 : end_request( ctx, now, FD_FAILOVER_HANDOFF_NOT_HOLDER );
1128 0 : return;
1129 0 : }
1130 0 : ctx->request.peer.boot_id = peer->boot_id;
1131 0 : }
1132 :
1133 : /* send the HANDOFF_REQUEST over the channel */
1134 0 : fd_failover_handoff_request_t request = { .handoff_id=ctx->request.id, .target_boot_id=ctx->request.peer.boot_id, .replay_slot=ctx->replay_slot };
1135 0 : if( FD_UNLIKELY( fd_failover_channel_send( ctx->channel, now, FD_FAILOVER_MSG_HANDOFF_REQUEST, (uchar const *)&request, sizeof(request) ) ) ) return;
1136 0 : ctx->request.sent = 1;
1137 0 : ctx->request.send_cnt = fd_ulong_sat_add( ctx->request.send_cnt, 1UL );
1138 0 : if( FD_UNLIKELY( ctx->request.quick ) ) ctx->request.until = fd_long_max( ctx->request.until, fd_long_sat_add( now, FD_FAILOVER_DEADLINE_NANOS ) );
1139 0 : LOG_CMD_STEP( ctx, "handoff %lu: asked peer boot %016lx for the identity (send %lu)", ctx->request.id, ctx->request.peer.boot_id, ctx->request.send_cnt );
1140 0 : *charge_busy = 1;
1141 0 : }
1142 :
1143 : /* Operator commands and status */
1144 :
1145 : static ulong
1146 0 : get_settled_result( fd_failover_tile_ctx_t const * ctx ) {
1147 0 : if( FD_LIKELY( holds_identity( ctx ) ) ) return FD_ADMINCTL_RESULT_SUCCESS;
1148 0 : return ctx->reply.type==(ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED ? get_reject_result( ctx->reply.reason ) : FD_FAILOVER_CONTROL_RESULT_HANDOFF_ABORTED;
1149 0 : }
1150 :
1151 : /* Stages the current request with this run's address, flags and
1152 : deadline, and starts dialing.
1153 :
1154 : Returns the address it will dial, zero when none is known yet. */
1155 : static uint
1156 : stage_request( fd_failover_tile_ctx_t * ctx,
1157 : fd_adminctl_failover_req_t const * req,
1158 0 : long now ) {
1159 0 : uint addr = req->addr ? req->addr : ctx->staked_addr;
1160 0 : ctx->cmd_addr = req->addr;
1161 0 : ctx->cmd_port = req->port;
1162 0 : ctx->request.addr = addr;
1163 0 : ctx->request.quick = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER );
1164 0 : ctx->request.until = fd_long_sat_add( now, ctx->request.quick && !ctx->request.send_cnt ? FD_FAILOVER_FORCE_WAIT_NANOS : FD_FAILOVER_DEADLINE_NANOS );
1165 0 : ctx->request.sent = 0;
1166 0 : ctx->stuck = 0;
1167 0 : fd_failover_channel_init_dialer( ctx->channel, addr, get_dial_port( ctx ) );
1168 0 : return addr;
1169 0 : }
1170 :
1171 : /* `failover promote` asks the machine running the identity for it.
1172 : Opens a handoff request, to be driven by poll_peer().
1173 :
1174 : Returns SUCCESS when the command is settled and
1175 : FD_FAILOVER_CMD_PENDING when it waits for the outcome. */
1176 : static ulong
1177 : run_handoff_cmd( fd_failover_tile_ctx_t * ctx,
1178 : fd_adminctl_failover_req_t const * req,
1179 0 : long now ) {
1180 0 : int paused = is_request_paused( ctx ) && !is_switch_pending( ctx );
1181 0 : if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) {
1182 0 : if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) {
1183 0 : FD_LOG_WARNING(( "handoff %lu: the authenticated peer changed before it confirmed our answer, ending the confirmation wait", ctx->request.id ));
1184 0 : ulong result = get_settled_result( ctx );
1185 0 : end_request( ctx, now, holds_identity( ctx ) ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED );
1186 0 : return result;
1187 0 : }
1188 0 : FD_LOG_NOTICE(( "handoff %lu was bound to peer boot %016lx, the authenticated peer changed, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id ));
1189 0 : end_request( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED );
1190 0 : paused = 0;
1191 0 : }
1192 0 : if( FD_UNLIKELY( paused ) ) {
1193 0 : uint addr = stage_request( ctx, req, now );
1194 0 : if( FD_LIKELY( addr ) ) {
1195 0 : LOG_CMD_STEP( ctx, "handoff %lu: resuming at `" FD_IP4_ADDR_FMT ":%hu` using %s", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), get_dial_port( ctx ),
1196 0 : ctx->cmd_addr ? "`--address`" : "gossip" );
1197 0 : } else {
1198 0 : LOG_CMD_STEP( ctx, "handoff %lu: resuming once gossip shows the machine running the identity, pass --address to dial one now", ctx->request.id );
1199 0 : }
1200 0 : return ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ? get_settled_result( ctx ) : FD_FAILOVER_CMD_PENDING;
1201 0 : }
1202 0 : if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && !is_switch_pending( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_HANDED_OFF;
1203 0 : if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || is_switch_pending( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
1204 0 : if( FD_UNLIKELY( holds_identity( ctx ) ) ) return FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS;
1205 0 : if( FD_UNLIKELY( ctx->installed_ranked ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDS_OTHER;
1206 0 : fd_memset( &ctx->request, 0, sizeof(ctx->request) );
1207 0 : ctx->request.id = ctx->handoff_base+(++ctx->handoff_cnt);
1208 0 : if( FD_UNLIKELY( !ctx->request.id ) ) ctx->request.id = ++ctx->handoff_cnt;
1209 0 : ctx->session.close_after_send = 0;
1210 0 : uint addr = stage_request( ctx, req, now );
1211 0 : if( FD_LIKELY( addr ) ) {
1212 0 : LOG_CMD_STEP( ctx, "handoff %lu: requesting handoff from the machine running the identity at `" FD_IP4_ADDR_FMT ":%hu` using %s",
1213 0 : ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), get_dial_port( ctx ), ctx->cmd_addr ? "`--address`" : "gossip" );
1214 0 : } else {
1215 0 : LOG_CMD_STEP( ctx, "handoff %lu: waiting for gossip to show the machine running the identity, or for `--address` once this command returns", ctx->request.id );
1216 0 : }
1217 :
1218 0 : set_action( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER );
1219 0 : return FD_FAILOVER_CMD_PENDING;
1220 0 : }
1221 :
1222 : /* `failover takeover` (after the handoff failed) takes the identity
1223 : without coordination, with --force even from a machine that may be
1224 : live. */
1225 : static ulong
1226 : run_promote_cmd( fd_failover_tile_ctx_t * ctx,
1227 : fd_adminctl_failover_req_t const * req,
1228 0 : long now ) {
1229 0 : int force = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE );
1230 0 : if( FD_UNLIKELY( !( req->flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER ) ) ) return FD_ADMINCTL_RESULT_UNKNOWN_COMMAND;
1231 0 : static uchar const no_target[ 32 ] = { 0 };
1232 0 : if( FD_UNLIKELY( !fd_memeq( req->target, no_target, 32UL ) && !fd_memeq( req->target, ctx->hello.staked_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_IDENTITY_CHANGED;
1233 0 : ulong result = check_promotion( ctx );
1234 0 : if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result;
1235 0 : if( FD_UNLIKELY( !force && ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) {
1236 0 : return FD_FAILOVER_CONTROL_RESULT_HANDED_OFF;
1237 0 : }
1238 0 : if( FD_UNLIKELY( !force && is_holder_live( ctx, now ) ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDER_LIVE;
1239 0 : if( ctx->request.id && is_requesting( ctx ) ) end_request( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED );
1240 0 : int gave_up = ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE;
1241 0 : if( gave_up ) {
1242 0 : ctx->peer_history.valid = 0;
1243 0 : resolve_handoff( ctx, FD_FAILOVER_HANDOFF_CANCELLED );
1244 0 : }
1245 0 : start_promotion( ctx, FD_FAILOVER_SOURCE_EMPTY, NULL, 0UL, req->flags & (FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER|FD_ADMINCTL_FAILOVER_FLAG_FORCE) );
1246 0 : ctx->promotion.gave_up = gave_up;
1247 0 : return FD_FAILOVER_CMD_PENDING;
1248 0 : }
1249 :
1250 : static ulong
1251 : run_preflight_cmd( fd_failover_tile_ctx_t const * ctx,
1252 0 : fd_adminctl_failover_req_t const * req ) {
1253 0 : if( fd_memeq( req->target, ctx->hello.staked_pubkey, 32UL ) ) return FD_ADMINCTL_RESULT_SUCCESS;
1254 0 : if( FD_UNLIKELY( is_switch_pending( ctx ) || ( ctx->action!=FD_FAILOVER_ACTION_IDLE && !is_request_paused( ctx ) ) ) ) {
1255 0 : return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS;
1256 0 : }
1257 0 : if( FD_UNLIKELY( holds_identity( ctx ) && ctx->own_voted ) ) return FD_FAILOVER_CONTROL_RESULT_HOLDS_OTHER;
1258 0 : return FD_ADMINCTL_RESULT_SUCCESS;
1259 0 : }
1260 :
1261 : /* Runs an operator command. Returns either its result or
1262 : FD_FAILOVER_CMD_PENDING. */
1263 : static ulong
1264 : apply_control( fd_failover_tile_ctx_t * ctx,
1265 : fd_adminctl_failover_req_t const * req,
1266 0 : long now ) {
1267 0 : if( FD_UNLIKELY( ctx->cmd_nonce ) ) return FD_FAILOVER_CONTROL_RESULT_BUSY;
1268 0 : switch( req->cmd ) {
1269 0 : case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: return run_handoff_cmd( ctx, req, now );
1270 0 : case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return run_promote_cmd( ctx, req, now );
1271 0 : case FD_ADMINCTL_FAILOVER_CMD_PREFLIGHT: return run_preflight_cmd( ctx, req );
1272 0 : default: return FD_ADMINCTL_RESULT_UNSUPPORTED;
1273 0 : }
1274 0 : }
1275 :
1276 : /* Answers one bus frame with the result for nonce. */
1277 : static void
1278 : publish_bus_result( fd_failover_tile_ctx_t * ctx,
1279 : fd_stem_context_t * stem,
1280 : ulong nonce,
1281 0 : ulong result ) {
1282 0 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk );
1283 0 : fd_memset( out, 0, sizeof(*out) );
1284 0 : out->nonce = nonce;
1285 0 : out->result = result;
1286 0 : FD_STORE( ulong, out->payload, ctx->last_vote_slot );
1287 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1288 0 : fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_RESPONSE, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub );
1289 0 : ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark );
1290 0 : }
1291 :
1292 : /* Publishes the outcome finish_cmd recorded. */
1293 : static void
1294 : publish_cmd_answer( fd_failover_tile_ctx_t * ctx,
1295 0 : fd_stem_context_t * stem ) {
1296 0 : publish_bus_result( ctx, stem, ctx->cmd_nonce, ctx->cmd_result );
1297 0 : ctx->cmd_nonce = 0UL;
1298 0 : ctx->cmd_done = 0;
1299 0 : ctx->cmd_step_tail = ctx->cmd_step_head;
1300 0 : }
1301 :
1302 : /* Publishes the oldest unsent step of the pending command. */
1303 : static void
1304 : publish_cmd_step( fd_failover_tile_ctx_t * ctx,
1305 0 : fd_stem_context_t * stem ) {
1306 0 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk );
1307 0 : fd_memset( out, 0, sizeof(*out) );
1308 0 : out->nonce = ctx->cmd_nonce;
1309 0 : fd_cstr_ncpy( (char *)out->payload, ctx->cmd_step[ ctx->cmd_step_tail%FD_FAILOVER_STEP_CNT ], FD_FAILOVER_BUS_STEP_MAX );
1310 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1311 0 : fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_STEP, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub );
1312 0 : ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark );
1313 0 : ctx->cmd_step_tail++;
1314 0 : }
1315 :
1316 : /* Runs one operator command. The admin tile validated the ABI already. */
1317 : static void
1318 : serve_bus_request( fd_failover_tile_ctx_t * ctx,
1319 : fd_stem_context_t * stem,
1320 0 : long now ) {
1321 0 : fd_adminctl_failover_req_t req;
1322 0 : fd_memcpy( &req, ctx->bus_req.payload, sizeof(req) );
1323 0 : char const * cmd_name = fd_adminctl_failover_cmd_typed( req.cmd, req.flags );
1324 0 : ulong step_head = ctx->cmd_step_head;
1325 0 : ulong result = apply_control( ctx, &req, now );
1326 0 : if( FD_UNLIKELY( result==FD_FAILOVER_CMD_PENDING ) ) {
1327 0 : ctx->cmd_nonce = ctx->bus_req.nonce;
1328 0 : ctx->cmd_step_tail = step_head;
1329 0 : return;
1330 0 : }
1331 0 : if( FD_UNLIKELY( result==FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS ) ) FD_LOG_NOTICE(( "`%s` done, this machine already runs the identity", cmd_name ));
1332 0 : else if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) FD_LOG_WARNING(( "`%s` refused (result %#lx)", cmd_name, result ));
1333 0 : publish_bus_result( ctx, stem, ctx->bus_req.nonce, result );
1334 0 : }
|
