Line data Source code
1 : #include "../../disco/topo/fd_topo.h"
2 : #include "../../disco/events/generated/fd_event_gen.h"
3 : #include "../../disco/keyguard/fd_keyswitch.h"
4 : #include "../../disco/keyguard/fd_keyload.h"
5 :
6 : #include "fd_adminctl.h"
7 : #include "../failover/fd_failover_bus.h"
8 : #include <linux/futex.h>
9 : #include "generated/fd_admin_tile_seccomp.h"
10 :
11 : struct fd_admin_tile_ctx {
12 : fd_topo_t const * topo;
13 : fd_adminctl_t * adminctl;
14 : uchar identity_pubkey[ 32UL ];
15 : fd_keyswitch_t * voter_av_keyswitch;
16 : fd_keyswitch_t * sign_av_keyswitch[ FD_TOPO_MAX_TILES ];
17 : ulong sign_av_keyswitch_cnt;
18 : fd_sha512_t sha512[ 1 ];
19 :
20 : ulong replay_out_idx; /* admin_replay stem out index */
21 : ulong snap_create_slot_idx; /* adminctl slot of snapshot-create command */
22 : ulong snap_create_target_slot; /* requested slot retained until Replay responds */
23 : ulong snap_create_start_time; /* command start retained until Replay responds */
24 :
25 : long poll_ticks;
26 : long idle_poll_ticks;
27 : long deadline_ticks; /* next adminctl poll */
28 :
29 : int failover_enabled; /* failover hands off the identity */
30 : int failover_switch; /* a failover identity switch is in progress */
31 : ulong failover_epoch; /* failover epoch, bumped by every set-identity under failover */
32 : int failover_pending; /* the failover tile has not been told about the last set-identity yet */
33 : uchar failover_pubkey[ 32 ]; /* the key failover hands off, [paths.identity_key] until set-identity or a prepare picks another */
34 :
35 : /* Failover commands go to the failover tile over the bus, one at a
36 : time like snapshot creation. */
37 : ulong failov_out_idx; /* admin_failov stem out index */
38 : fd_wksp_t * failov_out_mem;
39 : ulong failov_out_chunk0;
40 : ulong failov_out_wmark;
41 : ulong failov_out_chunk;
42 : ulong failov_in_idx; /* failov_admin stem in index */
43 : ulong in_cnt; /* stem ins, failov_admin is found among them */
44 : fd_wksp_t * failov_in_mem;
45 : ulong failov_in_chunk0;
46 : ulong failov_in_wmark;
47 : fd_failover_bus_msg_t failov_in; /* frame copied in during_frag */
48 : ulong failover_slot_idx; /* adminctl slot parked on the bus, or ULONG_MAX */
49 : ulong failover_slot_cmd; /* FD_ADMINCTL_FAILOVER_CMD_* of the parked command */
50 : ulong failover_nonce; /* nonce of the last command forwarded to the failover tile */
51 : ulong failover_answered; /* nonce of the last command the failover tile responded to */
52 : ulong failover_start_time; /* command start retained until the failover tile responds */
53 : char failover_args_json[ 96 ]; /* the parked command and its flags for the event */
54 : ulong failover_args_json_len;
55 : char failover_cmd_cstr[ 48 ]; /* the parked command and its flags for the log */
56 : int failover_retargeted; /* the parked command changed the failover target before it was forwarded */
57 : };
58 :
59 : typedef struct fd_admin_tile_ctx fd_admin_tile_ctx_t;
60 :
61 : static inline fd_event_admin_command_t
62 : prepare_admin_command( int type,
63 : void const * payload,
64 0 : ulong payload_sz ) {
65 0 : fd_event_admin_command_t event = {
66 0 : .type = type,
67 0 : .args_json = { '{', '}' },
68 0 : .args_json_len = 2UL,
69 0 : .start_time = (ulong)fd_log_wallclock(),
70 0 : .payload_size = payload_sz,
71 0 : .has_payload_version = 0,
72 0 : };
73 0 : if( FD_LIKELY( payload_sz>=sizeof(ulong) ) ) {
74 0 : event.payload_version = FD_LOAD( ulong, payload );
75 0 : event.has_payload_version = 1;
76 0 : }
77 0 : return event;
78 0 : }
79 :
80 : static inline void
81 : report_admin_command( fd_event_admin_command_t * event,
82 0 : int result ) {
83 0 : FD_TEST( result>=0 && result<=FD_EVENT_ADMIN_COMMAND_RESULT_CUSTOM );
84 0 : event->result = result;
85 0 : event->end_time = (ulong)fd_log_wallclock();
86 0 : fd_event_report_admin_command( event );
87 0 : }
88 :
89 : static inline void
90 : report_admin_command_custom_result( fd_event_admin_command_t * event,
91 0 : char const * custom_result ) {
92 0 : FD_TEST( fd_cstr_printf_check( (char *)event->custom_result,
93 0 : sizeof(event->custom_result),
94 0 : &event->custom_result_len,
95 0 : "%s",
96 0 : custom_result ) );
97 0 : report_admin_command( event, FD_EVENT_ADMIN_COMMAND_RESULT_CUSTOM );
98 0 : }
99 :
100 : FD_FN_CONST static inline ulong
101 0 : scratch_align( void ) {
102 0 : return alignof(fd_admin_tile_ctx_t);
103 0 : }
104 :
105 : FD_FN_PURE static inline ulong
106 0 : scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
107 0 : return sizeof(fd_admin_tile_ctx_t);
108 0 : }
109 :
110 : static void
111 : privileged_init( fd_topo_t const * topo,
112 0 : fd_topo_tile_t const * tile ) {
113 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
114 0 : fd_admin_tile_ctx_t * ctx = (fd_admin_tile_ctx_t *)scratch;
115 0 : fd_memset( ctx, 0, sizeof(fd_admin_tile_ctx_t) );
116 :
117 0 : if( FD_UNLIKELY( !strcmp( tile->admin.identity_key_path, "" ) ) )
118 0 : FD_LOG_ERR(( "identity_key_path not set" ));
119 :
120 0 : fd_memcpy( ctx->identity_pubkey, fd_keyload_load( tile->admin.identity_key_path, /* pubkey only: */ 1 ), 32UL );
121 :
122 : /* With failover, the configured identity is the key that failover
123 : hands off, whichever key this machine booted under. */
124 0 : ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL );
125 0 : if( FD_UNLIKELY( failov_idx!=ULONG_MAX ) ) {
126 0 : uchar const * target = fd_keyload_load( topo->tiles[ failov_idx ].failov.target_key_path, /* pubkey only: */ 1 );
127 0 : fd_memcpy( ctx->failover_pubkey, target, 32UL );
128 0 : fd_keyload_unload( target, 1 );
129 0 : }
130 0 : }
131 :
132 : static void
133 : unprivileged_init( fd_topo_t const * topo,
134 0 : fd_topo_tile_t const * tile ) {
135 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
136 0 : fd_admin_tile_ctx_t * ctx = (fd_admin_tile_ctx_t *)scratch;
137 0 : ctx->replay_out_idx = ULONG_MAX;
138 0 : ctx->snap_create_slot_idx = ULONG_MAX;
139 0 : ctx->poll_ticks = (long)(1e6*fd_tempo_tick_per_ns( NULL )); /* 1ms */
140 0 : ctx->idle_poll_ticks = (long)((double)FD_SLEEP_PARK_CAP_NS*fd_tempo_tick_per_ns( NULL ));
141 0 : ctx->deadline_ticks = 0L;
142 0 : ctx->failover_enabled = fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX;
143 0 : ctx->failover_slot_idx = ULONG_MAX;
144 0 : ctx->topo = topo;
145 :
146 0 : fd_topo_obj_t const * adminctl_obj = fd_topo_find_tile_obj( topo, tile, "adminctl" );
147 0 : FD_TEST( adminctl_obj );
148 :
149 0 : ctx->adminctl = fd_adminctl_join( fd_topo_obj_laddr( topo, adminctl_obj->id ) );
150 0 : FD_TEST( ctx->adminctl );
151 :
152 0 : ctx->replay_out_idx = fd_topo_find_tile_out_link( topo, tile, "admin_replay", 0UL );
153 :
154 : /* The failover bus links exist only when the failover tile does. */
155 0 : ctx->failov_out_idx = fd_topo_find_tile_out_link( topo, tile, "admin_failov", 0UL );
156 0 : ctx->failov_in_idx = fd_topo_find_tile_in_link ( topo, tile, "failov_admin", 0UL );
157 0 : ctx->in_cnt = tile->in_cnt;
158 0 : if( FD_UNLIKELY( (ctx->failov_out_idx==ULONG_MAX)!=(ctx->failov_in_idx==ULONG_MAX) ) ) {
159 0 : FD_LOG_ERR(( "admin tile needs both the admin_failov and failov_admin links or neither" ));
160 0 : }
161 0 : if( FD_UNLIKELY( ctx->failov_out_idx!=ULONG_MAX ) ) {
162 0 : fd_topo_link_t const * out_link = &topo->links[ tile->out_link_id[ ctx->failov_out_idx ] ];
163 0 : ctx->failov_out_mem = topo->workspaces[ topo->objs[ out_link->dcache_obj_id ].wksp_id ].wksp;
164 0 : ctx->failov_out_chunk0 = fd_dcache_compact_chunk0( ctx->failov_out_mem, out_link->dcache );
165 0 : ctx->failov_out_wmark = fd_dcache_compact_wmark ( ctx->failov_out_mem, out_link->dcache, out_link->mtu );
166 0 : ctx->failov_out_chunk = ctx->failov_out_chunk0;
167 :
168 0 : fd_topo_link_t const * in_link = &topo->links[ tile->in_link_id[ ctx->failov_in_idx ] ];
169 0 : ctx->failov_in_mem = topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ].wksp;
170 0 : ctx->failov_in_chunk0 = fd_dcache_compact_chunk0( ctx->failov_in_mem, in_link->dcache );
171 0 : ctx->failov_in_wmark = fd_dcache_compact_wmark ( ctx->failov_in_mem, in_link->dcache, in_link->mtu );
172 0 : }
173 :
174 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
175 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
176 : /* The stem numbers only the in links it polls, and we compare its
177 : numbers with topology in link indexes, so with failover every in
178 : link is polled. */
179 0 : if( FD_UNLIKELY( ctx->failov_in_idx!=ULONG_MAX ) ) FD_TEST( tile->in_link_poll[ i ] );
180 0 : if( FD_UNLIKELY( i==ctx->failov_in_idx ) ) continue;
181 0 : if( FD_UNLIKELY( strcmp( link->name, "replay_admin" ) ) ) {
182 0 : FD_LOG_ERR(( "unexpected input link name %s", link->name ));
183 0 : }
184 0 : }
185 :
186 0 : ulong voter_idx = fd_topo_find_tile( topo, "votor", 0UL );
187 0 : FD_TEST( voter_idx!=ULONG_MAX );
188 0 : FD_TEST( topo->tiles[ voter_idx ].av_keyswitch_obj_id!=ULONG_MAX );
189 0 : ctx->voter_av_keyswitch = fd_keyswitch_join( fd_topo_obj_laddr( topo, topo->tiles[ voter_idx ].av_keyswitch_obj_id ) );
190 0 : FD_TEST( ctx->voter_av_keyswitch );
191 :
192 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
193 0 : fd_topo_tile_t const * sign_tile = &topo->tiles[ i ];
194 0 : if( FD_LIKELY( strcmp( sign_tile->name, "sign" ) ) ) continue;
195 0 : FD_TEST( sign_tile->av_keyswitch_obj_id!=ULONG_MAX );
196 0 : ctx->sign_av_keyswitch[ ctx->sign_av_keyswitch_cnt ] = fd_keyswitch_join( fd_topo_obj_laddr( topo, sign_tile->av_keyswitch_obj_id ) );
197 0 : FD_TEST( ctx->sign_av_keyswitch[ ctx->sign_av_keyswitch_cnt ] );
198 0 : ctx->sign_av_keyswitch_cnt++;
199 0 : }
200 0 : FD_TEST( ctx->sign_av_keyswitch_cnt );
201 :
202 0 : FD_TEST( fd_sha512_join( fd_sha512_new( ctx->sha512 ) ) );
203 0 : }
204 :
205 : /* The process of switching identity of the validator is somewhat
206 : involved, to prevent it from producing torn data (for example,
207 : a block where half the shreds are signed by one private key, and half
208 : are signed by another).
209 :
210 : The process of switching is a state machine that progresses linearly
211 : through each of the states. Generally, no transitions are allowed
212 : except direct forward steps, except in emergency recovery cases an
213 : operator can force the state past the initial lock.
214 :
215 : Votor is referred to as the Voter tile for the purposes of the
216 : identity switch.
217 :
218 : The states follow, in order. */
219 :
220 : /* State 0: UNLOCKED.
221 : The validator is not currently in the process of switching keys. */
222 0 : #define FD_SET_IDENTITY_STATE_UNLOCKED (0UL)
223 :
224 : /* State 1: LOCKED
225 : Some client to the validator has requested a key switch. To do so,
226 : it acquired an exclusive lock on the validator to prevent the
227 : switch potentially being interleaved with another client. */
228 0 : #define FD_SET_IDENTITY_STATE_LOCKED (1UL)
229 :
230 : /* State 2: REPLAY_HALT_REQUESTED
231 : The first step in the key switch process is to pause Replay,
232 : preventing us from becoming leader or completing additional slots,
233 : but finishing any currently in progress leader slot if there is
234 : one. Replay continues polling incoming links while paused so
235 : downstream tiles can drain without a reliable-link cycle.
236 :
237 : In Firedancer, this halt request goes to the Replay tile, which
238 : causes the tile to switch the identity key it uses to determine the
239 : identity's balance as well as when the validator is the leader.
240 : Replay reports the producer sequence of replay_slot, the link votor
241 : reads, as it switches. */
242 0 : #define FD_SET_IDENTITY_STATE_REPLAY_HALT_REQUESTED (2UL)
243 :
244 : /* State 3: REPLAY_HALTED
245 : Replay has paused and the validator is no longer leader. No more
246 : slots will complete until Replay is unhalted. Replay has switched
247 : its own identity key and reported the voter's replay link sequence.
248 :
249 : At this point, we also have the guarantee that there are no more
250 : outstanding shreds that have to be signed with the old key. Any
251 : tiles related to the leader pipeline that rely on the identity key
252 : will not be used. */
253 0 : #define FD_SET_IDENTITY_STATE_REPLAY_HALTED (3UL)
254 :
255 : /* State 4: VOTER_HALT_REQUESTED
256 : Voter has been requested to consume its replay link through the
257 : sequence Replay reported, stop producing new vote transactions by
258 : backpressuring SLOT_COMPLETED, drain its local publish queue, and
259 : switch identity. */
260 0 : #define FD_SET_IDENTITY_STATE_VOTER_HALT_REQUESTED (4UL)
261 :
262 : /* State 5: VOTER_HALTED
263 : Voter has drained its pre-switch publish queue, switched identity,
264 : and reported the sequence after the final drained message. */
265 0 : #define FD_SET_IDENTITY_STATE_VOTER_HALTED (5UL)
266 :
267 : /* State 6: SIGNERS_HALT_REQUESTED
268 : Repair, Gossip, Bundle, and Rserve will stop sending requests
269 : downstream to the sign tile. This is done to avoid any mismatches
270 : with the identity key. Their identity keys will be switched during
271 : this step, except for Gossip, which switches during
272 : SIGNERS_UNHALT_REQUESTED.
273 :
274 : These tiles use the identity key to populate messages signed by the
275 : sign tile:
276 : (a) Repair. The repair tile uses the identity key as part of the
277 : repair protocol. The identity key is included in and used
278 : for signing requests. Because Repair uses an asynchronous
279 : signing mechanism, Repair will first wait until all
280 : outstanding sign requests have been received back from the
281 : sign tile before halting any new signing requests.
282 : (b) Gossip. The gossip tile sends out ContactInfo messages with
283 : our identity key, and also uses the identity key to sign
284 : outgoing gossip messages.
285 : (c) Bundle. The bundle tile uses the identity key to sign an
286 : authentication challenge from the bundle server.
287 : (d) Rserve. The rserve tile uses the identity key to sign
288 : outgoing pings.
289 : (e) Shred. The shred tile has the sign tile sign FEC sets
290 : asynchronously; it waits for its outstanding requests to be
291 : answered under the old key before switching. */
292 0 : #define FD_SET_IDENTITY_STATE_SIGNERS_HALT_REQUESTED (6UL)
293 :
294 : /* State 7: SIGNERS_HALTED
295 : Repair, Gossip, Bundle, and Rserve are no longer sending requests
296 : to the sign tile. */
297 0 : #define FD_SET_IDENTITY_STATE_SIGNERS_HALTED (7UL)
298 :
299 : /* State 8: ALL_SWITCH_REQUESTED
300 : The client now requests that all other tiles which consume the
301 : identity key in some way switch to the new key. The leader
302 : pipeline is still halted, although it doesn't strictly need to be,
303 : since outgoing shreds have been flushed. This is done to keep the
304 : control flow simpler. The sign tile's switch is requested first to
305 : avoid any potential mismatches with the identity key.
306 :
307 : The other tiles using the identity key are:
308 : (a) Sign. The sign tile is responsible for holding the private
309 : key and servicing signing requests from other tiles.
310 : (b) GUI. The GUI shows the validator identity key to the user,
311 : and uses the key to determine which blocks are ours for
312 : highlighting on the frontend.
313 : (c) Gossvf. The gossvf tile uses the identity key to detect
314 : duplicate running instances of the same validator node as
315 : well as other message handling.
316 : (d) Shred. The shred tile uses the identity key to determine the
317 : position of the validator in the Turbine tree and to sign
318 : outgoing shreds.
319 : (e) Event. Outgoing events to the event server are signed with
320 : the identity key to authenticate the sender. */
321 0 : #define FD_SET_IDENTITY_STATE_ALL_SWITCH_REQUESTED (8UL)
322 :
323 : /* State 9: ALL_SWITCHED
324 : All remaining tiles that use the identity key have confirmed that
325 : they have switched to the new key. Gossip has not yet updated its
326 : identity key. Repair, Gossip, Bundle, and Rserve remain halted. */
327 0 : #define FD_SET_IDENTITY_STATE_ALL_SWITCHED (9UL)
328 :
329 : /* State 10: SIGNERS_UNHALT_REQUESTED
330 : Now that the sign tile is using the switched identity key, the
331 : halted signers can be unhalted. Gossip switches its identity key
332 : during this step. These are the same tiles from
333 : SIGNERS_HALT_REQUESTED. */
334 0 : #define FD_SET_IDENTITY_STATE_SIGNERS_UNHALT_REQUESTED (10UL)
335 :
336 : /* State 11: SIGNERS_UNHALTED
337 : All tiles that rely on the sign tile have been unhalted. Replay
338 : remains paused. */
339 0 : #define FD_SET_IDENTITY_STATE_SIGNERS_UNHALTED (11UL)
340 :
341 : /* State 12: REPLAY_UNHALT_REQUESTED
342 : The final state, now that all tiles have switched, Replay can be
343 : unpaused and the validator can resume processing and producing
344 : blocks. The next state once Replay confirms it is unpaused is
345 : UNLOCKED. */
346 0 : #define FD_SET_IDENTITY_STATE_REPLAY_UNHALT_REQUESTED (12UL)
347 :
348 : static fd_keyswitch_t *
349 : find_identity_keyswitch( fd_admin_tile_ctx_t * ctx,
350 0 : char const * tile_name ) {
351 0 : fd_topo_t const * topo = ctx->topo;
352 0 : ulong tile_idx = fd_topo_find_tile( topo, tile_name, 0UL );
353 0 : FD_TEST( tile_idx!=ULONG_MAX );
354 0 : FD_TEST( topo->tiles[ tile_idx ].id_keyswitch_obj_id!=ULONG_MAX );
355 :
356 0 : fd_keyswitch_t * keyswitch = fd_topo_obj_laddr( topo, topo->tiles[ tile_idx ].id_keyswitch_obj_id );
357 0 : FD_TEST( keyswitch );
358 0 : return keyswitch;
359 0 : }
360 :
361 : static int FD_FN_SENSITIVE
362 : poll_set_identity( fd_admin_tile_ctx_t * ctx,
363 : ulong * state,
364 : ulong identity_outset,
365 : uchar * keypair,
366 0 : ulong vote_history_slot ) {
367 0 : fd_topo_t const * topo = ctx->topo;
368 :
369 0 : switch( *state ) {
370 0 : case FD_SET_IDENTITY_STATE_UNLOCKED: {
371 0 : fd_keyswitch_t * replay = find_identity_keyswitch( ctx, "replay" );
372 0 : if( FD_LIKELY( FD_KEYSWITCH_STATE_UNLOCKED==FD_ATOMIC_CAS( &replay->state, FD_KEYSWITCH_STATE_UNLOCKED, FD_KEYSWITCH_STATE_LOCKED ) ) ) {
373 0 : *state = FD_SET_IDENTITY_STATE_LOCKED;
374 0 : FD_LOG_INFO(( "Locking validator identity for key switch..." ));
375 0 : } else {
376 0 : FD_LOG_CRIT(( "identity keyswitch is in a locked state but should be unlocked" ));
377 0 : }
378 0 : break;
379 0 : }
380 0 : case FD_SET_IDENTITY_STATE_LOCKED: {
381 0 : fd_keyswitch_t * replay = find_identity_keyswitch( ctx, "replay" );
382 0 : memcpy( replay->bytes, keypair+32UL, 32UL );
383 :
384 0 : FD_COMPILER_MFENCE();
385 0 : replay->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
386 0 : FD_COMPILER_MFENCE();
387 0 : *state = FD_SET_IDENTITY_STATE_REPLAY_HALT_REQUESTED;
388 0 : FD_LOG_INFO(( "Pausing Replay for key switch..." ));
389 0 : break;
390 0 : }
391 0 : case FD_SET_IDENTITY_STATE_REPLAY_HALT_REQUESTED: {
392 0 : fd_keyswitch_t * replay = find_identity_keyswitch( ctx, "replay" );
393 0 : if( FD_LIKELY( replay->state==FD_KEYSWITCH_STATE_COMPLETED ) ) {
394 0 : fd_memzero_explicit( replay->bytes, 64UL );
395 0 : FD_COMPILER_MFENCE();
396 0 : *state = FD_SET_IDENTITY_STATE_REPLAY_HALTED;
397 0 : FD_LOG_INFO(( "Replay successfully paused..." ));
398 0 : } else if( FD_UNLIKELY( replay->state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) {
399 0 : FD_SPIN_PAUSE();
400 0 : } else {
401 0 : FD_LOG_ERR(( "Unexpected replay keyswitch state %lu", replay->state ));
402 0 : }
403 0 : break;
404 0 : }
405 0 : case FD_SET_IDENTITY_STATE_REPLAY_HALTED: {
406 0 : fd_keyswitch_t * replay = find_identity_keyswitch( ctx, "replay" );
407 0 : fd_keyswitch_t * voter = find_identity_keyswitch( ctx, "votor" );
408 0 : voter->param = replay->result;
409 0 : memcpy( voter->bytes, keypair+32UL, 32UL );
410 : /* Votor takes the slot to wait to vote past. */
411 0 : FD_STORE( ulong, voter->bytes+32UL, vote_history_slot );
412 0 : FD_COMPILER_MFENCE();
413 0 : voter->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
414 0 : FD_COMPILER_MFENCE();
415 :
416 0 : *state = FD_SET_IDENTITY_STATE_VOTER_HALT_REQUESTED;
417 0 : FD_LOG_INFO(( "Pausing votor and draining queued messages..." ));
418 0 : break;
419 0 : }
420 0 : case FD_SET_IDENTITY_STATE_VOTER_HALT_REQUESTED: {
421 0 : fd_keyswitch_t * voter = find_identity_keyswitch( ctx, "votor" );
422 0 : if( FD_LIKELY( voter->state==FD_KEYSWITCH_STATE_COMPLETED ) ) {
423 0 : fd_memzero_explicit( voter->bytes, 64UL );
424 0 : FD_COMPILER_MFENCE();
425 0 : *state = FD_SET_IDENTITY_STATE_VOTER_HALTED;
426 0 : FD_LOG_INFO(( "votor successfully paused..." ));
427 0 : } else if( FD_UNLIKELY( voter->state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) {
428 0 : FD_SPIN_PAUSE();
429 0 : } else {
430 0 : FD_LOG_ERR(( "Unexpected votor keyswitch state %lu", voter->state ));
431 0 : }
432 0 : break;
433 0 : }
434 0 : case FD_SET_IDENTITY_STATE_VOTER_HALTED: {
435 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
436 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
437 0 : if( FD_LIKELY( tile->id_keyswitch_obj_id==ULONG_MAX ) ) continue;
438 0 : if( strcmp( tile->name, "repair" ) &&
439 0 : strcmp( tile->name, "gossip" ) &&
440 0 : strcmp( tile->name, "bundle" ) &&
441 0 : strcmp( tile->name, "rserve" ) &&
442 0 : strcmp( tile->name, "shred" ) ) {
443 0 : continue;
444 0 : }
445 :
446 0 : fd_keyswitch_t * tile_ks = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
447 0 : tile_ks->param = 0UL;
448 0 : memcpy( tile_ks->bytes, keypair+32UL, 32UL );
449 0 : FD_COMPILER_MFENCE();
450 0 : tile_ks->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
451 0 : FD_COMPILER_MFENCE();
452 0 : }
453 0 : *state = FD_SET_IDENTITY_STATE_SIGNERS_HALT_REQUESTED;
454 0 : FD_LOG_INFO(( "Requesting to halt remaining signers..." ));
455 0 : break;
456 0 : }
457 0 : case FD_SET_IDENTITY_STATE_SIGNERS_HALT_REQUESTED: {
458 0 : int all_switched = 1;
459 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
460 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
461 0 : if( FD_LIKELY( tile->id_keyswitch_obj_id==ULONG_MAX ) ) continue;
462 0 : if( strcmp( tile->name, "repair" ) &&
463 0 : strcmp( tile->name, "gossip" ) &&
464 0 : strcmp( tile->name, "bundle" ) &&
465 0 : strcmp( tile->name, "rserve" ) &&
466 0 : strcmp( tile->name, "shred" ) ) {
467 0 : continue;
468 0 : }
469 :
470 0 : fd_keyswitch_t * tile_ks = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
471 0 : if( FD_LIKELY( tile_ks->state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) {
472 0 : all_switched = 0;
473 0 : break;
474 0 : }
475 0 : }
476 0 : if( FD_LIKELY( all_switched ) ) {
477 0 : FD_LOG_INFO(( "All remaining signers successfully halted..." ));
478 0 : *state = FD_SET_IDENTITY_STATE_SIGNERS_HALTED;
479 0 : } else {
480 0 : FD_SPIN_PAUSE();
481 0 : }
482 0 : break;
483 0 : }
484 0 : case FD_SET_IDENTITY_STATE_SIGNERS_HALTED: {
485 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
486 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
487 0 : if( strcmp( tile->name, "sign" ) ) continue;
488 0 : fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
489 0 : memcpy( sign->bytes, keypair, 64UL );
490 0 : if( FD_UNLIKELY( ctx->failover_switch ) ) {
491 : /* With failover, the sign tile selects the junk key or the
492 : failover identity by its public key. */
493 0 : sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY;
494 0 : fd_memcpy( sign->bytes, keypair+32UL, 32UL );
495 0 : fd_memset( sign->bytes+32UL, 0, 32UL );
496 0 : }
497 0 : FD_COMPILER_MFENCE();
498 0 : sign->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
499 0 : FD_COMPILER_MFENCE();
500 0 : }
501 :
502 0 : fd_memzero_explicit( keypair, 32UL ); /* Private key no longer needed by the admin tile. */
503 :
504 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
505 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
506 0 : if( FD_LIKELY( tile->id_keyswitch_obj_id==ULONG_MAX ) ) continue;
507 0 : if( FD_LIKELY( !strcmp( tile->name, "sign" ) ||
508 0 : !strcmp( tile->name, "replay" ) ||
509 0 : !strcmp( tile->name, "repair" ) ||
510 0 : !strcmp( tile->name, "gossip" ) ||
511 0 : !strcmp( tile->name, "votor" ) ||
512 0 : !strcmp( tile->name, "bundle" ) ||
513 0 : !strcmp( tile->name, "rserve" ) ||
514 0 : !strcmp( tile->name, "shred" ) ) ) continue;
515 :
516 0 : fd_keyswitch_t * tile_ks = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
517 0 : if( !strcmp( tile->name, "gossvf" ) ) tile_ks->param = identity_outset;
518 0 : memcpy( tile_ks->bytes, keypair+32UL, 32UL );
519 0 : FD_COMPILER_MFENCE();
520 0 : tile_ks->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
521 0 : FD_COMPILER_MFENCE();
522 0 : }
523 :
524 0 : FD_LOG_INFO(( "Requesting all remaining tiles switch identity key..." ));
525 0 : *state = FD_SET_IDENTITY_STATE_ALL_SWITCH_REQUESTED;
526 0 : break;
527 0 : }
528 0 : case FD_SET_IDENTITY_STATE_ALL_SWITCH_REQUESTED: {
529 0 : ulong all_switched = 1UL;
530 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
531 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
532 0 : if( FD_LIKELY( tile->id_keyswitch_obj_id==ULONG_MAX ) ) continue;
533 0 : if( FD_LIKELY( !strcmp( tile->name, "replay" ) ||
534 0 : !strcmp( tile->name, "repair" ) ||
535 0 : !strcmp( tile->name, "gossip" ) ||
536 0 : !strcmp( tile->name, "votor" ) ||
537 0 : !strcmp( tile->name, "bundle" ) ||
538 0 : !strcmp( tile->name, "rserve" ) ||
539 0 : !strcmp( tile->name, "shred" ) ) ) continue;
540 :
541 0 : fd_keyswitch_t * tile_ks = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
542 0 : if( FD_LIKELY( tile_ks->state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) {
543 0 : all_switched = 0UL;
544 0 : break;
545 0 : } else if( FD_UNLIKELY( tile_ks->state==FD_KEYSWITCH_STATE_COMPLETED ) ) {
546 0 : if( FD_LIKELY( !strcmp( tile->name, "sign" ) ) ) {
547 0 : FD_COMPILER_MFENCE();
548 0 : fd_memzero_explicit( tile_ks->bytes, 64UL );
549 0 : FD_COMPILER_MFENCE();
550 0 : }
551 0 : continue;
552 0 : } else {
553 0 : FD_LOG_ERR(( "Unexpected %s keyswitch state %lu", tile->name, tile_ks->state ));
554 0 : }
555 0 : }
556 :
557 0 : if( FD_LIKELY( all_switched ) ) {
558 0 : FD_LOG_INFO(( "All tiles successfully switched identity key..." ));
559 0 : *state = FD_SET_IDENTITY_STATE_ALL_SWITCHED;
560 0 : } else {
561 0 : FD_SPIN_PAUSE();
562 0 : }
563 0 : break;
564 0 : }
565 0 : case FD_SET_IDENTITY_STATE_ALL_SWITCHED: {
566 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
567 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
568 0 : if( FD_LIKELY( tile->id_keyswitch_obj_id==ULONG_MAX ) ) continue;
569 0 : if( strcmp( tile->name, "repair" ) &&
570 0 : strcmp( tile->name, "gossip" ) &&
571 0 : strcmp( tile->name, "votor" ) &&
572 0 : strcmp( tile->name, "bundle" ) &&
573 0 : strcmp( tile->name, "rserve" ) ) {
574 0 : continue;
575 0 : }
576 :
577 0 : fd_keyswitch_t * tile_ks = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
578 0 : if( !strcmp( tile->name, "gossip" ) ) tile_ks->param = identity_outset;
579 0 : FD_COMPILER_MFENCE();
580 0 : tile_ks->state = FD_KEYSWITCH_STATE_UNHALT_PENDING;
581 0 : FD_COMPILER_MFENCE();
582 0 : }
583 :
584 0 : FD_LOG_INFO(( "Requesting to unpause signers..." ));
585 0 : *state = FD_SET_IDENTITY_STATE_SIGNERS_UNHALT_REQUESTED;
586 0 : break;
587 0 : }
588 0 : case FD_SET_IDENTITY_STATE_SIGNERS_UNHALT_REQUESTED: {
589 0 : int all_switched = 1;
590 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
591 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
592 0 : if( FD_LIKELY( tile->id_keyswitch_obj_id==ULONG_MAX ) ) continue;
593 0 : if( strcmp( tile->name, "repair" ) &&
594 0 : strcmp( tile->name, "gossip" ) &&
595 0 : strcmp( tile->name, "votor" ) &&
596 0 : strcmp( tile->name, "bundle" ) &&
597 0 : strcmp( tile->name, "rserve" ) ) {
598 0 : continue;
599 0 : }
600 :
601 0 : fd_keyswitch_t * tile_ks = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
602 0 : if( FD_LIKELY( tile_ks->state==FD_KEYSWITCH_STATE_UNHALT_PENDING ) ) {
603 0 : all_switched = 0;
604 0 : break;
605 0 : }
606 0 : }
607 0 : if( FD_LIKELY( all_switched ) ) {
608 0 : FD_LOG_INFO(( "Successfully unpaused all non-leader signers..." ));
609 0 : *state = FD_SET_IDENTITY_STATE_SIGNERS_UNHALTED;
610 0 : } else {
611 0 : FD_SPIN_PAUSE();
612 0 : }
613 0 : break;
614 0 : }
615 0 : case FD_SET_IDENTITY_STATE_SIGNERS_UNHALTED: {
616 0 : fd_keyswitch_t * replay = find_identity_keyswitch( ctx, "replay" );
617 0 : replay->state = FD_KEYSWITCH_STATE_UNHALT_PENDING;
618 0 : FD_LOG_INFO(( "Requesting to unpause Replay..." ));
619 0 : *state = FD_SET_IDENTITY_STATE_REPLAY_UNHALT_REQUESTED;
620 0 : break;
621 0 : }
622 0 : case FD_SET_IDENTITY_STATE_REPLAY_UNHALT_REQUESTED: {
623 0 : fd_keyswitch_t * replay = find_identity_keyswitch( ctx, "replay" );
624 0 : if( FD_LIKELY( replay->state==FD_KEYSWITCH_STATE_COMPLETED ) ) {
625 0 : FD_LOG_INFO(( "Replay unpaused..." ));
626 0 : replay->state = FD_KEYSWITCH_STATE_UNLOCKED;
627 0 : *state = FD_SET_IDENTITY_STATE_UNLOCKED;
628 0 : } else if( FD_UNLIKELY( replay->state==FD_KEYSWITCH_STATE_UNHALT_PENDING ) ) {
629 0 : FD_SPIN_PAUSE();
630 0 : } else {
631 0 : FD_LOG_ERR(( "Unexpected replay keyswitch state %lu", replay->state ));
632 0 : }
633 0 : break;
634 0 : }
635 0 : default:
636 0 : FD_LOG_ERR(( "Unexpected set-identity state %lu", *state ));
637 0 : }
638 :
639 0 : return *state==FD_SET_IDENTITY_STATE_UNLOCKED;
640 0 : }
641 :
642 : /* Check every sign tile before changing the failover identity or halting
643 : any tiles. Admin serializes commands, so authorized voters cannot
644 : change between this check and the identity switch. */
645 :
646 : static void
647 : failover_identity_check_begin( fd_admin_tile_ctx_t * ctx,
648 0 : uchar const * identity ) {
649 0 : fd_topo_t const * topo = ctx->topo;
650 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
651 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
652 0 : if( strcmp( tile->name, "sign" ) ) continue;
653 0 : fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
654 0 : fd_memcpy( sign->bytes, identity, 32UL );
655 0 : sign->param = FD_KEYSWITCH_PARAM_IDENTITY_CHECK;
656 0 : fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_SWITCH_PENDING );
657 0 : }
658 0 : }
659 :
660 : /* Returns ULONG_MAX until every sign tile has answered. */
661 : static ulong
662 0 : failover_identity_check_poll( fd_admin_tile_ctx_t * ctx ) {
663 0 : fd_topo_t const * topo = ctx->topo;
664 0 : ulong result = FD_ADMINCTL_RESULT_SUCCESS;
665 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
666 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
667 0 : if( strcmp( tile->name, "sign" ) ) continue;
668 0 : fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
669 0 : ulong state = fd_keyswitch_state_query( sign );
670 0 : if( state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) return ULONG_MAX;
671 0 : FD_TEST( state==FD_KEYSWITCH_STATE_COMPLETED || state==FD_KEYSWITCH_STATE_FAILED );
672 0 : if( state==FD_KEYSWITCH_STATE_FAILED ) result = sign->result;
673 0 : }
674 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
675 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
676 0 : if( strcmp( tile->name, "sign" ) ) continue;
677 0 : fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id );
678 0 : sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR;
679 0 : fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_UNLOCKED );
680 0 : }
681 0 : return result;
682 0 : }
683 :
684 : /* Runs until every sign tile has responded. */
685 : static ulong
686 : failover_identity_check( fd_admin_tile_ctx_t * ctx,
687 0 : uchar const * identity ) {
688 0 : failover_identity_check_begin( ctx, identity );
689 0 : ulong result;
690 0 : while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE();
691 0 : return result;
692 0 : }
693 :
694 : /* With failover, set-identity still switches the same way. */
695 :
696 : static void
697 : failover_operator_begin( fd_admin_tile_ctx_t * ctx,
698 0 : uchar const * identity ) {
699 0 : ctx->failover_epoch++;
700 0 : fd_memcpy( ctx->failover_pubkey, identity, 32UL );
701 0 : FD_BASE58_ENCODE_32_BYTES( identity, new_identity );
702 0 : FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, failover_identity );
703 0 : FD_LOG_NOTICE(( "set-identity switches to `%s`, failover restarts on this machine for identity `%s`", new_identity, failover_identity ));
704 :
705 0 : fd_topo_t const * topo = ctx->topo;
706 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
707 0 : if( strcmp( topo->tiles[ i ].name, "sign" ) ) continue;
708 0 : fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, topo->tiles[ i ].id_keyswitch_obj_id );
709 0 : sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR;
710 0 : }
711 0 : find_identity_keyswitch( ctx, "votor" )->failover = 0UL;
712 0 : ctx->failover_pending = 1;
713 0 : }
714 :
715 : /* Sends a pending OPERATOR once the failover link has room to spare. */
716 :
717 : static void
718 : failover_operator_notify( fd_admin_tile_ctx_t * ctx,
719 0 : fd_stem_context_t * stem ) {
720 0 : if( FD_LIKELY( !ctx->failover_pending ) ) return;
721 0 : if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX || stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) return;
722 0 : fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk );
723 0 : fd_memset( msg, 0, sizeof(*msg) );
724 0 : fd_failover_operator_t operator = { .epoch=ctx->failover_epoch };
725 0 : fd_memcpy( operator.identity, ctx->identity_pubkey, 32UL );
726 0 : fd_memcpy( operator.failover_identity, ctx->failover_pubkey, 32UL );
727 0 : fd_memcpy( msg->payload, &operator, sizeof(operator) );
728 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
729 0 : fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_OPERATOR_ACTION, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub );
730 0 : ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark );
731 0 : ctx->failover_pending = 0;
732 0 : }
733 :
734 : static void FD_FN_SENSITIVE
735 : set_identity( fd_admin_tile_ctx_t * ctx,
736 : fd_stem_context_t * stem,
737 : ulong slot_idx,
738 : void * data,
739 0 : ulong data_sz ) {
740 :
741 0 : fd_adminctl_t * adminctl = ctx->adminctl;
742 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SET_IDENTITY, data, data_sz );
743 0 : FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity );
744 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"old_identity\":\"%s\"}", old_identity ) );
745 :
746 0 : if( FD_UNLIKELY( data_sz<sizeof(ulong) ) ) {
747 0 : FD_LOG_WARNING(( "adminctl set-identity payload too small: %lu", data_sz ));
748 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
749 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
750 0 : return;
751 0 : }
752 :
753 0 : ulong version = FD_LOAD( ulong, data );
754 0 : if( FD_UNLIKELY( version!=FD_ADMINCTL_SET_IDENTITY_PAYLOAD_VERSION ) ) {
755 0 : FD_LOG_WARNING(( "unsupported adminctl set-identity payload version %lu", version ));
756 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_VERSION_MISMATCH );
757 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH );
758 0 : return;
759 0 : }
760 :
761 0 : if( FD_UNLIKELY( data_sz!=sizeof(fd_adminctl_set_identity_t) ) ) {
762 0 : FD_LOG_WARNING(( "unexpected adminctl set-identity payload_sz %lu", data_sz ));
763 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
764 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
765 0 : return;
766 0 : }
767 :
768 0 : fd_adminctl_set_identity_t * req = fd_type_pun( data );
769 :
770 0 : fd_pubkey_t public_key;
771 0 : fd_ed25519_public_from_private( public_key.uc, req->keypair, ctx->sha512 );
772 0 : if( FD_UNLIKELY( memcmp( public_key.uc, req->keypair+32UL, 32UL ) ) ) {
773 0 : FD_LOG_WARNING(( "set-identity failed: public key in key file does not match private key" ));
774 0 : report_admin_command_custom_result( &event, "keypair_mismatch" );
775 0 : fd_adminctl_complete( adminctl, slot_idx, FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH );
776 0 : return;
777 0 : }
778 :
779 0 : if( FD_UNLIKELY( !memcmp( public_key.uc, ctx->identity_pubkey, 32UL ) ) ) {
780 0 : FD_LOG_WARNING(( "set-identity failed: the validator is already running as the identity" ));
781 0 : report_admin_command_custom_result( &event, "same_identity" );
782 0 : fd_adminctl_complete( adminctl, slot_idx, FD_SET_IDENTITY_RESULT_SAME_IDENTITY );
783 0 : return;
784 0 : }
785 :
786 0 : if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_operator_begin( ctx, req->keypair+32UL );
787 :
788 0 : ulong state = FD_SET_IDENTITY_STATE_UNLOCKED;
789 0 : ulong identity_outset = (ulong)fd_log_wallclock();
790 0 : for(;;) {
791 0 : if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, req->keypair, req->vote_history_slot ) ) ) break;
792 0 : }
793 :
794 0 : memcpy( ctx->identity_pubkey, req->keypair+32UL, 32UL );
795 0 : if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_operator_notify( ctx, stem );
796 :
797 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
798 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS );
799 0 : }
800 :
801 : static void
802 : get_identity( fd_admin_tile_ctx_t * ctx,
803 : ulong slot_idx,
804 : void * data,
805 0 : ulong data_sz ) {
806 :
807 0 : fd_adminctl_t * adminctl = ctx->adminctl;
808 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_GET_IDENTITY, data, data_sz );
809 0 : FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, identity );
810 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"identity\":\"%s\"}", identity ) );
811 :
812 0 : if( FD_UNLIKELY( data_sz<sizeof(ulong) ) ) {
813 0 : FD_LOG_WARNING(( "adminctl get-identity payload too small: %lu", data_sz ));
814 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
815 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
816 0 : return;
817 0 : }
818 :
819 0 : ulong version = FD_LOAD( ulong, data );
820 0 : if( FD_UNLIKELY( version!=FD_ADMINCTL_GET_IDENTITY_PAYLOAD_VERSION ) ) {
821 0 : FD_LOG_WARNING(( "unsupported adminctl get-identity payload version %lu", version ));
822 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_VERSION_MISMATCH );
823 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH );
824 0 : return;
825 0 : }
826 :
827 0 : if( FD_UNLIKELY( data_sz!=sizeof(fd_adminctl_get_identity_req_t) ) ) {
828 0 : FD_LOG_WARNING(( "unexpected adminctl get-identity payload_sz %lu", data_sz ));
829 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
830 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
831 0 : return;
832 0 : }
833 :
834 : /* Adminctl commands are serviced one at a time by this tile, which is
835 : the only driver of identity switches, so the tracked identity
836 : cannot be mid-switch here. */
837 0 : fd_adminctl_get_identity_resp_t resp;
838 0 : resp.version = FD_ADMINCTL_GET_IDENTITY_PAYLOAD_VERSION;
839 0 : memcpy( resp.identity_pubkey, ctx->identity_pubkey, 32UL );
840 :
841 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
842 0 : fd_adminctl_complete_response( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS, &resp, sizeof(resp) );
843 0 : }
844 :
845 : /* The process of adding an authorized voter to the validator must be
846 : done carefully in order to prevent votes being generated with an
847 : authorized voter that the sign tile is not yet aware of. The
848 : authorized voter must be added to the sign tile before it is added
849 : to the votor tile. All transitions must be
850 : linear and in forward order. */
851 :
852 : /* State 0: UNLOCKED
853 : The validator is not currently in the process of switching keys. */
854 0 : #define FD_ADD_AUTH_VOTER_STATE_UNLOCKED (0UL)
855 :
856 : /* State 1: LOCKED
857 : Some client to the validator has requested to add an authorized
858 : voter. To do so, it acquired an exclusive lock on the validator to
859 : prevent the switch potentially being interleaved with another
860 : client. */
861 0 : #define FD_ADD_AUTH_VOTER_STATE_LOCKED (1UL)
862 :
863 : /* State 2: SIGN_TILE_REQUESTED
864 : The first step to add an authorized voter is to notify the sign
865 : tile that an authorized voter is being added. */
866 0 : #define FD_ADD_AUTH_VOTER_STATE_SIGN_TILE_REQUESTED (2UL)
867 :
868 : /* State 3: SIGN_TILE_UPDATED
869 : The Sign tile has confirmed that it has updated its internal
870 : mapping for the set of supported authorized voters. At this point
871 : the sign tile is aware of the new authorized voter but the voter
872 : tile will not vote with the new authorized voter yet. */
873 0 : #define FD_ADD_AUTH_VOTER_STATE_SIGN_TILE_UPDATED (3UL)
874 :
875 : /* State 4: VOTER_TILE_REQUESTED
876 : Once the Sign tile is updated, now the voter tile must be notified
877 : that an authorized voter is being added so it can start voting with
878 : the new authorized voter. */
879 0 : #define FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_REQUESTED (4UL)
880 :
881 : /* State 5: VOTER_TILE_UPDATED
882 : The voter tile has confirmed that it has updated its internal
883 : mapping for the set of supported authorized voters. */
884 0 : #define FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_UPDATED (5UL)
885 :
886 : /* State 6: UNLOCK_REQUESTED
887 : The client now requests that the voter tile unpause the pipeline
888 : so the validator can start producing votes with the new authorized
889 : voter. */
890 0 : #define FD_ADD_AUTH_VOTER_STATE_UNLOCK_REQUESTED (6UL)
891 :
892 : static void FD_FN_SENSITIVE
893 : poll_add_authorized_voter( fd_admin_tile_ctx_t * ctx,
894 : ulong * state,
895 : uchar * keypair,
896 0 : ulong * result ) {
897 0 : fd_keyswitch_t * voter = ctx->voter_av_keyswitch;
898 :
899 0 : switch( *state ) {
900 0 : case FD_ADD_AUTH_VOTER_STATE_UNLOCKED: {
901 0 : if( FD_LIKELY( FD_KEYSWITCH_STATE_UNLOCKED==FD_ATOMIC_CAS( &voter->state, FD_KEYSWITCH_STATE_UNLOCKED, FD_KEYSWITCH_STATE_LOCKED ) ) ) {
902 0 : *state = FD_ADD_AUTH_VOTER_STATE_LOCKED;
903 0 : FD_LOG_INFO(( "Locking authorized voter set for authorized voter update..." ));
904 0 : } else {
905 : /* keyswitch changes should be guarded and ordered by adminctl.
906 : If the keyswitch is in a locked state means there is
907 : unexpected process state and the validator should crash. */
908 0 : FD_LOG_CRIT(( "keyswitch is in a locked state but should be unlocked" ));
909 0 : }
910 0 : break;
911 0 : }
912 0 : case FD_ADD_AUTH_VOTER_STATE_LOCKED: {
913 0 : for( ulong i=0UL; i<ctx->sign_av_keyswitch_cnt; i++ ) {
914 0 : fd_keyswitch_t * sign = ctx->sign_av_keyswitch[ i ];
915 0 : memcpy( sign->bytes, keypair, 64UL );
916 0 : sign->param = FD_KEYSWITCH_PARAM_AV_ADD;
917 0 : FD_COMPILER_MFENCE();
918 0 : sign->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
919 0 : FD_COMPILER_MFENCE();
920 0 : }
921 0 : fd_memzero_explicit( keypair, 32UL );
922 0 : *state = FD_ADD_AUTH_VOTER_STATE_SIGN_TILE_REQUESTED;
923 0 : FD_LOG_INFO(( "Requesting all sign tiles to update authorized voter key set..." ));
924 0 : break;
925 0 : }
926 0 : case FD_ADD_AUTH_VOTER_STATE_SIGN_TILE_REQUESTED: {
927 0 : int all_updated = 1;
928 0 : for( ulong i=0UL; i<ctx->sign_av_keyswitch_cnt; i++ ) {
929 0 : fd_keyswitch_t * sign = ctx->sign_av_keyswitch[ i ];
930 0 : if( FD_UNLIKELY( sign->state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) {
931 0 : all_updated = 0;
932 0 : } else if( FD_UNLIKELY( sign->state==FD_KEYSWITCH_STATE_FAILED ) ) {
933 : /* Recoverable error: the sign tile failed to update the set
934 : of authorized voters is a result of bad caller input. All
935 : the sign tiles should be in sync, which means that if one
936 : sign tile failed, we expect all of them to. */
937 0 : fd_memzero_explicit( sign->bytes, 64UL );
938 0 : if( FD_LIKELY( !*result ) ) *result = sign->result;
939 0 : } else { /* sign->state==FD_KEYSWITCH_STATE_COMPLETED */
940 0 : fd_memzero_explicit( sign->bytes, 64UL );
941 0 : }
942 0 : }
943 :
944 0 : if( FD_LIKELY( all_updated ) ) {
945 0 : if( FD_UNLIKELY( *result ) ) *state = FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_UPDATED;
946 0 : else *state = FD_ADD_AUTH_VOTER_STATE_SIGN_TILE_UPDATED;
947 0 : } else {
948 0 : FD_SPIN_PAUSE();
949 0 : }
950 0 : break;
951 0 : }
952 0 : case FD_ADD_AUTH_VOTER_STATE_SIGN_TILE_UPDATED: {
953 0 : memcpy( voter->bytes, keypair+32UL, 32UL );
954 0 : voter->param = FD_KEYSWITCH_PARAM_AV_ADD;
955 0 : FD_COMPILER_MFENCE();
956 0 : voter->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
957 0 : FD_COMPILER_MFENCE();
958 0 : *state = FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_REQUESTED;
959 0 : FD_LOG_INFO(( "Requesting votor tile to update authorized voter key set..." ));
960 0 : break;
961 0 : }
962 0 : case FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_REQUESTED: {
963 : /* There is a guarantee that the voter tile will be in sync with
964 : the set of authorized voters in the sign tile. At this point
965 : that means that the command should succeed because invariants
966 : such as not having duplicate authorized voter keys and too many
967 : authorized voters are upheld. If this doesn't hold true, the
968 : voter tile will detect any corruption and gracefully crash the
969 : validator. */
970 0 : if( FD_LIKELY( voter->state==FD_KEYSWITCH_STATE_COMPLETED ) ) {
971 0 : *state = FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_UPDATED;
972 0 : FD_LOG_INFO(( "votor tile key set successfully updated..." ));
973 0 : } else {
974 0 : FD_SPIN_PAUSE();
975 0 : }
976 0 : break;
977 0 : }
978 0 : case FD_ADD_AUTH_VOTER_STATE_VOTER_TILE_UPDATED: {
979 0 : voter->state = FD_KEYSWITCH_STATE_UNHALT_PENDING;
980 0 : *state = FD_ADD_AUTH_VOTER_STATE_UNLOCK_REQUESTED;
981 0 : FD_LOG_INFO(( "Requesting an unlock of the authorized voter key set..." ));
982 0 : break;
983 0 : }
984 0 : case FD_ADD_AUTH_VOTER_STATE_UNLOCK_REQUESTED: {
985 0 : if( FD_LIKELY( voter->state==FD_KEYSWITCH_STATE_UNLOCKED ) ) {
986 0 : *state = FD_ADD_AUTH_VOTER_STATE_UNLOCKED;
987 0 : FD_LOG_INFO(( "Authorized voter key set unlocked..." ));
988 0 : } else {
989 0 : FD_SPIN_PAUSE();
990 0 : }
991 0 : break;
992 0 : }
993 0 : default: {
994 0 : FD_LOG_CRIT(( "Unexpected add-authorized-voter state %lu", *state ));
995 0 : }
996 0 : }
997 0 : }
998 :
999 : static void FD_FN_SENSITIVE
1000 : add_authorized_voter( fd_admin_tile_ctx_t * ctx,
1001 : ulong slot_idx,
1002 : void * data,
1003 0 : ulong data_sz ) {
1004 :
1005 0 : fd_adminctl_t * adminctl = ctx->adminctl;
1006 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER, data, data_sz );
1007 :
1008 0 : if( FD_UNLIKELY( data_sz<sizeof(ulong) ) ) {
1009 0 : FD_LOG_WARNING(( "adminctl add-authorized-voter payload too small: %lu", data_sz ));
1010 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1011 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1012 0 : return;
1013 0 : }
1014 :
1015 0 : ulong version = FD_LOAD( ulong, data );
1016 0 : if( FD_UNLIKELY( version!=FD_ADMINCTL_ADD_AUTH_VOTER_PAYLOAD_VERSION ) ) {
1017 0 : FD_LOG_WARNING(( "unsupported adminctl add-authorized-voter payload version %lu", version ));
1018 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_VERSION_MISMATCH );
1019 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH );
1020 0 : return;
1021 0 : }
1022 :
1023 0 : if( FD_UNLIKELY( data_sz!=sizeof(fd_adminctl_add_auth_voter_t) ) ) {
1024 0 : FD_LOG_WARNING(( "unexpected adminctl add-authorized-voter payload_sz %lu", data_sz ));
1025 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1026 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1027 0 : return;
1028 0 : }
1029 :
1030 0 : fd_adminctl_add_auth_voter_t * req = fd_type_pun( data );
1031 0 : FD_BASE58_ENCODE_32_BYTES( req->keypair+32UL, authorized_voter );
1032 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"authorized_voter\":\"%s\"}", authorized_voter ) );
1033 :
1034 0 : uchar public_key[ 32UL ];
1035 0 : fd_ed25519_public_from_private( public_key, req->keypair, ctx->sha512 );
1036 0 : if( FD_UNLIKELY( memcmp( public_key, req->keypair+32UL, 32UL ) ) ) {
1037 0 : FD_LOG_WARNING(( "add-authorized-voter failed: public key in key file does not match private key" ));
1038 0 : report_admin_command_custom_result( &event, "keypair_mismatch" );
1039 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADD_AUTHORIZED_VOTER_RESULT_KEYPAIR_MISMATCH );
1040 0 : return;
1041 0 : }
1042 :
1043 : /* The identity failover hands off cannot also be an authorized voter. */
1044 0 : if( FD_UNLIKELY( ctx->failover_enabled && !memcmp( public_key, ctx->failover_pubkey, 32UL ) ) ) {
1045 0 : FD_LOG_WARNING(( "add-authorized-voter failed: the key is the identity failover hands off on this validator" ));
1046 0 : report_admin_command_custom_result( &event, "failover_identity" );
1047 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADD_AUTHORIZED_VOTER_RESULT_FAILOVER_IDENTITY );
1048 0 : return;
1049 0 : }
1050 :
1051 0 : ulong result = FD_ADMINCTL_RESULT_SUCCESS;
1052 0 : ulong state = FD_ADD_AUTH_VOTER_STATE_UNLOCKED;
1053 0 : for(;;) {
1054 0 : poll_add_authorized_voter( ctx, &state, req->keypair, &result );
1055 0 : if( FD_UNLIKELY( state==FD_ADD_AUTH_VOTER_STATE_UNLOCKED ) ) break;
1056 0 : }
1057 :
1058 0 : switch( result ) {
1059 0 : case FD_ADMINCTL_RESULT_SUCCESS:
1060 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
1061 0 : break;
1062 0 : case FD_ADD_AUTHORIZED_VOTER_RESULT_MAX_AUTH_VOTERS:
1063 0 : report_admin_command_custom_result( &event, "max_authorized_voters" );
1064 0 : break;
1065 0 : case FD_ADD_AUTHORIZED_VOTER_RESULT_DUPLICATE_AUTH_VOTER:
1066 0 : report_admin_command_custom_result( &event, "duplicate_authorized_voter" );
1067 0 : break;
1068 0 : default:
1069 0 : FD_LOG_ERR(( "unexpected add-authorized-voter result %lu", result ));
1070 0 : }
1071 0 : fd_adminctl_complete( adminctl, slot_idx, result );
1072 0 : }
1073 :
1074 : static void
1075 : snapshot_create( fd_admin_tile_ctx_t * ctx,
1076 : fd_stem_context_t * stem,
1077 : ulong slot_idx,
1078 : void const * payload,
1079 0 : ulong payload_sz ) {
1080 :
1081 0 : fd_adminctl_t * adminctl = ctx->adminctl;
1082 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE, payload, payload_sz );
1083 :
1084 0 : if( FD_UNLIKELY( payload_sz!=sizeof(fd_adminctl_snap_create_t) ) ) {
1085 0 : FD_LOG_WARNING(( "unexpected adminctl snapshot-create payload_sz %lu", payload_sz ));
1086 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1087 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1088 0 : return;
1089 0 : }
1090 0 : fd_adminctl_snap_create_t const * req = fd_type_pun_const( payload );
1091 0 : if( FD_UNLIKELY( req->version!=FD_ADMINCTL_SNAP_CREATE_PAYLOAD_VERSION ) ) {
1092 0 : FD_LOG_WARNING(( "unsupported adminctl snapshot-create payload version %lu", req->version ));
1093 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_VERSION_MISMATCH );
1094 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH );
1095 0 : return;
1096 0 : }
1097 0 : ulong target_slot = req->slot;
1098 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"target_slot\":%lu}", target_slot ) );
1099 :
1100 0 : if( FD_UNLIKELY( ctx->replay_out_idx==ULONG_MAX ) ) {
1101 0 : FD_LOG_WARNING(( "admin requested snapshot creation, but admin tile has no replay command link" ));
1102 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED );
1103 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED );
1104 0 : return;
1105 0 : }
1106 :
1107 0 : if( FD_UNLIKELY( ctx->snap_create_slot_idx!=ULONG_MAX ) ) {
1108 0 : FD_LOG_WARNING(( "admin requested snapshot creation, but another snapshot-create command is pending replay response" ));
1109 0 : report_admin_command_custom_result( &event, "busy" );
1110 0 : fd_adminctl_complete( adminctl, slot_idx, FD_SNAPSHOT_CREATE_RESULT_BUSY );
1111 0 : return;
1112 0 : }
1113 :
1114 0 : ulong ctl = fd_frag_meta_ctl( FD_ADMINCTL_CMD_SNAP_CREATE, 0, 0, 0 );
1115 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1116 0 : fd_stem_publish( stem, ctx->replay_out_idx, target_slot, 0UL, 0UL, ctl, 0UL, tspub );
1117 0 : ctx->snap_create_slot_idx = slot_idx;
1118 0 : ctx->snap_create_target_slot = target_slot;
1119 0 : ctx->snap_create_start_time = event.start_time;
1120 0 : }
1121 :
1122 : static void
1123 : snapshot_create_response( fd_admin_tile_ctx_t * ctx,
1124 : ulong sig,
1125 0 : ulong ctl ) {
1126 :
1127 0 : if( FD_UNLIKELY( fd_frag_meta_ctl_orig( ctl )!=FD_ADMINCTL_CMD_SNAP_CREATE ) ) {
1128 0 : FD_LOG_ERR(( "unexpected replay admin response orig %lu", fd_frag_meta_ctl_orig( ctl ) ));
1129 0 : }
1130 :
1131 0 : fd_event_admin_command_t event = {
1132 0 : .type = FD_EVENT_ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE,
1133 0 : .start_time = ctx->snap_create_start_time,
1134 0 : .payload_version = FD_ADMINCTL_SNAP_CREATE_PAYLOAD_VERSION,
1135 0 : .has_payload_version = 1,
1136 0 : .payload_size = sizeof(fd_adminctl_snap_create_t),
1137 0 : };
1138 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"target_slot\":%lu}", ctx->snap_create_target_slot ) );
1139 0 : switch( sig ) {
1140 0 : case FD_ADMINCTL_RESULT_SUCCESS:
1141 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
1142 0 : break;
1143 0 : case FD_SNAPSHOT_CREATE_RESULT_BUSY:
1144 0 : report_admin_command_custom_result( &event, "busy" );
1145 0 : break;
1146 0 : case FD_ADMINCTL_RESULT_UNSUPPORTED:
1147 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED );
1148 0 : break;
1149 0 : case FD_SNAPSHOT_CREATE_RESULT_NOT_READY:
1150 0 : report_admin_command_custom_result( &event, "not_ready" );
1151 0 : break;
1152 0 : case FD_SNAPSHOT_CREATE_RESULT_SLOT_IN_PAST:
1153 0 : report_admin_command_custom_result( &event, "slot_in_past" );
1154 0 : break;
1155 0 : default:
1156 0 : FD_LOG_ERR(( "unexpected snapshot-create result %lu", sig ));
1157 0 : }
1158 0 : fd_adminctl_complete( ctx->adminctl, ctx->snap_create_slot_idx, sig );
1159 0 : ctx->snap_create_slot_idx = ULONG_MAX;
1160 0 : ctx->snap_create_target_slot = 0UL;
1161 0 : ctx->snap_create_start_time = 0UL;
1162 0 : }
1163 :
1164 : /* Removing all authorized voters from the validator is the inverse of
1165 : add-authorized-voter, and must be done in the opposite order. When
1166 : adding, the sign tile is updated before the votor tile so that it never asks
1167 : the sign tile to sign a vote with an authority index the sign tile
1168 : does not yet know about. When removing, the voter tile must be
1169 : cleared before the sign tiles, so that the voter stops referencing an
1170 : authorized voter index before the sign tile drops the corresponding
1171 : key.
1172 :
1173 : All transitions are linear and in forward order.
1174 :
1175 : Unlike add-authorized-voter, removal cannot fail on the tile side: it
1176 : is unconditional and idempotent (clearing an empty set succeeds).
1177 :
1178 : The votor signs synchronously, so it has no signing requests in
1179 : flight once it confirms the clear. */
1180 :
1181 : /* State 0: UNLOCKED
1182 : The validator is not currently in the process of switching keys. */
1183 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCKED (0UL)
1184 :
1185 : /* State 1: LOCKED
1186 : Some client to the validator has requested to remove all authorized
1187 : voters. To do so, it acquired an exclusive lock on the validator to
1188 : prevent the removal potentially being interleaved with another
1189 : client. */
1190 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_LOCKED (1UL)
1191 :
1192 : /* State 2: VOTER_TILE_REQUESTED
1193 : The voter tile has been notified to clear its authorized voter set.
1194 : It is cleared first so it stops voting with any authorized voter
1195 : before the sign tiles drop the keys. */
1196 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_VOTER_TILE_REQUESTED (2UL)
1197 :
1198 : /* State 3: VOTER_TILE_CLEARED
1199 : The voter tile confirmed it cleared its authorized voter map. At
1200 : this point the validator will only prepare votes signed by the
1201 : identity key. */
1202 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_VOTER_TILE_CLEARED (3UL)
1203 :
1204 : /* State 4: SIGN_TILE_REQUESTED
1205 : All sign tiles have been notified to clear their authorized voter
1206 : keys. */
1207 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_SIGN_TILE_REQUESTED (4UL)
1208 :
1209 : /* State 5: SIGN_TILE_CLEARED
1210 : All sign tiles confirmed they cleared (and securely zeroed) their
1211 : authorized voter keys. */
1212 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_SIGN_TILE_CLEARED (5UL)
1213 :
1214 : /* State 6: UNLOCK_REQUESTED
1215 : The client requests that the voter tile release the lock. */
1216 0 : #define FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCK_REQUESTED (6UL)
1217 :
1218 : static void
1219 : poll_remove_all_authorized_voters( fd_admin_tile_ctx_t * ctx,
1220 0 : ulong * state ) {
1221 0 : fd_keyswitch_t * voter = ctx->voter_av_keyswitch;
1222 :
1223 0 : switch( *state ) {
1224 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCKED: {
1225 0 : if( FD_LIKELY( FD_KEYSWITCH_STATE_UNLOCKED==FD_ATOMIC_CAS( &voter->state, FD_KEYSWITCH_STATE_UNLOCKED, FD_KEYSWITCH_STATE_LOCKED ) ) ) {
1226 0 : *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_LOCKED;
1227 0 : FD_LOG_INFO(( "Locking authorized voter set for authorized voter update..." ));
1228 0 : } else {
1229 : /* keyswitch changes should be guarded and ordered by adminctl.
1230 : If the keyswitch is in a locked state means there is
1231 : unexpected process state and the validator should crash. */
1232 0 : FD_LOG_CRIT(( "keyswitch is in a locked state but should be unlocked" ));
1233 0 : }
1234 0 : break;
1235 0 : }
1236 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_LOCKED: {
1237 0 : voter->param = FD_KEYSWITCH_PARAM_AV_CLEAR;
1238 0 : FD_COMPILER_MFENCE();
1239 0 : voter->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
1240 0 : FD_COMPILER_MFENCE();
1241 0 : *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_VOTER_TILE_REQUESTED;
1242 0 : FD_LOG_INFO(( "Requesting votor tile to clear authorized voter key set..." ));
1243 0 : break;
1244 0 : }
1245 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_VOTER_TILE_REQUESTED: {
1246 0 : if( FD_LIKELY( voter->state==FD_KEYSWITCH_STATE_COMPLETED ) ) {
1247 0 : *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_VOTER_TILE_CLEARED;
1248 0 : FD_LOG_INFO(( "votor tile authorized voter key set cleared..." ));
1249 0 : } else {
1250 0 : FD_SPIN_PAUSE();
1251 0 : }
1252 0 : break;
1253 0 : }
1254 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_VOTER_TILE_CLEARED: {
1255 0 : for( ulong i=0UL; i<ctx->sign_av_keyswitch_cnt; i++ ) {
1256 0 : fd_keyswitch_t * sign = ctx->sign_av_keyswitch[ i ];
1257 0 : sign->param = FD_KEYSWITCH_PARAM_AV_CLEAR;
1258 0 : FD_COMPILER_MFENCE();
1259 0 : sign->state = FD_KEYSWITCH_STATE_SWITCH_PENDING;
1260 0 : FD_COMPILER_MFENCE();
1261 0 : }
1262 0 : *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_SIGN_TILE_REQUESTED;
1263 0 : FD_LOG_INFO(( "Requesting all sign tiles to clear authorized voter key set..." ));
1264 0 : break;
1265 0 : }
1266 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_SIGN_TILE_REQUESTED: {
1267 0 : int all_cleared = 1;
1268 0 : for( ulong i=0UL; i<ctx->sign_av_keyswitch_cnt; i++ ) {
1269 0 : fd_keyswitch_t * sign = ctx->sign_av_keyswitch[ i ];
1270 0 : if( FD_UNLIKELY( sign->state!=FD_KEYSWITCH_STATE_COMPLETED ) ) {
1271 0 : all_cleared = 0;
1272 0 : break;
1273 0 : }
1274 0 : }
1275 :
1276 0 : if( FD_LIKELY( all_cleared ) ) *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_SIGN_TILE_CLEARED;
1277 0 : else FD_SPIN_PAUSE();
1278 0 : break;
1279 0 : }
1280 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_SIGN_TILE_CLEARED: {
1281 0 : voter->state = FD_KEYSWITCH_STATE_UNHALT_PENDING;
1282 0 : *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCK_REQUESTED;
1283 0 : FD_LOG_INFO(( "Requesting an unlock of the authorized voter key set..." ));
1284 0 : break;
1285 0 : }
1286 0 : case FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCK_REQUESTED: {
1287 0 : if( FD_LIKELY( voter->state==FD_KEYSWITCH_STATE_UNLOCKED ) ) {
1288 0 : *state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCKED;
1289 0 : FD_LOG_INFO(( "Authorized voter key set unlocked..." ));
1290 0 : } else {
1291 0 : FD_SPIN_PAUSE();
1292 0 : }
1293 0 : break;
1294 0 : }
1295 0 : default: {
1296 0 : FD_LOG_CRIT(( "Unexpected remove-all-authorized-voters state %lu", *state ));
1297 0 : }
1298 0 : }
1299 0 : }
1300 :
1301 : static void
1302 : remove_all_authorized_voters( fd_admin_tile_ctx_t * ctx,
1303 : ulong slot_idx,
1304 : void * data,
1305 0 : ulong data_sz ) {
1306 :
1307 0 : fd_adminctl_t * adminctl = ctx->adminctl;
1308 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS, data, data_sz );
1309 :
1310 0 : if( FD_UNLIKELY( data_sz<sizeof(ulong) ) ) {
1311 0 : FD_LOG_WARNING(( "adminctl remove-all-authorized-voters payload too small: %lu", data_sz ));
1312 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1313 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1314 0 : return;
1315 0 : }
1316 :
1317 0 : ulong version = FD_LOAD( ulong, data );
1318 0 : if( FD_UNLIKELY( version!=FD_ADMINCTL_REMOVE_ALL_AUTH_VOTERS_PAYLOAD_VERSION ) ) {
1319 0 : FD_LOG_WARNING(( "unsupported adminctl remove-all-authorized-voters payload version %lu", version ));
1320 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_VERSION_MISMATCH );
1321 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH );
1322 0 : return;
1323 0 : }
1324 :
1325 0 : if( FD_UNLIKELY( data_sz!=sizeof(fd_adminctl_remove_all_auth_voters_t) ) ) {
1326 0 : FD_LOG_WARNING(( "unexpected adminctl remove-all-authorized-voters payload_sz %lu", data_sz ));
1327 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1328 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1329 0 : return;
1330 0 : }
1331 :
1332 0 : ulong state = FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCKED;
1333 0 : for(;;) {
1334 0 : poll_remove_all_authorized_voters( ctx, &state );
1335 0 : if( FD_UNLIKELY( state==FD_REMOVE_ALL_AUTH_VOTERS_STATE_UNLOCKED ) ) break;
1336 0 : }
1337 :
1338 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
1339 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS );
1340 0 : }
1341 :
1342 : static inline long
1343 0 : next_deadline( fd_admin_tile_ctx_t const * ctx ) {
1344 0 : return ctx->deadline_ticks;
1345 0 : }
1346 :
1347 : /* The failover tile responds to commands. We check the ABI, forward
1348 : the command over the bus and park the adminctl slot until the
1349 : response comes back, similar to how snapshot creation waits on
1350 : replay. */
1351 :
1352 : static fd_event_admin_command_t *
1353 : failover_event_args( fd_admin_tile_ctx_t const * ctx,
1354 0 : fd_event_admin_command_t * event ) {
1355 0 : if( FD_LIKELY( ctx->failover_args_json_len ) ) {
1356 0 : fd_memcpy( event->args_json, ctx->failover_args_json, ctx->failover_args_json_len );
1357 0 : event->args_json_len = ctx->failover_args_json_len;
1358 0 : }
1359 0 : return event;
1360 0 : }
1361 :
1362 : static void
1363 : failover_complete( fd_admin_tile_ctx_t * ctx,
1364 : ulong result,
1365 0 : ulong slot ) {
1366 0 : int preflight = ctx->failover_slot_cmd==FD_ADMINCTL_FAILOVER_CMD_PREFLIGHT;
1367 0 : if( FD_LIKELY( !preflight ) ) {
1368 0 : fd_event_admin_command_t event = {
1369 0 : .type = FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE,
1370 0 : .args_json = { '{', '}' },
1371 0 : .args_json_len = 2UL,
1372 0 : .start_time = ctx->failover_start_time,
1373 0 : .payload_version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION,
1374 0 : .has_payload_version = 1,
1375 0 : .payload_size = sizeof(fd_adminctl_failover_req_t),
1376 0 : };
1377 0 : if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) {
1378 0 : report_admin_command( failover_event_args( ctx, &event ), FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
1379 0 : } else {
1380 0 : char custom[ 24 ];
1381 0 : FD_TEST( fd_cstr_printf_check( custom, sizeof(custom), NULL, "%#lx", result ) );
1382 0 : report_admin_command_custom_result( failover_event_args( ctx, &event ), custom );
1383 0 : }
1384 0 : }
1385 0 : fd_adminctl_failover_resp_t resp = {
1386 0 : .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION,
1387 0 : .slot=slot
1388 0 : };
1389 0 : fd_memcpy( resp.identity, ctx->identity_pubkey, 32UL );
1390 0 : fd_adminctl_complete_response( ctx->adminctl, ctx->failover_slot_idx, result, &resp, sizeof(resp) );
1391 0 : FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, identity );
1392 0 : if( FD_UNLIKELY( preflight ) ) {
1393 0 : if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) {
1394 0 : FD_LOG_WARNING(( "`%s` refused before it changed the failover identity (result %#lx), nothing was done", ctx->failover_cmd_cstr, result ));
1395 0 : }
1396 0 : } else if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) {
1397 0 : FD_LOG_NOTICE(( "`%s` done, this machine runs `%s`", ctx->failover_cmd_cstr, identity ));
1398 0 : } else if( FD_UNLIKELY( result==FD_FAILOVER_CONTROL_RESULT_ALREADY_HOLDS ) ) {
1399 0 : FD_LOG_NOTICE(( "`%s` done, this machine already runs `%s`", ctx->failover_cmd_cstr, identity ));
1400 0 : } else if( FD_UNLIKELY( ctx->failover_retargeted ) ) {
1401 0 : FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, target );
1402 0 : FD_LOG_WARNING(( "`%s` failed (result %#lx), this machine runs `%s` and the failover target is now `%s`", ctx->failover_cmd_cstr, result, identity, target ));
1403 0 : } else {
1404 0 : FD_LOG_WARNING(( "`%s` failed (result %#lx), this machine runs `%s`", ctx->failover_cmd_cstr, result, identity ));
1405 0 : }
1406 0 : ctx->failover_retargeted = 0;
1407 0 : ctx->failover_slot_idx = ULONG_MAX;
1408 0 : ctx->failover_start_time = 0UL;
1409 0 : ctx->failover_args_json_len = 0UL;
1410 0 : ctx->failover_cmd_cstr[ 0 ] = '\0';
1411 0 : }
1412 :
1413 : /* Load a target key without changing the installed identity. */
1414 : static ulong FD_FN_SENSITIVE
1415 : failover_prepare( fd_admin_tile_ctx_t * ctx,
1416 0 : uchar const * keypair ) {
1417 0 : uchar pubkey[ 32 ];
1418 0 : fd_ed25519_public_from_private( pubkey, keypair, ctx->sha512 );
1419 0 : if( FD_UNLIKELY( !fd_memeq( pubkey, keypair+32UL, 32UL ) ) ) return FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH;
1420 0 : ulong result = failover_identity_check( ctx, pubkey );
1421 0 : if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result;
1422 0 : if( fd_memeq( pubkey, ctx->failover_pubkey, 32UL ) ) return FD_ADMINCTL_RESULT_SUCCESS;
1423 :
1424 0 : for( ulong i=0UL; i<ctx->topo->tile_cnt; i++ ) {
1425 0 : fd_topo_tile_t const * tile = &ctx->topo->tiles[ i ];
1426 0 : if( strcmp( tile->name, "sign" ) ) continue;
1427 0 : fd_keyswitch_t * sign = fd_topo_obj_laddr( ctx->topo, tile->id_keyswitch_obj_id );
1428 0 : fd_memcpy( sign->bytes, keypair, 64UL );
1429 0 : sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PREPARE;
1430 0 : fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_SWITCH_PENDING );
1431 0 : }
1432 0 : while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE();
1433 : /* Preflight checked every sign tile before any target was loaded. */
1434 0 : FD_TEST( result==FD_ADMINCTL_RESULT_SUCCESS );
1435 0 : fd_memcpy( ctx->failover_pubkey, pubkey, 32UL );
1436 0 : ctx->failover_epoch++;
1437 0 : ctx->failover_pending = 1;
1438 0 : ctx->failover_retargeted = 1;
1439 0 : return FD_ADMINCTL_RESULT_SUCCESS;
1440 0 : }
1441 :
1442 : static void FD_FN_SENSITIVE
1443 : failover_request( fd_admin_tile_ctx_t * ctx,
1444 : fd_stem_context_t * stem,
1445 : ulong slot_idx,
1446 : ulong cmd_id,
1447 : void * data,
1448 0 : ulong data_sz ) {
1449 0 : (void)cmd_id;
1450 :
1451 0 : fd_adminctl_t * adminctl = ctx->adminctl;
1452 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE, data, data_sz );
1453 :
1454 0 : if( FD_UNLIKELY( data_sz<sizeof(ulong) ) ) {
1455 0 : FD_LOG_WARNING(( "adminctl failover payload too small: %lu", data_sz ));
1456 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1457 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1458 0 : return;
1459 0 : }
1460 :
1461 0 : ulong version = FD_LOAD( ulong, data );
1462 0 : if( FD_UNLIKELY( version!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION ) ) {
1463 0 : FD_LOG_WARNING(( "unsupported adminctl failover payload version %lu", version ));
1464 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_VERSION_MISMATCH );
1465 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH );
1466 0 : return;
1467 0 : }
1468 :
1469 0 : int prepare = data_sz==sizeof(fd_adminctl_failover_prepare_t);
1470 0 : if( FD_UNLIKELY( data_sz!=sizeof(fd_adminctl_failover_req_t) && !prepare ) ) {
1471 0 : FD_LOG_WARNING(( "unexpected adminctl failover payload_sz %lu", data_sz ));
1472 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH );
1473 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH );
1474 0 : return;
1475 0 : }
1476 :
1477 0 : fd_adminctl_failover_req_t req;
1478 0 : fd_memcpy( &req, data, sizeof(req) );
1479 0 : int force = !!(req.flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE);
1480 0 : int takeover = !!(req.flags & FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER);
1481 0 : int preflight = req.cmd==FD_ADMINCTL_FAILOVER_CMD_PREFLIGHT;
1482 0 : char const * invalid = NULL;
1483 0 : if( req.cmd>=FD_ADMINCTL_FAILOVER_CMD_CNT ) invalid = "unknown command";
1484 0 : else if( req.flags & ~(FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE|FD_ADMINCTL_FAILOVER_FLAG_TAKEOVER) ) invalid = "unknown flags";
1485 0 : else if( force && !takeover ) invalid = "--force goes with a takeover";
1486 0 : else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_PROMOTE && !takeover ) invalid = "a promote request is only sent by a takeover";
1487 0 : else if( req.cmd!=FD_ADMINCTL_FAILOVER_CMD_HANDOFF && (req.addr || req.port) ) invalid = "an address or port goes only with a handoff";
1488 0 : else if( preflight && prepare ) invalid = "a preflight carries no keypair";
1489 0 : if( FD_UNLIKELY( invalid ) ) {
1490 0 : FD_LOG_WARNING(( "refusing adminctl failover cmd %lu flags %lu, %s", req.cmd, req.flags, invalid ));
1491 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND );
1492 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND );
1493 0 : return;
1494 0 : }
1495 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len,
1496 0 : "{\"yes\":%s,\"takeover\":%s,\"force\":%s}",
1497 0 : (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? "true" : "false",
1498 0 : takeover ? "true" : "false",
1499 0 : force ? "true" : "false" ) );
1500 0 : char cmd_cstr[ 48 ];
1501 0 : FD_TEST( fd_cstr_printf_check( cmd_cstr, sizeof(cmd_cstr), NULL, "%s%s", fd_adminctl_failover_cmd_typed( req.cmd, req.flags ),
1502 0 : (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? " --yes" : "" ) );
1503 :
1504 0 : if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX ) ) {
1505 0 : FD_LOG_WARNING(( "`%s` refused, the admin tile has no failover bus link", cmd_cstr ));
1506 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED );
1507 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED );
1508 0 : return;
1509 0 : }
1510 :
1511 : /* Keep one command outstanding until the failover tile responds. */
1512 0 : if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce || ctx->failover_pending ) ) {
1513 0 : FD_LOG_WARNING(( "`%s` refused, another failover command is still running on the failover tile", cmd_cstr ));
1514 0 : report_admin_command_custom_result( &event, "busy" );
1515 0 : fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY );
1516 0 : return;
1517 0 : }
1518 :
1519 : /* The sign tiles cannot install an authorized voter as the identity. */
1520 0 : if( FD_LIKELY( !prepare ) ) {
1521 0 : ulong result = failover_identity_check( ctx, preflight ? req.target : ctx->failover_pubkey );
1522 0 : if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) {
1523 0 : char custom[ 24 ];
1524 0 : FD_TEST( fd_cstr_printf_check( custom, sizeof(custom), NULL, "%#lx", result ) );
1525 0 : FD_LOG_WARNING(( "`%s` refused before it reached the failover tile (result %s), nothing was done", cmd_cstr, custom ));
1526 0 : report_admin_command_custom_result( &event, custom );
1527 0 : fd_adminctl_complete( adminctl, slot_idx, result );
1528 0 : return;
1529 0 : }
1530 0 : }
1531 :
1532 0 : if( prepare ) {
1533 0 : if( FD_UNLIKELY( stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) {
1534 0 : report_admin_command_custom_result( &event, "busy" );
1535 0 : fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY );
1536 0 : return;
1537 0 : }
1538 0 : fd_adminctl_failover_prepare_t * payload = data;
1539 0 : ctx->failover_retargeted = 0;
1540 0 : ulong result = failover_prepare( ctx, payload->keypair );
1541 0 : fd_memzero_explicit( payload->keypair, sizeof(payload->keypair) );
1542 0 : if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) {
1543 0 : FD_LOG_WARNING(( "`%s` refused before it reached the failover tile (result %#lx), nothing was done", cmd_cstr, result ));
1544 0 : report_admin_command_custom_result( &event, "target_refused" );
1545 0 : fd_adminctl_complete( adminctl, slot_idx, result );
1546 0 : return;
1547 0 : }
1548 : /* The public target and epoch must reach the controller before the
1549 : command. after_credit reserves enough credits for both frames. */
1550 0 : failover_operator_notify( ctx, stem );
1551 0 : FD_TEST( !ctx->failover_pending );
1552 0 : }
1553 :
1554 : /* Kept for the event and the log of the response. */
1555 0 : FD_TEST( event.args_json_len<=sizeof(ctx->failover_args_json) );
1556 0 : fd_memcpy( ctx->failover_args_json, event.args_json, event.args_json_len );
1557 0 : ctx->failover_args_json_len = event.args_json_len;
1558 0 : fd_cstr_ncpy( ctx->failover_cmd_cstr, cmd_cstr, sizeof(ctx->failover_cmd_cstr) );
1559 0 : if( FD_LIKELY( !preflight ) ) FD_LOG_NOTICE(( "forwarding `%s` to the failover tile", cmd_cstr ));
1560 :
1561 0 : fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk );
1562 0 : fd_memset( msg, 0, sizeof(*msg) );
1563 0 : msg->nonce = ++ctx->failover_nonce;
1564 0 : fd_memcpy( msg->payload, &req, sizeof(req) );
1565 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1566 0 : fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_REQUEST, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub );
1567 0 : ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark );
1568 :
1569 0 : ctx->failover_slot_idx = slot_idx;
1570 0 : ctx->failover_slot_cmd = req.cmd;
1571 0 : ctx->failover_start_time = event.start_time;
1572 0 : }
1573 :
1574 : static void
1575 0 : failover_step( fd_admin_tile_ctx_t * ctx ) {
1576 0 : fd_failover_bus_msg_t * msg = &ctx->failov_in;
1577 0 : if( FD_UNLIKELY( ctx->failover_slot_idx==ULONG_MAX || msg->nonce!=ctx->failover_nonce ) ) return;
1578 0 : msg->payload[ FD_FAILOVER_BUS_STEP_MAX-1UL ] = '\0';
1579 0 : fd_adminctl_step( ctx->adminctl, ctx->failover_slot_idx, (char const *)msg->payload );
1580 0 : }
1581 :
1582 : static void
1583 0 : failover_response( fd_admin_tile_ctx_t * ctx ) {
1584 0 : fd_failover_bus_msg_t const * msg = &ctx->failov_in;
1585 0 : if( FD_LIKELY( msg->nonce==ctx->failover_nonce ) ) ctx->failover_answered = msg->nonce;
1586 : /* A response for an older command is dropped. */
1587 0 : if( FD_UNLIKELY( ctx->failover_slot_idx==ULONG_MAX || msg->nonce!=ctx->failover_nonce ) ) {
1588 0 : FD_LOG_WARNING(( "dropping a stale failover command response" ));
1589 0 : return;
1590 0 : }
1591 0 : failover_complete( ctx, msg->result, FD_LOAD( ulong, msg->payload ) );
1592 0 : }
1593 :
1594 : /* The failover tile asked for an identity switch. We only pass the
1595 : public key on, the sign tile selects the junk key or the failover
1596 : identity.
1597 : Like set-identity this blocks until every tile has switched. We poll
1598 : no adminctl command meanwhile, so every command blocks on it. */
1599 :
1600 : static void
1601 : failover_switch_request( fd_admin_tile_ctx_t * ctx,
1602 0 : fd_stem_context_t * stem ) {
1603 0 : ulong nonce = ctx->failov_in.nonce;
1604 0 : fd_failover_switch_req_t req;
1605 0 : fd_memcpy( &req, ctx->failov_in.payload, sizeof(req) );
1606 :
1607 0 : fd_failover_switch_resp_t response;
1608 0 : fd_memset( &response, 0, sizeof(response) );
1609 0 : response.result = FD_FAILOVER_SWITCH_ERR_DISABLED;
1610 0 : if( FD_UNLIKELY( ctx->failover_enabled && req.epoch!=ctx->failover_epoch ) ) {
1611 0 : FD_LOG_WARNING(( "refusing the failover tile's identity switch, set-identity ran since it asked" ));
1612 0 : response.result = FD_FAILOVER_SWITCH_ERR_STALE;
1613 0 : response.operator.epoch = ctx->failover_epoch;
1614 0 : fd_memcpy( response.operator.identity, ctx->identity_pubkey, 32UL );
1615 0 : fd_memcpy( response.operator.failover_identity, ctx->failover_pubkey, 32UL );
1616 0 : } else if( FD_UNLIKELY( ctx->failover_enabled && failover_identity_check( ctx, req.identity )!=FD_ADMINCTL_RESULT_SUCCESS ) ) {
1617 0 : FD_BASE58_ENCODE_32_BYTES( req.identity, target );
1618 0 : FD_LOG_WARNING(( "refusing the failover tile's identity switch to `%s`, it is one of the authorized voters, remove it with remove-all-authorized-voters first", target ));
1619 0 : response.result = FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER;
1620 0 : } else if( FD_LIKELY( ctx->failover_enabled ) ) {
1621 0 : fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SET_IDENTITY, NULL, 0UL );
1622 0 : FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity );
1623 0 : FD_BASE58_ENCODE_32_BYTES( req.identity, new_identity );
1624 0 : FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len,
1625 0 : "{\"old_identity\":\"%s\",\"identity\":\"%s\",\"source\":\"failover\"}", old_identity, new_identity ) );
1626 0 : FD_LOG_NOTICE(( "failover: switching the identity from `%s` to `%s`, failover commands wait until every tile has switched", old_identity, new_identity ));
1627 :
1628 : /* No private half, only the public key reaches the keyswitches. */
1629 0 : uchar keypair[ 64 ];
1630 0 : fd_memset( keypair, 0, 32UL );
1631 0 : fd_memcpy( keypair+32UL, req.identity, 32UL );
1632 :
1633 0 : find_identity_keyswitch( ctx, "votor" )->failover = 1UL; /* votor enforces adoption before target participation */
1634 0 : ctx->failover_switch = 1;
1635 0 : ulong state = FD_SET_IDENTITY_STATE_UNLOCKED;
1636 0 : ulong identity_outset = (ulong)fd_log_wallclock();
1637 0 : for(;;) {
1638 0 : if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, keypair, 0UL ) ) ) break;
1639 0 : }
1640 0 : ctx->failover_switch = 0;
1641 0 : fd_memcpy( ctx->identity_pubkey, req.identity, 32UL );
1642 :
1643 : /* Votor returns its final votor_hist sequence after halting signing. */
1644 0 : response.result = FD_FAILOVER_SWITCH_OK;
1645 0 : response.history_watermark = find_identity_keyswitch( ctx, "votor" )->result;
1646 0 : report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS );
1647 0 : FD_LOG_NOTICE(( "failover: every tile switched to `%s`, votor halted signing at sequence %lu", new_identity, response.history_watermark ));
1648 0 : } else {
1649 0 : FD_LOG_WARNING(( "the failover tile asked for an identity switch but failover is off, refusing" ));
1650 0 : }
1651 :
1652 0 : fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk );
1653 0 : fd_memset( out, 0, sizeof(*out) );
1654 0 : out->nonce = nonce;
1655 0 : fd_memcpy( out->payload, &response, sizeof(response) );
1656 0 : ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() );
1657 0 : fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_SWITCH_RESP, ctx->failov_out_chunk, sizeof(*out), 0UL, tspub, tspub );
1658 0 : ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*out), ctx->failov_out_chunk0, ctx->failov_out_wmark );
1659 0 : }
1660 :
1661 : static inline void FD_FN_SENSITIVE
1662 : after_credit( fd_admin_tile_ctx_t * ctx,
1663 : fd_stem_context_t * stem,
1664 : int * opt_poll_in,
1665 0 : int * charge_busy ) {
1666 :
1667 0 : fd_adminctl_t * adminctl = ctx->adminctl;
1668 0 : ulong slot_idx = ULONG_MAX;
1669 0 : void * payload = NULL;
1670 0 : ulong payload_sz = 0UL;
1671 :
1672 0 : failover_operator_notify( ctx, stem );
1673 :
1674 0 : if( FD_LIKELY( stem->now<ctx->deadline_ticks ) ) return;
1675 0 : ctx->deadline_ticks = stem->now + fd_long_if( ( ctx->snap_create_slot_idx!=ULONG_MAX || ctx->failover_slot_idx!=ULONG_MAX ), ctx->poll_ticks, ctx->idle_poll_ticks );
1676 :
1677 0 : ulong cmd_id = FD_ADMINCTL_CMD_IDLE;
1678 0 : for( ulong i=0UL; i<FD_ADMINCTL_SLOT_CNT && cmd_id==FD_ADMINCTL_CMD_IDLE; i++ ) cmd_id = fd_adminctl_poll( adminctl, &slot_idx, &payload, &payload_sz );
1679 0 : switch( cmd_id ) {
1680 0 : case FD_ADMINCTL_CMD_IDLE:
1681 0 : break;
1682 0 : case FD_ADMINCTL_CMD_ADD_AUTH_VOTER:
1683 0 : add_authorized_voter( ctx, slot_idx, payload, payload_sz );
1684 0 : *charge_busy = 1;
1685 0 : break;
1686 0 : case FD_ADMINCTL_CMD_SET_IDENTITY:
1687 0 : set_identity( ctx, stem, slot_idx, payload, payload_sz );
1688 0 : *charge_busy = 1;
1689 0 : if( FD_UNLIKELY( ctx->failover_enabled ) ) *opt_poll_in = 0; /* it may have published OPERATOR */
1690 0 : break;
1691 0 : case FD_ADMINCTL_CMD_REMOVE_ALL_AUTH_VOTERS:
1692 0 : remove_all_authorized_voters( ctx, slot_idx, payload, payload_sz );
1693 0 : *charge_busy = 1;
1694 0 : break;
1695 0 : case FD_ADMINCTL_CMD_GET_IDENTITY:
1696 0 : get_identity( ctx, slot_idx, payload, payload_sz );
1697 0 : *charge_busy = 1;
1698 0 : break;
1699 0 : case FD_ADMINCTL_CMD_SNAP_CREATE:
1700 0 : snapshot_create( ctx, stem, slot_idx, payload, payload_sz );
1701 0 : if( FD_LIKELY( ctx->snap_create_slot_idx!=ULONG_MAX ) ) ctx->deadline_ticks = stem->now + ctx->poll_ticks; /* poll fast while replay responds */
1702 0 : *charge_busy = 1;
1703 0 : *opt_poll_in = 0;
1704 0 : break;
1705 0 : case FD_ADMINCTL_CMD_FAILOVER_PROMOTE:
1706 0 : failover_request( ctx, stem, slot_idx, cmd_id, payload, payload_sz );
1707 0 : *charge_busy = 1;
1708 0 : *opt_poll_in = 0;
1709 0 : break;
1710 0 : default:
1711 0 : FD_LOG_WARNING(( "unexpected adminctl cmd %lu", cmd_id ));
1712 0 : fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND );
1713 0 : }
1714 0 : }
1715 :
1716 : static void
1717 : during_frag( fd_admin_tile_ctx_t * ctx,
1718 : ulong in_idx FD_PARAM_UNUSED,
1719 : ulong seq FD_PARAM_UNUSED,
1720 : ulong sig,
1721 : ulong chunk FD_PARAM_UNUSED,
1722 : ulong sz FD_PARAM_UNUSED,
1723 0 : ulong ctl ) {
1724 0 : if( FD_UNLIKELY( in_idx==ctx->failov_in_idx ) ) {
1725 0 : if( FD_UNLIKELY( chunk<ctx->failov_in_chunk0 || chunk>ctx->failov_in_wmark || sz!=sizeof(fd_failover_bus_msg_t) ) ) {
1726 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->failov_in_chunk0, ctx->failov_in_wmark ));
1727 0 : }
1728 0 : fd_memcpy( &ctx->failov_in, fd_chunk_to_laddr_const( ctx->failov_in_mem, chunk ), sizeof(fd_failover_bus_msg_t) );
1729 0 : return;
1730 0 : }
1731 :
1732 0 : if( FD_UNLIKELY( ctx->snap_create_slot_idx==ULONG_MAX ) ) {
1733 0 : FD_LOG_ERR(( "unexpected replay snapshot-create response with no pending adminctl command" ));
1734 0 : return;
1735 0 : }
1736 0 : snapshot_create_response( ctx, sig, ctl );
1737 0 : }
1738 :
1739 : static void
1740 : after_frag( fd_admin_tile_ctx_t * ctx,
1741 : ulong in_idx,
1742 : ulong seq FD_PARAM_UNUSED,
1743 : ulong sig,
1744 : ulong sz FD_PARAM_UNUSED,
1745 : ulong tsorig FD_PARAM_UNUSED,
1746 : ulong tspub FD_PARAM_UNUSED,
1747 0 : fd_stem_context_t * stem ) {
1748 0 : if( FD_LIKELY( in_idx!=ctx->failov_in_idx ) ) return;
1749 0 : switch( sig ) {
1750 0 : case FD_FAILOVER_BUS_SWITCH_REQ:
1751 0 : failover_switch_request( ctx, stem );
1752 0 : break;
1753 0 : case FD_FAILOVER_BUS_RESPONSE:
1754 0 : failover_response( ctx );
1755 0 : break;
1756 0 : case FD_FAILOVER_BUS_STEP:
1757 0 : failover_step( ctx );
1758 0 : break;
1759 0 : default:
1760 0 : FD_LOG_WARNING(( "unexpected failover bus frame %lu", sig ));
1761 0 : break;
1762 0 : }
1763 0 : }
1764 :
1765 : static ulong
1766 : populate_allowed_seccomp( fd_topo_t const * topo FD_PARAM_UNUSED,
1767 : fd_topo_tile_t const * tile FD_PARAM_UNUSED,
1768 : ulong out_cnt,
1769 0 : struct sock_filter * out ) {
1770 :
1771 0 : populate_sock_filter_policy_fd_admin_tile( out_cnt, out, (uint)fd_log_private_logfile_fd() );
1772 0 : return sock_filter_policy_fd_admin_tile_instr_cnt;
1773 0 : }
1774 :
1775 : static ulong
1776 : populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED,
1777 : fd_topo_tile_t const * tile FD_PARAM_UNUSED,
1778 : ulong out_fds_cnt,
1779 0 : int * out_fds ) {
1780 :
1781 0 : if( FD_UNLIKELY( out_fds_cnt<2UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
1782 :
1783 0 : ulong out_cnt = 0UL;
1784 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
1785 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
1786 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
1787 0 : return out_cnt;
1788 0 : }
1789 :
1790 0 : #define STEM_BURST (2UL)
1791 0 : #define STEM_LAZY ((long)1e6) /* 1ms */
1792 : #define STEM_ALWAYS_PARK 1
1793 :
1794 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_admin_tile_ctx_t
1795 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_admin_tile_ctx_t)
1796 :
1797 0 : #define STEM_CALLBACK_NEXT_DEADLINE next_deadline
1798 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
1799 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
1800 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
1801 :
1802 : #include "../../disco/stem/fd_stem.c"
1803 :
1804 : static ulong
1805 0 : max_event_sz( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) {
1806 0 : return sizeof(fd_event_admin_command_t);
1807 0 : }
1808 :
1809 : fd_topo_run_tile_t fd_tile_admin = {
1810 : .name = "admin",
1811 : .max_event_sz = max_event_sz,
1812 : .populate_allowed_seccomp = populate_allowed_seccomp,
1813 : .populate_allowed_fds = populate_allowed_fds,
1814 : .scratch_align = scratch_align,
1815 : .scratch_footprint = scratch_footprint,
1816 : .privileged_init = privileged_init,
1817 : .unprivileged_init = unprivileged_init,
1818 : .run = stem_run,
1819 : };
|
