Line data Source code
1 : /* force-mismatch attaches to a running validator (or backtest) and
2 : forces a genuine bank hash mismatch at a chosen slot, without any
3 : support from the validator itself.
4 :
5 : It works by injecting a hard fork into the bank state in shared
6 : memory. A hard fork at slot T is mixed into the bank hash of the
7 : first block with parent_slot<T<=slot, and also moves the
8 : LastRestartSlot sysvar to T, so the validator computes a bank hash
9 : the rest of the cluster (or the ledger) disagrees with. This is the
10 : same mechanism an operator's --hard-fork uses, so every downstream
11 : mismatch path (backtest ledger comparison, Alpenglow footer check,
12 : tower hard fork detection) is exercised for real.
13 :
14 : Hard forks are carried in bank->f and copied from parent to child
15 : when a bank becomes replayable, so the fork is appended to every live
16 : bank with slot<T. Descendants inherit it from there. The shred
17 : version is computed from the replay tile's private copy of the
18 : snapshot hard forks, so it is unaffected. */
19 :
20 : #include "../../shared/fd_config.h"
21 : #include "../../shared/fd_action.h"
22 : #include "../../shared/fd_bootinfo.h"
23 : #include "../../../flamenco/runtime/fd_bank.h"
24 : #include "../../../util/pod/fd_pod.h"
25 : #include "../../../util/wksp/fd_wksp_private.h"
26 :
27 : #include <errno.h>
28 : #include <fcntl.h>
29 : #include <unistd.h>
30 : #include <sys/stat.h>
31 :
32 : extern action_t * ACTIONS[];
33 :
34 : static void
35 : force_mismatch_cmd_args( int * pargc,
36 : char *** pargv,
37 0 : args_t * args ) {
38 0 : char const * topo_name = fd_env_strip_cmdline_cstr( pargc, pargv, "--topo", NULL, "" );
39 :
40 0 : ulong topo_name_len = strlen( topo_name );
41 0 : if( FD_UNLIKELY( topo_name_len > sizeof(args->force_mismatch.topo)-1 ) ) FD_LOG_ERR(( "Unknown --topo %s", topo_name ));
42 0 : fd_cstr_fini( fd_cstr_append_text( fd_cstr_init( args->force_mismatch.topo ), topo_name, topo_name_len ) );
43 :
44 0 : args->force_mismatch.at_slot = fd_env_strip_cmdline_ulong( pargc, pargv, "--at-slot", NULL, 0UL );
45 0 : args->force_mismatch.after = fd_env_strip_cmdline_ulong( pargc, pargv, "--after", NULL, 0UL );
46 :
47 0 : if( FD_UNLIKELY( !args->force_mismatch.at_slot == !args->force_mismatch.after ) ) {
48 0 : FD_LOG_ERR(( "exactly one of --at-slot <slot> or --after <slots> must be given" ));
49 0 : }
50 0 : }
51 :
52 : static void
53 : reconstruct_topo( config_t * config,
54 0 : char const * topo_name ) {
55 0 : if( !topo_name[0] ) return; /* keep default action topo */
56 :
57 0 : action_t const * selected = NULL;
58 0 : for( action_t ** a=ACTIONS; *a; a++ ) {
59 0 : action_t const * action = *a;
60 0 : if( 0==strcmp( action->name, topo_name ) ) {
61 0 : selected = action;
62 0 : break;
63 0 : }
64 0 : }
65 :
66 0 : if( !selected ) FD_LOG_ERR(( "Unknown --topo %s", topo_name ));
67 0 : if( !selected->topo ) FD_LOG_ERR(( "Cannot recover topology for --topo %s", topo_name ));
68 :
69 0 : selected->topo( config );
70 0 : }
71 :
72 : static int
73 : bank_has_fork( fd_bank_t const * bank,
74 0 : ulong slot ) {
75 0 : ulong cnt = fd_ulong_min( FD_VOLATILE_CONST( bank->f.hard_fork_cnt ), FD_HARD_FORKS_MAX );
76 0 : for( ulong i=0UL; i<cnt; i++ ) {
77 0 : if( FD_VOLATILE_CONST( bank->f.hard_forks[ i ].slot )==slot ) return 1;
78 0 : }
79 0 : return 0;
80 0 : }
81 :
82 : /* bank_add_fork appends the hard fork before publishing the new count,
83 : so a concurrent reader sees either the old or the new list. */
84 :
85 : static void
86 : bank_add_fork( fd_bank_t * bank,
87 0 : ulong slot ) {
88 0 : ulong cnt = FD_VOLATILE_CONST( bank->f.hard_fork_cnt );
89 0 : if( FD_UNLIKELY( cnt>=FD_HARD_FORKS_MAX ) ) FD_LOG_ERR(( "bank for slot %lu already has %lu hard forks", bank->f.slot, cnt ));
90 0 : bank->f.hard_forks[ cnt ].slot = slot;
91 0 : bank->f.hard_forks[ cnt ].cnt = 1UL;
92 0 : FD_COMPILER_MFENCE();
93 0 : FD_VOLATILE( bank->f.hard_fork_cnt ) = cnt+1UL;
94 0 : }
95 :
96 : /* max_replayed_slot returns the highest slot among replayable and
97 : frozen banks, or 0 if there are none yet. */
98 :
99 : static ulong
100 : max_replayed_slot( fd_bank_t const * pool,
101 0 : ulong bank_max ) {
102 0 : ulong max = 0UL;
103 0 : for( ulong i=0UL; i<bank_max; i++ ) {
104 0 : ulong state = FD_VOLATILE_CONST( pool[ i ].state );
105 0 : if( state!=FD_BANK_STATE_REPLAYABLE && state!=FD_BANK_STATE_FROZEN ) continue;
106 0 : max = fd_ulong_max( max, FD_VOLATILE_CONST( pool[ i ].f.slot ) );
107 0 : }
108 0 : return max;
109 0 : }
110 :
111 : static int
112 0 : any_live_validator( void ) {
113 0 : fd_bootinfo_instance_t instances[ FD_BOOTINFO_INSTANCE_MAX ];
114 0 : ulong cnt = fd_bootinfo_discover( instances, FD_BOOTINFO_INSTANCE_MAX );
115 0 : for( ulong i=0UL; i<cnt; i++ ) if( instances[ i ].live ) return 1;
116 0 : return 0;
117 0 : }
118 :
119 : /* wksp_ready returns 1 if the workspace file exists, was created no
120 : earlier than min_ctime_nanos, and has been formatted. */
121 :
122 : static int
123 : wksp_ready( config_t const * config,
124 : fd_topo_wksp_t const * wksp,
125 0 : long min_ctime_nanos ) {
126 0 : char const * mount_path;
127 0 : switch( wksp->page_sz ) {
128 0 : case FD_SHMEM_GIGANTIC_PAGE_SZ: mount_path = config->hugetlbfs.gigantic_page_mount_path; break;
129 0 : case FD_SHMEM_HUGE_PAGE_SZ: mount_path = config->hugetlbfs.huge_page_mount_path; break;
130 0 : default: mount_path = config->hugetlbfs.normal_page_mount_path; break;
131 0 : }
132 0 : char path[ PATH_MAX ];
133 0 : FD_TEST( fd_cstr_printf_check( path, PATH_MAX, NULL, "%s/%s_%s.wksp", mount_path, config->name, wksp->name ) );
134 :
135 0 : int fd = open( path, O_RDONLY );
136 0 : if( FD_UNLIKELY( -1==fd ) ) return 0;
137 0 : struct stat st;
138 0 : ulong magic = 0UL;
139 0 : int ready = !fstat( fd, &st ) &&
140 0 : (long)st.st_ctim.tv_sec*(long)1e9 + st.st_ctim.tv_nsec>=min_ctime_nanos &&
141 0 : pread( fd, &magic, sizeof(ulong), 0L )==(long)sizeof(ulong) &&
142 0 : magic==FD_WKSP_MAGIC;
143 0 : if( FD_UNLIKELY( -1==close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
144 0 : return ready;
145 0 : }
146 :
147 : static void
148 : force_mismatch_cmd_fn( args_t * args,
149 0 : config_t * config ) {
150 0 : long start_nanos = fd_log_wallclock();
151 :
152 : /* A short backtest can replay past the target within milliseconds of
153 : booting, so allow starting this before the validator. */
154 0 : if( FD_LIKELY( !config->has_user_config ) ) {
155 0 : FD_LOG_NOTICE(( "waiting for a running validator" ));
156 0 : while( !any_live_validator() ) fd_log_sleep( (long)1e6 );
157 0 : }
158 :
159 0 : fd_bootinfo_adopt( config );
160 0 : reconstruct_topo( config, args->force_mismatch.topo );
161 0 : fd_bootinfo_check_layout( config );
162 :
163 0 : fd_topo_t * topo = &config->topo;
164 0 : ulong banks_obj_id = fd_pod_query_ulong( topo->props, "banks", ULONG_MAX );
165 0 : if( FD_UNLIKELY( banks_obj_id==ULONG_MAX ) ) FD_LOG_ERR(( "banks object not found in topology" ));
166 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->objs[ banks_obj_id ].wksp_id ];
167 :
168 : /* A backtest publishes no bootinfo and leaves its workspaces behind
169 : on exit, so unless a live validator was found, only attach to a
170 : workspace created after this command started. */
171 0 : long min_ctime_nanos = config->boot_timestamp_nanos ? 0L : start_nanos;
172 0 : if( min_ctime_nanos ) FD_LOG_NOTICE(( "waiting for %s_%s.wksp to be created", topo->app_name, wksp->name ));
173 0 : while( !wksp_ready( config, wksp, min_ctime_nanos ) ) fd_log_sleep( (long)1e6 );
174 0 : fd_topo_join_workspace( topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
175 :
176 : /* Banks are accessed raw rather than with fd_banks_join, which would
177 : also attach the file backed per-fork stores. */
178 0 : fd_banks_t * banks = fd_topo_obj_laddr( topo, banks_obj_id );
179 0 : while( FD_VOLATILE_CONST( banks->magic )!=FD_BANKS_MAGIC ) fd_log_sleep( (long)1e6 );
180 0 : fd_bank_t * pool = fd_type_pun( (uchar *)banks + banks->pool_offset );
181 0 : ulong bank_max = banks->max_total_banks;
182 :
183 : /* Wait for the snapshot to load so there is a replayed slot to count
184 : from, and to refuse a target that has already been replayed. */
185 0 : ulong replayed;
186 0 : while( !(replayed=max_replayed_slot( pool, bank_max )) ) FD_SPIN_PAUSE();
187 :
188 0 : ulong target = args->force_mismatch.at_slot;
189 0 : if( args->force_mismatch.after ) target = replayed + args->force_mismatch.after;
190 0 : if( FD_UNLIKELY( target<=replayed ) ) {
191 0 : FD_LOG_ERR(( "--at-slot %lu has already been replayed (highest replayed slot %lu)", target, replayed ));
192 0 : }
193 0 : FD_LOG_NOTICE(( "highest replayed slot %lu, injecting hard fork at slot %lu", replayed, target ));
194 :
195 : /* Keep every bank below the target carrying the fork until a block
196 : crossing the target is replayed. Rescanning repairs banks that
197 : were cloned from a parent before it was patched (or rewritten by
198 : the snapshot loader). */
199 0 : ulong patched = 0UL;
200 0 : for(;;) {
201 0 : for( ulong i=0UL; i<bank_max; i++ ) {
202 0 : fd_bank_t * bank = &pool[ i ];
203 0 : ulong state = FD_VOLATILE_CONST( bank->state );
204 0 : if( state!=FD_BANK_STATE_REPLAYABLE && state!=FD_BANK_STATE_FROZEN && state!=FD_BANK_STATE_PRUNABLE ) continue;
205 :
206 0 : ulong slot = FD_VOLATILE_CONST( bank->f.slot );
207 0 : if( slot<target ) {
208 0 : if( !bank_has_fork( bank, target ) ) { bank_add_fork( bank, target ); patched++; }
209 0 : continue;
210 0 : }
211 :
212 : /* A prunable bank may never have been replayable, and banks past
213 : the first crossing are judged by their ancestor. */
214 0 : ulong parent_slot = FD_VOLATILE_CONST( bank->f.parent_slot );
215 0 : if( state==FD_BANK_STATE_PRUNABLE || parent_slot>=target ) continue;
216 :
217 0 : if( FD_UNLIKELY( !bank_has_fork( bank, target ) ) ) {
218 0 : if( state==FD_BANK_STATE_FROZEN ) {
219 0 : FD_LOG_ERR(( "slot %lu (parent %lu) froze before the hard fork could be injected", slot, parent_slot ));
220 0 : }
221 : /* Cloned from an unpatched parent but still executing. The
222 : bank hash still picks up the fork when the bank freezes. */
223 0 : FD_LOG_WARNING(( "slot %lu (parent %lu) is already replaying, injecting late", slot, parent_slot ));
224 0 : bank_add_fork( bank, target );
225 0 : patched++;
226 0 : }
227 :
228 0 : ulong bank_seq = FD_VOLATILE_CONST( bank->bank_seq );
229 0 : FD_LOG_NOTICE(( "slot %lu (parent %lu) replays with the hard fork (%lu banks patched), waiting for it to complete", slot, parent_slot, patched ));
230 0 : while( FD_VOLATILE_CONST( bank->state )==FD_BANK_STATE_REPLAYABLE && FD_VOLATILE_CONST( bank->bank_seq )==bank_seq ) FD_SPIN_PAUSE();
231 0 : state = FD_VOLATILE_CONST( bank->state );
232 0 : if( FD_UNLIKELY( FD_VOLATILE_CONST( bank->bank_seq )!=bank_seq || (state!=FD_BANK_STATE_FROZEN && state!=FD_BANK_STATE_DEAD) ) ) {
233 0 : FD_LOG_WARNING(( "slot %lu was pruned before completing", slot ));
234 0 : return;
235 0 : }
236 : /* Under Alpenglow the footer check marks the mismatched block
237 : dead, otherwise it freezes and the mismatch surfaces later
238 : (backtest comparison, tower hard fork detection). */
239 0 : FD_BASE58_ENCODE_32_BYTES( bank->f.bank_hash.uc, bank_hash_b58 );
240 0 : FD_LOG_NOTICE(( "slot %lu %s with forced bank hash %s", slot, state==FD_BANK_STATE_DEAD ? "was marked dead" : "froze", bank_hash_b58 ));
241 0 : return;
242 0 : }
243 0 : FD_SPIN_PAUSE();
244 0 : }
245 0 : }
246 :
247 : static void
248 0 : force_mismatch_args_help( fd_action_help_t * help ) {
249 0 : fd_action_help_arg( help, "--at-slot", "<slot>", "Force the mismatch on the first block at or after this absolute slot" );
250 0 : fd_action_help_arg( help, "--after", "<slots>", "Force the mismatch this many slots past the highest replayed slot" );
251 0 : fd_action_help_arg( help, "--topo", "<command>", "Build the topology from another subcommand (e.g. `backtest`) instead of\n"
252 0 : "the default validator topology" );
253 0 : }
254 :
255 : action_t fd_action_force_mismatch = {
256 : .name = "force-mismatch",
257 : .args = force_mismatch_cmd_args,
258 : .fn = force_mismatch_cmd_fn,
259 : .perm = NULL,
260 : .description = "Force a bank hash mismatch in a running validator",
261 : .detail = "Attaches to a running validator or backtest and injects a hard fork into\n"
262 : "its bank state, so the first block at or after the target slot computes\n"
263 : "a bank hash that disagrees with the cluster or ledger. Exactly one of\n"
264 : "--at-slot or --after must be given.",
265 : .usage = "force-mismatch (--at-slot <slot> | --after <slots>) [--topo <command>]",
266 : .args_help = force_mismatch_args_help,
267 : .is_immediate = 0,
268 : };
|
