Line data Source code
1 : #include "../../shared/commands/configure/configure.h"
2 : #include "../../shared/commands/run/run.h" /* initialize_workspaces */
3 : #include "../../shared/fd_config.h" /* config_t */
4 : #include "../../../disco/topo/fd_topob.h"
5 : #include "../../../disco/net/fd_net_tile.h" /* fd_topos_net_tiles */
6 : #include "../../../disco/fd_clock_tile.h"
7 : #include "../../../discof/gossip/fd_gossip_tile.h"
8 :
9 : #include "../../firedancer/commands/monitor_gossip/gossip_diag.h"
10 :
11 : #include "core_subtopo.h"
12 : #include "gossip.h"
13 :
14 : #include <stdio.h> /* printf */
15 : #include <stdlib.h>
16 : #include <unistd.h> /* isatty */
17 : #include <sys/ioctl.h>
18 :
19 : extern fd_topo_obj_callbacks_t * CALLBACKS[];
20 :
21 : fd_topo_run_tile_t
22 : fdctl_tile_run( fd_topo_tile_t const * tile );
23 :
24 : static void
25 0 : gossip_cmd_topo( config_t * config ) {
26 : /* Disable non-gossip listen ports */
27 0 : config->tiles.shred.shred_listen_port = 0U;
28 0 : config->tiles.quic.quic_transaction_listen_port = 0U;
29 0 : config->tiles.quic.regular_transaction_listen_port = 0U;
30 0 : config->tiles.repair.repair_client_listen_port = 0U;
31 0 : config->tiles.rserve.repair_serve_listen_port = 0U;
32 :
33 0 : static ulong tile_to_cpu[ FD_TILE_MAX ] = {0}; /* TODO */
34 :
35 0 : ulong net_tile_cnt = config->layout.net_tile_count;
36 :
37 : /* Reset topology from scratch */
38 0 : fd_topo_t * topo = &config->topo;
39 0 : fd_topob_new( &config->topo, config->name );
40 0 : topo->max_page_size = fd_cstr_to_shmem_page_sz( config->hugetlbfs.max_page_size );
41 :
42 0 : fd_core_subtopo( config, tile_to_cpu );
43 0 : fd_gossip_subtopo( config, tile_to_cpu );
44 :
45 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "sign_gossip", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
46 0 : for( ulong i=0UL; i<net_tile_cnt; i++ ) fd_topos_net_tile_finish( topo, i );
47 0 : fd_topob_auto_layout( topo );
48 0 : fd_topob_waker( topo );
49 0 : fd_topob_finish( topo, CALLBACKS );
50 0 : }
51 :
52 : void
53 0 : fd_gossip_subtopo( config_t * config, ulong tile_to_cpu[ FD_TILE_MAX ] FD_PARAM_UNUSED ) {
54 0 : fd_topo_t * topo = &config->topo;
55 :
56 0 : ulong gossvf_tile_count = config->firedancer.layout.gossvf_tile_count;
57 0 : ulong net_tile_cnt = config->layout.net_tile_count;
58 :
59 0 : static char* const tiles_to_add[] = {
60 0 : "gossvf",
61 0 : "ipecho",
62 0 : "gossip",
63 0 : };
64 0 : for( int i=0; i<3; ++i) FD_TEST( fd_topo_find_tile( topo, tiles_to_add[i], 0UL ) == ULONG_MAX );
65 :
66 0 : fd_topob_wksp( topo, "gossip" );
67 0 : fd_topo_tile_t * gossip_tile = fd_topob_tile( topo, "gossip", "gossip", "metric_in", 0UL, 1, 0, 0 );
68 0 : fd_cstr_ncpy( gossip_tile->gossip.identity_key_path, config->paths.identity_key, sizeof(gossip_tile->gossip.identity_key_path) );
69 0 : gossip_tile->gossip.entrypoints_cnt = config->gossip.entrypoints_cnt;
70 0 : for( ulong i=0UL; i<config->gossip.entrypoints_cnt; i++ ) {
71 0 : fd_cstr_ncpy( gossip_tile->gossip.entrypoints[ i ], config->gossip.entrypoints[ i ], sizeof(gossip_tile->gossip.entrypoints[ i ]) );
72 0 : }
73 0 : fd_cstr_ncpy( gossip_tile->gossip.gossip_host, config->firedancer.gossip.host, sizeof(gossip_tile->gossip.gossip_host) );
74 0 : gossip_tile->gossip.net_ip_addr = config->net.ip_addr;
75 0 : gossip_tile->gossip.ip_addr = config->net.ip_addr;
76 0 : gossip_tile->gossip.expected_shred_version = config->consensus.expected_shred_version;
77 0 : gossip_tile->gossip.max_entries = config->tiles.gossip.max_entries;
78 0 : gossip_tile->gossip.ports.gossip = config->gossip.port;
79 0 : gossip_tile->gossip.ports.votor = 0;
80 0 : gossip_tile->gossip.ports.repair = 0;
81 0 : gossip_tile->gossip.ports.rserve = 0;
82 0 : gossip_tile->gossip.ports.tpu = 0;
83 0 : gossip_tile->gossip.ports.tpu_quic = 0;
84 0 : gossip_tile->gossip.ports.tvu = 0;
85 0 : gossip_tile->gossip.ports.tvu_quic = 0;
86 0 : gossip_tile->gossip.boot_timestamp_nanos = config->boot_timestamp_nanos;
87 :
88 0 : gossip_tile->gossip.wait_for_supermajority_at_slot = 0UL;
89 0 : gossip_tile->gossip.wait_for_supermajority_hash_is_zero = 1;
90 :
91 0 : fd_topob_wksp( topo, "gossvf" );
92 0 : for( ulong i=0UL; i<gossvf_tile_count; i++ ) {
93 0 : fd_topo_tile_t * gossvf_tile = fd_topob_tile( topo, "gossvf", "gossvf", "metric_in", 0UL, 1, 0, 0 );
94 0 : fd_cstr_ncpy( gossvf_tile->gossvf.identity_key_path, config->paths.identity_key, sizeof(gossvf_tile->gossvf.identity_key_path) );
95 0 : gossvf_tile->gossvf.tcache_depth = 1UL<<22UL;
96 0 : gossvf_tile->gossvf.shred_version = config->consensus.expected_shred_version;
97 0 : gossvf_tile->gossvf.allow_private_address = config->development.gossip.allow_private_address;
98 0 : gossvf_tile->gossvf.entrypoints_cnt = config->gossip.entrypoints_cnt;
99 0 : gossvf_tile->gossvf.boot_timestamp_nanos = config->boot_timestamp_nanos;
100 0 : fd_cstr_ncpy( gossvf_tile->gossvf.gossip_host, config->firedancer.gossip.host, sizeof(gossvf_tile->gossvf.gossip_host) );
101 0 : gossvf_tile->gossvf.gossip_addr.addr = config->net.ip_addr;
102 0 : gossvf_tile->gossvf.gossip_addr.port = fd_ushort_bswap( config->gossip.port );
103 0 : gossvf_tile->gossvf.src_addr.addr = config->net.ip_addr;
104 0 : gossvf_tile->gossvf.src_addr.port = fd_ushort_bswap( config->gossip.port );
105 0 : for( ulong i=0UL; i<config->gossip.entrypoints_cnt; i++ ) {
106 0 : fd_cstr_ncpy( gossvf_tile->gossvf.entrypoints[ i ], config->gossip.entrypoints[ i ], sizeof(gossvf_tile->gossvf.entrypoints[ i ]) );
107 0 : }
108 0 : }
109 0 : for( ulong i=0UL; i<net_tile_cnt; i++ ) {
110 0 : fd_topos_net_rx_link( topo, "net_gossvf", i, config->net.ingress_buffer_size );
111 0 : }
112 0 : for( ulong i=0UL; i<gossvf_tile_count; i++ ) {
113 0 : for( ulong j=0UL; j<net_tile_cnt; j++ ) {
114 0 : fd_topob_tile_in( topo, "gossvf", i, "metric_in", "net_gossvf", j, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
115 0 : }
116 0 : }
117 :
118 0 : fd_topob_wksp( topo, "gossip_net" );
119 0 : fd_topob_link( topo, "gossip_net", "gossip_net", 65536*4UL, FD_NET_MTU, 1UL );
120 0 : fd_topos_tile_in_net( topo, "metric_in", "gossip_net", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
121 0 : fd_topob_tile_out( topo, "gossip", 0UL, "gossip_net", 0UL );
122 :
123 0 : fd_topob_wksp( topo, "ipecho" );
124 0 : fd_topo_tile_t * ipecho_tile = fd_topob_tile( topo, "ipecho", "ipecho", "metric_in", 0UL, 0, 0, 1 );
125 0 : ipecho_tile->ipecho.expected_shred_version = config->consensus.expected_shred_version;
126 0 : ipecho_tile->ipecho.bind_address = config->net.ip_addr;
127 0 : ipecho_tile->ipecho.bind_port = config->gossip.port;
128 0 : ipecho_tile->ipecho.entrypoints_cnt = config->gossip.entrypoints_cnt;
129 0 : for( ulong i=0UL; i<config->gossip.entrypoints_cnt; i++ ) {
130 0 : fd_cstr_ncpy( ipecho_tile->ipecho.entrypoints[ i ], config->gossip.entrypoints[ i ], sizeof(ipecho_tile->ipecho.entrypoints[ i ]) );
131 0 : }
132 :
133 0 : fd_topob_wksp( topo, "ipecho_out" );
134 0 : fd_topob_link( topo, "ipecho_out", "ipecho_out", 4UL, 0UL, 1UL );
135 0 : fd_topob_tile_out( topo, "ipecho", 0UL, "ipecho_out", 0UL );
136 :
137 0 : for( ulong i=0UL; i<gossvf_tile_count; i++ ) {
138 0 : fd_topob_tile_in( topo, "gossvf", i, "metric_in", "ipecho_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
139 0 : }
140 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "ipecho_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
141 :
142 0 : fd_topob_wksp( topo, "gossvf_gossip" );
143 0 : fd_topob_wksp( topo, "gossip_gossvf" );
144 0 : fd_topob_wksp( topo, "gossip_ciaddr" );
145 0 : fd_topob_wksp( topo, "gossip_misc" );
146 :
147 0 : fd_topob_link( topo, "gossip_gossvf", "gossip_gossvf", 65536UL*4, sizeof(fd_gossip_ping_update_t), 1UL );
148 0 : fd_topob_tile_out( topo, "gossip", 0UL, "gossip_gossvf", 0UL );
149 :
150 0 : fd_topob_link( topo, "gossip_ciaddr", "gossip_ciaddr", 65536UL*2, sizeof(fd_gossip_update_message_t), 1UL );
151 0 : fd_topob_link( topo, "gossip_misc", "gossip_misc", 65536UL*2, sizeof(fd_gossip_update_message_t), 1UL )->permit_no_consumers = 1;
152 0 : fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciaddr", 0UL );
153 0 : fd_topob_tile_out( topo, "gossip", 0UL, "gossip_misc", 0UL );
154 0 : for( ulong i=0UL; i<gossvf_tile_count; i++ ) {
155 0 : fd_topob_link( topo, "gossvf_gossip", "gossvf_gossip", 65536UL*4, FD_GOSSIP_GOSSVF_MTU, 1UL );
156 0 : fd_topob_tile_out( topo, "gossvf", i, "gossvf_gossip", i );
157 0 : fd_topob_tile_in( topo, "gossip", 0UL, "metric_in", "gossvf_gossip", i, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
158 :
159 : /* Only one link_kind for gossip_ciaddr broadcast link */
160 0 : fd_topob_tile_in( topo, "gossvf", i, "metric_in", "gossip_gossvf", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
161 0 : fd_topob_tile_in( topo, "gossvf", i, "metric_in", "gossip_ciaddr", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
162 0 : }
163 :
164 0 : fd_topob_wksp( topo, "gossip_sign" );
165 0 : fd_topob_link( topo, "gossip_sign", "gossip_sign", 128UL, 2048UL, 1UL );
166 0 : fd_topob_tile_in( topo, "sign", 0UL, "metric_in", "gossip_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
167 0 : fd_topob_wksp( topo, "sign_gossip" );
168 0 : fd_topob_link( topo, "sign_gossip", "sign_gossip", 128UL, 64UL, 1UL );
169 0 : fd_topob_tile_out( topo, "sign", 0UL, "sign_gossip", 0UL );
170 0 : fd_topob_tile_out( topo, "gossip", 0UL, "gossip_sign", 0UL );
171 0 : }
172 :
173 : static args_t
174 0 : configure_args( void ) {
175 0 : args_t args = {
176 0 : .configure.command = CONFIGURE_CMD_INIT,
177 0 : };
178 :
179 0 : ulong stage_idx = 0UL;
180 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_mlx5;
181 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_hugetlbfs;
182 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_sysctl;
183 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_bonding;
184 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_ethtool_channels;
185 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_ethtool_offloads;
186 0 : args.configure.stages[ stage_idx++ ] = &fd_cfg_stage_ethtool_loopback;
187 0 : args.configure.stages[ stage_idx++ ] = NULL;
188 :
189 0 : return args;
190 0 : }
191 :
192 : void
193 : gossip_cmd_perm( args_t * args FD_PARAM_UNUSED,
194 : fd_cap_chk_t * chk,
195 0 : config_t const * config ) {
196 0 : args_t c_args = configure_args();
197 0 : configure_cmd_perm( &c_args, chk, config );
198 0 : run_cmd_perm( NULL, chk, config );
199 0 : }
200 :
201 : /* Display helper functions and types have been extracted to
202 : gossip_diag.h / gossip_diag.c in the shared commands directory. */
203 :
204 : static void
205 : gossip_args( int * pargc,
206 : char *** pargv,
207 0 : args_t * args ) {
208 0 : args->gossip.max_entries = fd_env_strip_cmdline_ulong ( pargc, pargv, "--max-entries", NULL, ULONG_MAX );
209 0 : args->gossip.max_contact = fd_env_strip_cmdline_ulong ( pargc, pargv, "--max-contact-infos", NULL, ULONG_MAX );
210 0 : args->gossip.compact_mode = fd_env_strip_cmdline_contains( pargc, pargv, "--compact" );
211 0 : }
212 :
213 : void
214 : gossip_cmd_fn( args_t * args,
215 0 : config_t * config ) {
216 0 : args_t c_args = configure_args();
217 0 : configure_cmd_fn( &c_args, config );
218 :
219 0 : run_firedancer_init( config, 1, 1 );
220 :
221 0 : int const is_xdp = ( 0==strcmp( config->net.provider, "xdp" ) );
222 0 : if( is_xdp ) fd_topo_install_xdp_simple( &config->topo, config->net.bind_address_parsed );
223 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_WRITE, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
224 0 : if( 0==strcmp( config->net.provider, "mlx5" ) ) {
225 0 : fd_topo_install_mlx5( &config->topo, NULL );
226 0 : }
227 0 : fd_topo_fill( &config->topo );
228 :
229 : /* FIXME allow running sandboxed/multiprocess */
230 0 : fd_topo_run_single_process( &config->topo, config->uid, config->gid, fdctl_tile_run );
231 :
232 0 : fd_gossip_diag_ctx_t diag_ctx[1];
233 0 : if( FD_UNLIKELY( fd_gossip_diag_init( diag_ctx, &config->topo, config ) ) )
234 0 : FD_LOG_ERR(( "Failed to initialize gossip diagnostics" ));
235 :
236 0 : fd_clock_tile_t clock[1];
237 0 : fd_clock_tile_init( clock );
238 :
239 0 : long start_time = fd_clock_tile_now( clock );
240 0 : long next_report_time = start_time + 1000000000L;
241 :
242 0 : for(;;) {
243 0 : long current_time = fd_clock_tile_now( clock );
244 :
245 0 : if( FD_LIKELY( current_time < next_report_time ) ) {
246 0 : continue;
247 0 : }
248 0 : next_report_time += 1000000000L;
249 :
250 0 : fd_gossip_diag_render( diag_ctx, args->gossip.compact_mode );
251 :
252 0 : if( FD_UNLIKELY( diag_ctx->last_total_crds >= args->gossip.max_entries ||
253 0 : diag_ctx->last_total_contact_infos >= args->gossip.max_contact ) ) {
254 0 : long elapsed = current_time - start_time;
255 0 : double elapsed_secs = (double)elapsed / 1000000000.0;
256 0 : printf( "User defined thresholds reached in %.2fs\n"
257 0 : " Table Size : %lu\n"
258 0 : " Contact Infos: %lu\n",
259 0 : elapsed_secs, diag_ctx->last_total_crds, diag_ctx->last_total_contact_infos );
260 0 : break;
261 0 : }
262 0 : fd_clock_tile_recal( clock );
263 0 : }
264 0 : }
265 :
266 : static void
267 0 : gossip_args_help( fd_action_help_t * help ) {
268 0 : fd_action_help_arg( help, "--max-entries", "<num>", "Exit once we see <num> CRDS entries in the table" );
269 0 : fd_action_help_arg( help, "--max-contact-infos", "<num>", "Exit once we see <num> contact infos in the table" );
270 0 : fd_action_help_arg( help, "--compact", NULL, "Use a denser output format" );
271 0 : }
272 :
273 : action_t fd_action_gossip = {
274 : .name = "gossip",
275 : .args = gossip_args,
276 : .fn = gossip_cmd_fn,
277 : .perm = gossip_cmd_perm,
278 : .topo = gossip_cmd_topo,
279 : .description = "Run a reduced topology that joins gossip and prints diagnostics",
280 : .detail = "Boots a minimal Firedancer topology containing the gossip tile, joins the\n"
281 : "cluster's gossip network, and periodically prints CRDS table and contact\n"
282 : "info statistics until the configured thresholds are reached.",
283 : .usage = "gossip [OPTIONS]",
284 : .args_help = gossip_args_help,
285 : };
|
