Line data Source code
1 : #include "configure.h"
2 :
3 : #include <errno.h>
4 : #include <unistd.h>
5 :
6 : #include "fd_ethtool_ioctl.h"
7 : #include "../../../../disco/net/fd_linux_bond.h"
8 :
9 0 : #define NAME "ethtool-channels"
10 :
11 : static int
12 0 : enabled( fd_config_t const * config ) {
13 :
14 : /* only enable if network stack is XDP */
15 0 : if( 0!=strcmp( config->net.provider, "xdp" ) ) return 0;
16 :
17 0 : return 1;
18 0 : }
19 :
20 : static void
21 : init_perm( fd_cap_chk_t * chk,
22 0 : fd_config_t const * config FD_PARAM_UNUSED ) {
23 0 : fd_cap_chk_root( chk, NAME, "modify network device configuration with ethtool" );
24 0 : }
25 :
26 : static void
27 : fini_perm( fd_cap_chk_t * chk,
28 0 : fd_config_t const * config FD_PARAM_UNUSED ) {
29 0 : fd_cap_chk_root( chk, NAME, "modify network device configuration with ethtool" );
30 0 : }
31 :
32 : /* FIXME: Centrally define listen port list to avoid this configure
33 : stage from going out of sync with port mappings. */
34 : static uint
35 : get_ports( fd_config_t const * config,
36 0 : ushort * ports ) {
37 0 : uint port_cnt = 0U;
38 :
39 0 : #define ADD_PORT( p ) do { \
40 0 : ushort __port = ( p ); \
41 0 : if( FD_UNLIKELY( __port==0U ) ) break; \
42 0 : int __dupe = 0; \
43 0 : for( uint __p=0U; !__dupe && __p<port_cnt; ++__p ) __dupe = (ports[ __p ]==__port); \
44 0 : if( FD_UNLIKELY( __dupe ) ) break; \
45 0 : ports[ port_cnt ] = __port; \
46 0 : port_cnt++; \
47 0 : } while(0)
48 :
49 0 : ADD_PORT( config->tiles.shred.shred_listen_port );
50 0 : ADD_PORT( config->tiles.quic.quic_transaction_listen_port );
51 0 : ADD_PORT( config->tiles.quic.regular_transaction_listen_port );
52 0 : ADD_PORT( config->gossip.port );
53 0 : ADD_PORT( config->tiles.repair.repair_client_listen_port );
54 0 : ADD_PORT( config->tiles.rserve.repair_serve_listen_port );
55 0 : ADD_PORT( config->firedancer.development.votor.quic_client_listen_port );
56 0 : ADD_PORT( config->firedancer.development.votor.quic_server_listen_port );
57 0 : #undef ADD_PORT
58 :
59 0 : return port_cnt;
60 0 : }
61 :
62 : /* Attempts to initialize the device in simple or dedicated mode. If
63 : strict is true, FD_LOG_ERR's on failure. Otherwise, returns 1 on
64 : failure. Returns 0 on success. */
65 : static int
66 : init_device( char const * device,
67 : fd_config_t const * config,
68 : int dedicated_mode,
69 : int listen_gre,
70 : int strict,
71 0 : uint device_cnt ) {
72 0 : FD_TEST( dedicated_mode || strict );
73 :
74 0 : uint const net_tile_cnt = config->layout.net_tile_count;
75 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
76 0 : FD_LOG_ERR(( "net tile count %u must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
77 0 : }
78 0 : uint const queue_cnt = net_tile_cnt / device_cnt;
79 :
80 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
81 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
82 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
83 :
84 : /* This should happen first, otherwise changing the number of channels may fail */
85 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_set_default( &ioc ) );
86 :
87 0 : uint const num_channels = !dedicated_mode ? queue_cnt : 0 /* maximum allowed */;
88 0 : int ret = fd_ethtool_ioctl_channels_set_num( &ioc, num_channels );
89 0 : if( FD_UNLIKELY( 0!=ret ) ) {
90 0 : if( strict ) {
91 0 : if( FD_LIKELY( ret == EBUSY ) )
92 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to set number of channels. "
93 0 : "This is most commonly caused by an issue with the Intel ice driver on certain versions "
94 0 : "of Ubuntu. If you are using the ice driver, `sudo dmesg | grep %s` contains "
95 0 : "messages about RDMA, and you do not need RDMA, try running `rmmod irdma` and/or "
96 0 : "blacklisting the irdma kernel module.", device, device ));
97 0 : else
98 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to set number of channels", device ));
99 0 : }
100 0 : return 1;
101 0 : }
102 :
103 : /* Some drivers (e.g. igb) put the RXFH table into an incorrect state
104 : after changing the channel count. So in simple mode we reset it
105 : to the default again. */
106 0 : if( !dedicated_mode ) {
107 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_set_default( &ioc ) );
108 :
109 : /* Configure the NIC to include UDP source and destination ports in
110 : the RSS hash for UDP/IPv4 flows. Without this, most NICs default
111 : to hashing only on IP addresses, which causes severe RX queue
112 : imbalance when all traffic targets the same destination IP and
113 : port (e.g. gossip). Not needed in dedicated mode since ntuple
114 : rules bypass RSS for Firedancer traffic. */
115 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_rxfh_set_flow_hash_udp4( &ioc ) ) )
116 0 : FD_LOG_WARNING(( "failed to set UDP flow hash on device %s, RSS distribution may be poor", device ));
117 0 : }
118 :
119 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_clear( &ioc ) );
120 :
121 0 : if( dedicated_mode ) {
122 : /* Some drivers (e.g. ixgbe) reset the RXFH table upon activation
123 : of the ntuple feature, so we do this first. */
124 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_feature_set( &ioc, FD_ETHTOOL_FEATURE_NTUPLE, 1 ) ) ) {
125 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to enable ntuple feature. Try `net.xdp.rss_queue_mode=\"simple\"`", device ));
126 0 : else return 1;
127 0 : }
128 :
129 : /* Remove a queue from the rxfh table for each net tile. */
130 0 : uint rxfh_queue_cnt;
131 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
132 0 : if( FD_UNLIKELY( queue_cnt>=rxfh_queue_cnt ) ) {
133 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), too many net tiles %u for queue count %u. "
134 0 : "Try `net.xdp.rss_queue_mode=\"simple\"` or reduce net tile count",
135 0 : device, net_tile_cnt, rxfh_queue_cnt ));
136 0 : else return 1;
137 0 : }
138 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_rxfh_set_suffix( &ioc, queue_cnt ) ) ) {
139 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to isolate queues. Try `net.xdp.rss_queue_mode=\"simple\"`", device ));
140 0 : else return 1;
141 0 : }
142 :
143 : /* Add a ntuple rule for each listening destination port. If there
144 : are multiple net tiles, create a group of rules for each tile. */
145 0 : int ntuple_error = 0;
146 0 : ushort ports[ 32 ];
147 0 : uint port_cnt = get_ports( config, ports );
148 0 : uint rule_idx = 0;
149 0 : uint const rule_group_cnt = fd_uint_pow2_up( queue_cnt );
150 0 : for( uint r=0U; !ntuple_error && r<rule_group_cnt; r++ ) {
151 0 : for( uint p=0U; !ntuple_error && p<port_cnt; p++ ) {
152 0 : ntuple_error = 0!=fd_ethtool_ioctl_ntuple_set_udp_dport( &ioc, rule_idx++, ports[ p ], r, rule_group_cnt, r%queue_cnt );
153 0 : }
154 0 : }
155 0 : if( FD_UNLIKELY( ntuple_error ) ) {
156 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to install ntuple rules. "
157 0 : "Try `net.xdp.rss_queue_mode=\"simple\"` or `layout.net_tile_count=1`", device ));
158 0 : else return 1;
159 0 : }
160 :
161 : /* It's rather unfortunate, but given the APIs that ethtool exposes,
162 : we can't do much better than sending all GRE packets to a single
163 : queue. At least more recent Mellanox NICs certainly do support
164 : RSS based on the inner header, but it's not possible to configure
165 : this in a generic way. */
166 0 : int gre_ntuple_error = 0;
167 0 : if( listen_gre ) gre_ntuple_error = fd_ethtool_ioctl_ntuple_set_gre( &ioc, rule_idx++, 0U );
168 0 : if( FD_UNLIKELY( gre_ntuple_error ) ) {
169 0 : if( strict ) {
170 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to install ntuple rule "
171 0 : "to route GRE packets to Firedancer. If you require GRE-wrapped "
172 0 : "traffic (e.g. with DoubleZero), set `net.xdp.rss_queue_mode=\"simple\"`. "
173 0 : "Otherwise, set `net.xdp.listen_gre=false`.", device ));
174 0 : } else {
175 0 : return 1;
176 0 : }
177 0 : }
178 0 : }
179 :
180 0 : return 0;
181 0 : }
182 :
183 : static void
184 0 : init( fd_config_t const * config ) {
185 0 : int only_dedicated =
186 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "dedicated" ));
187 0 : int try_dedicated = only_dedicated ||
188 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "auto" ) );
189 :
190 : /* if using a bonded device, we need to set channels on the
191 : underlying devices. */
192 0 : int is_bonded = fd_bonding_is_master( config->net.interface );
193 0 : uint device_cnt = 1U;
194 0 : if( is_bonded && config->net.xdp.native_bond ) {
195 0 : device_cnt = fd_bonding_slave_cnt( config->net.interface );
196 0 : }
197 :
198 0 : int listen_gre = config->net.xdp.listen_gre;
199 :
200 : /* If the mode was auto, we will try to init in dedicated mode but will
201 : not fail the stage if this is not successful. If the mode was
202 : dedicated, we will require success. */
203 0 : if( try_dedicated ) {
204 0 : int failed = 0;
205 0 : if( is_bonded ) {
206 0 : fd_bonding_slave_iter_t iter_[1];
207 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
208 0 : for( ; !failed && !fd_bonding_slave_iter_done( iter );
209 0 : fd_bonding_slave_iter_next( iter ) ) {
210 0 : failed = init_device( fd_bonding_slave_iter_ele( iter ), config, 1, listen_gre, only_dedicated, device_cnt );
211 0 : }
212 0 : } else {
213 0 : failed = init_device( config->net.interface, config, 1, listen_gre, only_dedicated, device_cnt );
214 0 : }
215 0 : if( !failed ) return;
216 0 : FD_TEST( !only_dedicated );
217 0 : FD_LOG_WARNING(( "error configuring network device (%s), rss_queue_mode \"auto\" attempted"
218 0 : " \"dedicated\" configuration but falling back to \"simple\".", config->net.interface ));
219 : /* Wipe partial dedicated configuration before simple init */
220 0 : fd_ethtool_ioctl_reset( config->net.interface );
221 0 : }
222 :
223 : /* Require success for simple mode, either configured or as fallback */
224 0 : if( is_bonded ) {
225 0 : fd_bonding_slave_iter_t iter_[1];
226 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
227 0 : for( ; !fd_bonding_slave_iter_done( iter );
228 0 : fd_bonding_slave_iter_next( iter ) ) {
229 0 : init_device( fd_bonding_slave_iter_ele( iter ), config, 0, listen_gre, 1, device_cnt );
230 0 : }
231 0 : } else {
232 0 : init_device( config->net.interface, config, 0, listen_gre, 1, device_cnt );
233 0 : }
234 0 : }
235 :
236 : static int
237 : check_device_is_configured( char const * device,
238 : fd_config_t const * config,
239 : int dedicated_mode,
240 0 : uint device_cnt ) {
241 0 : uint const net_tile_cnt = config->layout.net_tile_count;
242 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
243 0 : FD_LOG_ERR(( "net tile count %u must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
244 0 : }
245 0 : uint const queue_cnt = net_tile_cnt / device_cnt;
246 :
247 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
248 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
249 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
250 :
251 0 : fd_ethtool_ioctl_channels_t channels;
252 0 : FD_TEST( 0==fd_ethtool_ioctl_channels_get_num( &ioc, &channels ) );
253 0 : if( channels.current!=(dedicated_mode ? channels.max : queue_cnt) ) return 0;
254 :
255 0 : uint rxfh_queue_cnt;
256 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
257 0 : rxfh_queue_cnt = fd_uint_min( rxfh_queue_cnt, channels.current );
258 :
259 0 : uint rxfh_table[ FD_ETHTOOL_MAX_RXFH_TABLE_CNT ] = { 0 };
260 0 : uint rxfh_table_ele_cnt;
261 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_table( &ioc, rxfh_table, &rxfh_table_ele_cnt ) );
262 0 : int rxfh_error = (dedicated_mode && 0U==rxfh_table_ele_cnt);
263 0 : uint const start_queue = dedicated_mode ? queue_cnt : 0U;
264 0 : for( uint j=0U, q=start_queue; !rxfh_error && j<rxfh_table_ele_cnt; j++) {
265 0 : rxfh_error = (rxfh_table[ j ]!=q++);
266 0 : if( FD_UNLIKELY( q>=rxfh_queue_cnt ) ) q = start_queue;
267 0 : }
268 0 : if( rxfh_error ) return 0;
269 :
270 0 : if( dedicated_mode ) {
271 0 : int ntuple_feature_active;
272 0 : FD_TEST( 0==fd_ethtool_ioctl_feature_test( &ioc, FD_ETHTOOL_FEATURE_NTUPLE, &ntuple_feature_active ) );
273 0 : if( !ntuple_feature_active ) return 0;
274 0 : }
275 :
276 0 : if( !dedicated_mode ) {
277 0 : int ntuple_rules_empty;
278 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, NULL, 0, 0, UINT_MAX, &ntuple_rules_empty ) );
279 0 : if( !ntuple_rules_empty ) return 0;
280 0 : } else {
281 0 : int ports_valid;
282 0 : ushort ports[ 32 ];
283 0 : uint port_cnt = get_ports( config, ports );
284 0 : int listen_gre = config->net.xdp.listen_gre;
285 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, ports, port_cnt, queue_cnt, listen_gre ? 0U : UINT_MAX, &ports_valid ));
286 0 : if( !ports_valid ) return 0;
287 0 : }
288 :
289 0 : return 1;
290 0 : }
291 :
292 : static configure_result_t
293 : check( fd_config_t const * config,
294 0 : int check_type FD_PARAM_UNUSED ) {
295 0 : int only_dedicated =
296 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "dedicated" ));
297 0 : int check_dedicated = only_dedicated ||
298 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "auto" ));
299 :
300 0 : int is_bonded = fd_bonding_is_master( config->net.interface );
301 0 : uint device_cnt = 1U;
302 0 : if( is_bonded && config->net.xdp.native_bond ) {
303 0 : device_cnt = fd_bonding_slave_cnt( config->net.interface );
304 0 : }
305 :
306 0 : if( check_dedicated ) {
307 0 : int is_configured = 1;
308 0 : if( is_bonded ) {
309 0 : fd_bonding_slave_iter_t iter_[1];
310 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
311 0 : for( ; is_configured && !fd_bonding_slave_iter_done( iter );
312 0 : fd_bonding_slave_iter_next( iter ) ) {
313 0 : is_configured = check_device_is_configured( fd_bonding_slave_iter_ele( iter ), config, 1, device_cnt );
314 0 : }
315 0 : } else {
316 0 : is_configured = check_device_is_configured( config->net.interface, config, 1, device_cnt );
317 0 : }
318 0 : if( is_configured ) CONFIGURE_OK();
319 0 : }
320 :
321 0 : if( !only_dedicated ) {
322 0 : int is_configured = 1;
323 0 : if( is_bonded ) {
324 0 : fd_bonding_slave_iter_t iter_[1];
325 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
326 0 : for( ; is_configured && !fd_bonding_slave_iter_done( iter );
327 0 : fd_bonding_slave_iter_next( iter ) ) {
328 0 : is_configured = check_device_is_configured( fd_bonding_slave_iter_ele( iter ), config, 0, device_cnt );
329 0 : }
330 0 : } else {
331 0 : is_configured = check_device_is_configured( config->net.interface, config, 0, device_cnt );
332 0 : }
333 0 : if( is_configured ) CONFIGURE_OK();
334 0 : }
335 :
336 0 : int is_modified = fd_ethtool_ioctl_is_modified( config->net.interface );
337 0 : if( is_modified )
338 0 : PARTIALLY_CONFIGURED( "device `%s` has partial ethtool-channels network configuration", config->net.interface );
339 :
340 0 : NOT_CONFIGURED( "device `%s` missing ethtool-channels network configuration", config->net.interface );
341 0 : }
342 :
343 : static int
344 : fini( fd_config_t const * config,
345 0 : int pre_init FD_PARAM_UNUSED ) {
346 0 : return fd_ethtool_ioctl_reset( config->net.interface );
347 0 : }
348 :
349 : configure_stage_t fd_cfg_stage_ethtool_channels = {
350 : .name = NAME,
351 : .always_recreate = 0,
352 : .enabled = enabled,
353 : .init_perm = init_perm,
354 : .fini_perm = fini_perm,
355 : .init = init,
356 : .fini = fini,
357 : .check = check,
358 : };
359 :
360 : #undef NAME
|
