Line data Source code
1 : #include "configure.h"
2 :
3 : #include <errno.h>
4 : #include <unistd.h>
5 :
6 : #include "fd_ethtool_ioctl.h"
7 : #include "../../../../disco/net/fd_linux_bond.h"
8 :
9 0 : #define NAME "ethtool-channels"
10 :
11 : static int
12 0 : enabled( fd_config_t const * config ) {
13 :
14 : /* only enable if network stack is XDP */
15 0 : if( 0!=strcmp( config->net.provider, "xdp" ) ) return 0;
16 :
17 0 : return 1;
18 0 : }
19 :
20 : static void
21 : init_perm( fd_cap_chk_t * chk,
22 0 : fd_config_t const * config FD_PARAM_UNUSED ) {
23 0 : fd_cap_chk_root( chk, NAME, "modify network device configuration with ethtool" );
24 0 : }
25 :
26 : static void
27 : fini_perm( fd_cap_chk_t * chk,
28 0 : fd_config_t const * config FD_PARAM_UNUSED ) {
29 0 : fd_cap_chk_root( chk, NAME, "modify network device configuration with ethtool" );
30 0 : }
31 :
32 : /* Attempts to initialize the device in simple or dedicated mode. If
33 : strict is true, FD_LOG_ERR's on failure. Otherwise, returns 1 on
34 : failure. Returns 0 on success. */
35 : static int
36 : init_device( char const * device,
37 : fd_config_t const * config,
38 : int dedicated_mode,
39 : int listen_gre,
40 : int strict,
41 0 : uint device_cnt ) {
42 0 : FD_TEST( dedicated_mode || strict );
43 :
44 0 : uint const net_tile_cnt = config->layout.net_tile_count;
45 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
46 0 : FD_LOG_ERR(( "net tile count %u must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
47 0 : }
48 0 : uint const queue_cnt = net_tile_cnt / device_cnt;
49 :
50 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
51 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
52 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
53 :
54 : /* This should happen first, otherwise changing the number of channels may fail */
55 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_set_default( &ioc ) );
56 :
57 0 : uint const num_channels = !dedicated_mode ? queue_cnt : 0 /* maximum allowed */;
58 0 : int ret = fd_ethtool_ioctl_channels_set_num( &ioc, num_channels );
59 0 : if( FD_UNLIKELY( 0!=ret ) ) {
60 0 : if( strict ) {
61 0 : if( FD_LIKELY( ret == EBUSY ) )
62 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to set number of channels. "
63 0 : "This is most commonly caused by an issue with the Intel ice driver on certain versions "
64 0 : "of Ubuntu. If you are using the ice driver, `sudo dmesg | grep %s` contains "
65 0 : "messages about RDMA, and you do not need RDMA, try running `rmmod irdma` and/or "
66 0 : "blacklisting the irdma kernel module.", device, device ));
67 0 : else
68 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to set number of channels", device ));
69 0 : }
70 0 : return 1;
71 0 : }
72 :
73 : /* Some drivers (e.g. igb) put the RXFH table into an incorrect state
74 : after changing the channel count. So in simple mode we reset it
75 : to the default again. */
76 0 : if( !dedicated_mode ) {
77 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_set_default( &ioc ) );
78 :
79 : /* Configure the NIC to include UDP source and destination ports in
80 : the RSS hash for UDP/IPv4 flows. Without this, most NICs default
81 : to hashing only on IP addresses, which causes severe RX queue
82 : imbalance when all traffic targets the same destination IP and
83 : port (e.g. gossip). Not needed in dedicated mode since ntuple
84 : rules bypass RSS for Firedancer traffic. */
85 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_rxfh_set_flow_hash_udp4( &ioc ) ) )
86 0 : FD_LOG_WARNING(( "failed to set UDP flow hash on device %s, RSS distribution may be poor", device ));
87 0 : }
88 :
89 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_clear( &ioc ) );
90 :
91 0 : if( dedicated_mode ) {
92 : /* Some drivers (e.g. ixgbe) reset the RXFH table upon activation
93 : of the ntuple feature, so we do this first. */
94 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_feature_set( &ioc, FD_ETHTOOL_FEATURE_NTUPLE, 1 ) ) ) {
95 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to enable ntuple feature. Try `net.xdp.rss_queue_mode=\"simple\"`", device ));
96 0 : else return 1;
97 0 : }
98 :
99 : /* Remove a queue from the rxfh table for each net tile. */
100 0 : uint rxfh_queue_cnt;
101 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
102 0 : if( FD_UNLIKELY( queue_cnt>=rxfh_queue_cnt ) ) {
103 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), too many net tiles %u for queue count %u. "
104 0 : "Try `net.xdp.rss_queue_mode=\"simple\"` or reduce net tile count",
105 0 : device, net_tile_cnt, rxfh_queue_cnt ));
106 0 : else return 1;
107 0 : }
108 0 : if( FD_UNLIKELY( 0!=fd_ethtool_ioctl_rxfh_set_suffix( &ioc, queue_cnt ) ) ) {
109 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to isolate queues. Try `net.xdp.rss_queue_mode=\"simple\"`", device ));
110 0 : else return 1;
111 0 : }
112 :
113 : /* Add a ntuple rule for each listening destination port. If there
114 : are multiple net tiles, create a group of rules for each tile. */
115 0 : int ntuple_error = 0;
116 0 : ushort ports[ FD_NET_RX_DEST_CNT ];
117 0 : uint port_cnt = configure_udp_ports( config, ports );
118 0 : uint rule_idx = 0;
119 0 : uint const rule_group_cnt = fd_uint_pow2_up( queue_cnt );
120 0 : for( uint r=0U; !ntuple_error && r<rule_group_cnt; r++ ) {
121 0 : for( uint p=0U; !ntuple_error && p<port_cnt; p++ ) {
122 0 : ntuple_error = 0!=fd_ethtool_ioctl_ntuple_set_udp_dport( &ioc, rule_idx++, ports[ p ], r, rule_group_cnt, r%queue_cnt );
123 0 : }
124 0 : }
125 0 : if( FD_UNLIKELY( ntuple_error ) ) {
126 0 : if( strict ) FD_LOG_ERR(( "error configuring network device (%s), failed to install ntuple rules. "
127 0 : "Try `net.xdp.rss_queue_mode=\"simple\"` or `layout.net_tile_count=1`", device ));
128 0 : else return 1;
129 0 : }
130 :
131 : /* It's rather unfortunate, but given the APIs that ethtool exposes,
132 : we can't do much better than sending all GRE packets to a single
133 : queue. At least more recent Mellanox NICs certainly do support
134 : RSS based on the inner header, but it's not possible to configure
135 : this in a generic way. */
136 0 : int gre_ntuple_error = 0;
137 0 : if( listen_gre ) gre_ntuple_error = fd_ethtool_ioctl_ntuple_set_gre( &ioc, rule_idx++, 0U );
138 0 : if( FD_UNLIKELY( gre_ntuple_error ) ) {
139 0 : if( strict ) {
140 0 : FD_LOG_ERR(( "error configuring network device (%s), failed to install ntuple rule "
141 0 : "to route GRE packets to Firedancer. If you require GRE-wrapped "
142 0 : "traffic (e.g. with DoubleZero), set `net.xdp.rss_queue_mode=\"simple\"`. "
143 0 : "Otherwise, set `net.xdp.listen_gre=false`.", device ));
144 0 : } else {
145 0 : return 1;
146 0 : }
147 0 : }
148 0 : }
149 :
150 0 : return 0;
151 0 : }
152 :
153 : static void
154 0 : init( fd_config_t const * config ) {
155 0 : int only_dedicated =
156 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "dedicated" ));
157 0 : int try_dedicated = only_dedicated ||
158 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "auto" ) );
159 :
160 : /* if using a bonded device, we need to set channels on the
161 : underlying devices. */
162 0 : int is_bonded = fd_bonding_is_master( config->net.interface );
163 0 : uint device_cnt = 1U;
164 0 : if( is_bonded && config->net.xdp.native_bond ) {
165 0 : device_cnt = fd_bonding_slave_cnt( config->net.interface );
166 0 : }
167 :
168 0 : int listen_gre = config->net.xdp.listen_gre;
169 :
170 : /* If the mode was auto, we will try to init in dedicated mode but will
171 : not fail the stage if this is not successful. If the mode was
172 : dedicated, we will require success. */
173 0 : if( try_dedicated ) {
174 0 : int failed = 0;
175 0 : if( is_bonded ) {
176 0 : fd_bonding_slave_iter_t iter_[1];
177 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
178 0 : for( ; !failed && !fd_bonding_slave_iter_done( iter );
179 0 : fd_bonding_slave_iter_next( iter ) ) {
180 0 : failed = init_device( fd_bonding_slave_iter_ele( iter ), config, 1, listen_gre, only_dedicated, device_cnt );
181 0 : }
182 0 : } else {
183 0 : failed = init_device( config->net.interface, config, 1, listen_gre, only_dedicated, device_cnt );
184 0 : }
185 0 : if( !failed ) return;
186 0 : FD_TEST( !only_dedicated );
187 0 : FD_LOG_WARNING(( "error configuring network device (%s), rss_queue_mode \"auto\" attempted"
188 0 : " \"dedicated\" configuration but falling back to \"simple\".", config->net.interface ));
189 : /* Wipe partial dedicated configuration before simple init */
190 0 : fd_ethtool_ioctl_reset( config->net.interface );
191 0 : }
192 :
193 : /* Require success for simple mode, either configured or as fallback */
194 0 : if( is_bonded ) {
195 0 : fd_bonding_slave_iter_t iter_[1];
196 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
197 0 : for( ; !fd_bonding_slave_iter_done( iter );
198 0 : fd_bonding_slave_iter_next( iter ) ) {
199 0 : init_device( fd_bonding_slave_iter_ele( iter ), config, 0, listen_gre, 1, device_cnt );
200 0 : }
201 0 : } else {
202 0 : init_device( config->net.interface, config, 0, listen_gre, 1, device_cnt );
203 0 : }
204 0 : }
205 :
206 : static int
207 : check_device_is_configured( char const * device,
208 : fd_config_t const * config,
209 : int dedicated_mode,
210 0 : uint device_cnt ) {
211 0 : uint const net_tile_cnt = config->layout.net_tile_count;
212 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
213 0 : FD_LOG_ERR(( "net tile count %u must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
214 0 : }
215 0 : uint const queue_cnt = net_tile_cnt / device_cnt;
216 :
217 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
218 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
219 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
220 :
221 0 : fd_ethtool_ioctl_channels_t channels;
222 0 : FD_TEST( 0==fd_ethtool_ioctl_channels_get_num( &ioc, &channels ) );
223 0 : if( channels.current!=(dedicated_mode ? channels.max : queue_cnt) ) return 0;
224 :
225 0 : uint rxfh_queue_cnt;
226 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
227 0 : rxfh_queue_cnt = fd_uint_min( rxfh_queue_cnt, channels.current );
228 :
229 0 : uint rxfh_table[ FD_ETHTOOL_MAX_RXFH_TABLE_CNT ] = { 0 };
230 0 : uint rxfh_table_ele_cnt;
231 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_table( &ioc, rxfh_table, &rxfh_table_ele_cnt ) );
232 0 : int rxfh_error = (dedicated_mode && 0U==rxfh_table_ele_cnt);
233 0 : uint const start_queue = dedicated_mode ? queue_cnt : 0U;
234 0 : for( uint j=0U, q=start_queue; !rxfh_error && j<rxfh_table_ele_cnt; j++) {
235 0 : rxfh_error = (rxfh_table[ j ]!=q++);
236 0 : if( FD_UNLIKELY( q>=rxfh_queue_cnt ) ) q = start_queue;
237 0 : }
238 0 : if( rxfh_error ) return 0;
239 :
240 0 : if( dedicated_mode ) {
241 0 : int ntuple_feature_active;
242 0 : FD_TEST( 0==fd_ethtool_ioctl_feature_test( &ioc, FD_ETHTOOL_FEATURE_NTUPLE, &ntuple_feature_active ) );
243 0 : if( !ntuple_feature_active ) return 0;
244 0 : }
245 :
246 0 : if( !dedicated_mode ) {
247 0 : int ntuple_rules_empty;
248 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, NULL, 0, 0, UINT_MAX, &ntuple_rules_empty ) );
249 0 : if( !ntuple_rules_empty ) return 0;
250 0 : } else {
251 0 : int ports_valid;
252 0 : ushort ports[ FD_NET_RX_DEST_CNT ];
253 0 : uint port_cnt = configure_udp_ports( config, ports );
254 0 : int listen_gre = config->net.xdp.listen_gre;
255 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, ports, port_cnt, queue_cnt, listen_gre ? 0U : UINT_MAX, &ports_valid ));
256 0 : if( !ports_valid ) return 0;
257 0 : }
258 :
259 0 : return 1;
260 0 : }
261 :
262 : static configure_result_t
263 : check( fd_config_t const * config,
264 0 : int check_type FD_PARAM_UNUSED ) {
265 0 : int only_dedicated =
266 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "dedicated" ));
267 0 : int check_dedicated = only_dedicated ||
268 0 : (0==strcmp( config->net.xdp.rss_queue_mode, "auto" ));
269 :
270 0 : int is_bonded = fd_bonding_is_master( config->net.interface );
271 0 : uint device_cnt = 1U;
272 0 : if( is_bonded && config->net.xdp.native_bond ) {
273 0 : device_cnt = fd_bonding_slave_cnt( config->net.interface );
274 0 : }
275 :
276 0 : if( check_dedicated ) {
277 0 : int is_configured = 1;
278 0 : if( is_bonded ) {
279 0 : fd_bonding_slave_iter_t iter_[1];
280 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
281 0 : for( ; is_configured && !fd_bonding_slave_iter_done( iter );
282 0 : fd_bonding_slave_iter_next( iter ) ) {
283 0 : is_configured = check_device_is_configured( fd_bonding_slave_iter_ele( iter ), config, 1, device_cnt );
284 0 : }
285 0 : } else {
286 0 : is_configured = check_device_is_configured( config->net.interface, config, 1, device_cnt );
287 0 : }
288 0 : if( is_configured ) CONFIGURE_OK();
289 0 : }
290 :
291 0 : if( !only_dedicated ) {
292 0 : int is_configured = 1;
293 0 : if( is_bonded ) {
294 0 : fd_bonding_slave_iter_t iter_[1];
295 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, config->net.interface );
296 0 : for( ; is_configured && !fd_bonding_slave_iter_done( iter );
297 0 : fd_bonding_slave_iter_next( iter ) ) {
298 0 : is_configured = check_device_is_configured( fd_bonding_slave_iter_ele( iter ), config, 0, device_cnt );
299 0 : }
300 0 : } else {
301 0 : is_configured = check_device_is_configured( config->net.interface, config, 0, device_cnt );
302 0 : }
303 0 : if( is_configured ) CONFIGURE_OK();
304 0 : }
305 :
306 0 : int is_modified = fd_ethtool_ioctl_is_modified( config->net.interface );
307 0 : if( is_modified )
308 0 : PARTIALLY_CONFIGURED( "device `%s` has partial ethtool-channels network configuration", config->net.interface );
309 :
310 0 : NOT_CONFIGURED( "device `%s` missing ethtool-channels network configuration", config->net.interface );
311 0 : }
312 :
313 : static int
314 : fini( fd_config_t const * config,
315 0 : int pre_init FD_PARAM_UNUSED ) {
316 0 : return fd_ethtool_ioctl_reset( config->net.interface );
317 0 : }
318 :
319 : configure_stage_t fd_cfg_stage_ethtool_channels = {
320 : .name = NAME,
321 : .always_recreate = 0,
322 : .enabled = enabled,
323 : .init_perm = init_perm,
324 : .fini_perm = fini_perm,
325 : .init = init,
326 : .fini = fini,
327 : .check = check,
328 : };
329 :
330 : #undef NAME
|
