Line data Source code
1 : #include <errno.h>
2 : #include <unistd.h>
3 : #include <linux/ethtool.h>
4 : #include <linux/sockios.h>
5 : #include <net/if.h>
6 : #include <sys/ioctl.h>
7 : #include <sys/socket.h>
8 : #include <sys/sysinfo.h>
9 : #include <netinet/in.h>
10 :
11 : #include "fd_ethtool_ioctl.h"
12 : #include "../../../../disco/net/fd_linux_bond.h"
13 : #include "../../../../util/fd_util.h"
14 : #include "../../../../util/net/fd_ip4.h"
15 :
16 : #define MAX_RXFH_KEY_SIZE (1024)
17 0 : #define MAX_FEATURES (2048)
18 : #define MAX_NTUPLE_RULES (8192)
19 :
20 : #define ETHTOOL_CMD_SIZE( base_t, data_t, data_len ) ( sizeof(base_t) + (sizeof(data_t)*(data_len)) )
21 :
22 : static int
23 : run_ioctl( fd_ethtool_ioctl_t * ioc,
24 : char const * cmd,
25 : void * data,
26 : int log,
27 0 : int log_notsupp ) {
28 0 : ioc->ifr.ifr_data = data;
29 0 : if( FD_UNLIKELY( ioctl( ioc->fd, SIOCETHTOOL, &ioc->ifr ) ) ) {
30 0 : int error = errno;
31 0 : if( (!!log) & ((errno!=EOPNOTSUPP) | (!!log_notsupp)) )
32 0 : FD_LOG_WARNING(( "error configuring network device (%s), ioctl(SIOCETHTOOL,%s) failed (%i-%s)",
33 0 : ioc->ifr.ifr_name, cmd, errno, fd_io_strerror( errno ) ));
34 0 : return error;
35 0 : }
36 0 : return 0;
37 0 : }
38 :
39 : #define TRY_RUN_IOCTL( ioc, cmd, data ) \
40 0 : do { int __ret__ = run_ioctl( (ioc), (cmd), (data), 1, 1 ); \
41 0 : if( FD_UNLIKELY( __ret__ != 0 ) ) { return __ret__; } } while(0)
42 :
43 : fd_ethtool_ioctl_t *
44 : fd_ethtool_ioctl_init( fd_ethtool_ioctl_t * ioc,
45 0 : char const * device ) {
46 0 : if( FD_UNLIKELY( strlen( device ) >= IF_NAMESIZE ) ) {
47 0 : FD_LOG_WARNING(( "device name `%s` is too long", device ));
48 0 : return NULL;
49 0 : }
50 0 : if( FD_UNLIKELY( strlen( device ) == 0 ) ) {
51 0 : FD_LOG_WARNING(( "device name `%s` is empty", device ));
52 0 : return NULL;
53 0 : }
54 :
55 0 : ioc->fd = socket( AF_INET, SOCK_DGRAM, 0 );
56 0 : if( FD_UNLIKELY( ioc->fd < 0 ) ) {
57 0 : FD_LOG_WARNING(( "error configuring network device (%s), socket(AF_INET,SOCK_DGRAM,0) failed (%i-%s)",
58 0 : device, errno, fd_io_strerror( errno ) ));
59 0 : return NULL;
60 0 : }
61 :
62 0 : fd_memset( &ioc->ifr, 0, sizeof(struct ifreq) );
63 0 : fd_cstr_fini( fd_cstr_append_cstr( fd_cstr_init( ioc->ifr.ifr_name ), device ) );
64 :
65 0 : return ioc;
66 0 : }
67 :
68 : void
69 0 : fd_ethtool_ioctl_fini( fd_ethtool_ioctl_t * ioc ) {
70 0 : if( FD_UNLIKELY( close( ioc->fd ) ) ) {
71 0 : FD_LOG_WARNING(( "error configuring network device (%s), close() socket failed (%i-%s)",
72 0 : ioc->ifr.ifr_name, errno, fd_io_strerror( errno ) ));
73 0 : }
74 0 : ioc->fd = -1;
75 0 : fd_memset( &ioc->ifr, 0, sizeof(struct ifreq) );
76 0 : }
77 :
78 : int
79 : fd_ethtool_ioctl_channels_set_num( fd_ethtool_ioctl_t * ioc,
80 0 : uint num ) {
81 0 : struct ethtool_channels ech = { .cmd = ETHTOOL_GCHANNELS };
82 0 : int ret = run_ioctl( ioc, "ETHTOOL_GCHANNELS", &ech, 1, 0 );
83 0 : if( FD_UNLIKELY( ret!=0 ) ) {
84 0 : if( (ret==EOPNOTSUPP) & ((num==0) | (num==1))) return 0;
85 0 : return ret;
86 0 : }
87 :
88 0 : ech.cmd = ETHTOOL_SCHANNELS;
89 0 : if( num == 0 ) {
90 0 : uint max_queue_count = ech.max_combined ? ech.max_combined : ech.max_rx;
91 0 : num = fd_uint_min( max_queue_count, (uint)get_nprocs() );
92 0 : }
93 0 : if( ech.max_combined ) {
94 0 : ech.combined_count = num;
95 0 : ech.rx_count = 0;
96 0 : ech.tx_count = 0;
97 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --set-channels %s combined %u`%s", fd_log_style_dim(), ioc->ifr.ifr_name, num , fd_log_style_normal() ));
98 0 : } else {
99 0 : ech.combined_count = 0;
100 0 : ech.rx_count = num;
101 0 : ech.tx_count = num;
102 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --set-channels %s rx %u tx %u`%s", fd_log_style_dim(), ioc->ifr.ifr_name, num, num , fd_log_style_normal() ));
103 0 : }
104 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_SCHANNELS", &ech );
105 0 : return 0;
106 0 : }
107 :
108 : int
109 : fd_ethtool_ioctl_channels_get_num( fd_ethtool_ioctl_t * ioc,
110 0 : fd_ethtool_ioctl_channels_t * channels ) {
111 0 : struct ethtool_channels ech = { .cmd = ETHTOOL_GCHANNELS };
112 0 : int ret = run_ioctl( ioc, "ETHTOOL_GCHANNELS", &ech, 1, 0 );
113 0 : if( FD_UNLIKELY( ret!=0 ) ) {
114 0 : if( FD_LIKELY( ret==EOPNOTSUPP ) ) {
115 : /* network device doesn't support getting number of channels, so
116 : it must always be 1 */
117 0 : channels->supported = 0;
118 0 : channels->current = 1;
119 0 : channels->max = 1;
120 0 : return 0;
121 0 : }
122 0 : return ret;
123 0 : }
124 0 : channels->supported = 1;
125 :
126 0 : if( FD_LIKELY( ech.combined_count ) ) {
127 0 : channels->current = ech.combined_count;
128 0 : channels->max = fd_uint_min( ech.max_combined, (uint)get_nprocs() );
129 0 : return 0;
130 0 : }
131 0 : if( ech.rx_count || ech.tx_count ) {
132 0 : if( FD_UNLIKELY( ech.rx_count != ech.tx_count ) )
133 0 : FD_LOG_WARNING(( "device `%s` has unbalanced channel count: (got %u rx, %u tx)",
134 0 : ioc->ifr.ifr_name, ech.rx_count, ech.tx_count ));
135 0 : channels->current = ech.rx_count;
136 0 : channels->max = fd_uint_min( ech.max_rx, (uint)get_nprocs() );
137 0 : return 0;
138 0 : }
139 0 : return EINVAL;
140 0 : }
141 :
142 : int
143 0 : fd_ethtool_ioctl_rxfh_set_default( fd_ethtool_ioctl_t * ioc ) {
144 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --set-rxfh-indir %s default`%s", fd_log_style_dim(), ioc->ifr.ifr_name , fd_log_style_normal() ));
145 0 : struct ethtool_rxfh_indir rxfh = {
146 0 : .cmd = ETHTOOL_SRXFHINDIR,
147 0 : .size = 0, /* default indirection table */
148 0 : };
149 0 : int ret = run_ioctl( ioc, "ETHTOOL_SRXFHINDIR", &rxfh, 1, 0 );
150 0 : if( FD_UNLIKELY( ret==EOPNOTSUPP ) ) return 0;
151 0 : return ret;
152 0 : }
153 :
154 : int
155 : fd_ethtool_ioctl_rxfh_set_suffix( fd_ethtool_ioctl_t * ioc,
156 0 : uint start_idx ) {
157 : /* Get current channel count */
158 0 : struct ethtool_channels ech = { .cmd = ETHTOOL_GCHANNELS };
159 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GCHANNELS", &ech );
160 0 : uint const channels_cnt = ech.combined_count ? ech.combined_count : ech.rx_count;
161 :
162 : /* Get current RXFH queue count
163 :
164 : Note: One would expect that ethtool can always configure the RXFH
165 : indirection table to target all channels / queues supported by the
166 : device. This is not the case. Some drivers limit the max queue
167 : index in the table to less than the current channel count. For
168 : example, see ixgbe_rss_indir_tbl_max(). */
169 0 : struct ethtool_rxnfc nfc = { .cmd = ETHTOOL_GRXRINGS };
170 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXRINGS", &nfc );
171 0 : uint const queue_cnt = (uint)nfc.data;
172 0 : if( FD_UNLIKELY( start_idx>=queue_cnt || queue_cnt>channels_cnt ) ) return EINVAL;
173 :
174 0 : union {
175 0 : struct ethtool_rxfh_indir m;
176 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_rxfh_indir, uint, FD_ETHTOOL_MAX_RXFH_TABLE_CNT ) ];
177 0 : } rxfh = { 0 };
178 :
179 : /* Get count of rx indirection table */
180 0 : rxfh.m.cmd = ETHTOOL_GRXFHINDIR;
181 0 : rxfh.m.size = 0;
182 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXFHINDIR", &rxfh );
183 0 : uint const table_ele_cnt = rxfh.m.size;
184 0 : if( FD_UNLIKELY( (table_ele_cnt == 0) | (table_ele_cnt > FD_ETHTOOL_MAX_RXFH_TABLE_CNT) ) )
185 0 : return EINVAL;
186 :
187 : /* Set table to round robin over all channels from [start_idx, queue_cnt) */
188 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --set-rxfh-indir %s start %u equal %u`%s", fd_log_style_dim(),
189 0 : ioc->ifr.ifr_name, start_idx, queue_cnt - start_idx , fd_log_style_normal() ));
190 0 : rxfh.m.cmd = ETHTOOL_SRXFHINDIR;
191 0 : rxfh.m.size = table_ele_cnt;
192 0 : for( uint j=0u, q=start_idx; j<table_ele_cnt; j++ ) {
193 0 : rxfh.m.ring_index[ j ] = q++;
194 0 : if( FD_UNLIKELY( q>=queue_cnt ) ) q = start_idx;
195 0 : }
196 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_SRXFHINDIR", &rxfh );
197 :
198 0 : return 0;
199 0 : }
200 :
201 : int
202 : fd_ethtool_ioctl_rxfh_get_queue_cnt( fd_ethtool_ioctl_t * ioc,
203 : uint * queue_cnt )
204 0 : {
205 0 : struct ethtool_rxnfc nfc = { .cmd = ETHTOOL_GRXRINGS };
206 0 : int ret = run_ioctl( ioc, "ETHTOOL_GRXRINGS", &nfc, 1, 0 );
207 0 : if( FD_UNLIKELY( ret!=0 ) ) {
208 0 : if( FD_LIKELY( ret==EOPNOTSUPP ) ) {
209 0 : *queue_cnt = 1;
210 0 : return 0;
211 0 : }
212 0 : return ret;
213 0 : }
214 0 : *queue_cnt = (uint)nfc.data;
215 0 : FD_TEST( *queue_cnt>0U );
216 0 : return 0;
217 0 : }
218 :
219 : int
220 : fd_ethtool_ioctl_rxfh_get_table( fd_ethtool_ioctl_t * ioc,
221 : uint * table,
222 0 : uint * table_ele_cnt ) {
223 : /* Note: A simpler implementation of this would use ETHTOOL_GRXFHINDIR
224 : as we are only concerned with the indirection table and do not need
225 : the other information. However, it appears that the ice driver has
226 : a bugged implementation of this command. */
227 :
228 0 : union {
229 0 : struct ethtool_rxfh m;
230 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_rxfh, uint, FD_ETHTOOL_MAX_RXFH_TABLE_CNT ) + MAX_RXFH_KEY_SIZE ];
231 0 : } rxfh = { 0 };
232 :
233 : /* First get the count of the indirection table and hash key */
234 0 : rxfh.m.cmd = ETHTOOL_GRSSH;
235 0 : int ret = run_ioctl( ioc, "ETHTOOL_GRSSH", &rxfh, 1, 0 );
236 0 : if( FD_UNLIKELY( ret!=0 ) ) {
237 0 : if( FD_LIKELY( ret==EOPNOTSUPP ) ) {
238 0 : *table_ele_cnt = 0;
239 0 : return 0;
240 0 : }
241 0 : return ret;
242 0 : }
243 0 : if( FD_UNLIKELY( (rxfh.m.indir_size > FD_ETHTOOL_MAX_RXFH_TABLE_CNT) |
244 0 : (rxfh.m.key_size > MAX_RXFH_KEY_SIZE) ) )
245 0 : return EINVAL;
246 0 : *table_ele_cnt = rxfh.m.indir_size;
247 :
248 0 : if( 0!=*table_ele_cnt ) {
249 : /* Now get the table contents itself. We also get the key bytes. */
250 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRSSH", &rxfh );
251 0 : fd_memcpy( table, rxfh.m.rss_config, *table_ele_cnt * sizeof(uint) );
252 0 : }
253 0 : return 0;
254 0 : }
255 :
256 : static int
257 : get_feature_idx( fd_ethtool_ioctl_t * ioc,
258 : char const * name,
259 : uint * feature_idx,
260 0 : uint * feature_cnt ) {
261 : /* Check size of features string set is not too large (prevent overflow) */
262 0 : union {
263 0 : struct ethtool_sset_info m;
264 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_sset_info, uint, 1 ) ];
265 0 : } esi = { .m = {
266 0 : .cmd = ETHTOOL_GSSET_INFO,
267 0 : .sset_mask = fd_ulong_mask_bit( ETH_SS_FEATURES )
268 0 : } };
269 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GSSET_INFO", &esi );
270 0 : if( FD_UNLIKELY( (esi.m.data[0] == 0) | (esi.m.data[0] > MAX_FEATURES) ) )
271 0 : return EINVAL;
272 0 : *feature_cnt = esi.m.data[0];
273 :
274 : /* Get strings from features string set */
275 0 : union {
276 0 : struct ethtool_gstrings m;
277 0 : uchar _[ sizeof(struct ethtool_gstrings) + (MAX_FEATURES * ETH_GSTRING_LEN) ];
278 0 : } egs = { 0 };
279 0 : egs.m.cmd = ETHTOOL_GSTRINGS;
280 0 : egs.m.string_set = ETH_SS_FEATURES;
281 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GSTRINGS", &egs );
282 :
283 : /* Find index of matching string from the set */
284 0 : for( uint i=0U; i<egs.m.len; i++) {
285 0 : uchar const * gstring = egs.m.data + (i * ETH_GSTRING_LEN);
286 0 : if( 0==strncmp( (char const *)gstring, name, ETH_GSTRING_LEN ) ) {
287 0 : *feature_idx = i;
288 0 : return 0;
289 0 : }
290 0 : }
291 0 : return -1;
292 0 : }
293 :
294 : int
295 : fd_ethtool_ioctl_feature_set( fd_ethtool_ioctl_t * ioc,
296 : char const * name,
297 0 : int enabled ) {
298 0 : uint feature_idx;
299 0 : uint feature_cnt;
300 0 : int ret = get_feature_idx( ioc, name, &feature_idx, &feature_cnt );
301 0 : if( FD_UNLIKELY( ret!=0 ) ) {
302 0 : if( (ret==-1) & (!enabled) ) return 0;
303 0 : return EINVAL;
304 0 : }
305 0 : uint feature_block = feature_idx / 32U;
306 0 : uint feature_offset = feature_idx % 32U;
307 :
308 0 : union {
309 0 : struct ethtool_gfeatures m;
310 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_gfeatures, struct ethtool_get_features_block, MAX_FEATURES / 32U ) ];
311 0 : } egf = { 0 };
312 0 : egf.m.cmd = ETHTOOL_GFEATURES;
313 0 : egf.m.size = MAX_FEATURES / 32U;
314 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GFEATURES", &egf );
315 0 : if( enabled == !!(egf.m.features[ feature_block ].active & fd_uint_mask_bit( (int)feature_offset )) )
316 0 : return 0;
317 :
318 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --features %s %s %s`%s", fd_log_style_dim(),
319 0 : ioc->ifr.ifr_name, name, enabled ? "on" : "off" , fd_log_style_normal() ));
320 0 : union {
321 0 : struct ethtool_sfeatures m;
322 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_sfeatures, struct ethtool_set_features_block, MAX_FEATURES / 32U ) ];
323 0 : } esf = { 0 };
324 0 : esf.m.cmd = ETHTOOL_SFEATURES;
325 0 : esf.m.size = fd_uint_align_up( feature_cnt, 32U ) / 32U;
326 0 : esf.m.features[ feature_block ].valid = fd_uint_mask_bit( (int)feature_offset );
327 0 : esf.m.features[ feature_block ].requested = enabled ? fd_uint_mask_bit( (int)feature_offset ) : 0;
328 :
329 : /* Note: ETHTOOL_SFEATURES has special behavior where it returns a
330 : positive nonzero number with flags set for specific things.
331 : ETHTOOL_F_UNSUPPORTED is set if the feature is not able to be
332 : changed, i.e. it is forever fixed on or fixed off. */
333 0 : ioc->ifr.ifr_data = &esf;
334 0 : ret = ioctl( ioc->fd, SIOCETHTOOL, &ioc->ifr );
335 0 : if( FD_UNLIKELY( ret < 0 ) ) {
336 0 : int error = errno;
337 0 : FD_LOG_WARNING(( "error configuring network device (%s), ioctl(SIOCETHTOOL,ETHTOOL_SFEATURES) failed (%i-%s)",
338 0 : ioc->ifr.ifr_name, errno, fd_io_strerror( errno ) ));
339 0 : return error;
340 0 : }
341 0 : if( FD_UNLIKELY( ret==ETHTOOL_F_UNSUPPORTED ) ) {
342 0 : FD_LOG_WARNING(( "error configuring network device (%s), unable to change fixed feature (%s)",
343 0 : ioc->ifr.ifr_name, name ));
344 0 : return EINVAL;
345 0 : }
346 0 : if( FD_UNLIKELY( ret!=0 ) ) {
347 0 : FD_LOG_WARNING(( "error configuring network device (%s), ioctl(SIOCETHTOOL,ETHTOOL_SFEATURES) failed (%d)",
348 0 : ioc->ifr.ifr_name, ret ));
349 0 : return EINVAL;
350 0 : }
351 0 : return 0;
352 0 : }
353 :
354 : int
355 : fd_ethtool_ioctl_feature_test( fd_ethtool_ioctl_t * ioc,
356 : char const * name,
357 0 : int * enabled ) {
358 0 : uint feature_idx;
359 0 : uint feature_cnt;
360 0 : int ret = get_feature_idx( ioc, name, &feature_idx, &feature_cnt );
361 0 : if( FD_UNLIKELY( ret!=0 ) ) {
362 0 : if( ret==-1 ) {
363 0 : *enabled = 0;
364 0 : return 0;
365 0 : }
366 0 : return EINVAL;
367 0 : }
368 0 : uint feature_block = feature_idx / 32U;
369 0 : uint feature_offset = feature_idx % 32U;
370 :
371 0 : union {
372 0 : struct ethtool_gfeatures m;
373 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_gfeatures, struct ethtool_get_features_block, MAX_FEATURES / 32U ) ];
374 0 : } egf = { 0 };
375 0 : egf.m.cmd = ETHTOOL_GFEATURES;
376 0 : egf.m.size = MAX_FEATURES / 32U;
377 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GFEATURES", &egf );
378 :
379 0 : *enabled = !!(egf.m.features[ feature_block ].active & fd_uint_mask_bit( (int)feature_offset ));
380 0 : return 0;
381 0 : }
382 :
383 : int
384 : fd_ethtool_ioctl_feature_gro_set( fd_ethtool_ioctl_t * ioc,
385 0 : int enabled ) {
386 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --offload %s generic-receive-offload %s`%s", fd_log_style_dim(),
387 0 : ioc->ifr.ifr_name, enabled ? "on" : "off" , fd_log_style_normal() ));
388 0 : struct ethtool_value gro = {
389 0 : .cmd = ETHTOOL_SGRO,
390 0 : .data = !!enabled
391 0 : };
392 0 : int ret = run_ioctl( ioc, "ETHTOOL_SGRO", &gro, 1, 0 );
393 0 : if( FD_UNLIKELY( (ret==EOPNOTSUPP) & (!enabled) ) ) return 0;
394 0 : return ret;
395 0 : }
396 :
397 : int
398 : fd_ethtool_ioctl_feature_gro_test( fd_ethtool_ioctl_t * ioc,
399 : int * enabled,
400 0 : int * supported ) {
401 0 : struct ethtool_value gro = { .cmd = ETHTOOL_GGRO };
402 0 : int ret = run_ioctl( ioc, "ETHTOOL_GGRO", &gro, 1, 0 );
403 0 : *supported = (ret!=EOPNOTSUPP);
404 0 : if( FD_UNLIKELY( ret!=0 ) ) {
405 0 : if( FD_LIKELY( ret==EOPNOTSUPP ) ) {
406 0 : *enabled = 0;
407 0 : return 0;
408 0 : }
409 0 : return ret;
410 0 : }
411 0 : *enabled = !!gro.data;
412 0 : return 0;
413 0 : }
414 :
415 : int
416 0 : fd_ethtool_ioctl_ntuple_clear( fd_ethtool_ioctl_t * ioc ) {
417 0 : union {
418 0 : struct ethtool_rxnfc m;
419 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_rxnfc, uint, MAX_NTUPLE_RULES ) ];
420 0 : } efc = { 0 };
421 :
422 : /* Get count of currently defined rules, return if none exist */
423 0 : efc.m.cmd = ETHTOOL_GRXCLSRLCNT;
424 0 : int ret = run_ioctl( ioc, "ETHTOOL_GRXCLSRLCNT", &efc, 1, 0 );
425 0 : if( FD_UNLIKELY( ret!=0 ) ) {
426 0 : if( ret==EOPNOTSUPP ) return 0;
427 0 : return ret;
428 0 : }
429 0 : uint const rule_cnt = efc.m.rule_cnt;
430 0 : if( FD_UNLIKELY( rule_cnt > MAX_NTUPLE_RULES ) )
431 0 : return EINVAL;
432 0 : if( rule_cnt == 0 )
433 0 : return 0;
434 :
435 : /* Get location indices of all rules */
436 0 : efc.m.cmd = ETHTOOL_GRXCLSRLALL;
437 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXCLSRLALL", &efc );
438 :
439 : /* Delete all rules */
440 0 : for( uint i=0u; i<rule_cnt; i++) {
441 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --config-ntuple %s delete %u`%s", fd_log_style_dim(), ioc->ifr.ifr_name, efc.m.rule_locs[ i ] , fd_log_style_normal() ));
442 0 : struct ethtool_rxnfc del = {
443 0 : .cmd = ETHTOOL_SRXCLSRLDEL,
444 0 : .fs = { .location = efc.m.rule_locs[ i ] }
445 0 : };
446 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_SRXCLSRLDEL", &del );
447 0 : }
448 :
449 0 : return 0;
450 0 : }
451 :
452 : int
453 : fd_ethtool_ioctl_ntuple_set_udp_dport( fd_ethtool_ioctl_t * ioc,
454 : uint rule_idx,
455 : ushort dport,
456 : uint rule_group_idx,
457 : uint rule_group_cnt,
458 0 : uint queue_idx ) {
459 : /* Note: Most drivers do not support RX_CLS_LOC_ANY, instead requiring
460 : an explicit location index. However, some drivers require it
461 : (e.g. bnxt). We can tell based on the RX_CLS_LOC_SPECIAL flag. */
462 0 : struct ethtool_rxnfc get = { .cmd = ETHTOOL_GRXCLSRLCNT };
463 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXCLSRLCNT", &get );
464 0 : int const any_location = !!(get.data & RX_CLS_LOC_SPECIAL);
465 :
466 0 : FD_TEST( rule_group_idx<rule_group_cnt && fd_uint_is_pow2( rule_group_cnt ) && rule_group_cnt>0U );
467 0 : uint const src_addr = fd_uint_bswap( rule_group_idx );
468 0 : uint const src_addr_mask = fd_uint_bswap( rule_group_cnt-1U );
469 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --config-ntuple %s flow-type udp4 dst-port %hu src-ip "
470 0 : FD_IP4_ADDR_FMT " m " FD_IP4_ADDR_FMT " queue %u`%s",
471 0 : fd_log_style_dim(), ioc->ifr.ifr_name, dport, FD_IP4_ADDR_FMT_ARGS( src_addr ),
472 0 : FD_IP4_ADDR_FMT_ARGS( ~src_addr_mask ), queue_idx, fd_log_style_normal() ));
473 0 : struct ethtool_rxnfc efc = {
474 0 : .cmd = ETHTOOL_SRXCLSRLINS,
475 0 : .fs = {
476 0 : .flow_type = UDP_V4_FLOW,
477 0 : .h_u = { .udp_ip4_spec = {
478 0 : .pdst = fd_ushort_bswap( dport ),
479 0 : .ip4src = src_addr } },
480 0 : .m_u = { .udp_ip4_spec = {
481 0 : .pdst = 0xFFFF,
482 0 : .ip4src = src_addr_mask } },
483 0 : .ring_cookie = queue_idx,
484 0 : .location = any_location ? RX_CLS_LOC_ANY : rule_idx,
485 0 : }
486 0 : };
487 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_SRXCLSRLINS", &efc );
488 0 : return 0;
489 0 : }
490 :
491 : int
492 : fd_ethtool_ioctl_ntuple_set_gre( fd_ethtool_ioctl_t * ioc,
493 : uint rule_idx,
494 0 : uint queue_idx ) {
495 : /* See above for any_location */
496 0 : struct ethtool_rxnfc get = { .cmd = ETHTOOL_GRXCLSRLCNT };
497 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXCLSRLCNT", &get );
498 0 : int const any_location = !!(get.data & RX_CLS_LOC_SPECIAL);
499 :
500 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --config-ntuple %s flow-type ip4 l4proto 47 queue %u`%s", fd_log_style_dim(),
501 0 : ioc->ifr.ifr_name, queue_idx , fd_log_style_normal() ));
502 0 : struct ethtool_rxnfc efc = {
503 0 : .cmd = ETHTOOL_SRXCLSRLINS,
504 0 : .fs = {
505 0 : .flow_type = IPV4_USER_FLOW,
506 0 : .h_u = { .usr_ip4_spec = {
507 0 : .ip_ver = ETH_RX_NFC_IP4,
508 0 : .proto = IPPROTO_GRE } },
509 0 : .m_u = { .usr_ip4_spec = {
510 0 : .ip_ver = 0xFF,
511 0 : .proto = 0xFF } },
512 0 : .ring_cookie = queue_idx,
513 0 : .location = any_location ? RX_CLS_LOC_ANY : rule_idx,
514 0 : }
515 0 : };
516 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_SRXCLSRLINS", &efc );
517 0 : return 0;
518 0 : }
519 :
520 : int
521 : fd_ethtool_ioctl_ntuple_validate( fd_ethtool_ioctl_t * ioc,
522 : ushort const * dports,
523 : uint dports_cnt,
524 : uint queue_cnt,
525 : uint gre_queue,
526 0 : int * valid ) {
527 0 : union {
528 0 : struct ethtool_rxnfc m;
529 0 : uchar _[ ETHTOOL_CMD_SIZE( struct ethtool_rxnfc, uint, MAX_NTUPLE_RULES ) ];
530 0 : } efc = { 0 };
531 :
532 0 : int gre_rule = (gre_queue!=UINT_MAX);
533 :
534 : /* Get count of currently defined rules */
535 0 : efc.m.cmd = ETHTOOL_GRXCLSRLCNT;
536 0 : int ret = run_ioctl( ioc, "ETHTOOL_GRXCLSRLCNT", &efc, 1, 0 );
537 0 : if( FD_UNLIKELY( ret!=0 ) ) {
538 0 : if( FD_LIKELY( ret==EOPNOTSUPP ) ) {
539 0 : *valid = (dports_cnt==0U) & !gre_rule;
540 0 : return 0;
541 0 : }
542 0 : return ret;
543 0 : }
544 0 : uint const rule_cnt = efc.m.rule_cnt;
545 0 : if( FD_UNLIKELY( rule_cnt>MAX_NTUPLE_RULES ) )
546 0 : return EINVAL;
547 0 : if( (dports_cnt==0U) & !gre_rule ) {
548 0 : *valid = (rule_cnt==0U);
549 0 : return 0;
550 0 : }
551 0 : FD_TEST( queue_cnt>0U );
552 0 : uint const rule_group_cnt = fd_uint_pow2_up( queue_cnt );
553 0 : if( rule_cnt!=(dports_cnt*rule_group_cnt + (uint)gre_rule) ) {
554 0 : *valid = 0;
555 0 : return 0;
556 0 : }
557 :
558 0 : ushort expected[ MAX_NTUPLE_RULES ] = { 0 };
559 0 : int found_gre = 0;
560 0 : for( uint r=0U; r<rule_group_cnt; r++ ) {
561 0 : for( uint p=0U; p<dports_cnt; p++ ) {
562 0 : expected[ (r*dports_cnt)+p ] = dports[ p ];
563 0 : }
564 0 : }
565 :
566 : /* Get location indices of all rules */
567 0 : efc.m.cmd = ETHTOOL_GRXCLSRLALL;
568 0 : efc.m.rule_cnt = rule_cnt;
569 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXCLSRLALL", &efc );
570 :
571 : /* Loop over all rules, returning early if any are invalid */
572 0 : static const struct ethtool_flow_ext EXPECTED_EXT_MASK = { 0 };
573 0 : const union ethtool_flow_union expected_mask = { .udp_ip4_spec = {
574 0 : .pdst = 0xFFFF,
575 0 : .ip4src = fd_uint_bswap( rule_group_cnt-1U ),
576 0 : } };
577 0 : static const union ethtool_flow_union expected_gre_mask = { .usr_ip4_spec = {
578 0 : .ip_ver = 0xFF,
579 0 : .proto = 0xFF
580 0 : } };
581 0 : for( uint i=0u; i<efc.m.rule_cnt; i++) {
582 0 : struct ethtool_rxnfc get = {
583 0 : .cmd = ETHTOOL_GRXCLSRULE,
584 0 : .fs = { .location = efc.m.rule_locs[ i ] }
585 0 : };
586 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_GRXCLSRULE", &get );
587 0 : uint flow_type = get.fs.flow_type & ~(uint)FLOW_RSS & ~(uint)FLOW_EXT & ~(uint)FLOW_MAC_EXT;
588 0 : uint rule_group_idx = fd_uint_bswap( get.fs.h_u.udp_ip4_spec.ip4src );
589 0 : ushort dport = fd_ushort_bswap( get.fs.h_u.udp_ip4_spec.pdst );
590 :
591 : /* Is this the GRE rule? */
592 0 : if( FD_UNLIKELY( ((!found_gre) & (flow_type==IPV4_USER_FLOW) & (get.fs.ring_cookie==gre_queue) &
593 0 : (get.fs.h_u.usr_ip4_spec.proto==IPPROTO_GRE))&&
594 0 : !memcmp( &get.fs.m_u, &expected_gre_mask, sizeof(expected_gre_mask) ) &&
595 0 : !memcmp( &get.fs.m_ext, &EXPECTED_EXT_MASK, sizeof(EXPECTED_EXT_MASK)) ) ) {
596 0 : found_gre = 1;
597 0 : continue;
598 0 : } else if( FD_UNLIKELY( ((flow_type!=UDP_V4_FLOW) | (rule_group_idx>=rule_group_cnt) |
599 0 : (get.fs.ring_cookie!=(rule_group_idx%queue_cnt)) | (dport==0)) ||
600 0 : 0!=memcmp( &get.fs.m_u, &expected_mask, sizeof(expected_mask) ) ||
601 0 : 0!=memcmp( &get.fs.m_ext, &EXPECTED_EXT_MASK, sizeof(EXPECTED_EXT_MASK)) ) ) {
602 0 : *valid = 0;
603 0 : return 0;
604 0 : }
605 : /* This is a valid udp rule, find the expected port it matches in
606 : the given rule group, or return an error */
607 0 : int found = 0;
608 0 : for( uint j=0u; j<dports_cnt; j++) {
609 0 : if( expected[ (rule_group_idx*dports_cnt)+j ] == dport ) {
610 0 : expected[ (rule_group_idx*dports_cnt)+j ] = 0U;
611 0 : found = 1;
612 0 : break;
613 0 : }
614 0 : }
615 0 : if( !found ) {
616 0 : *valid = 0;
617 0 : return 0;
618 0 : }
619 0 : }
620 :
621 : /* All UDP rules are valid and matched expected ports. It's fully
622 : valid if GRE matched. */
623 0 : *valid = found_gre==gre_rule;
624 0 : return 0;
625 0 : }
626 :
627 : int
628 0 : fd_ethtool_ioctl_rxfh_set_flow_hash_udp4( fd_ethtool_ioctl_t * ioc ) {
629 0 : FD_LOG_NOTICE(( "%sRUN: `ethtool --config-nfc %s rx-flow-hash udp4 sdfn`%s", fd_log_style_dim(), ioc->ifr.ifr_name , fd_log_style_normal() ));
630 0 : struct ethtool_rxnfc nfc = {
631 0 : .cmd = ETHTOOL_SRXFH,
632 0 : .flow_type = UDP_V4_FLOW,
633 0 : .data = RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3,
634 0 : };
635 0 : TRY_RUN_IOCTL( ioc, "ETHTOOL_SRXFH", &nfc );
636 0 : return 0;
637 0 : }
638 :
639 : static int
640 0 : is_modified_device( char const * device ) {
641 0 : fd_ethtool_ioctl_t ioc __attribute__((cleanup(fd_ethtool_ioctl_fini)));
642 0 : if( FD_UNLIKELY( &ioc!=fd_ethtool_ioctl_init( &ioc, device ) ) )
643 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
644 :
645 0 : fd_ethtool_ioctl_channels_t channels;
646 0 : FD_TEST( 0==fd_ethtool_ioctl_channels_get_num( &ioc, &channels ) );
647 0 : if( channels.current!=channels.max ) return 1;
648 :
649 0 : uint rxfh_queue_cnt;
650 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_queue_cnt( &ioc, &rxfh_queue_cnt ) );
651 :
652 0 : uint rxfh_table[ FD_ETHTOOL_MAX_RXFH_TABLE_CNT ] = { 0 };
653 0 : uint rxfh_table_ele_cnt;
654 0 : FD_TEST( 0==fd_ethtool_ioctl_rxfh_get_table( &ioc, rxfh_table, &rxfh_table_ele_cnt ) );
655 0 : for( uint j=0U, q=0U; j<rxfh_table_ele_cnt; j++) {
656 0 : if( rxfh_table[ j ]!=q++ ) return 1;
657 0 : if( q>=rxfh_queue_cnt ) q = 0;
658 0 : }
659 :
660 0 : int ntuple_rules_empty;
661 0 : FD_TEST( 0==fd_ethtool_ioctl_ntuple_validate( &ioc, NULL, 0, 0, UINT_MAX, &ntuple_rules_empty ) );
662 0 : if( !ntuple_rules_empty ) return 1;
663 :
664 0 : return 0;
665 0 : }
666 :
667 : static int
668 0 : reset_device( char const * device ) {
669 0 : if( !is_modified_device( device ) ) return 0;
670 :
671 0 : int error = 0;
672 :
673 0 : fd_ethtool_ioctl_t ioc;
674 0 : if( FD_UNLIKELY( &ioc != fd_ethtool_ioctl_init( &ioc, device ) ) )
675 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to init ethtool ioctl", device ));
676 :
677 : /* Compare states to report whether the reset changed the device. */
678 0 : fd_ethtool_ioctl_channels_t channels_orig;
679 0 : error |= (0!=fd_ethtool_ioctl_channels_get_num( &ioc, &channels_orig ));
680 0 : uint rxfh_table_orig[ FD_ETHTOOL_MAX_RXFH_TABLE_CNT ] = { 0 };
681 0 : uint rxfh_table_orig_ele_cnt;
682 0 : error |= (0!=fd_ethtool_ioctl_rxfh_get_table( &ioc, rxfh_table_orig, &rxfh_table_orig_ele_cnt ));
683 0 : int ntuple_rules_empty_orig;
684 0 : error |= (0!=fd_ethtool_ioctl_ntuple_validate( &ioc, NULL, 0, 0, UINT_MAX, &ntuple_rules_empty_orig ));
685 0 : if( FD_UNLIKELY( error ) )
686 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to determine initial state", device ));
687 :
688 : /* Leave the ntuple feature flag unchanged. */
689 0 : error |= (0!=fd_ethtool_ioctl_ntuple_clear( &ioc ));
690 :
691 : /* This should happen first, otherwise changing the number of channels may fail */
692 0 : error |= (0!=fd_ethtool_ioctl_rxfh_set_default( &ioc ));
693 :
694 0 : int channels_error = fd_ethtool_ioctl_channels_set_num( &ioc, 0 /* max */ );
695 0 : error |= (0!=channels_error);
696 :
697 : /* Some drivers (i40e) do not always evenly redistribute the RXFH table
698 : when increasing the channel count, so we run this again just in case. */
699 0 : error |= (0!=fd_ethtool_ioctl_rxfh_set_default( &ioc ));
700 :
701 0 : if( FD_UNLIKELY( error ) ) {
702 0 : if( FD_LIKELY( channels_error==EBUSY ) )
703 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to set to default state. "
704 0 : "This is most commonly caused by an issue with the Intel ice driver on certain versions "
705 0 : "of Ubuntu. If you are using the ice driver, `sudo dmesg | grep %s` contains "
706 0 : "messages about RDMA, and you do not need RDMA, try running `rmmod irdma` and/or "
707 0 : "blacklisting the irdma kernel module.", device, device ));
708 0 : else
709 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to set to default state", device ));
710 0 : }
711 :
712 0 : fd_ethtool_ioctl_channels_t channels_new;
713 0 : error |= (0!=fd_ethtool_ioctl_channels_get_num( &ioc, &channels_new ));
714 0 : uint rxfh_table_new[ FD_ETHTOOL_MAX_RXFH_TABLE_CNT ] = { 0 };
715 0 : uint rxfh_table_new_ele_cnt;
716 0 : error |= (0!=fd_ethtool_ioctl_rxfh_get_table( &ioc, rxfh_table_new, &rxfh_table_new_ele_cnt ));
717 0 : int ntuple_rules_empty_new;
718 0 : error |= (0!=fd_ethtool_ioctl_ntuple_validate( &ioc, NULL, 0, 0, UINT_MAX, &ntuple_rules_empty_new ));
719 0 : if( FD_UNLIKELY( error ) )
720 0 : FD_LOG_ERR(( "error configuring network device (%s), unable to determine final state", device ));
721 :
722 0 : fd_ethtool_ioctl_fini( &ioc );
723 :
724 0 : int modified = (0!=memcmp( &channels_orig, &channels_new, sizeof(fd_ethtool_ioctl_channels_t) )) ||
725 0 : (rxfh_table_orig_ele_cnt != rxfh_table_new_ele_cnt) ||
726 0 : (0!=memcmp( rxfh_table_orig, rxfh_table_new, rxfh_table_orig_ele_cnt * sizeof(uint) )) ||
727 0 : (ntuple_rules_empty_orig!=ntuple_rules_empty_new);
728 0 : return modified;
729 0 : }
730 :
731 : int
732 0 : fd_ethtool_ioctl_is_modified( char const * device ) {
733 0 : if( !fd_bonding_is_master( device ) ) return is_modified_device( device );
734 :
735 0 : fd_bonding_slave_iter_t iter_[1];
736 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, device );
737 0 : for( ; !fd_bonding_slave_iter_done( iter ); fd_bonding_slave_iter_next( iter ) ) {
738 0 : if( is_modified_device( fd_bonding_slave_iter_ele( iter ) ) ) return 1;
739 0 : }
740 0 : return 0;
741 0 : }
742 :
743 : int
744 0 : fd_ethtool_ioctl_reset( char const * device ) {
745 0 : if( !fd_bonding_is_master( device ) ) return reset_device( device );
746 :
747 0 : int modified = 0;
748 0 : fd_bonding_slave_iter_t iter_[1];
749 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, device );
750 0 : for( ; !fd_bonding_slave_iter_done( iter ); fd_bonding_slave_iter_next( iter ) ) {
751 0 : modified |= reset_device( fd_bonding_slave_iter_ele( iter ) );
752 0 : }
753 0 : return modified;
754 0 : }
|
