Line data Source code
1 : /* Topology support routines for the net tile */
2 :
3 : #define _GNU_SOURCE
4 : #include "fd_net_tile.h"
5 : #include "../topo/fd_topob.h"
6 : #include "../netlink/fd_netlink_tile.h"
7 : #include "../../app/shared/fd_config.h" /* FIXME layering violation */
8 : #include "../../util/pod/fd_pod_format.h"
9 : #include "fd_linux_bond.h"
10 : #include "iavf/fd_iavf.h"
11 : #include "../../waltz/ip/fd_iproute.h"
12 :
13 : #include <errno.h>
14 : #include <limits.h>
15 : #include <net/if.h>
16 : #include <unistd.h>
17 :
18 : static void
19 : fd_net_read_device_driver( char * driver,
20 : ulong driver_sz,
21 0 : char const * if_name ) {
22 0 : FD_TEST( driver && driver_sz>=4UL );
23 0 : char path[ PATH_MAX ], target[ PATH_MAX ];
24 0 : fd_memcpy( driver, "n/a", 4UL );
25 0 : if( FD_UNLIKELY( !fd_cstr_printf_check( path, sizeof(path), NULL,
26 0 : "/sys/class/net/%s/device/driver", if_name ) ) ) {
27 0 : FD_LOG_WARNING(( "cannot read driver for interface `%s`: sysfs path too long", if_name ));
28 0 : return;
29 0 : }
30 :
31 0 : long n = readlink( path, target, sizeof(target)-1UL );
32 0 : if( FD_UNLIKELY( n<0L ) ) {
33 0 : if( errno!=ENOENT ) FD_LOG_WARNING(( "readlink(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
34 0 : return;
35 0 : }
36 0 : if( FD_UNLIKELY( (ulong)n==sizeof(target)-1UL ) ) {
37 0 : FD_LOG_WARNING(( "device driver link `%s` is too long", path ));
38 0 : return;
39 0 : }
40 0 : target[ n ] = '\0';
41 :
42 0 : char const * base = strrchr( target, '/' );
43 0 : base = base ? base+1 : target;
44 0 : if( FD_UNLIKELY( !base[0] ) ) {
45 0 : FD_LOG_WARNING(( "device driver link `%s` has an empty driver name", path ));
46 0 : return;
47 0 : }
48 0 : ulong name_sz = strlen( base )+1UL;
49 0 : if( FD_UNLIKELY( name_sz>driver_sz ) ) {
50 0 : FD_LOG_WARNING(( "driver name in `%s` exceeds output buffer size %lu", path, driver_sz ));
51 0 : return;
52 0 : }
53 0 : fd_memcpy( driver, base, name_sz );
54 0 : }
55 :
56 : void
57 : fd_net_get_driver( char * driver,
58 : ulong driver_sz,
59 0 : char const * if_name ) {
60 0 : FD_TEST( driver && driver_sz>=4UL );
61 0 : if( !fd_bonding_is_master( if_name ) ) {
62 0 : fd_net_read_device_driver( driver, driver_sz, if_name );
63 0 : return;
64 0 : }
65 :
66 0 : driver[0] = '\0';
67 0 : fd_bonding_slave_iter_t iter[1];
68 0 : for( fd_bonding_slave_iter_init( iter, if_name );
69 0 : !fd_bonding_slave_iter_done( iter );
70 0 : fd_bonding_slave_iter_next( iter ) ) {
71 0 : char member_driver[ NAME_MAX+1UL ];
72 0 : fd_net_read_device_driver( member_driver, fd_ulong_min( sizeof(member_driver), driver_sz ),
73 0 : fd_bonding_slave_iter_ele( iter ) );
74 0 : if( FD_UNLIKELY( !strcmp( member_driver, "n/a" ) || ( driver[0] && strcmp( driver, member_driver ) ) ) ) {
75 0 : driver[0] = '\0';
76 0 : break;
77 0 : }
78 0 : fd_cstr_ncpy( driver, member_driver, driver_sz );
79 0 : }
80 0 : if( FD_UNLIKELY( !driver[0] ) ) fd_memcpy( driver, "n/a", 4UL );
81 0 : }
82 :
83 : void
84 : fd_net_rx_dest_list( fd_net_rx_dest_t out[ FD_NET_RX_DEST_CNT ],
85 0 : fd_topo_net_tile_t const * net ) {
86 0 : fd_net_rx_dest_t const rx_dests[ FD_NET_RX_DEST_CNT ] = {
87 0 : { DST_PROTO_TPU_UDP, "net_quic", net->legacy_transaction_listen_port, 1 },
88 0 : { DST_PROTO_TPU_QUIC, "net_quic", net->quic_transaction_listen_port, 1 },
89 0 : { DST_PROTO_SHRED, "net_shred", net->shred_listen_port, 1 },
90 0 : { DST_PROTO_GOSSIP, "net_gossvf", net->gossip_listen_port, 1 },
91 0 : { DST_PROTO_REPAIR, "net_shred", net->repair_client_listen_port, 1 },
92 0 : { DST_PROTO_RSERVE, "net_rserve", net->repair_serve_listen_port, 0 },
93 0 : { DST_PROTO_VOTOR, "net_votor", net->votor_quic_client_listen_port, 1 },
94 0 : { DST_PROTO_VOTOR, "net_votor", net->votor_quic_server_listen_port, 1 }
95 0 : };
96 0 : fd_memcpy( out, rx_dests, sizeof(rx_dests) );
97 0 : }
98 :
99 : char const *
100 0 : fd_net_tile_name( char const * provider ) {
101 0 : if( 0==strcmp( provider, "xdp" ) ) {
102 0 : return "net";
103 0 : } else if( 0==strcmp( provider, "mlx5" ) ) {
104 0 : return "mlx5";
105 0 : } else if( 0==strcmp( provider, "iavf" ) ) {
106 0 : return "iavf";
107 0 : } else if( 0==strcmp( provider, "socket" ) ) {
108 0 : return "sock";
109 0 : }
110 0 : FD_LOG_ERR(( "invalid net provider: %s", provider ));
111 0 : }
112 :
113 : static void
114 : setup_mlx5_tile( fd_topo_t * topo,
115 : ulong tile_kind_id,
116 : fd_topo_tile_t * netlink_tile,
117 : ulong const * tile_to_cpu,
118 : fd_config_net_t const * net_cfg,
119 : ulong route_max,
120 0 : ulong route_peer_max ) {
121 0 : fd_topo_tile_t * tile = fd_topob_tile( topo, "mlx5", "mlx5", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
122 0 : tile->sleep_eventfd = topo->sleep_obj_id!=ULONG_MAX;
123 0 : fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
124 0 : fd_topob_tile_in( topo, "netlnk", 0UL, "metric_in", "net_netlnk", tile_kind_id, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
125 0 : fd_topob_tile_out( topo, "mlx5", tile_kind_id, "net_netlnk", tile_kind_id );
126 0 : fd_topob_tile_in( topo, "mlx5", tile_kind_id, "metric_in", "iproute_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
127 0 : fd_netlink_topo_join( topo, netlink_tile, tile );
128 :
129 0 : fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
130 0 : fd_topob_tile_uses( topo, tile, umem_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
131 0 : fd_pod_insertf_ulong( topo->props, umem_obj->id, "net.%lu.umem", tile_kind_id );
132 :
133 0 : FD_STATIC_ASSERT( sizeof(tile->mlx5.if_name)==IF_NAMESIZE, str_bounds );
134 0 : fd_cstr_ncpy( tile->mlx5.if_name, net_cfg->interface, IF_NAMESIZE );
135 :
136 0 : tile->mlx5.net.bind_address = net_cfg->bind_address_parsed;
137 0 : tile->mlx5.rx_queue_size = net_cfg->mlx5.rx_queue_size;
138 0 : tile->mlx5.tx_queue_size = net_cfg->mlx5.tx_queue_size;
139 0 : tile->mlx5.batch_size = FD_MLX5_BATCH_SIZE;
140 :
141 0 : tile->net.umem_dcache_obj_id = umem_obj->id;
142 0 : tile->mlx5.netdev_tbl_obj_id = netlink_tile->netlink.netdev_tbl_obj_id;
143 0 : tile->mlx5.route_max = route_max;
144 0 : tile->mlx5.route_peer_max = route_peer_max;
145 0 : tile->mlx5.route_peer_seed = 1UL + tile_kind_id;
146 0 : tile->mlx5.neigh4_obj_id = netlink_tile->netlink.neigh4_obj_id;
147 0 : }
148 :
149 : static void
150 : setup_iavf_tile( fd_topo_t * topo,
151 : ulong tile_kind_id,
152 : fd_topo_tile_t * netlink_tile,
153 : ulong const * tile_to_cpu,
154 : fd_config_net_t const * net_cfg,
155 : ulong route_max,
156 0 : ulong route_peer_max ) {
157 0 : fd_topo_tile_t * tile = fd_topob_tile( topo, "iavf", "iavf", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
158 0 : fd_topo_tile_t const * first_tile = NULL;
159 0 : if( 0!=tile_kind_id ) {
160 0 : ulong first_tile_id = fd_topo_find_tile( topo, "iavf", 0UL );
161 0 : FD_TEST( first_tile_id!=ULONG_MAX );
162 0 : first_tile = &topo->tiles[ first_tile_id ];
163 : /* Only tile 0 consumes PF link events from the VF Admin Queues.
164 : The other iavf tiles copy its vf_link_usable array to track each
165 : VF's link state. */
166 0 : fd_topob_tile_uses( topo, tile, &topo->objs[ first_tile->tile_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
167 0 : }
168 0 : fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
169 0 : fd_topob_tile_in( topo, "netlnk", 0UL, "metric_in", "net_netlnk", tile_kind_id, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
170 0 : fd_topob_tile_out( topo, "iavf", tile_kind_id, "net_netlnk", tile_kind_id );
171 0 : fd_topob_tile_in( topo, "iavf", tile_kind_id, "metric_in", "iproute_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
172 0 : fd_netlink_topo_join( topo, netlink_tile, tile );
173 :
174 0 : fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
175 0 : fd_topob_tile_uses( topo, tile, umem_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
176 0 : fd_pod_insertf_ulong( topo->props, umem_obj->id, "net.%lu.umem", tile_kind_id );
177 :
178 0 : FD_STATIC_ASSERT( sizeof(tile->iavf.if_name)==IF_NAMESIZE, str_bounds );
179 0 : fd_cstr_ncpy( tile->iavf.if_name, net_cfg->interface, IF_NAMESIZE );
180 :
181 0 : if( first_tile!=NULL ) {
182 0 : tile->iavf.member_cnt = first_tile->iavf.member_cnt;
183 0 : fd_memcpy( tile->iavf.members, first_tile->iavf.members, sizeof(tile->iavf.members) );
184 0 : } else if( FD_UNLIKELY( -1==fd_iavf_member_interfaces( net_cfg->interface, tile->iavf.members, &tile->iavf.member_cnt ) ) ) {
185 0 : FD_LOG_ERR(( "IAVF member discovery failed (%i-%s)", errno, fd_io_strerror( errno ) ));
186 0 : }
187 :
188 0 : tile->iavf.net.bind_address = net_cfg->bind_address_parsed;
189 0 : tile->iavf.rx_queue_size = net_cfg->iavf.rx_queue_size;
190 0 : tile->iavf.tx_queue_size = net_cfg->iavf.tx_queue_size;
191 :
192 0 : tile->net.umem_dcache_obj_id = umem_obj->id;
193 0 : tile->iavf.netdev_tbl_obj_id = netlink_tile->netlink.netdev_tbl_obj_id;
194 0 : tile->iavf.route_max = route_max;
195 0 : tile->iavf.route_peer_max = route_peer_max;
196 0 : tile->iavf.route_peer_seed = 1UL + tile_kind_id;
197 0 : tile->iavf.neigh4_obj_id = netlink_tile->netlink.neigh4_obj_id;
198 0 : }
199 :
200 : static void
201 : setup_xdp_tile( fd_topo_t * topo,
202 : ulong tile_kind_id,
203 : fd_topo_tile_t * netlink_tile,
204 : ulong const * tile_to_cpu,
205 : fd_config_net_t const * net_cfg,
206 : ulong route_max,
207 : ulong route_peer_max,
208 : char const * if_phys,
209 0 : ulong if_queue ) {
210 0 : fd_topo_tile_t * tile = fd_topob_tile( topo, "net", "net", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, topo->sleep_obj_id!=ULONG_MAX );
211 0 : fd_topob_link( topo, "net_netlnk", "net_netlnk", 128UL, 0UL, 0UL );
212 0 : fd_topob_tile_in( topo, "netlnk", 0UL, "metric_in", "net_netlnk", tile_kind_id, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED );
213 0 : fd_topob_tile_out( topo, "net", tile_kind_id, "net_netlnk", tile_kind_id );
214 0 : fd_topob_tile_in( topo, "net", tile_kind_id, "metric_in", "iproute_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED );
215 0 : fd_netlink_topo_join( topo, netlink_tile, tile );
216 :
217 0 : fd_topo_obj_t * umem_obj = fd_topob_obj( topo, "dcache", "net_umem" );
218 0 : fd_topob_tile_uses( topo, tile, umem_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
219 0 : fd_pod_insertf_ulong( topo->props, umem_obj->id, "net.%lu.umem", tile_kind_id );
220 :
221 0 : FD_STATIC_ASSERT( sizeof(tile->xdp.if_virt)==IF_NAMESIZE, str_bounds );
222 0 : fd_cstr_ncpy( tile->xdp.if_virt, net_cfg->interface, IF_NAMESIZE );
223 0 : tile->net.bind_address = net_cfg->bind_address_parsed;
224 :
225 0 : FD_STATIC_ASSERT( sizeof(tile->xdp.if_phys)==IF_NAMESIZE, str_bounds );
226 0 : fd_cstr_ncpy( tile->xdp.if_phys, if_phys, IF_NAMESIZE );
227 0 : tile->xdp.if_queue = (uint)if_queue;
228 :
229 0 : tile->xdp.tx_flush_timeout_ns = (long)net_cfg->xdp.flush_timeout_micros * 1000L;
230 0 : tile->xdp.xdp_rx_queue_size = net_cfg->xdp.xdp_rx_queue_size;
231 0 : tile->xdp.xdp_tx_queue_size = net_cfg->xdp.xdp_tx_queue_size;
232 0 : tile->xdp.zero_copy = net_cfg->xdp.xdp_zero_copy;
233 0 : fd_cstr_ncpy( tile->xdp.xdp_mode, net_cfg->xdp.xdp_mode, sizeof(tile->xdp.xdp_mode) );
234 :
235 0 : fd_cstr_ncpy( tile->xdp.poll_mode, net_cfg->xdp.poll_mode, sizeof(tile->xdp.poll_mode) );
236 :
237 0 : tile->xdp.net.umem_dcache_obj_id = umem_obj->id;
238 0 : tile->xdp.netdev_tbl_obj_id = netlink_tile->netlink.netdev_tbl_obj_id;
239 0 : tile->xdp.route_max = route_max;
240 0 : tile->xdp.route_peer_max = route_peer_max;
241 0 : tile->xdp.route_peer_seed = 1UL + tile_kind_id;
242 0 : tile->xdp.neigh4_obj_id = netlink_tile->netlink.neigh4_obj_id;
243 :
244 : /* Allocate free ring */
245 :
246 0 : tile->xdp.free_ring_depth = tile->xdp.xdp_tx_queue_size;
247 0 : }
248 :
249 : static void
250 : setup_sock_tile( fd_topo_t * topo,
251 : ulong const * tile_to_cpu,
252 0 : fd_config_net_t const * net_cfg ) {
253 0 : fd_topo_tile_t * tile = fd_topob_tile( topo, "sock", "sock", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 );
254 0 : tile->sock.net.bind_address = net_cfg->bind_address_parsed;
255 :
256 0 : if( FD_UNLIKELY( net_cfg->socket.receive_buffer_size>INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.receive_buffer_size]" ));
257 0 : if( FD_UNLIKELY( net_cfg->socket.send_buffer_size >INT_MAX ) ) FD_LOG_ERR(( "invalid [net.socket.send_buffer_size]" ));
258 0 : tile->sock.so_rcvbuf = (int)net_cfg->socket.receive_buffer_size;
259 0 : tile->sock.so_sndbuf = (int)net_cfg->socket.send_buffer_size ;
260 0 : }
261 :
262 : void
263 : fd_topos_net_tiles( fd_topo_t * topo,
264 : ulong net_tile_cnt,
265 : fd_config_net_t const * net_cfg,
266 : ulong netlnk_max_routes,
267 : ulong netlnk_max_peer_routes,
268 : ulong netlnk_max_neighbors,
269 0 : ulong const tile_to_cpu[ FD_TILE_MAX ] ) {
270 : /* net_umem: Packet buffers */
271 0 : fd_topob_wksp( topo, "net_umem" );
272 :
273 0 : fd_pod_insert_cstr( topo->props, "net.provider", net_cfg->provider );
274 :
275 : /* Create workspaces */
276 :
277 0 : if( 0==strcmp( net_cfg->provider, "xdp" ) ) {
278 :
279 : /* net: private working memory of the net tiles */
280 0 : fd_topob_wksp( topo, "net" );
281 : /* netlnk: private working memory of the netlnk tile */
282 0 : fd_topob_wksp( topo, "netlnk" );
283 : /* netbase: shared network config (config plane) */
284 0 : fd_topob_wksp( topo, "netbase" );
285 : /* net_netlnk: net->netlnk ARP requests */
286 0 : fd_topob_wksp( topo, "net_netlnk" );
287 0 : fd_topob_wksp( topo, "iproute" );
288 :
289 0 : fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 1 );
290 : /* A dump is emitted synchronously. Keep room for one complete dump in
291 : addition to an already outstanding complete dump. */
292 0 : ulong iproute_depth = fd_ulong_pow2_up( 4UL*(netlnk_max_routes+netlnk_max_peer_routes)+8UL );
293 0 : fd_topob_link( topo, "iproute_out", "iproute", iproute_depth, sizeof(fd_iproute_msg_t), 1UL );
294 0 : fd_topob_tile_out( topo, "netlnk", 0UL, "iproute_out", 0UL );
295 0 : fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );
296 :
297 : /* Enumerate network devices to attach to */
298 0 : uint devices[ FD_NET_BOND_SLAVE_MAX ] = {0};
299 0 : uint device_cnt = 1U;
300 0 : if( net_cfg->xdp.native_bond && fd_bonding_is_master( net_cfg->interface ) ) {
301 0 : fd_bonding_slave_iter_t iter_[1];
302 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_, net_cfg->interface );
303 0 : uint slave_cnt;
304 0 : for( slave_cnt=0U;
305 0 : /* */ !fd_bonding_slave_iter_done( iter );
306 0 : slave_cnt++, fd_bonding_slave_iter_next( iter ) ) {
307 0 : if( FD_UNLIKELY( slave_cnt>=FD_NET_BOND_SLAVE_MAX ) ) {
308 0 : FD_LOG_ERR(( "bond interface %s has too many slave devices; max is %u (see [net.xdp.native_bond])",
309 0 : net_cfg->interface, FD_NET_BOND_SLAVE_MAX ));
310 0 : }
311 0 : uint if_idx = if_nametoindex( fd_bonding_slave_iter_ele( iter ) );
312 0 : if( FD_UNLIKELY( !if_idx ) ) FD_LOG_ERR(( "if_nametoindex(%s) failed", fd_bonding_slave_iter_ele( iter ) ));
313 0 : devices[ slave_cnt ] = if_idx;
314 0 : }
315 0 : if( slave_cnt==0 ) {
316 0 : FD_LOG_ERR(( "no bond slave devices detected on interface %s (see [net.xdp.native_bond])", net_cfg->interface ));
317 0 : }
318 0 : device_cnt = (uint)slave_cnt;
319 0 : } else {
320 0 : devices[ 0 ] = if_nametoindex( net_cfg->interface );
321 0 : if( FD_UNLIKELY( !devices[ 0 ] ) ) FD_LOG_ERR(( "unsupported [net.interface]: `%s`", net_cfg->interface ));
322 0 : device_cnt = 1U;
323 0 : }
324 :
325 : /* Verify that net_tile_cnt is a multiple of device_cnt */
326 0 : if( FD_UNLIKELY( net_tile_cnt%device_cnt!=0 ) ) {
327 0 : FD_LOG_ERR(( "net tile count %lu must be a multiple of the number of slave devices %u (incompatible settings [layout.net_tile_count] and [net.xdp.native_bond])", net_tile_cnt, device_cnt ));
328 0 : }
329 0 : uint dev_queue_cnt = (uint)net_tile_cnt/device_cnt;
330 :
331 : /* Assign XDP tiles to device queues */
332 0 : ulong tile_kind_id = 0UL;
333 0 : for( uint i=0UL; i<device_cnt; i++ ) {
334 0 : char if_name[ IF_NAMESIZE ];
335 0 : if( FD_UNLIKELY( !if_indextoname( devices[ i ], if_name ) ) ) {
336 0 : FD_LOG_ERR(( "error initializing network stack: if_indextoname(%u) failed (try disabling [net.xdp.native_bond]?)", i ));
337 0 : }
338 0 : for( ulong j=0UL; j<dev_queue_cnt; j++ ) {
339 0 : setup_xdp_tile( topo, tile_kind_id++, netlink_tile, tile_to_cpu, net_cfg, netlnk_max_routes, netlnk_max_peer_routes, if_name, (uint)j );
340 0 : }
341 0 : }
342 0 : FD_TEST( tile_kind_id==net_tile_cnt );
343 :
344 0 : } else if( 0==strcmp( net_cfg->provider, "socket" ) ) {
345 :
346 : /* sock: private working memory of the sock tiles */
347 0 : fd_topob_wksp( topo, "sock" );
348 :
349 0 : for( ulong i=0UL; i<net_tile_cnt; i++ ) {
350 0 : setup_sock_tile( topo, tile_to_cpu, net_cfg );
351 0 : }
352 0 : } else if( 0==strcmp( net_cfg->provider, "mlx5" ) ) {
353 :
354 : /* mlx5: private working memory of the mlx5 tiles */
355 0 : fd_topob_wksp( topo, "mlx5" );
356 : /* netlnk: private working memory of the netlnk tile */
357 0 : fd_topob_wksp( topo, "netlnk" );
358 : /* netbase: shared network config (config plane) */
359 0 : fd_topob_wksp( topo, "netbase" );
360 : /* net_netlnk: net->netlnk ARP requests */
361 0 : fd_topob_wksp( topo, "net_netlnk" );
362 0 : fd_topob_wksp( topo, "iproute" );
363 :
364 0 : fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 1 );
365 0 : ulong iproute_depth = fd_ulong_pow2_up( 4UL*(netlnk_max_routes+netlnk_max_peer_routes)+8UL );
366 0 : fd_topob_link( topo, "iproute_out", "iproute", iproute_depth, sizeof(fd_iproute_msg_t), 1UL );
367 0 : fd_topob_tile_out( topo, "netlnk", 0UL, "iproute_out", 0UL );
368 0 : fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );
369 :
370 0 : for( ulong i=0UL; i<net_tile_cnt; i++ ) {
371 0 : setup_mlx5_tile( topo, i, netlink_tile, tile_to_cpu, net_cfg, netlnk_max_routes, netlnk_max_peer_routes );
372 0 : }
373 :
374 0 : } else if( 0==strcmp( net_cfg->provider, "iavf" ) ) {
375 :
376 : /* iavf: private working memory of the iavf tiles */
377 0 : fd_topob_wksp( topo, "iavf" );
378 : /* netlnk: private working memory of the netlnk tile */
379 0 : fd_topob_wksp( topo, "netlnk" );
380 : /* netbase: shared network config (config plane) */
381 0 : fd_topob_wksp( topo, "netbase" );
382 : /* net_netlnk: net->netlnk ARP requests */
383 0 : fd_topob_wksp( topo, "net_netlnk" );
384 0 : fd_topob_wksp( topo, "iproute" );
385 :
386 0 : fd_topo_tile_t * netlink_tile = fd_topob_tile( topo, "netlnk", "netlnk", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 1 );
387 0 : ulong iproute_depth = fd_ulong_pow2_up( 4UL*(netlnk_max_routes+netlnk_max_peer_routes)+8UL );
388 0 : fd_topob_link( topo, "iproute_out", "iproute", iproute_depth, sizeof(fd_iproute_msg_t), 1UL );
389 0 : fd_topob_tile_out( topo, "netlnk", 0UL, "iproute_out", 0UL );
390 0 : fd_netlink_topo_create( netlink_tile, topo, netlnk_max_routes, netlnk_max_peer_routes, netlnk_max_neighbors, net_cfg->interface );
391 :
392 0 : if( FD_UNLIKELY( 0==net_tile_cnt || net_tile_cnt>FD_IAVF_TILE_MAX ) ) {
393 0 : FD_LOG_ERR(( "IAVF net tile count %lu must be in [1,%lu]", net_tile_cnt, FD_IAVF_TILE_MAX ));
394 0 : }
395 0 : for( ulong i=0UL; i<net_tile_cnt; i++ ) {
396 0 : setup_iavf_tile( topo, i, netlink_tile, tile_to_cpu, net_cfg, netlnk_max_routes, netlnk_max_peer_routes );
397 0 : }
398 :
399 0 : } else {
400 0 : FD_LOG_ERR(( "invalid `net.provider`" ));
401 0 : }
402 0 : }
403 :
404 : static void
405 : add_umem_rx_link( fd_topo_t * topo,
406 : char const * link_name,
407 : ulong net_kind_id,
408 : ulong depth,
409 0 : ulong burst ) {
410 0 : if( FD_UNLIKELY( !topo || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
411 0 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
412 0 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
413 :
414 0 : ulong kind_id = 0UL;
415 0 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
416 0 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
417 0 : }
418 :
419 0 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
420 0 : strncpy( link->name, link_name, sizeof(link->name) );
421 0 : link->id = topo->link_cnt;
422 0 : link->kind_id = kind_id;
423 0 : link->depth = depth;
424 0 : link->mtu = FD_NET_MTU;
425 0 : link->burst = burst;
426 :
427 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", "net_umem" );
428 0 : link->mcache_obj_id = obj->id;
429 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
430 :
431 0 : link->dcache_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_kind_id );
432 0 : if( FD_UNLIKELY( link->dcache_obj_id==ULONG_MAX ) ) FD_LOG_ERR(( "umem dcache not found for net %lu", net_kind_id ));
433 :
434 0 : topo->link_cnt++;
435 0 : }
436 :
437 : void
438 : fd_topos_net_rx_link( fd_topo_t * topo,
439 : char const * link_name,
440 : ulong net_kind_id,
441 0 : ulong depth ) {
442 0 : char const * provider = fd_pod_query_cstr( topo->props, "net.provider", "" );
443 0 : if( 0==strcmp( provider, "xdp" ) ) {
444 0 : add_umem_rx_link( topo, link_name, net_kind_id, depth, 0UL );
445 0 : fd_topob_tile_out( topo, "net", net_kind_id, link_name, net_kind_id );
446 0 : } else if( 0==strcmp( provider, "mlx5" ) ) {
447 0 : ulong tile_id = fd_topo_find_tile( topo, "mlx5", net_kind_id );
448 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile mlx5:%lu not found", net_kind_id ));
449 0 : add_umem_rx_link( topo, link_name, net_kind_id, depth, topo->tiles[ tile_id ].mlx5.batch_size );
450 0 : fd_topob_tile_out( topo, "mlx5", net_kind_id, link_name, net_kind_id );
451 0 : } else if( 0==strcmp( provider, "iavf" ) ) {
452 0 : ulong tile_id = fd_topo_find_tile( topo, "iavf", net_kind_id );
453 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile iavf:%lu not found", net_kind_id ));
454 0 : add_umem_rx_link( topo, link_name, net_kind_id, depth, topo->tiles[ tile_id ].iavf.member_cnt*FD_IAVF_BATCH_SIZE );
455 0 : fd_topob_tile_out( topo, "iavf", net_kind_id, link_name, net_kind_id );
456 0 : } else if( 0==strcmp( provider, "socket" ) ) {
457 0 : fd_topob_link( topo, link_name, "net_umem", depth, FD_NET_MTU, 64 );
458 0 : fd_topob_tile_out( topo, "sock", net_kind_id, link_name, net_kind_id );
459 0 : }
460 0 : }
461 :
462 : void
463 : fd_topos_tile_in_net( fd_topo_t * topo,
464 : char const * fseq_wksp,
465 : char const * link_name,
466 : ulong link_kind_id,
467 : int reliable,
468 0 : int polled ) {
469 0 : for( ulong j=0UL; j<(topo->tile_cnt); j++ ) {
470 0 : if( 0==strcmp( topo->tiles[ j ].name, "net" ) ||
471 0 : 0==strcmp( topo->tiles[ j ].name, "sock" ) ||
472 0 : 0==strcmp( topo->tiles[ j ].name, "mlx5" ) ||
473 0 : 0==strcmp( topo->tiles[ j ].name, "iavf" ) ) {
474 0 : fd_topob_tile_in( topo, topo->tiles[ j ].name, topo->tiles[ j ].kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled );
475 0 : }
476 0 : }
477 0 : }
478 :
479 : static void
480 : fd_topos_xdp_setup_mem( fd_topo_t * topo,
481 0 : fd_topo_tile_t * net_tile ) {
482 0 : ulong rx_depth = net_tile->xdp.xdp_rx_queue_size;
483 0 : ulong tx_depth = net_tile->xdp.xdp_tx_queue_size;
484 0 : rx_depth += (rx_depth/2UL);
485 0 : tx_depth += (tx_depth/2UL);
486 :
487 0 : if( net_tile->kind_id==0 ) {
488 : /* Double it for loopback XSK */
489 0 : rx_depth *= 2UL;
490 0 : tx_depth *= 2UL;
491 0 : }
492 :
493 0 : ulong cum_frame_cnt = rx_depth + tx_depth;
494 :
495 : /* Count up the depth of all RX mcaches */
496 :
497 0 : for( ulong j=0UL; j<(net_tile->out_cnt); j++ ) {
498 0 : ulong link_id = net_tile->out_link_id[ j ];
499 0 : ulong mcache_obj_id = topo->links[ link_id ].mcache_obj_id;
500 0 : ulong depth = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.depth", mcache_obj_id );
501 0 : if( FD_UNLIKELY( depth==ULONG_MAX ) ) FD_LOG_ERR(( "Didn't find depth for mcache %s", topo->links[ link_id ].name ));
502 0 : cum_frame_cnt += depth + 1UL;
503 0 : }
504 :
505 : /* Create a dcache object */
506 :
507 0 : ulong umem_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", net_tile->kind_id );
508 0 : FD_TEST( umem_obj_id!=ULONG_MAX );
509 :
510 0 : FD_TEST( net_tile->net.umem_dcache_obj_id > 0 );
511 0 : fd_pod_insertf_ulong( topo->props, cum_frame_cnt, "obj.%lu.depth", umem_obj_id );
512 0 : fd_pod_insertf_ulong( topo->props, 2UL, "obj.%lu.burst", umem_obj_id ); /* 4096 byte padding */
513 0 : fd_pod_insertf_ulong( topo->props, 2048UL, "obj.%lu.mtu", umem_obj_id );
514 0 : }
515 :
516 : void
517 : fd_topo_install_xdp( fd_topo_t const * topo,
518 : fd_xdp_fds_t * fds,
519 : uint * fds_cnt,
520 : uint bind_addr,
521 0 : int dry_run ) {
522 0 : uint fds_max = *fds_cnt;
523 0 : memset( fds, 0, fds_max*sizeof(fd_xdp_fds_t) );
524 :
525 0 : uint if_cnt = 0U;
526 0 : # define ADD_IF_IDX( idx_ ) do { \
527 0 : uint idx = (idx_); \
528 0 : int found = 0; \
529 0 : for( uint i=0U; i<if_cnt; i++ ) { \
530 0 : if( fds[ i ].if_idx==idx ) { \
531 0 : found = 1; \
532 0 : break; \
533 0 : } \
534 0 : } \
535 0 : if( !found ) { \
536 0 : FD_TEST( if_cnt<FD_NET_BOND_SLAVE_MAX+1 ); \
537 0 : fds[ if_cnt++ ].if_idx = idx; \
538 0 : } \
539 0 : } while(0)
540 :
541 : /* Create a list of unique fds */
542 :
543 0 : ulong net_tile_cnt = fd_topo_tile_name_cnt( topo, "net" );
544 0 : for( ulong tile_kind_id=0UL; tile_kind_id<net_tile_cnt; tile_kind_id++ ) {
545 0 : ulong net_tile_id = fd_topo_find_tile( topo, "net", tile_kind_id );
546 0 : FD_TEST( net_tile_id!=ULONG_MAX );
547 0 : fd_topo_tile_t const * tile = &topo->tiles[ net_tile_id ];
548 0 : uint if_idx = if_nametoindex( tile->xdp.if_phys ); FD_TEST( if_idx );
549 0 : ADD_IF_IDX( if_idx );
550 0 : }
551 :
552 : /* Add loopback unless found */
553 :
554 0 : uint lo_idx = if_nametoindex( "lo" ); FD_TEST( lo_idx );
555 0 : ADD_IF_IDX( lo_idx );
556 :
557 : /* Done with config discovery */
558 :
559 0 : *fds_cnt = if_cnt;
560 0 : int next_fd = 123462;
561 0 : for( uint i=0U; i<if_cnt; i++ ) {
562 0 : fds[ i ].xsk_map_fd = next_fd++;
563 0 : fds[ i ].prog_link_fd = next_fd++;
564 0 : }
565 0 : if( dry_run ) return;
566 :
567 : /* Install */
568 :
569 0 : ulong net0_tile_idx = fd_topo_find_tile( topo, "net", 0UL );
570 0 : FD_TEST( net0_tile_idx!=ULONG_MAX );
571 0 : fd_topo_tile_t const * net0_tile = &topo->tiles[ net0_tile_idx ];
572 :
573 : /* Get UDP ports to listen on */
574 0 : fd_net_rx_dest_t rx_dests[ FD_NET_RX_DEST_CNT ];
575 0 : fd_net_rx_dest_list( rx_dests, &net0_tile->net );
576 0 : ushort udp_port_candidates[ FD_NET_RX_DEST_CNT ];
577 0 : for( ulong i=0UL; i<FD_NET_RX_DEST_CNT; i++ ) udp_port_candidates[ i ] = rx_dests[ i ].listen_port;
578 :
579 0 : for( uint i=0U; i<if_cnt; i++ ) {
580 : /* Override XDP mode for loopback */
581 0 : char const * xdp_mode = net0_tile->xdp.xdp_mode;
582 0 : if( fds[ i ].if_idx==1U ) xdp_mode = "skb";
583 :
584 0 : fd_xdp_fds_t xdp_fds = fd_xdp_install(
585 0 : fds[ i ].if_idx,
586 0 : bind_addr,
587 0 : FD_NET_RX_DEST_CNT,
588 0 : udp_port_candidates,
589 0 : xdp_mode );
590 0 : if( FD_UNLIKELY( -1==dup2( xdp_fds.xsk_map_fd, fds[ i ].xsk_map_fd ) ) ) {
591 0 : FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
592 0 : }
593 0 : if( FD_UNLIKELY( -1==close( xdp_fds.xsk_map_fd ) ) ) {
594 0 : FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
595 0 : }
596 0 : if( FD_UNLIKELY( -1==dup2( xdp_fds.prog_link_fd, fds[ i ].prog_link_fd ) ) ) {
597 0 : FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
598 0 : }
599 0 : if( FD_UNLIKELY( -1==close( xdp_fds.prog_link_fd ) ) ) {
600 0 : FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
601 0 : }
602 0 : }
603 0 : }
604 :
605 : # undef ADD_IF_IDX
606 :
607 : void
608 : fd_topos_sock_lo( fd_topo_t * topo,
609 0 : fd_topo_tile_t const * net_tile ) {
610 0 : fd_topob_wksp( topo, "sock" );
611 0 : fd_topo_tile_t * sock = fd_topob_tile( topo, "sock", "sock", "metric_in", ULONG_MAX, 0, 0, 0 );
612 0 : sock->net = net_tile->net;
613 0 : sock->sock.only_recv_lo = 1;
614 0 : sock->sock.so_rcvbuf = 32*1024*1024;
615 0 : sock->sock.net_tile_id = net_tile->id;
616 :
617 0 : for( ulong i=0UL; i<net_tile->out_cnt; i++ ) {
618 0 : fd_topo_link_t const * net_out_link = &topo->links[ net_tile->out_link_id[ i ] ];
619 :
620 : /* Which links of net to duplicate */
621 0 : if( 0!=strncmp( net_out_link->name, "net_", 4UL ) || 0==strcmp( net_out_link->name, "net_netlnk" ) ) continue;
622 :
623 : /* Duplicate link */
624 0 : fd_topo_link_t * sock_out_link = fd_topob_link( topo, net_out_link->name, "sock", 4096UL, FD_NET_MTU, 64UL );
625 0 : fd_topob_tile_out( topo, "sock", sock->kind_id, sock_out_link->name, sock_out_link->kind_id );
626 :
627 : /* Loop through all tiles, check their in links for net */
628 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
629 0 : fd_topo_tile_t * consumer = &topo->tiles[ j ];
630 0 : ulong const in_cnt = consumer->in_cnt;
631 0 : for( ulong k=0UL; k<in_cnt; k++ ) {
632 0 : if( consumer->in_link_id[ k ]!=net_out_link->id ) continue;
633 :
634 : /* Add as sock link consumer */
635 0 : char const * fseq_wksp = topo->workspaces[ topo->objs[ consumer->in_link_fseq_obj_id[ k ] ].wksp_id ].name;
636 0 : fd_topob_tile_in( topo, consumer->name, consumer->kind_id, fseq_wksp, sock_out_link->name, sock_out_link->kind_id,
637 0 : consumer->in_link_reliable[ k ], consumer->in_link_poll[ k ] );
638 0 : }
639 0 : }
640 0 : }
641 0 : }
642 :
643 : ulong
644 0 : sock_lo_net_tile_id( fd_topo_tile_t const * tile ) {
645 0 : if( 0==strcmp( tile->name, "sock" ) && tile->sock.only_recv_lo ) return tile->sock.net_tile_id;
646 0 : return ULONG_MAX;
647 0 : }
648 :
649 : void
650 0 : sock_lo_set_affinity( fd_topo_t * topo ) {
651 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
652 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
653 0 : ulong net_tile_id = sock_lo_net_tile_id( tile );
654 0 : if( net_tile_id==ULONG_MAX ) continue;
655 0 : FD_TEST( net_tile_id<topo->tile_cnt );
656 0 : fd_topo_tile_t const * net_tile = &topo->tiles[ net_tile_id ];
657 0 : tile->cpu_idx = net_tile->cpu_idx;
658 0 : tile->floats = net_tile->floats;
659 0 : }
660 0 : }
661 :
662 : static void
663 : fd_topos_mlx5_setup_mem( fd_topo_t * topo,
664 0 : fd_topo_tile_t * mlx5_tile ) {
665 0 : ulong cum_frame_cnt = 0UL;
666 :
667 0 : ulong const rx_depth = mlx5_tile->mlx5.rx_queue_size;
668 0 : ulong const tx_depth = mlx5_tile->mlx5.tx_queue_size;
669 0 : cum_frame_cnt += rx_depth + tx_depth;
670 :
671 : /* Count up the depth of all RX mcaches */
672 :
673 0 : for( ulong j=0UL; j<(mlx5_tile->out_cnt); j++ ) {
674 0 : ulong link_id = mlx5_tile->out_link_id[ j ];
675 0 : ulong mcache_obj_id = topo->links[ link_id ].mcache_obj_id;
676 0 : ulong depth = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.depth", mcache_obj_id );
677 0 : if( FD_UNLIKELY( depth==ULONG_MAX ) ) FD_LOG_ERR(( "Didn't find depth for mcache %s", topo->links[ link_id ].name ));
678 0 : cum_frame_cnt += depth + 1UL;
679 0 : }
680 :
681 : /* Create a dcache object */
682 :
683 0 : ulong umem_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", mlx5_tile->kind_id );
684 0 : FD_TEST( umem_obj_id!=ULONG_MAX );
685 :
686 0 : FD_TEST( mlx5_tile->net.umem_dcache_obj_id > 0 );
687 0 : fd_pod_insertf_ulong( topo->props, cum_frame_cnt, "obj.%lu.depth", umem_obj_id );
688 0 : fd_pod_insertf_ulong( topo->props, 2UL, "obj.%lu.burst", umem_obj_id ); /* 4096 byte padding */
689 0 : fd_pod_insertf_ulong( topo->props, FD_NET_MTU, "obj.%lu.mtu", umem_obj_id );
690 0 : }
691 :
692 : static void
693 : fd_topos_iavf_setup_mem( fd_topo_t * topo,
694 0 : fd_topo_tile_t * iavf_tile ) {
695 0 : ulong cum_frame_cnt = 0UL;
696 :
697 0 : ulong const rx_depth = iavf_tile->iavf.rx_queue_size;
698 0 : ulong const tx_depth = iavf_tile->iavf.tx_queue_size;
699 0 : cum_frame_cnt += iavf_tile->iavf.member_cnt*(rx_depth + tx_depth);
700 :
701 : /* Count up the depth of all RX mcaches */
702 :
703 0 : for( ulong j=0UL; j<(iavf_tile->out_cnt); j++ ) {
704 0 : ulong link_id = iavf_tile->out_link_id[ j ];
705 0 : ulong mcache_obj_id = topo->links[ link_id ].mcache_obj_id;
706 0 : ulong depth = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.depth", mcache_obj_id );
707 0 : if( FD_UNLIKELY( depth==ULONG_MAX ) ) FD_LOG_ERR(( "Didn't find depth for mcache %s", topo->links[ link_id ].name ));
708 0 : cum_frame_cnt += depth + 1UL;
709 0 : }
710 :
711 : /* Create a dcache object */
712 :
713 0 : ulong umem_obj_id = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "net.%lu.umem", iavf_tile->kind_id );
714 0 : FD_TEST( umem_obj_id!=ULONG_MAX );
715 :
716 0 : FD_TEST( iavf_tile->net.umem_dcache_obj_id > 0 );
717 0 : fd_pod_insertf_ulong( topo->props, cum_frame_cnt, "obj.%lu.depth", umem_obj_id );
718 0 : fd_pod_insertf_ulong( topo->props, 2UL, "obj.%lu.burst", umem_obj_id ); /* 4096 byte padding */
719 0 : fd_pod_insertf_ulong( topo->props, FD_NET_MTU, "obj.%lu.mtu", umem_obj_id );
720 0 : }
721 :
722 : void
723 : fd_topos_net_tile_finish( fd_topo_t * topo,
724 0 : ulong net_kind_id ) {
725 0 : char const * provider = fd_pod_query_cstr( topo->props, "net.provider", "" );
726 0 : if( 0==strcmp( provider, "xdp" ) ) {
727 0 : ulong tile_id = fd_topo_find_tile( topo, "net", net_kind_id );
728 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) {
729 0 : FD_LOG_ERR(( "tile net:%lu not found", net_kind_id ));
730 0 : }
731 0 : fd_topos_xdp_setup_mem( topo, &topo->tiles[ tile_id ] );
732 0 : } else if( 0==strcmp( provider, "mlx5" ) ) {
733 0 : ulong tile_id = fd_topo_find_tile( topo, "mlx5", net_kind_id );
734 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) {
735 0 : FD_LOG_ERR(( "tile mlx5:%lu not found", net_kind_id ));
736 0 : }
737 0 : fd_topos_mlx5_setup_mem( topo, &topo->tiles[ tile_id ] );
738 0 : } else if( 0==strcmp( provider, "iavf" ) ) {
739 0 : ulong tile_id = fd_topo_find_tile( topo, "iavf", net_kind_id );
740 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) {
741 0 : FD_LOG_ERR(( "tile iavf:%lu not found", net_kind_id ));
742 0 : }
743 0 : fd_topos_iavf_setup_mem( topo, &topo->tiles[ tile_id ] );
744 0 : } else {
745 0 : return;
746 0 : }
747 0 : }
|
