Line data Source code
1 : #ifndef HEADER_fd_src_disco_net_fd_net_tile_private_h
2 : #define HEADER_fd_src_disco_net_fd_net_tile_private_h
3 :
4 : /* fd_net_tile_private.h defines private APIs used by network tiles. */
5 :
6 : #include "fd_net_tile.h"
7 :
8 : #include <net/if.h>
9 : #include <linux/if_arp.h>
10 :
11 : #include "../metrics/fd_metrics.h"
12 : #include "../netlink/fd_netlink_tile.h"
13 : #include "../stem/fd_stem.h"
14 : #include "../topo/fd_topo.h"
15 : #include "../../waltz/mib/fd_netdev_tbl.h"
16 : #include "../../waltz/ip/fd_fib4.h"
17 : #include "../../waltz/neigh/fd_neigh4_map.h"
18 :
19 : #include "../../waltz/ip/fd_iproute.h"
20 : #include "../../util/net/fd_eth.h"
21 : #include "../../util/net/fd_gre.h"
22 : #include "../../util/net/fd_ip4.h"
23 : #include "../../util/net/fd_udp.h"
24 : #include "fd_net_sock.h"
25 :
26 0 : #define FD_NET_IN_KIND_TX (0U)
27 0 : #define FD_NET_IN_KIND_IPROUTE (1U)
28 :
29 : /* FD_NET_IN_MAX is the max number of input links per net tile */
30 : #define FD_NET_IN_MAX (32UL)
31 :
32 : /* FD_NET_GRE_MAX is the max number of GRE tunnels the net tile will
33 : monitor. If a packet comes in with a source IP that doesn't match
34 : the endpoint of one of the first FD_NET_GRE_MAX tunnels (in the order
35 : the OS enumerates them), it will be dropped. This is limited for
36 : performance reasons. */
37 0 : #define FD_NET_GRE_MAX (4UL)
38 :
39 : /* fd_net_router_t stores userland route and neighbor state. */
40 : struct fd_net_router {
41 : ulong netlnk_out_idx;
42 : uint solicit_ip;
43 : uint solicit_if_idx;
44 :
45 : /* Route and neighbor tables */
46 : fd_fib4_t fib_local[1];
47 : fd_fib4_t fib_main[1];
48 : fd_neigh4_hmap_t neigh4[1];
49 :
50 : /* Netdev table */
51 : fd_netdev_tbl_join_t netdev_tbl; /* local copy in scratch */
52 : fd_netdev_tbl_join_t netdev_shared; /* shared seqlock-protected table */
53 :
54 : uint if_virt;
55 : uint default_address;
56 : };
57 : typedef struct fd_net_router fd_net_router_t;
58 :
59 : /* fd_net_tx_route_t stores a resolved TX route. It includes src_ip to fill
60 : an unset source address in the original IPv4 header, but no original
61 : destination address because packet preparation leaves that field
62 : unchanged. IP addresses are in network byte order. */
63 : struct fd_net_tx_route {
64 : uchar mac_addrs[12];
65 : ushort mtu;
66 : uint src_ip;
67 : uint if_idx;
68 : uint use_loopback;
69 : uint use_gre;
70 : uint gre_outer_src_ip;
71 : uint gre_outer_dst_ip;
72 : /* routing_failure is set if userland routing failed. The packet is
73 : then sent on fd_net_tile_t::sock and routed by the kernel. For a
74 : GRE route whose outer endpoint failed routing, use_gre is also set
75 : and the encapsulated packet is sent to gre_outer_dst_ip. */
76 : int routing_failure;
77 : };
78 : typedef struct fd_net_tx_route fd_net_tx_route_t;
79 :
80 : /* fd_net_in_dcache_ctx_t stores info needed to validate and locate a frag
81 : in an input link's dcache. */
82 : typedef struct {
83 : void * wksp_base;
84 : ulong chunk0; /* Lowest allowed starting chunk */
85 : ulong wmark; /* Highest allowed starting chunk */
86 : } fd_net_in_dcache_ctx_t;
87 :
88 : /* fd_net_tile_t stores provider agnostic private tile state */
89 : struct fd_net_tile {
90 : ulong kind_id;
91 : ulong tile_cnt;
92 :
93 : fd_net_in_dcache_ctx_t in_dcache_ctx[ FD_NET_IN_MAX ];
94 : uchar in_kind[ FD_NET_IN_MAX ];
95 :
96 : fd_net_router_t router;
97 :
98 : /* iproute_msg stages route updates between during_frag and after_frag */
99 : fd_iproute_msg_t iproute_msg;
100 : fd_net_tx_route_t tx_route;
101 :
102 : void * pkt_buf_wksp_base;
103 : ulong pkt_buf_chunk0;
104 : ulong pkt_buf_wmark;
105 : uint bind_address;
106 :
107 : ulong rx_dest_cnt;
108 : ushort rx_listen_ports[ FD_NET_RX_DEST_CNT ];
109 : ulong rx_proto_ids [ FD_NET_RX_DEST_CNT ];
110 : ulong rx_out_link_idx[ FD_NET_RX_DEST_CNT ];
111 : ulong rx_out_cnt;
112 : uint gre_tunnel_ip[ FD_NET_GRE_MAX ];
113 :
114 : /* sock is a raw socket for packets that failed userland routing */
115 : fd_net_sock_t sock;
116 :
117 : struct {
118 : ulong rx_pkt_cnt;
119 : ulong rx_bytes_total;
120 : ulong rx_gre_cnt;
121 : ulong rx_vlan_cnt;
122 : ulong rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_CNT ];
123 :
124 : ulong tx_pkt_cnt;
125 : ulong tx_bytes_total;
126 : ulong tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_CNT ];
127 : ulong tx_route_result_cnt[ FD_METRICS_ENUM_PKT_TX_ROUTE_RESULT_CNT ];
128 : ulong tx_slow_path_reason_cnt[ FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_CNT ];
129 : ulong tx_gre_cnt;
130 : ulong tx_slow_path_submit_cnt;
131 : } metrics;
132 : };
133 : typedef struct fd_net_tile fd_net_tile_t;
134 :
135 : FD_PROTOTYPES_BEGIN
136 :
137 : /* fd_net_tx_route_lookup resolves one destination and returns 1 on
138 : success, 0 on failure. For GRE, returns the tunnel settings without
139 : resolving the outer route. */
140 : static inline int
141 : fd_net_tx_route_lookup( fd_net_tile_t * net,
142 : uint dst_ip,
143 0 : fd_net_tx_route_t * out ) {
144 :
145 0 : if( FD_UNLIKELY( !out ) ) return 0;
146 0 : *out = (fd_net_tx_route_t) {0};
147 0 : fd_net_router_t * router = &net->router;
148 :
149 : /* Route lookup */
150 :
151 0 : fd_fib4_hop_t hop[2] = {0};
152 0 : hop[0] = fd_fib4_lookup( router->fib_local, dst_ip, 0UL );
153 0 : hop[1] = fd_fib4_lookup( router->fib_main, dst_ip, 0UL );
154 0 : fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
155 :
156 0 : uint rtype = next_hop->rtype;
157 0 : uint if_idx = next_hop->if_idx;
158 0 : uint ip4_src = fd_uint_if( !!net->bind_address, net->bind_address, next_hop->ip4_src );
159 : /* if the provider uses a socket fallback it'll need the src ip.
160 : Success paths below overwrite it. */
161 0 : out->src_ip = fd_uint_if( !ip4_src, router->default_address, ip4_src );
162 :
163 0 : if( FD_UNLIKELY( rtype==FD_FIB4_RTYPE_LOCAL ) ) {
164 0 : rtype = FD_FIB4_RTYPE_UNICAST;
165 0 : if_idx = 1;
166 0 : }
167 :
168 0 : if( FD_UNLIKELY( rtype!=FD_FIB4_RTYPE_UNICAST ) ) {
169 0 : uint const reason = fd_uint_if( rtype==FD_FIB4_RTYPE_THROW,
170 0 : FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_NO_ROUTE_IDX,
171 0 : FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_UNSUPPORTED_ROUTE_IDX );
172 0 : net->metrics.tx_slow_path_reason_cnt[ reason ]++;
173 0 : out->routing_failure = 1;
174 0 : return 0;
175 0 : }
176 :
177 0 : fd_netdev_t * netdev = fd_netdev_tbl_query( &router->netdev_tbl, if_idx );
178 0 : if( !netdev ) {
179 0 : net->metrics.tx_slow_path_reason_cnt[ FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_INTERFACE_MISSING_IDX ]++;
180 0 : out->routing_failure = 1;
181 0 : return 0;
182 0 : }
183 :
184 0 : out->mtu = netdev->mtu;
185 0 : out->if_idx = if_idx;
186 :
187 0 : if( netdev->dev_type==ARPHRD_LOOPBACK ) {
188 0 : memset( out->mac_addrs, 0, sizeof(out->mac_addrs) );
189 0 : out->src_ip = fd_uint_if( !ip4_src, FD_IP4_ADDR( 127,0,0,1 ), ip4_src );
190 0 : out->use_loopback = 1U;
191 0 : return 1;
192 0 : } else if( netdev->dev_type==ARPHRD_IPGRE ) {
193 0 : out->src_ip = ip4_src;
194 0 : if( FD_UNLIKELY( !ip4_src || !netdev->gre_dst_ip ) ) {
195 0 : net->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
196 0 : return 0;
197 0 : }
198 : /* skip MAC addrs lookup for GRE inner dst ip */
199 0 : out->gre_outer_src_ip = netdev->gre_src_ip;
200 0 : out->gre_outer_dst_ip = netdev->gre_dst_ip;
201 0 : out->use_gre = 1U;
202 0 : return 1;
203 0 : }
204 :
205 0 : if( FD_UNLIKELY( netdev->dev_type!=ARPHRD_ETHER ) ) {
206 0 : net->metrics.tx_slow_path_reason_cnt[ FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_UNSUPPORTED_INTERFACE_IDX ]++;
207 0 : out->routing_failure = 1;
208 0 : return 0;
209 0 : }
210 :
211 0 : if( FD_UNLIKELY( if_idx!=router->if_virt ) ) {
212 0 : net->metrics.tx_slow_path_reason_cnt[ FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_UNSUPPORTED_INTERFACE_IDX ]++;
213 0 : out->routing_failure = 1;
214 0 : return 0;
215 0 : }
216 :
217 : /* Neighbor resolve */
218 0 : uint neigh_ip = next_hop->ip4_gw;
219 0 : if( !neigh_ip ) neigh_ip = dst_ip;
220 :
221 0 : fd_neigh4_entry_t neigh[1];
222 0 : int neigh_res = fd_neigh4_hmap_query_entry( router->neigh4, neigh_ip, neigh );
223 0 : if( FD_UNLIKELY( neigh_res!=FD_MAP_SUCCESS ) ) {
224 : /* Neighbor not found */
225 0 : router->solicit_ip = neigh_ip;
226 0 : router->solicit_if_idx = if_idx;
227 0 : net->metrics.tx_slow_path_reason_cnt[ FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_NO_NEIGHBOR_IDX ]++;
228 0 : out->routing_failure = 1;
229 0 : return 0;
230 0 : }
231 0 : if( FD_UNLIKELY( neigh->state != FD_NEIGH4_STATE_ACTIVE ) ) {
232 0 : net->metrics.tx_slow_path_reason_cnt[ FD_METRICS_ENUM_PKT_TX_SLOW_PATH_REASON_V_NO_NEIGHBOR_IDX ]++;
233 0 : out->routing_failure = 1;
234 0 : return 0;
235 0 : }
236 0 : ip4_src = fd_uint_if( !ip4_src, router->default_address, ip4_src );
237 0 : out->src_ip = ip4_src;
238 0 : memcpy( out->mac_addrs+0, neigh->mac_addr, 6 );
239 0 : memcpy( out->mac_addrs+6, netdev->mac_addr, 6 );
240 :
241 0 : return 1;
242 0 : }
243 :
244 : /* fd_net_tx_route resolves dst_ip into out, including the remote endpoint
245 : for GRE tunnels. src_ip remains the inner source IP, while if_idx, mtu,
246 : and mac_addrs come from the outer route. Rejects GRE or loopback outer
247 : routes. Returns 1 on success, 0 on failure. If kernel socket fallback
248 : is allowed, sets routing_failure and retains the source and tunnel
249 : addresses needed to prepare the packet. */
250 : static inline int
251 : fd_net_tx_route( fd_net_tile_t * net,
252 : uint dst_ip,
253 0 : fd_net_tx_route_t * out ) {
254 0 : if( FD_UNLIKELY( !fd_net_tx_route_lookup( net, dst_ip, out ) ) ) return 0;
255 0 : if( FD_LIKELY( !out->use_gre ) ) return 1;
256 :
257 0 : fd_net_tx_route_t outer_route;
258 0 : if( FD_UNLIKELY( !fd_net_tx_route_lookup( net, out->gre_outer_dst_ip, &outer_route ) &&
259 0 : !outer_route.routing_failure ) ) return 0;
260 0 : if( FD_UNLIKELY( outer_route.use_gre || outer_route.use_loopback ) ) {
261 0 : net->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
262 0 : return 0;
263 0 : }
264 :
265 0 : memcpy( out->mac_addrs, outer_route.mac_addrs, sizeof(out->mac_addrs) );
266 0 : out->mtu = outer_route.mtu;
267 0 : out->if_idx = outer_route.if_idx;
268 0 : out->gre_outer_src_ip = fd_uint_if( !out->gre_outer_src_ip, outer_route.src_ip, out->gre_outer_src_ip );
269 0 : out->routing_failure = outer_route.routing_failure;
270 0 : return !out->routing_failure;
271 0 : }
272 :
273 : /* fd_net_tx_probe_loopback returns 1 if dst_ip routes to a loopback
274 : device and 0 otherwise. It updates no metrics and requests no
275 : neighbor solicit, so tile 0 can use it to find loopback packets
276 : owned by another net tile without counting that tile's routing
277 : failures. */
278 : static inline int
279 : fd_net_tx_probe_loopback( fd_fib4_t const * fib_local,
280 : fd_fib4_t const * fib_main,
281 : fd_netdev_tbl_join_t * netdev_tbl,
282 0 : uint dst_ip ) {
283 0 : fd_fib4_hop_t hop[2];
284 0 : hop[0] = fd_fib4_lookup( fib_local, dst_ip, 0UL );
285 0 : hop[1] = fd_fib4_lookup( fib_main, dst_ip, 0UL );
286 0 : fd_fib4_hop_t const * next_hop = fd_fib4_hop_or( hop+0, hop+1 );
287 0 : if( next_hop->rtype!=FD_FIB4_RTYPE_LOCAL && next_hop->rtype!=FD_FIB4_RTYPE_UNICAST ) return 0;
288 0 : uint const if_idx = next_hop->rtype==FD_FIB4_RTYPE_LOCAL ? 1U : next_hop->if_idx;
289 0 : fd_netdev_t const * netdev = fd_netdev_tbl_query( netdev_tbl, if_idx );
290 0 : return NULL!=netdev && netdev->dev_type==ARPHRD_LOOPBACK;
291 0 : }
292 :
293 : /* fd_net_tx_select expects a TX input and returns 1 if this tile should
294 : transmit the packet described by sig, storing its route in
295 : net->tx_route, or 0 otherwise. The caller must handle route updates
296 : separately. */
297 : static inline int
298 : fd_net_tx_select( fd_net_tile_t * net,
299 0 : ulong sig ) {
300 0 : ulong proto_id = fd_disco_netmux_sig_proto( sig );
301 0 : if( FD_UNLIKELY( proto_id!=DST_PROTO_OUTGOING ) ) return 0;
302 :
303 0 : ulong const hash = fd_disco_netmux_sig_hash( sig );
304 0 : ulong target_idx = hash % net->tile_cnt;
305 0 : ulong const kind_id = net->kind_id;
306 0 : if( kind_id!=0UL && kind_id!=target_idx ) return 0;
307 :
308 0 : uint dst_ip = fd_disco_netmux_sig_ip( sig );
309 : /* Tile 0 probes for loopback without counting another tile's route failures. */
310 0 : if( kind_id!=target_idx &&
311 0 : !fd_net_tx_probe_loopback( net->router.fib_local, net->router.fib_main, &net->router.netdev_tbl, dst_ip ) ) return 0;
312 :
313 0 : fd_net_tx_route_t * route = &net->tx_route;
314 0 : if( FD_UNLIKELY( !fd_net_tx_route( net, dst_ip, route ) &&
315 0 : !route->routing_failure ) ) return 0;
316 :
317 0 : if( route->use_loopback ) target_idx = 0UL;
318 0 : return kind_id==target_idx;
319 0 : }
320 :
321 : static inline void
322 : fd_net_router_solicit( fd_net_router_t * router,
323 0 : fd_stem_context_t * stem ) {
324 0 : if( FD_LIKELY( !router->solicit_ip ) ) return;
325 :
326 0 : fd_stem_publish( stem, router->netlnk_out_idx, fd_netlink_neigh4_solicit_sig( router->solicit_ip, router->solicit_if_idx ), 0UL, 0UL, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
327 0 : router->solicit_ip = 0U;
328 0 : }
329 :
330 : /* fd_net_gre_tunnels_refresh fills ctx->gre_tunnel_ip with the IP
331 : addresses of (up to) the first FD_NET_GRE_MAX remote GRE tunnel
332 : endpoints. Any remaining entries are set to zero. */
333 : static inline void
334 : fd_net_gre_tunnels_refresh( fd_net_tile_t * ctx,
335 0 : fd_netdev_tbl_join_t const * netdev_tbl ) {
336 0 : fd_netdev_t const * dev_tbl = netdev_tbl->dev_tbl;
337 0 : ushort dev_cnt = netdev_tbl->hdr->dev_cnt;
338 :
339 0 : fd_memset( ctx->gre_tunnel_ip, 0, sizeof(ctx->gre_tunnel_ip) );
340 :
341 0 : ulong gre_tunnel_cnt = 0UL;
342 0 : for( ushort if_idx=0U; if_idx<dev_cnt && gre_tunnel_cnt<FD_NET_GRE_MAX; if_idx++ ) {
343 0 : fd_netdev_t const * netdev = dev_tbl+if_idx;
344 0 : if( netdev->dev_type==ARPHRD_IPGRE && netdev->gre_dst_ip ) {
345 0 : ctx->gre_tunnel_ip[ gre_tunnel_cnt++ ] = netdev->gre_dst_ip;
346 0 : }
347 0 : }
348 0 : }
349 :
350 : /* fd_net_tx_route_result_increment counts a transmit job this tile
351 : accepted after route resolution. Packets that failed userland routing
352 : take the slow path through the kernel, all others take the fast path
353 : (sent by the provider). */
354 : static inline void
355 0 : fd_net_tx_route_result_increment( fd_net_tile_t * ctx ) {
356 0 : ulong const idx = fd_ulong_if( !!ctx->tx_route.routing_failure,
357 0 : FD_METRICS_ENUM_PKT_TX_ROUTE_RESULT_V_SLOW_PATH_IDX,
358 0 : FD_METRICS_ENUM_PKT_TX_ROUTE_RESULT_V_FAST_PATH_IDX );
359 0 : ctx->metrics.tx_route_result_cnt[ idx ]++;
360 0 : }
361 :
362 : /* fd_net_in_frag_validate checks input dcache bounds and TX packet length. */
363 : static inline void
364 : fd_net_in_frag_validate( fd_net_tile_t const * ctx,
365 : ulong in_idx,
366 : ulong chunk,
367 0 : ulong frame_sz ) {
368 0 : fd_net_in_dcache_ctx_t const * input_ctx = &ctx->in_dcache_ctx[ in_idx ];
369 0 : if( FD_UNLIKELY( chunk<input_ctx->chunk0 ||
370 0 : chunk>input_ctx->wmark ||
371 0 : frame_sz>FD_NET_MTU ) ) {
372 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]",
373 0 : chunk, frame_sz, input_ctx->chunk0, input_ctx->wmark ));
374 0 : }
375 :
376 0 : if( FD_UNLIKELY( ctx->in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) return;
377 0 : if( FD_UNLIKELY( frame_sz<sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t) ) ) {
378 0 : FD_LOG_ERR(( "packet too small %lu (in_idx=%lu)", frame_sz, in_idx ));
379 0 : }
380 0 : if( FD_UNLIKELY( frame_sz>FD_ETH_PAYLOAD_MAX ) ) {
381 0 : FD_LOG_ERR(( "packet too big %lu (in_idx=%lu)", frame_sz, in_idx ));
382 0 : }
383 0 : }
384 :
385 : /* fd_net_iproute_msg_stage returns 1 if it staged an IPROUTE message, 0 otherwise. */
386 : static inline int
387 : fd_net_iproute_msg_stage( fd_net_tile_t * ctx,
388 : ulong in_idx,
389 : ulong chunk,
390 0 : ulong frame_sz ) {
391 0 : if( ctx->in_kind[ in_idx ]!=FD_NET_IN_KIND_IPROUTE ) return 0;
392 0 : if( FD_UNLIKELY( frame_sz!=sizeof(fd_iproute_msg_t) ) ) {
393 0 : FD_LOG_ERR(( "invalid iproute message size %lu", frame_sz ));
394 0 : }
395 0 : fd_memcpy( &ctx->iproute_msg, fd_chunk_to_laddr_const( ctx->in_dcache_ctx[ in_idx ].wksp_base, chunk ), sizeof(fd_iproute_msg_t) );
396 0 : return 1;
397 0 : }
398 :
399 : /* fd_net_tx_pkt_cpy copies a validated TX frag from an input dcache into
400 : the passed dst buffer, using chunk to locate the src frag. */
401 : static inline void
402 : fd_net_tx_pkt_cpy( fd_net_tile_t const * ctx,
403 : uchar * dst,
404 : ulong chunk,
405 : ulong frame_sz,
406 0 : ulong in_idx ) {
407 0 : uchar const * src = fd_chunk_to_laddr_const( ctx->in_dcache_ctx[ in_idx ].wksp_base, chunk );
408 0 : if( FD_UNLIKELY( ctx->tx_route.use_gre ) ) {
409 0 : ulong const inner_ip_off = sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t)+sizeof(fd_gre_hdr_t);
410 0 : fd_memcpy( dst, src, sizeof(fd_eth_hdr_t) );
411 0 : fd_memcpy( dst+inner_ip_off, src+sizeof(fd_eth_hdr_t), frame_sz-sizeof(fd_eth_hdr_t) );
412 0 : } else {
413 0 : fd_memcpy( dst, src, frame_sz );
414 0 : }
415 0 : }
416 :
417 : /* fd_net_tx_ip4_fill fills a missing source address with src_ip and
418 : updates the checksum. An existing source address is left unchanged. */
419 : static inline void
420 : fd_net_tx_ip4_fill( fd_ip4_hdr_t * ip4_hdr,
421 0 : uint src_ip ) {
422 0 : if( ip4_hdr->saddr==0U ) {
423 0 : ip4_hdr->saddr = src_ip;
424 0 : ip4_hdr->check = 0U;
425 0 : ip4_hdr->check = fd_ip4_hdr_check( ip4_hdr );
426 0 : }
427 0 : }
428 :
429 : /* fd_net_tx_pkt_prep checks for IPv4 EtherType, IP version 4, and an IPv4
430 : header length of at least 20 bytes that fits within the packet.
431 : Sets Ethernet src and dst MAC addresses from ctx->tx_route.mac_addrs.
432 : If the IPv4 src address is zero, sets it from ctx->tx_route.src_ip
433 : and recomputes the IPv4 checksum.
434 :
435 : For GRE, that source address update applies to the inner IPv4 header.
436 : Constructs an outer IPv4 header using ctx->tx_route.gre_outer_src_ip
437 : and ctx->tx_route.gre_outer_dst_ip, computes its checksum, and writes
438 : a GRE header. Updates *frame_sz to include the outer IPv4 and GRE
439 : headers.
440 :
441 : Returns 1 on success, 0 on failure. */
442 : static inline int
443 : fd_net_tx_pkt_prep( fd_net_tile_t * ctx,
444 : uchar * frame,
445 : ulong * frame_sz,
446 0 : ulong in_idx ) {
447 0 : fd_net_tx_route_t const * route = &ctx->tx_route;
448 0 : fd_eth_hdr_t * eth_hdr = (fd_eth_hdr_t *)frame;
449 0 : if( FD_UNLIKELY( eth_hdr->net_type!=fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) ) ) {
450 0 : FD_LOG_CRIT(( "in link %lu attempted to send packet with invalid ethertype %04x",
451 0 : in_idx, fd_ushort_bswap( eth_hdr->net_type ) ));
452 0 : }
453 :
454 0 : ulong const ip4_off = sizeof(fd_eth_hdr_t) + (route->use_gre ? sizeof(fd_ip4_hdr_t)+sizeof(fd_gre_hdr_t) : 0UL);
455 0 : fd_ip4_hdr_t * ip4_hdr = (fd_ip4_hdr_t *)(frame+ip4_off);
456 0 : ulong const ip4_sz = *frame_sz-sizeof(fd_eth_hdr_t);
457 0 : ulong const ip4_hdr_sz = FD_IP4_GET_LEN( *ip4_hdr );
458 0 : if( FD_UNLIKELY( FD_IP4_GET_VERSION( *ip4_hdr )!=4U ||
459 0 : ip4_hdr_sz<sizeof(fd_ip4_hdr_t) ||
460 0 : ip4_hdr_sz>ip4_sz ) ) {
461 0 : ctx->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
462 0 : return 0;
463 0 : }
464 0 : if( FD_UNLIKELY( !ip4_hdr->saddr && !route->src_ip ) ) {
465 0 : ctx->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_SOURCE_IP_MISSING_IDX ]++;
466 0 : return 0;
467 0 : }
468 :
469 0 : memcpy( eth_hdr->dst, route->mac_addrs, sizeof(eth_hdr->dst)+sizeof(eth_hdr->src) );
470 0 : fd_net_tx_ip4_fill( ip4_hdr, route->src_ip );
471 0 : if( FD_LIKELY( !route->use_gre ) ) return 1;
472 :
473 0 : if( FD_UNLIKELY( fd_ushort_bswap( ip4_hdr->net_tot_len )!=ip4_sz ) ) {
474 0 : ctx->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
475 0 : return 0;
476 0 : }
477 0 : if( FD_UNLIKELY( !route->gre_outer_src_ip || !route->gre_outer_dst_ip ) ) {
478 0 : ctx->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
479 0 : return 0;
480 0 : }
481 :
482 0 : ulong const outer_ip_sz = sizeof(fd_ip4_hdr_t)+sizeof(fd_gre_hdr_t)+ip4_sz;
483 0 : if( FD_UNLIKELY( sizeof(fd_eth_hdr_t)+outer_ip_sz>FD_NET_MTU || (route->mtu && outer_ip_sz>route->mtu) ) ) {
484 0 : ctx->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
485 0 : return 0;
486 0 : }
487 :
488 0 : fd_ip4_hdr_t outer_ip4 = {
489 0 : .verihl = FD_IP4_VERIHL( 4,5 ),
490 0 : .net_tot_len = fd_ushort_bswap( (ushort)outer_ip_sz ),
491 0 : .net_frag_off = fd_ushort_bswap( FD_IP4_HDR_FRAG_OFF_DF ),
492 0 : .ttl = 64U,
493 0 : .protocol = FD_IP4_HDR_PROTOCOL_GRE,
494 0 : .daddr = route->gre_outer_dst_ip
495 0 : };
496 0 : fd_net_tx_ip4_fill( &outer_ip4, route->gre_outer_src_ip );
497 0 : FD_STORE( fd_ip4_hdr_t, frame+sizeof(fd_eth_hdr_t), outer_ip4 );
498 :
499 0 : fd_gre_hdr_t const gre_hdr = {
500 0 : .flags_version = FD_GRE_HDR_FLG_VER_BASIC,
501 0 : .protocol = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP )
502 0 : };
503 0 : FD_STORE( fd_gre_hdr_t, frame+sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t), gre_hdr );
504 0 : *frame_sz = sizeof(fd_eth_hdr_t)+outer_ip_sz;
505 0 : return 1;
506 0 : }
507 :
508 : /* fd_net_tx_sock_send sends a prepared frame on ctx->sock. frame
509 : points to the Ethernet header and frame_sz is the size of the whole
510 : frame. The Ethernet header is not sent: the kernel routes the packet
511 : to the daddr of the first IPv4 header and builds its own L2 header.
512 : For GRE that is the outer header built by fd_net_tx_pkt_prep, so the
513 : packet goes to the tunnel endpoint. Returns 1 if the packet was sent and 0
514 : after counting a dropped packet. The caller updates the packet and
515 : byte counters, because they differ between providers. */
516 : static inline int
517 : fd_net_tx_sock_send( fd_net_tile_t * ctx,
518 : uchar const * frame,
519 0 : ulong frame_sz ) {
520 0 : fd_ip4_hdr_t const * ip4 = (fd_ip4_hdr_t const *)( frame+sizeof(fd_eth_hdr_t) );
521 0 : if( FD_UNLIKELY( !fd_net_sock_send( &ctx->sock, ip4, frame_sz-sizeof(fd_eth_hdr_t) ) ) ) {
522 0 : ctx->metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_SOCKET_SEND_FAILURE_IDX ]++;
523 0 : return 0;
524 0 : }
525 0 : ctx->metrics.tx_slow_path_submit_cnt++;
526 0 : return 1;
527 0 : }
528 :
529 : /* fd_net_rx_dest_add fills in ctx's mappings between a UDP port, protocol
530 : and output link for one rx dest. */
531 : static inline void
532 : fd_net_rx_dest_add( fd_net_tile_t * ctx,
533 : fd_topo_t const * topo,
534 : fd_topo_tile_t const * tile,
535 : ulong proto_id,
536 : char const * out_link,
537 : ushort listen_port,
538 0 : int out_link_required ) {
539 0 : if( FD_UNLIKELY( !listen_port ) ) return;
540 0 : ulong out_idx = fd_topo_find_tile_out_link( topo, tile, out_link, tile->kind_id );
541 :
542 0 : if( FD_UNLIKELY( out_idx==ULONG_MAX ) ) {
543 0 : if( FD_UNLIKELY( out_link_required ) ) {
544 0 : FD_LOG_ERR(( "net output link `%s` is missing for UDP port %hu", out_link, listen_port ));
545 0 : }
546 0 : return;
547 0 : }
548 :
549 0 : if( FD_UNLIKELY( ctx->rx_dest_cnt>=FD_NET_RX_DEST_CNT ) ) {
550 0 : FD_LOG_ERR(( "net tile RX destination count exceeds max of %lu", FD_NET_RX_DEST_CNT ));
551 0 : }
552 :
553 0 : ctx->rx_proto_ids [ ctx->rx_dest_cnt ] = proto_id;
554 0 : ctx->rx_listen_ports[ ctx->rx_dest_cnt ] = listen_port;
555 0 : ctx->rx_out_link_idx[ ctx->rx_dest_cnt ] = out_idx;
556 0 : ctx->rx_dest_cnt++;
557 0 : }
558 :
559 : static inline void
560 : fd_net_rx_dests_init( fd_net_tile_t * ctx,
561 : fd_topo_t const * topo,
562 0 : fd_topo_tile_t const * tile ) {
563 0 : ctx->rx_out_cnt = tile->out_cnt;
564 :
565 0 : fd_net_rx_dest_t dests[ FD_NET_RX_DEST_CNT ];
566 0 : fd_net_rx_dest_list( dests, &tile->net );
567 0 : for( ulong i=0UL; i<FD_NET_RX_DEST_CNT; i++ ) {
568 0 : fd_net_rx_dest_add( ctx, topo, tile, dests[ i ].proto_id, dests[ i ].out_link, dests[ i ].listen_port, dests[ i ].out_link_required );
569 0 : }
570 0 : }
571 :
572 : static inline int
573 : fd_net_rx_dest_lookup( fd_net_tile_t const * ctx,
574 : ushort dst_port,
575 : ulong * out_idx,
576 0 : ulong * proto_id ) {
577 0 : ulong rule_idx = ULONG_MAX;
578 0 : for( ulong i=0UL; i<ctx->rx_dest_cnt; i++ ) {
579 0 : if( ctx->rx_listen_ports[ i ]==dst_port ) {
580 0 : rule_idx = i;
581 0 : break;
582 0 : }
583 0 : }
584 0 : if( FD_UNLIKELY( rule_idx==ULONG_MAX ) ) return 0;
585 :
586 0 : *out_idx = ctx->rx_out_link_idx[ rule_idx ];
587 0 : *proto_id = ctx->rx_proto_ids [ rule_idx ];
588 0 : return *out_idx<ctx->rx_out_cnt;
589 0 : }
590 :
591 : /* fd_net_rx_pkt validates and publishes an RX packet. It returns whether
592 : publication succeeded and sets freed_chunk to a reusable buffer. */
593 : static inline int
594 : fd_net_rx_pkt( fd_net_tile_t * ctx,
595 : fd_stem_context_t * stem,
596 : ulong chunk, /* offset high bits */
597 : ulong ctl, /* offset low bits */
598 : ulong byte_len,
599 : ulong tspub,
600 : ulong * freed_chunk,
601 0 : ulong * iavf_non_fd_pkts_rx_total ) {
602 0 : *freed_chunk = chunk;
603 :
604 0 : ulong const min_udp_frame_sz = sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t)+sizeof(fd_udp_hdr_t);
605 0 : ulong const min_frame_sz = iavf_non_fd_pkts_rx_total ? sizeof(fd_eth_hdr_t) : min_udp_frame_sz;
606 0 : if( FD_UNLIKELY( byte_len<min_frame_sz || byte_len>FD_NET_MTU ) ) {
607 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
608 0 : return 0;
609 0 : }
610 :
611 0 : uchar * frame = (uchar *)fd_chunk_to_laddr( ctx->pkt_buf_wksp_base, chunk ) + ctl;
612 0 : fd_eth_hdr_t * eth_hdr = (fd_eth_hdr_t *)frame;
613 :
614 : /* Strip a single 802.1Q tag in place. The Ethernet header is moved
615 : forward so that it abuts the next header and takes its net_type
616 : from the tag. The regions overlap, so the header is staged in a
617 : temporary. Frames whose tag was already stripped by the NIC are
618 : not counted in rx_vlan_cnt. */
619 0 : int is_vlan = fd_ushort_bswap( eth_hdr->net_type )==FD_ETH_HDR_TYPE_VLAN;
620 0 : if( FD_UNLIKELY( is_vlan ) ) {
621 0 : if( FD_UNLIKELY( byte_len<sizeof(fd_eth_hdr_t)+sizeof(fd_vlan_tag_t) ) ) {
622 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
623 0 : return 0;
624 0 : }
625 0 : fd_vlan_tag_t const * vlan_tag = (fd_vlan_tag_t const *)(eth_hdr+1);
626 0 : fd_eth_hdr_t tmp_eth_hdr = *eth_hdr;
627 0 : tmp_eth_hdr.net_type = vlan_tag->net_type;
628 0 : FD_STORE( fd_eth_hdr_t, frame+sizeof(fd_vlan_tag_t), tmp_eth_hdr );
629 0 : frame += sizeof(fd_vlan_tag_t);
630 0 : ctl += sizeof(fd_vlan_tag_t);
631 0 : byte_len -= sizeof(fd_vlan_tag_t);
632 0 : eth_hdr = (fd_eth_hdr_t *)frame;
633 0 : }
634 :
635 0 : if( FD_UNLIKELY( fd_ushort_bswap( eth_hdr->net_type )!=FD_ETH_HDR_TYPE_IP ) ) {
636 0 : if( iavf_non_fd_pkts_rx_total ) goto iavf_non_fd_pkt;
637 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
638 0 : return 0;
639 0 : }
640 0 : if( FD_UNLIKELY( byte_len<sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t) ) ) {
641 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
642 0 : return 0;
643 0 : }
644 :
645 0 : fd_ip4_hdr_t * ip4_hdr = (fd_ip4_hdr_t *)(eth_hdr+1);
646 0 : ulong const ip4_hdr_sz = sizeof(fd_ip4_hdr_t);
647 0 : ulong ip4_total_sz = fd_ushort_bswap( ip4_hdr->net_tot_len );
648 0 : if( FD_UNLIKELY( ip4_hdr->verihl!=FD_IP4_VERIHL( 4,5 ) ) ) {
649 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
650 0 : return 0;
651 0 : }
652 0 : if( FD_UNLIKELY( ip4_total_sz<ip4_hdr_sz ||
653 0 : sizeof(fd_eth_hdr_t)+ip4_total_sz>byte_len ) ) {
654 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
655 0 : return 0;
656 0 : }
657 :
658 0 : int is_gre = ip4_hdr->protocol==FD_IP4_HDR_PROTOCOL_GRE;
659 0 : if( FD_UNLIKELY( is_gre ) ) {
660 0 : if( FD_UNLIKELY( !ctx->gre_tunnel_ip[0] ) ) {
661 0 : if( iavf_non_fd_pkts_rx_total ) goto iavf_non_fd_pkt;
662 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_GRE_UNCONFIGURED_IDX ]++;
663 0 : return 0;
664 0 : }
665 :
666 0 : int tunnel_found = 0;
667 0 : for( ulong i=0UL; i<FD_NET_GRE_MAX; i++ ) tunnel_found |= ip4_hdr->saddr==ctx->gre_tunnel_ip[ i ];
668 0 : ulong const overhead = ip4_hdr_sz+sizeof(fd_gre_hdr_t);
669 0 : if( FD_UNLIKELY( !tunnel_found || !ip4_hdr->saddr ||
670 0 : overhead+sizeof(fd_ip4_hdr_t)+sizeof(fd_udp_hdr_t)>ip4_total_sz ) ) {
671 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_GRE_INVALID_IDX ]++;
672 0 : return 0;
673 0 : }
674 :
675 0 : fd_gre_hdr_t const * gre_hdr = (fd_gre_hdr_t const *)((uchar *)ip4_hdr+ip4_hdr_sz);
676 0 : if( FD_UNLIKELY( gre_hdr->flags_version!=FD_GRE_HDR_FLG_VER_BASIC ||
677 0 : gre_hdr->protocol!=fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) ) ) {
678 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_GRE_INVALID_IDX ]++;
679 0 : return 0;
680 0 : }
681 :
682 0 : frame += overhead;
683 0 : fd_memcpy( frame, eth_hdr, sizeof(fd_eth_hdr_t) );
684 :
685 0 : byte_len -= overhead;
686 0 : ctl += overhead;
687 0 : eth_hdr = (fd_eth_hdr_t *)frame;
688 0 : ip4_hdr = (fd_ip4_hdr_t *)(eth_hdr+1);
689 0 : ip4_total_sz = fd_ushort_bswap( ip4_hdr->net_tot_len );
690 0 : }
691 :
692 0 : if( FD_UNLIKELY( ip4_hdr->verihl!=FD_IP4_VERIHL( 4,5 ) ||
693 0 : ip4_total_sz<ip4_hdr_sz ||
694 0 : sizeof(fd_eth_hdr_t)+ip4_total_sz>byte_len ) ) {
695 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
696 0 : return 0;
697 0 : }
698 0 : if( FD_UNLIKELY( ip4_hdr->protocol!=FD_IP4_HDR_PROTOCOL_UDP ) ) {
699 0 : if( iavf_non_fd_pkts_rx_total ) goto iavf_non_fd_pkt;
700 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
701 0 : return 0;
702 0 : }
703 0 : if( FD_UNLIKELY( ctx->bind_address && ip4_hdr->daddr!=ctx->bind_address ) ) {
704 0 : if( iavf_non_fd_pkts_rx_total ) goto iavf_non_fd_pkt;
705 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_DST_IP_MISMATCH_IDX ]++;
706 0 : return 0;
707 0 : }
708 :
709 0 : ulong const udp_off = sizeof(fd_eth_hdr_t)+ip4_hdr_sz;
710 0 : ulong const dgram_off = udp_off+sizeof(fd_udp_hdr_t);
711 0 : if( FD_UNLIKELY( dgram_off>byte_len ) ) {
712 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
713 0 : return 0;
714 0 : }
715 :
716 0 : fd_udp_hdr_t const * udp_hdr = (fd_udp_hdr_t const *)((uchar const *)eth_hdr+udp_off);
717 0 : ulong const udp_sz = fd_ushort_bswap( udp_hdr->net_len );
718 0 : if( FD_UNLIKELY( udp_sz<sizeof(fd_udp_hdr_t) || udp_sz>ip4_total_sz-ip4_hdr_sz ||
719 0 : fd_ip4_addr_is_mcast( ip4_hdr->saddr ) ) ) {
720 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
721 0 : return 0;
722 0 : }
723 :
724 0 : ushort const dst_port = fd_ushort_bswap( udp_hdr->net_dport );
725 0 : ulong out_idx;
726 0 : ulong proto_id;
727 0 : if( FD_UNLIKELY( !fd_net_rx_dest_lookup( ctx, dst_port, &out_idx, &proto_id ) ) ) {
728 : /* Missing output links for configured ports still count as drops. */
729 0 : if( iavf_non_fd_pkts_rx_total ) {
730 0 : ulong dest_idx = 0UL;
731 0 : for( ; dest_idx<ctx->rx_dest_cnt; dest_idx++ ) {
732 0 : if( ctx->rx_listen_ports[ dest_idx ]==dst_port ) break;
733 0 : }
734 0 : if( dest_idx==ctx->rx_dest_cnt ) goto iavf_non_fd_pkt;
735 0 : }
736 0 : ctx->metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_DST_PORT_UNBOUND_IDX ]++;
737 0 : return 0;
738 0 : }
739 :
740 0 : fd_frag_meta_t * mcache = stem->mcaches[ out_idx ];
741 0 : ulong const depth = stem->depths [ out_idx ];
742 0 : ulong const seq = stem->seqs[ out_idx ];
743 :
744 0 : *freed_chunk = mcache[ fd_mcache_line_idx( seq, depth ) ].chunk;
745 :
746 0 : ushort const src_port = fd_ushort_bswap( udp_hdr->net_sport );
747 0 : ulong const sig = fd_disco_netmux_sig( ip4_hdr->saddr, src_port, ip4_hdr->saddr, proto_id, dgram_off );
748 0 : ulong const tsorig = 0UL;
749 :
750 0 : fd_stem_publish( stem, out_idx, sig, chunk, byte_len, ctl, tsorig, tspub );
751 :
752 0 : ctx->metrics.rx_gre_cnt += (ulong)is_gre;
753 0 : ctx->metrics.rx_vlan_cnt += (ulong)is_vlan;
754 0 : ctx->metrics.rx_pkt_cnt++;
755 0 : ctx->metrics.rx_bytes_total += byte_len;
756 :
757 0 : return 1;
758 :
759 0 : iavf_non_fd_pkt:
760 0 : (*iavf_non_fd_pkts_rx_total)++;
761 0 : return 0;
762 0 : }
763 :
764 : FD_PROTOTYPES_END
765 :
766 : #endif /* HEADER_fd_src_disco_net_fd_net_tile_private_h */
|
