Line data Source code
1 : #define _GNU_SOURCE /* dup3 */
2 : #include "fd_sock_tile_private.h"
3 : #include "../fd_net_tile.h"
4 : #include "../../fd_disco_base.h"
5 : #include "../../topo/fd_topo.h"
6 : #include "../../../util/net/fd_eth.h"
7 : #include "../../../util/net/fd_ip4.h"
8 : #include "../../../util/net/fd_udp.h"
9 :
10 : #include <stdalign.h> /* alignof */
11 : #include <errno.h>
12 : #include <fcntl.h> /* fcntl */
13 : #include <unistd.h> /* dup3, close */
14 : #include <netinet/in.h> /* sockaddr_in */
15 : #include <net/if.h> /* if_nametoindex */
16 : #include <linux/filter.h> /* SO_ATTACH_FILTER */
17 : #include <sys/socket.h> /* socket */
18 : #include "../../metrics/fd_metrics.h"
19 :
20 : #include "generated/fd_sock_tile_seccomp.h"
21 :
22 : /* recv/sendmmsg packet count in batch and tango burst depth
23 : FIXME make configurable in the future?
24 : FIXME keep in sync with fd_net_tile_topo.c */
25 0 : #define STEM_BURST (64UL)
26 :
27 : /* Place RX socket file descriptors in contiguous integer range. */
28 0 : #define RX_SOCK_FD_MIN (128)
29 :
30 : /* Controls max ancillary data size.
31 : Must be aligned by alignof(struct cmsghdr) */
32 0 : #define FD_SOCK_CMSG_MAX (64UL)
33 :
34 : /* Musl's CMSG_NXTHDR macro compares size_t against ptrdiff_t,
35 : triggering -Wsign-compare. */
36 : #pragma GCC diagnostic push
37 : #pragma GCC diagnostic ignored "-Wsign-compare"
38 : #pragma GCC diagnostic ignored "-Wsign-conversion"
39 : static inline struct cmsghdr *
40 : fd_cmsg_nxthdr( struct msghdr * mhdr,
41 0 : struct cmsghdr * cmsg ) {
42 0 : return CMSG_NXTHDR( mhdr, cmsg );
43 0 : }
44 : #pragma GCC diagnostic pop
45 :
46 : static ulong
47 : populate_allowed_seccomp( fd_topo_t const * topo,
48 : fd_topo_tile_t const * tile,
49 : ulong out_cnt,
50 0 : struct sock_filter * out ) {
51 0 : FD_SCRATCH_ALLOC_INIT( l, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
52 0 : fd_sock_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sock_tile_t), sizeof(fd_sock_tile_t) );
53 :
54 0 : populate_sock_filter_policy_fd_sock_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->tx_sock, RX_SOCK_FD_MIN, RX_SOCK_FD_MIN+(uint)ctx->sock_cnt );
55 0 : return sock_filter_policy_fd_sock_tile_instr_cnt;
56 0 : }
57 :
58 : static ulong
59 : populate_allowed_fds( fd_topo_t const * topo,
60 : fd_topo_tile_t const * tile,
61 : ulong out_fds_cnt,
62 0 : int * out_fds ) {
63 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
64 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
65 0 : fd_sock_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sock_tile_t), sizeof(fd_sock_tile_t) );
66 :
67 0 : ulong sock_cnt = ctx->sock_cnt;
68 0 : if( FD_UNLIKELY( out_fds_cnt<sock_cnt+3UL ) ) {
69 0 : FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
70 0 : }
71 :
72 0 : ulong out_cnt = 0UL;
73 :
74 0 : out_fds[ out_cnt++ ] = 2; /* stderr */
75 0 : if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) ) {
76 0 : out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
77 0 : }
78 0 : if( -1!=ctx->tx_sock ) out_fds[ out_cnt++ ] = ctx->tx_sock;
79 0 : for( ulong j=0UL; j<sock_cnt; j++ ) {
80 0 : out_fds[ out_cnt++ ] = ctx->pollfd[ j ].fd;
81 0 : }
82 0 : return out_cnt;
83 0 : }
84 :
85 : FD_FN_CONST static inline ulong
86 0 : tx_scratch_footprint( void ) {
87 0 : return STEM_BURST * fd_ulong_align_up( FD_NET_MTU, FD_CHUNK_ALIGN );
88 0 : }
89 :
90 : FD_FN_CONST static inline ulong
91 0 : scratch_align( void ) {
92 0 : return 4096UL;
93 0 : }
94 :
95 : FD_FN_PURE static inline ulong
96 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
97 0 : ulong l = FD_LAYOUT_INIT;
98 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_sock_tile_t), sizeof(fd_sock_tile_t) );
99 0 : l = FD_LAYOUT_APPEND( l, alignof(struct iovec), STEM_BURST*sizeof(struct iovec) );
100 0 : l = FD_LAYOUT_APPEND( l, alignof(struct cmsghdr), STEM_BURST*FD_SOCK_CMSG_MAX );
101 0 : l = FD_LAYOUT_APPEND( l, alignof(struct sockaddr_in), STEM_BURST*sizeof(struct sockaddr_in) );
102 0 : l = FD_LAYOUT_APPEND( l, alignof(struct mmsghdr), STEM_BURST*sizeof(struct mmsghdr) );
103 0 : l = FD_LAYOUT_APPEND( l, alignof(ulong), STEM_BURST*sizeof(ulong) );
104 0 : if( !tile->sock.only_recv_lo ) l = FD_LAYOUT_APPEND( l, FD_CHUNK_ALIGN, tx_scratch_footprint() );
105 0 : return FD_LAYOUT_FINI( l, scratch_align() );
106 0 : }
107 :
108 : /* create_udp_socket creates and configures a new UDP socket for the
109 : sock tile at the given file descriptor ID. */
110 :
111 : static void
112 : create_udp_socket( int sock_fd,
113 : uint bind_addr,
114 : ushort listen_port,
115 : int so_rcvbuf,
116 0 : uint lo_ifindex ) {
117 :
118 0 : if( fcntl( sock_fd, F_GETFD, 0 )!=-1 ) {
119 0 : FD_LOG_ERR(( "file descriptor %d already exists", sock_fd ));
120 0 : } else if( errno!=EBADF ) {
121 0 : FD_LOG_ERR(( "fcntl(F_GETFD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
122 0 : }
123 :
124 0 : int orig_fd = socket( AF_INET, SOCK_DGRAM, IPPROTO_UDP );
125 0 : if( FD_UNLIKELY( orig_fd<0 ) ) {
126 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_DGRAM,IPPROTO_UDP) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
127 0 : }
128 :
129 0 : int reuseport = 1;
130 0 : if( FD_UNLIKELY( setsockopt( orig_fd, SOL_SOCKET, SO_REUSEPORT, &reuseport, sizeof(int) )<0 ) ) {
131 0 : FD_LOG_ERR(( "setsockopt(SOL_SOCKET,SO_REUSEPORT,1) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
132 0 : }
133 :
134 0 : int ip_pktinfo = 1;
135 0 : if( FD_UNLIKELY( setsockopt( orig_fd, IPPROTO_IP, IP_PKTINFO, &ip_pktinfo, sizeof(int) )<0 ) ) {
136 0 : FD_LOG_ERR(( "setsockopt(IPPROTO_IP,IP_PKTINFO,1) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
137 0 : }
138 :
139 0 : if( FD_UNLIKELY( 0!=setsockopt( orig_fd, SOL_SOCKET, SO_RCVBUF, &so_rcvbuf, sizeof(int) ) ) ) {
140 0 : FD_LOG_ERR(( "setsockopt(SOL_SOCKET,SO_RCVBUF,%i) failed (%i-%s)", so_rcvbuf, errno, fd_io_strerror( errno ) ));
141 0 : }
142 :
143 0 : struct sockaddr_in saddr = {
144 0 : .sin_family = AF_INET,
145 0 : .sin_addr.s_addr = bind_addr,
146 0 : .sin_port = fd_ushort_bswap( listen_port ),
147 0 : };
148 0 : if( lo_ifindex ) {
149 : /* SKF_AD_IFINDEX identifies the receive device, including local
150 : traffic addressed to a non loopback interface's IP address. */
151 0 : struct sock_filter filter[] = {
152 0 : BPF_STMT( BPF_LD|BPF_W|BPF_ABS, (uint)(SKF_AD_OFF+SKF_AD_IFINDEX) ),
153 0 : BPF_JUMP( BPF_JMP|BPF_JEQ|BPF_K, lo_ifindex, 0, 1 ),
154 0 : BPF_STMT( BPF_RET|BPF_K, UINT_MAX ),
155 0 : BPF_STMT( BPF_RET|BPF_K, 0 )
156 0 : };
157 0 : struct sock_fprog prog = { .len = 4, .filter = filter };
158 0 : if( FD_UNLIKELY( 0!=setsockopt( orig_fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog) ) ) ) {
159 0 : FD_LOG_ERR(( "setsockopt(SO_ATTACH_FILTER) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
160 0 : }
161 0 : }
162 0 : if( FD_UNLIKELY( 0!=bind( orig_fd, fd_type_pun_const( &saddr ), sizeof(struct sockaddr_in) ) ) ) {
163 0 : FD_LOG_ERR(( "bind(0.0.0.0:%i) failed (%i-%s)", listen_port, errno, fd_io_strerror( errno ) ));
164 0 : }
165 :
166 0 : if( orig_fd==sock_fd ) {
167 0 : if( FD_UNLIKELY( fcntl( sock_fd, F_SETFD, FD_CLOEXEC ) ) ) {
168 0 : FD_LOG_ERR(( "fcntl(F_SETFD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
169 0 : }
170 0 : return;
171 0 : }
172 0 : int dup_res = dup3( orig_fd, sock_fd, O_CLOEXEC );
173 0 : if( FD_UNLIKELY( dup_res!=sock_fd ) ) {
174 0 : FD_LOG_ERR(( "dup3 returned %i (%i-%s)", dup_res, errno, fd_io_strerror( errno ) ));
175 0 : }
176 :
177 0 : if( FD_UNLIKELY( 0!=close( orig_fd ) ) ) {
178 0 : FD_LOG_ERR(( "close(%d) failed (%i-%s)", orig_fd, errno, fd_io_strerror( errno ) ));
179 0 : }
180 0 : }
181 :
182 : static void
183 : privileged_init( fd_topo_t const * topo,
184 0 : fd_topo_tile_t const * tile ) {
185 0 : void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id );
186 0 : FD_SCRATCH_ALLOC_INIT( l, scratch );
187 0 : fd_sock_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_sock_tile_t), sizeof(fd_sock_tile_t) );
188 0 : struct iovec * batch_iov = FD_SCRATCH_ALLOC_APPEND( l, alignof(struct iovec), STEM_BURST*sizeof(struct iovec) );
189 0 : void * batch_cmsg = FD_SCRATCH_ALLOC_APPEND( l, alignof(struct cmsghdr), STEM_BURST*FD_SOCK_CMSG_MAX );
190 0 : struct sockaddr_in * batch_sa = FD_SCRATCH_ALLOC_APPEND( l, alignof(struct sockaddr_in), STEM_BURST*sizeof(struct sockaddr_in) );
191 0 : struct mmsghdr * batch_msg = FD_SCRATCH_ALLOC_APPEND( l, alignof(struct mmsghdr), STEM_BURST*sizeof(struct mmsghdr) );
192 0 : ulong * batch_fsz = FD_SCRATCH_ALLOC_APPEND( l, alignof(ulong), STEM_BURST*sizeof(ulong) );
193 0 : ulong tx_sz = tile->sock.only_recv_lo ? 0UL : tx_scratch_footprint();
194 0 : uchar * tx_scratch = FD_SCRATCH_ALLOC_APPEND( l, FD_CHUNK_ALIGN, tx_sz );
195 0 : FD_DCHECK_CRIT( scratch==ctx, "invalid layout" );
196 :
197 0 : fd_memset( ctx, 0, sizeof(fd_sock_tile_t) );
198 0 : fd_memset( batch_iov, 0, STEM_BURST*sizeof(struct iovec) );
199 0 : fd_memset( batch_sa, 0, STEM_BURST*sizeof(struct sockaddr_in) );
200 0 : fd_memset( batch_msg, 0, STEM_BURST*sizeof(struct mmsghdr) );
201 :
202 0 : ctx->batch_cnt = 0UL;
203 0 : ctx->batch_iov = batch_iov;
204 0 : ctx->batch_cmsg = batch_cmsg;
205 0 : ctx->batch_sa = batch_sa;
206 0 : ctx->batch_msg = batch_msg;
207 0 : ctx->batch_frame_sz = batch_fsz;
208 0 : ctx->tx_scratch0 = tx_scratch;
209 0 : ctx->tx_scratch1 = tx_scratch + tx_sz;
210 0 : ctx->tx_ptr = tx_scratch;
211 0 : ctx->tx_sock = -1;
212 0 : ctx->bind_address = tile->sock.net.bind_address;
213 0 : ctx->tick_per_ns = fd_tempo_tick_per_ns( NULL );
214 0 : if( tile->sock.only_recv_lo ) {
215 0 : ctx->lo_ifindex = if_nametoindex( "lo" );
216 0 : if( FD_UNLIKELY( !ctx->lo_ifindex ) ) {
217 0 : FD_LOG_ERR(( "if_nametoindex(lo) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
218 0 : }
219 0 : }
220 :
221 : /* Create receive sockets. Incrementally assign them to file
222 : descriptors starting at sock_fd_min. */
223 :
224 0 : int sock_fd_min = RX_SOCK_FD_MIN;
225 0 : fd_net_rx_dest_t rx_dests[ FD_NET_RX_DEST_CNT ];
226 0 : fd_net_rx_dest_list( rx_dests, &tile->net );
227 0 : for( ulong dest_idx=0UL; dest_idx<FD_NET_RX_DEST_CNT; dest_idx++ ) {
228 0 : if( !rx_dests[ dest_idx ].listen_port ) continue;
229 0 : uint sock_idx = ctx->sock_cnt;
230 0 : if( sock_idx>=FD_SOCK_TILE_MAX_SOCKETS ) FD_LOG_ERR(( "too many sockets" ));
231 0 : ushort listen_port = rx_dests[ dest_idx ].listen_port;
232 :
233 0 : char const * target_link = rx_dests[ dest_idx ].out_link;
234 0 : ctx->link_rx_map[ sock_idx ] = 0xFF;
235 0 : for( ulong j=0UL; j<(tile->out_cnt); j++ ) {
236 0 : if( 0==strcmp( topo->links[ tile->out_link_id[ j ] ].name, target_link ) ) {
237 0 : ctx->proto_id [ sock_idx ] = (uchar)rx_dests[ dest_idx ].proto_id;
238 0 : ctx->link_rx_map [ sock_idx ] = (uchar)j;
239 0 : ctx->rx_listen_ports[ sock_idx ] = (ushort)listen_port;
240 0 : break;
241 0 : }
242 0 : }
243 0 : if( ctx->link_rx_map[ sock_idx ]==0xFF ) {
244 : /* listen port number has no associated links,
245 : i.e. the repair server is disabled, then no net_rserve link. */
246 0 : continue;
247 0 : }
248 :
249 0 : int sock_fd = sock_fd_min + (int)sock_idx;
250 0 : create_udp_socket( sock_fd, tile->sock.net.bind_address, listen_port, tile->sock.so_rcvbuf, ctx->lo_ifindex );
251 0 : ctx->pollfd[ sock_idx ].fd = sock_fd;
252 0 : ctx->pollfd[ sock_idx ].events = POLLIN;
253 0 : ctx->sock_cnt++;
254 0 : }
255 :
256 0 : if( tile->sock.only_recv_lo ) return;
257 :
258 : /* Create transmit socket */
259 :
260 0 : int tx_sock = socket( AF_INET, SOCK_RAW|SOCK_CLOEXEC, FD_IP4_HDR_PROTOCOL_UDP );
261 0 : if( FD_UNLIKELY( tx_sock<0 ) ) {
262 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_RAW|SOCK_CLOEXEC,17) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
263 0 : }
264 :
265 0 : if( FD_UNLIKELY( 0!=setsockopt( tx_sock, SOL_SOCKET, SO_SNDBUF, &tile->sock.so_sndbuf, sizeof(int) ) ) ) {
266 0 : FD_LOG_ERR(( "setsockopt(SOL_SOCKET,SO_SNDBUF,%i) failed (%i-%s)", tile->sock.so_sndbuf, errno, fd_io_strerror( errno ) ));
267 0 : }
268 :
269 0 : uchar mcast_ttl = 64;
270 0 : if( FD_UNLIKELY( 0!=setsockopt( tx_sock, IPPROTO_IP, IP_MULTICAST_TTL, &mcast_ttl, sizeof(mcast_ttl) ) ) ) {
271 0 : FD_LOG_ERR(( "setsockopt(IPPROTO_IP,IP_MULTICAST_TTL,%u) failed (%i-%s)", (uint)mcast_ttl, errno, fd_io_strerror( errno ) ));
272 0 : }
273 :
274 0 : ctx->tx_sock = tx_sock;
275 0 : ctx->bind_address = tile->sock.net.bind_address;
276 0 : }
277 :
278 : static void
279 : unprivileged_init( fd_topo_t const * topo,
280 0 : fd_topo_tile_t const * tile ) {
281 0 : fd_sock_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
282 :
283 0 : if( FD_UNLIKELY( tile->out_cnt > MAX_NET_OUTS ) ) {
284 0 : FD_LOG_ERR(( "sock tile has %lu out links which exceeds the max (%lu)", tile->out_cnt, MAX_NET_OUTS ));
285 0 : }
286 :
287 0 : ctx->link_rx_cnt = tile->out_cnt;
288 0 : for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
289 0 : if( 0!=strncmp( topo->links[ tile->out_link_id[ i ] ].name, "net_", 4 ) ) {
290 0 : FD_LOG_ERR(( "out link %lu is not a net RX link", i ));
291 0 : }
292 0 : fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ i ] ];
293 0 : ctx->link_rx[ i ].base = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
294 0 : ctx->link_rx[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->link_rx[ i ].base, link->dcache );
295 0 : ctx->link_rx[ i ].wmark = fd_dcache_compact_wmark( ctx->link_rx[ i ].base, link->dcache, link->mtu );
296 0 : ctx->link_rx[ i ].chunk = ctx->link_rx[ i ].chunk0;
297 0 : if( FD_UNLIKELY( link->burst < STEM_BURST ) ) {
298 0 : FD_LOG_ERR(( "link %lu dcache burst is too low (%lu<%lu)",
299 0 : tile->out_link_id[ i ], link->burst, STEM_BURST ));
300 0 : }
301 0 : }
302 :
303 0 : FD_CHECK_ERR( tile->in_cnt<=MAX_NET_INS, "too many input links" );
304 :
305 : /* The loopback-only sock tile has no TX links and is not counted */
306 0 : ctx->tx_tile_idx = 0UL;
307 0 : ctx->tx_tile_cnt = 0UL;
308 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
309 0 : fd_topo_tile_t const * t = &topo->tiles[ i ];
310 0 : if( strcmp( t->name, "sock" ) || t->sock.only_recv_lo ) continue;
311 0 : if( t->id==tile->id ) ctx->tx_tile_idx = ctx->tx_tile_cnt;
312 0 : ctx->tx_tile_cnt++;
313 0 : }
314 0 : ctx->tx_tile_cnt = fd_ulong_max( ctx->tx_tile_cnt, 1UL );
315 :
316 0 : for( ulong i=0UL; i<(tile->in_cnt); i++ ) {
317 0 : if( !strstr( topo->links[ tile->in_link_id[ i ] ].name, "_net" ) ) {
318 0 : FD_LOG_ERR(( "in link %lu is not a net TX link", i ));
319 0 : }
320 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
321 0 : ctx->link_tx[ i ].base = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
322 0 : ctx->link_tx[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->link_tx[ i ].base, link->dcache );
323 0 : ctx->link_tx[ i ].wmark = fd_dcache_compact_wmark( ctx->link_tx[ i ].base, link->dcache, link->mtu );
324 0 : }
325 :
326 0 : }
327 :
328 : /* RX PATH (socket->tango) ********************************************/
329 :
330 : /* FIXME Pace RX polling and interleave it with TX jobs to reduce TX
331 : tail latency */
332 :
333 : /* poll_rx_socket does one recvmmsg batch receive on the given socket
334 : index. Returns the number of packets returned by recvmmsg. */
335 :
336 : static ulong
337 : poll_rx_socket( fd_sock_tile_t * ctx,
338 : fd_stem_context_t * stem,
339 : uint sock_idx,
340 : int sock_fd,
341 0 : ushort proto_id ) {
342 0 : ulong hdr_sz = sizeof(fd_eth_hdr_t) + sizeof(fd_ip4_hdr_t) + sizeof(fd_udp_hdr_t);
343 0 : ulong payload_max = FD_NET_MTU-hdr_sz;
344 0 : uchar rx_link = ctx->link_rx_map[ sock_idx ];
345 0 : ushort listen_port = ctx->rx_listen_ports[ sock_idx ];
346 :
347 0 : fd_sock_link_rx_t * link = ctx->link_rx + rx_link;
348 0 : void * const base = link->base;
349 0 : ulong const chunk0 = link->chunk0;
350 0 : ulong const wmark = link->wmark;
351 0 : ulong chunk_next = link->chunk;
352 0 : uchar * cmsg_next = ctx->batch_cmsg;
353 :
354 0 : for( ulong j=0UL; j<STEM_BURST; j++ ) {
355 0 : ctx->batch_iov[ j ].iov_base = (uchar *)fd_chunk_to_laddr( base, chunk_next ) + hdr_sz;
356 0 : ctx->batch_iov[ j ].iov_len = payload_max;
357 0 : ctx->batch_msg[ j ].msg_hdr = (struct msghdr) {
358 0 : .msg_iov = ctx->batch_iov+j,
359 0 : .msg_iovlen = 1,
360 0 : .msg_name = ctx->batch_sa+j,
361 0 : .msg_namelen = sizeof(struct sockaddr_in),
362 0 : .msg_control = cmsg_next,
363 0 : .msg_controllen = FD_SOCK_CMSG_MAX,
364 0 : };
365 0 : cmsg_next += FD_SOCK_CMSG_MAX;
366 : /* Speculatively prepare all chunk indexes for a receive.
367 : At function exit, chunks into which a packet was received are
368 : committed, all others are freed. */
369 0 : chunk_next = fd_dcache_compact_next( chunk_next, FD_NET_MTU, chunk0, wmark );
370 0 : }
371 :
372 0 : int msg_cnt = recvmmsg( sock_fd, ctx->batch_msg, STEM_BURST, MSG_DONTWAIT, NULL );
373 0 : if( FD_UNLIKELY( msg_cnt<0 ) ) {
374 0 : if( FD_UNLIKELY( errno==EINTR ) ) return 0UL;
375 0 : if( FD_LIKELY( errno==EAGAIN ) ) return 0UL;
376 : /* unreachable if socket is in a valid state */
377 0 : FD_LOG_ERR(( "recvmmsg failed (%i-%s)", errno, fd_io_strerror( errno ) ));
378 0 : }
379 0 : long ts = fd_tickcount();
380 0 : ctx->metrics.sys_recvmmsg_cnt++;
381 :
382 0 : if( FD_UNLIKELY( msg_cnt==0 ) ) return 0UL;
383 :
384 : /* Only frags published to rx_link may consume rx_link dcache space.
385 : Packets that are dropped or redirected leave a hole, so subsequent
386 : packets are compacted down to chunk_pub. */
387 0 : ulong chunk_pub = link->chunk;
388 :
389 0 : for( ulong j=0; j<(ulong)msg_cnt; j++ ) {
390 0 : uchar * payload = ctx->batch_iov[ j ].iov_base;
391 0 : ulong payload_sz = ctx->batch_msg[ j ].msg_len;
392 0 : struct sockaddr_in * sa = ctx->batch_msg[ j ].msg_hdr.msg_name;
393 0 : ulong frame_sz = payload_sz + hdr_sz;
394 0 : if( FD_UNLIKELY( ctx->lo_ifindex &&
395 0 : (ctx->batch_msg[ j ].msg_hdr.msg_flags & (MSG_TRUNC|MSG_CTRUNC)) ) ) continue;
396 0 : ctx->metrics.rx_bytes_total += frame_sz;
397 0 : if( FD_UNLIKELY( sa->sin_family!=AF_INET ) ) {
398 : /* unreachable */
399 0 : FD_LOG_ERR(( "Received packet with unexpected sin_family %i", sa->sin_family ));
400 0 : }
401 :
402 0 : long daddr = -1;
403 0 : struct cmsghdr * cmsg = CMSG_FIRSTHDR( &ctx->batch_msg[ j ].msg_hdr );
404 0 : if( FD_LIKELY( cmsg ) ) {
405 0 : do {
406 0 : if( FD_LIKELY( (cmsg->cmsg_level==IPPROTO_IP) &
407 0 : (cmsg->cmsg_type ==IP_PKTINFO) ) ) {
408 0 : struct in_pktinfo const * pi = (struct in_pktinfo const *)CMSG_DATA( cmsg );
409 0 : daddr = pi->ipi_addr.s_addr;
410 0 : }
411 0 : cmsg = fd_cmsg_nxthdr( &ctx->batch_msg[ j ].msg_hdr, cmsg );
412 0 : } while( FD_UNLIKELY( cmsg ) ); /* optimize for 1 cmsg */
413 0 : }
414 0 : if( FD_UNLIKELY( daddr<0L ) ) {
415 : /* unreachable because IP_PKTINFO was set */
416 0 : FD_LOG_ERR(( "Missing IP_PKTINFO on incoming packet" ));
417 0 : }
418 :
419 0 : fd_eth_hdr_t * eth_hdr = (fd_eth_hdr_t *)( payload-42UL );
420 0 : fd_ip4_hdr_t * ip_hdr = (fd_ip4_hdr_t *)( payload-28UL );
421 0 : fd_udp_hdr_t * udp_hdr = (fd_udp_hdr_t *)( payload- 8UL );
422 0 : memset( eth_hdr->dst, 0, 6 );
423 0 : memset( eth_hdr->src, 0, 6 );
424 0 : eth_hdr->net_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP );
425 0 : *ip_hdr = (fd_ip4_hdr_t) {
426 0 : .verihl = FD_IP4_VERIHL( 4, 5 ),
427 0 : .net_tot_len = fd_ushort_bswap( (ushort)( payload_sz+28UL ) ),
428 0 : .ttl = 1,
429 0 : .protocol = FD_IP4_HDR_PROTOCOL_UDP,
430 0 : };
431 0 : uint daddr_ = (uint)(ulong)daddr;
432 0 : memcpy( ip_hdr->saddr_c, &sa->sin_addr.s_addr, 4 );
433 0 : memcpy( ip_hdr->daddr_c, &daddr_, 4 );
434 0 : *udp_hdr = (fd_udp_hdr_t) {
435 0 : .net_sport = sa->sin_port,
436 0 : .net_dport = (ushort)fd_ushort_bswap( (ushort)listen_port ),
437 0 : .net_len = (ushort)fd_ushort_bswap( (ushort)( payload_sz+8UL ) ),
438 0 : .check = 0
439 0 : };
440 :
441 0 : ctx->metrics.rx_pkt_cnt++;
442 0 : ulong chunk = fd_laddr_to_chunk( base, eth_hdr );
443 0 : ulong sig = fd_disco_netmux_sig( sa->sin_addr.s_addr, fd_ushort_bswap( sa->sin_port ), sa->sin_addr.s_addr, proto_id, hdr_sz );
444 0 : ulong tspub = fd_frag_meta_ts_comp( ts );
445 :
446 0 : if( FD_UNLIKELY( chunk!=chunk_pub ) ) {
447 0 : memcpy( fd_chunk_to_laddr( base, chunk_pub ), eth_hdr, frame_sz );
448 0 : }
449 0 : fd_stem_publish( stem, rx_link, sig, chunk_pub, frame_sz, 0UL, 0UL, tspub );
450 0 : chunk_pub = fd_dcache_compact_next( chunk_pub, FD_NET_MTU, chunk0, wmark );
451 0 : }
452 :
453 : /* Rewind the chunk index to the first free index. */
454 0 : link->chunk = chunk_pub;
455 0 : return (ulong)msg_cnt;
456 0 : }
457 :
458 : static ulong
459 : poll_rx( fd_sock_tile_t * ctx,
460 0 : fd_stem_context_t * stem ) {
461 0 : ulong pkt_cnt = 0UL;
462 0 : if( FD_UNLIKELY( ctx->batch_cnt ) ) {
463 0 : FD_LOG_ERR(( "Batch is not clean" ));
464 0 : }
465 0 : ctx->tx_idle_cnt = 0; /* restart TX polling */
466 0 : long wait_ns = ctx->lo_ifindex ? (long)((double)fd_long_max( stem->then-fd_tickcount(), 0L )/ctx->tick_per_ns) : 0L;
467 0 : struct timespec ts = { .tv_sec = wait_ns/1000000000L, .tv_nsec = wait_ns%1000000000L };
468 0 : if( FD_UNLIKELY( ppoll( ctx->pollfd, ctx->sock_cnt, &ts, NULL )<0 ) ) {
469 0 : if( FD_UNLIKELY( errno==EINTR ) ) return 0UL;
470 0 : FD_LOG_ERR(( "ppoll failed (%i-%s)", errno, fd_io_strerror( errno ) ));
471 0 : }
472 0 : for( uint j=0U; j<ctx->sock_cnt; j++ ) {
473 0 : if( ctx->pollfd[ j ].revents & (POLLIN|POLLERR) ) {
474 0 : pkt_cnt += poll_rx_socket(
475 0 : ctx,
476 0 : stem,
477 0 : j,
478 0 : ctx->pollfd[ j ].fd,
479 0 : ctx->proto_id[ j ]
480 0 : );
481 0 : }
482 0 : ctx->pollfd[ j ].revents = 0;
483 0 : }
484 0 : return pkt_cnt;
485 0 : }
486 :
487 : /* TX PATH (tango->socket) ********************************************/
488 :
489 : static void
490 0 : flush_tx_batch( fd_sock_tile_t * ctx ) {
491 0 : ulong batch_cnt = ctx->batch_cnt;
492 0 : for( int j = 0; j < (int)batch_cnt; /* incremented in loop */ ) {
493 0 : int remain = (int)batch_cnt - j;
494 0 : int send_cnt = sendmmsg( ctx->tx_sock, ctx->batch_msg + j, (uint)remain, MSG_DONTWAIT );
495 0 : if( send_cnt>=0 ) {
496 0 : ctx->metrics.sys_sendmmsg_cnt[ FD_METRICS_ENUM_SENDMMSG_RESULT_V_SUCCESS_IDX ]++;
497 0 : }
498 0 : if( FD_UNLIKELY( send_cnt < remain ) ) {
499 0 : ctx->metrics.tx_drop_cnt++;
500 0 : if( FD_UNLIKELY( send_cnt < 0 ) ) {
501 0 : switch( errno ) {
502 0 : case EAGAIN:
503 0 : case ENOBUFS:
504 0 : ctx->metrics.sys_sendmmsg_cnt[ FD_METRICS_ENUM_SENDMMSG_RESULT_V_BUFFER_FULL_IDX ]++;
505 0 : break;
506 0 : case EPERM:
507 0 : ctx->metrics.sys_sendmmsg_cnt[ FD_METRICS_ENUM_SENDMMSG_RESULT_V_PERMISSION_IDX ]++;
508 0 : break;
509 0 : case ENETUNREACH:
510 0 : case EHOSTUNREACH:
511 0 : ctx->metrics.sys_sendmmsg_cnt[ FD_METRICS_ENUM_SENDMMSG_RESULT_V_UNREACHABLE_IDX ]++;
512 0 : break;
513 0 : case ENONET:
514 0 : case ENETDOWN:
515 0 : case EHOSTDOWN:
516 0 : ctx->metrics.sys_sendmmsg_cnt[ FD_METRICS_ENUM_SENDMMSG_RESULT_V_DOWN_IDX ]++;
517 0 : break;
518 0 : default:
519 0 : ctx->metrics.sys_sendmmsg_cnt[ FD_METRICS_ENUM_SENDMMSG_RESULT_V_OTHER_IDX ]++;
520 : /* log with NOTICE, since flushing has a significant negative performance impact */
521 0 : FD_LOG_NOTICE(( "sendmmsg failed (%i-%s)", errno, fd_io_strerror( errno ) ));
522 0 : }
523 :
524 : /* first message failed, so skip failing message and continue */
525 0 : j++;
526 0 : } else {
527 : /* send_cnt succeeded, so skip those and also the failing message */
528 0 : j += send_cnt + 1;
529 :
530 : /* add the successful count */
531 0 : ctx->metrics.tx_pkt_cnt += (ulong)send_cnt;
532 0 : for( int k=j-send_cnt-1; k<j-1; k++ ) ctx->metrics.tx_bytes_total += ctx->batch_frame_sz[ k ];
533 0 : }
534 :
535 0 : continue;
536 0 : }
537 :
538 : /* send_cnt == batch_cnt, so we sent everything */
539 0 : ctx->metrics.tx_pkt_cnt += (ulong)send_cnt;
540 0 : for( int k=j; k<j+send_cnt; k++ ) ctx->metrics.tx_bytes_total += ctx->batch_frame_sz[ k ];
541 0 : break;
542 0 : }
543 :
544 0 : ctx->tx_ptr = ctx->tx_scratch0;
545 0 : ctx->batch_cnt = 0;
546 0 : }
547 :
548 : /* before_frag is called when a new frag has been detected. Every sock
549 : tile reads every TX link, so each sends only its share of packets. */
550 :
551 : static inline int
552 : before_frag( fd_sock_tile_t * ctx,
553 : ulong in_idx FD_PARAM_UNUSED,
554 : ulong seq FD_PARAM_UNUSED,
555 0 : ulong sig ) {
556 0 : ulong proto_id = fd_disco_netmux_sig_proto( sig );
557 0 : if( FD_UNLIKELY( proto_id!=DST_PROTO_OUTGOING ) ) return 1;
558 0 : if( fd_disco_netmux_sig_hash( sig )%ctx->tx_tile_cnt!=ctx->tx_tile_idx ) return 1;
559 0 : return 0; /* continue */
560 0 : }
561 :
562 : /* during_frag is called when a new frag passed early filtering.
563 : Speculatively copies data into a sendmmsg buffer. (If all tiles
564 : respect backpressure could eliminate this copy) */
565 :
566 : static inline void
567 : during_frag( fd_sock_tile_t * ctx,
568 : ulong in_idx,
569 : ulong seq FD_PARAM_UNUSED,
570 : ulong sig,
571 : ulong chunk,
572 : ulong sz,
573 0 : ulong ctl FD_PARAM_UNUSED ) {
574 0 : if( FD_UNLIKELY( chunk<ctx->link_tx[ in_idx ].chunk0 || chunk>ctx->link_tx[ in_idx ].wmark || sz>FD_NET_MTU ) ) {
575 0 : FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->link_tx[ in_idx ].chunk0, ctx->link_tx[ in_idx ].wmark ));
576 0 : }
577 :
578 0 : ctx->parsed.invalid = 0;
579 :
580 0 : ulong const hdr_min = sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t)+sizeof(fd_udp_hdr_t);
581 0 : if( FD_UNLIKELY( sz<hdr_min ) ) {
582 : /* FIXME support ICMP messages in the future?
583 : Defer the error to after_frag, where we know we weren't
584 : overrun. */
585 0 : ctx->parsed.invalid = 1;
586 0 : return;
587 0 : }
588 :
589 0 : uchar const * frame = fd_chunk_to_laddr_const( ctx->link_tx[ in_idx ].base, chunk );
590 0 : ulong hdr_sz = fd_disco_netmux_sig_hdr_sz( sig );
591 0 : uchar const * payload = frame+hdr_sz;
592 0 : if( FD_UNLIKELY( hdr_sz>sz || hdr_sz<hdr_min ) ) {
593 0 : FD_LOG_ERR(( "packet from in_idx=%lu corrupt: hdr_sz=%lu total_sz=%lu",
594 0 : in_idx, hdr_sz, sz ));
595 0 : }
596 0 : ulong payload_sz = sz-hdr_sz;
597 :
598 0 : fd_ip4_hdr_t const * ip_hdr = (fd_ip4_hdr_t const *)( frame +sizeof(fd_eth_hdr_t) );
599 0 : fd_udp_hdr_t const * udp_hdr = (fd_udp_hdr_t const *)( payload-sizeof(fd_udp_hdr_t) );
600 0 : ctx->parsed.ip_version = FD_IP4_GET_VERSION( *ip_hdr );
601 0 : ctx->parsed.ip_protocol = ip_hdr->protocol;
602 0 : if( FD_UNLIKELY( ( ctx->parsed.ip_version !=4 ) |
603 0 : ( ctx->parsed.ip_protocol!=FD_IP4_HDR_PROTOCOL_UDP ) ) ) {
604 : /* sock tile only supports IPv4 UDP for now. Defer the error to
605 : after_frag, where we know we weren't overrun. */
606 0 : ctx->parsed.invalid = 1;
607 0 : return;
608 0 : }
609 :
610 0 : ulong msg_sz = sizeof(fd_udp_hdr_t) + payload_sz;
611 :
612 0 : ulong batch_idx = ctx->batch_cnt;
613 0 : FD_DCHECK_CRIT( batch_idx<STEM_BURST, "flow control error" );
614 0 : struct mmsghdr * msg = ctx->batch_msg + batch_idx;
615 0 : struct sockaddr_in * sa = ctx->batch_sa + batch_idx;
616 0 : struct iovec * iov = ctx->batch_iov + batch_idx;
617 0 : struct cmsghdr * cmsg = (void *)( (ulong)ctx->batch_cmsg + batch_idx*FD_SOCK_CMSG_MAX );
618 0 : uchar * buf = ctx->tx_ptr;
619 :
620 0 : *iov = (struct iovec) {
621 0 : .iov_base = buf,
622 0 : .iov_len = msg_sz,
623 0 : };
624 0 : sa->sin_family = AF_INET;
625 0 : sa->sin_addr.s_addr = FD_LOAD( uint, ip_hdr->daddr_c );
626 0 : sa->sin_port = 0; /* ignored */
627 :
628 0 : cmsg->cmsg_level = IPPROTO_IP;
629 0 : cmsg->cmsg_type = IP_PKTINFO;
630 0 : cmsg->cmsg_len = CMSG_LEN( sizeof(struct in_pktinfo) );
631 0 : struct in_pktinfo * pi = (struct in_pktinfo *)CMSG_DATA( cmsg );
632 0 : pi->ipi_ifindex = 0;
633 0 : pi->ipi_addr.s_addr = 0;
634 0 : pi->ipi_spec_dst.s_addr = fd_uint_if( !!ip_hdr->saddr, ip_hdr->saddr, ctx->bind_address );
635 :
636 0 : *msg = (struct mmsghdr) {
637 0 : .msg_hdr = {
638 0 : .msg_name = sa,
639 0 : .msg_namelen = sizeof(struct sockaddr_in),
640 0 : .msg_iov = iov,
641 0 : .msg_iovlen = 1,
642 0 : .msg_control = cmsg,
643 0 : .msg_controllen = CMSG_LEN( sizeof(struct in_pktinfo) )
644 0 : }
645 0 : };
646 :
647 0 : memcpy( buf, udp_hdr, sizeof(fd_udp_hdr_t) );
648 0 : fd_memcpy( buf+sizeof(fd_udp_hdr_t), payload, payload_sz );
649 0 : ctx->batch_frame_sz[ batch_idx ] = sz;
650 0 : }
651 :
652 : /* after_frag is called when a frag was copied into a sendmmsg buffer. */
653 :
654 : static void
655 : after_frag( fd_sock_tile_t * ctx,
656 : ulong in_idx,
657 : ulong seq FD_PARAM_UNUSED,
658 : ulong sig FD_PARAM_UNUSED,
659 : ulong sz,
660 : ulong tsorig FD_PARAM_UNUSED,
661 : ulong tspub FD_PARAM_UNUSED,
662 0 : fd_stem_context_t * stem FD_PARAM_UNUSED ) {
663 : /* Commit the packet added in during_frag. during_frag defers fatal
664 : errors to here (where we know we weren't overrun) by setting the
665 : invalid flag. */
666 :
667 0 : if( FD_UNLIKELY( ctx->parsed.invalid ) ) {
668 0 : ulong const hdr_min = sizeof(fd_eth_hdr_t)+sizeof(fd_ip4_hdr_t)+sizeof(fd_udp_hdr_t);
669 0 : if( FD_UNLIKELY( sz<hdr_min ) ) {
670 : /* FIXME support ICMP messages in the future? */
671 0 : FD_LOG_ERR(( "packet too small %lu (in_idx=%lu)", sz, in_idx ));
672 0 : }
673 0 : if( FD_UNLIKELY( ( ctx->parsed.ip_version !=4 ) |
674 0 : ( ctx->parsed.ip_protocol!=FD_IP4_HDR_PROTOCOL_UDP ) ) ) {
675 0 : FD_LOG_ERR(( "packet from in_idx=%lu: sock tile only supports IPv4 UDP for now", in_idx ));
676 0 : }
677 0 : }
678 :
679 0 : ctx->tx_idle_cnt = 0;
680 0 : ctx->batch_cnt++;
681 : /* Technically leaves a gap. sz is always larger than the payload
682 : written to tx_ptr because Ethernet & IPv4 headers were stripped. */
683 0 : ctx->tx_ptr += fd_ulong_align_up( sz, FD_CHUNK_ALIGN );
684 :
685 0 : if( ctx->batch_cnt >= STEM_BURST ) {
686 0 : flush_tx_batch( ctx );
687 0 : }
688 0 : }
689 :
690 : /* End TX path ********************************************************/
691 :
692 : static void
693 0 : metrics_write( fd_sock_tile_t * ctx ) {
694 0 : FD_MCNT_SET( SOCK, SYSCALL_RX, ctx->metrics.sys_recvmmsg_cnt );
695 0 : FD_MCNT_ENUM_COPY( SOCK, SYSCALL_TX, ctx->metrics.sys_sendmmsg_cnt );
696 0 : FD_MCNT_SET( SOCK, PKT_RX, ctx->metrics.rx_pkt_cnt );
697 0 : FD_MCNT_SET( SOCK, PKT_TX, ctx->metrics.tx_pkt_cnt );
698 0 : FD_MCNT_SET( SOCK, PKT_TX_FAILED, ctx->metrics.tx_drop_cnt );
699 0 : FD_MCNT_SET( SOCK, PKT_TX_BYTES, ctx->metrics.tx_bytes_total );
700 0 : FD_MCNT_SET( SOCK, PKT_RX_BYTES, ctx->metrics.rx_bytes_total );
701 0 : }
702 :
703 : /* after_credit is called every stem iteration when there are enough
704 : flow control credits to publish a burst of fragments. */
705 :
706 : static inline void
707 : after_credit( fd_sock_tile_t * ctx,
708 : fd_stem_context_t * stem,
709 : int * poll_in FD_PARAM_UNUSED,
710 0 : int * charge_busy ) {
711 0 : if( ctx->lo_ifindex ) {
712 : /* Housekeeping cannot publish progress while the tile sleeps. */
713 0 : FD_COMPILER_MFENCE();
714 0 : metrics_write( ctx );
715 0 : FD_MGAUGE_SET( TILE, HEARTBEAT_TIME, (ulong)fd_log_wallclock() );
716 0 : for( ulong i=0UL; i<ctx->link_rx_cnt; i++ ) {
717 0 : fd_mcache_seq_update( fd_mcache_seq_laddr( stem->mcaches[ i ] ), stem->seqs[ i ] );
718 0 : }
719 0 : FD_COMPILER_MFENCE();
720 0 : *charge_busy |= poll_rx( ctx, stem )!=0UL;
721 0 : return;
722 0 : }
723 0 : if( ctx->tx_idle_cnt > 512 ) {
724 0 : if( ctx->batch_cnt ) {
725 0 : flush_tx_batch( ctx );
726 0 : }
727 0 : ulong pkt_cnt = poll_rx( ctx, stem );
728 0 : *charge_busy = pkt_cnt!=0;
729 0 : }
730 0 : ctx->tx_idle_cnt++;
731 0 : }
732 :
733 : static ulong
734 : rlimit_file_cnt( fd_topo_t const * topo,
735 0 : fd_topo_tile_t const * tile ) {
736 0 : fd_sock_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
737 0 : return RX_SOCK_FD_MIN + ctx->sock_cnt;
738 0 : }
739 :
740 : #define STEM_NEVER_PARK 1
741 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_sock_tile_t
742 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_sock_tile_t)
743 :
744 0 : #define STEM_LAZY ((long)10e6) /* 10ms */
745 :
746 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
747 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
748 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
749 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
750 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
751 :
752 : #include "../../stem/fd_stem.c"
753 :
754 : fd_topo_run_tile_t fd_tile_sock = {
755 : .name = "sock",
756 : .rlimit_file_cnt_fn = rlimit_file_cnt,
757 : .populate_allowed_seccomp = populate_allowed_seccomp,
758 : .populate_allowed_fds = populate_allowed_fds,
759 : .scratch_align = scratch_align,
760 : .scratch_footprint = scratch_footprint,
761 : .privileged_init = privileged_init,
762 : .unprivileged_init = unprivileged_init,
763 : .run = stem_run,
764 : };
|
