Line data Source code
1 : #define _GNU_SOURCE
2 : /* The iavf tile translates Ethernet frames between Intel VF descriptor
3 : rings and fd_tango traffic. Each tile owns one RX/TX pair per VF, with
4 : completions written into the descriptor rings themselves.
5 :
6 : Separate Admin Queues carry configuration requests to the PF driver.
7 : Tile 0 consumes PF events for all tiles. */
8 :
9 : #include "../fd_net_tile.h"
10 : #include "../fd_net_tile_private.h"
11 : #include "fd_iavf_private.h"
12 : #include "../../topo/fd_topo.h"
13 :
14 : #include <errno.h>
15 : #include <net/if.h>
16 : #include <netinet/in.h>
17 : #include <stdlib.h>
18 : #include <sys/ioctl.h>
19 : #include <sys/socket.h>
20 : #include <unistd.h>
21 : #include <linux/rtnetlink.h>
22 :
23 : #include "../../../util/pod/fd_pod_format.h"
24 : #define FD_SOCKADDR_IN_SZ sizeof(struct sockaddr_in)
25 : #include "generated/fd_iavf_tile_seccomp.h"
26 :
27 0 : #define FD_IAVF_TX_FLUSH_TIMEOUT_NS ( 20000L)
28 0 : #define FD_IAVF_LO_TX_TIMEOUT_NS ( 500000L)
29 0 : #define FD_IAVF_VF_HOUSEKEEPING_INTERVAL_NS (1000000L)
30 :
31 : #define FD_IAVF_TX_DESC_TYPE_MASK (0xfUL) /* IAVF_TXD_QW1_DTYPE_MASK */
32 0 : #define FD_IAVF_TX_DESC_DONE (0xfUL) /* IAVF_TX_DESC_DTYPE_DESC_DONE */
33 0 : #define FD_IAVF_TX_DESC_CMD_SHIFT (4) /* IAVF_TXD_QW1_CMD_SHIFT */
34 0 : #define FD_IAVF_TX_DESC_CMD_EOP (1UL) /* IAVF_TX_DESC_CMD_EOP */
35 0 : #define FD_IAVF_TX_DESC_CMD_REPORT_STATUS (2UL) /* IAVF_TX_DESC_CMD_RS */
36 0 : #define FD_IAVF_TX_DESC_CMD_ICRC (4UL) /* IAVF_TX_DESC_CMD_ICRC */
37 0 : #define FD_IAVF_TX_DESC_BUFFER_SHIFT (34) /* IAVF_TXD_QW1_TX_BUF_SZ_SHIFT */
38 : #define FD_IAVF_TX_DESC_BUFFER_MAX (0x3fffUL) /* IAVF_TXD_QW1_TX_BUF_SZ_MASK >> 34 */
39 :
40 0 : #define FD_IAVF_RX_DESC_DONE (1UL<<0) /* IAVF_RXD_LEGACY_DD_M */
41 0 : #define FD_IAVF_RX_DESC_END_OF_PACKET (1UL<<1) /* IAVF_RXD_LEGACY_EOP_M */
42 0 : #define FD_IAVF_RX_DESC_MAC_ERR (1UL<<19) /* IAVF_RXD_LEGACY_RXE_M */
43 0 : #define FD_IAVF_RX_DESC_LENGTH_SHIFT (38) /* Low bit of IAVF_RXD_LEGACY_LENGTH_M */
44 0 : #define FD_IAVF_RX_DESC_LENGTH_MASK (0x3fffUL) /* IAVF_RXD_LEGACY_LENGTH_M >> 38 */
45 :
46 : FD_STATIC_ASSERT( FD_NET_MTU<=FD_IAVF_TX_DESC_BUFFER_MAX, iavf_tx_frame_sz );
47 :
48 : struct fd_iavf_tile_rx_comp {
49 : uint chunk;
50 : uint discard;
51 : uint end_of_packet;
52 : ulong frame_sz;
53 : };
54 : typedef struct fd_iavf_tile_rx_comp fd_iavf_tile_rx_comp_t;
55 :
56 : /* fd_iavf_tile_vf is private tile state for a single VF */
57 : struct fd_iavf_tile_vf {
58 : fd_iavf_vfio_t vfio;
59 : fd_iavf_adminq_t adminq;
60 : fd_iavf_vf_info_t vf_info;
61 : fd_iavf_queue_t rx_queue;
62 : fd_iavf_queue_t tx_queue;
63 : void * queue_memory;
64 : uint * rx_desc_buf_chunk;
65 : uint * tx_desc_buf_chunk;
66 : /* A Report Status completion can cover several frames. Keep each frame's
67 : length to count the bytes in a completed batch. */
68 : ushort * tx_desc_frame_sz;
69 : uint rx_pending_chunk[ FD_IAVF_BATCH_SIZE ];
70 : uint rx_pending_cnt;
71 : long tx_flush_deadline_ticks;
72 : };
73 : typedef struct fd_iavf_tile_vf fd_iavf_tile_vf_t;
74 :
75 : /* fd_iavf_tile is private tile state */
76 : struct __attribute__((aligned(64UL))) fd_iavf_tile {
77 : fd_iavf_tile_vf_t vfs[ FD_IAVF_MEMBER_MAX ];
78 : ulong vf_cnt;
79 : ulong tx_vf_idx;
80 : ulong rx_next_vf;
81 : /* vf_link_usable shares tile 0's PF link decision with the other tiles. */
82 : ulong vf_link_usable[ FD_IAVF_MEMBER_MAX ];
83 : ulong const * vf_link_usable_shared;
84 : long vf_housekeeping_interval_ticks;
85 : long vf_housekeeping_deadline_ticks;
86 : uint prepared;
87 : uint batch_size;
88 : long tx_flush_timeout_ticks;
89 : long lo_tx_timeout_ticks;
90 : long lo_tx_deadline_ticks;
91 : int lo_tx_sock;
92 : uint lo_tx_cnt;
93 : fd_net_tile_t net;
94 : ulong netdev_seq;
95 :
96 : struct mmsghdr lo_tx_msg [ FD_IAVF_BATCH_SIZE ];
97 : struct iovec lo_tx_iov [ FD_IAVF_BATCH_SIZE ];
98 : struct sockaddr_in lo_tx_addr[ FD_IAVF_BATCH_SIZE ];
99 : uchar lo_tx_buf [ FD_IAVF_BATCH_SIZE ][ FD_NET_MTU ];
100 :
101 : struct {
102 : ulong tx_no_buffer_cnt;
103 : ulong tx_no_link_cnt;
104 : ulong tx_loopback_fail_cnt;
105 : ulong rx_no_link_cnt;
106 : ulong non_fd_pkt_cnt;
107 : } metrics;
108 : };
109 : typedef struct fd_iavf_tile fd_iavf_tile_t;
110 :
111 : static void *
112 : fd_iavf_hw_join_queues( fd_iavf_tile_t * ctx,
113 0 : fd_topo_tile_t const * tile ) {
114 0 : FD_SCRATCH_ALLOC_INIT( scratch, ctx );
115 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_iavf_tile_t), sizeof(fd_iavf_tile_t) );
116 0 : ulong const queue_sz = fd_iavf_queue_footprint( tile->iavf.tx_queue_size, tile->iavf.rx_queue_size );
117 0 : ulong const tx_ring_sz = fd_ulong_align_up( tile->iavf.tx_queue_size*sizeof(fd_iavf_tx_desc_t), FD_IAVF_PAGE_SZ );
118 0 : ulong const rx_ring_sz = fd_ulong_align_up( tile->iavf.rx_queue_size*sizeof(fd_iavf_hw_rx_desc_t), FD_IAVF_PAGE_SZ );
119 :
120 0 : for( ulong vf_idx=0UL; vf_idx<tile->iavf.member_cnt; vf_idx++ ) {
121 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[vf_idx];
122 : /* Only tile 0 needs Admin Queue memory */
123 0 : vf->adminq.dma_memory = NULL;
124 0 : if( 0==tile->kind_id ) {
125 0 : vf->adminq.dma_memory = FD_SCRATCH_ALLOC_APPEND( scratch, FD_IAVF_PAGE_SZ, fd_iavf_adminq_footprint() );
126 0 : }
127 :
128 0 : vf->queue_memory = FD_SCRATCH_ALLOC_APPEND( scratch, FD_IAVF_PAGE_SZ, queue_sz );
129 0 : vf->tx_queue.ring = vf->queue_memory;
130 0 : vf->rx_queue.ring = (uchar *)vf->queue_memory + tx_ring_sz;
131 0 : vf->tx_queue.depth = tile->iavf.tx_queue_size;
132 0 : vf->rx_queue.depth = tile->iavf.rx_queue_size;
133 0 : vf->tx_queue.comp_ring = (ulong *)((uchar *)vf->queue_memory + tx_ring_sz + rx_ring_sz);
134 :
135 0 : vf->rx_desc_buf_chunk = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->iavf.rx_queue_size*sizeof(uint) );
136 0 : vf->tx_desc_buf_chunk = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->iavf.tx_queue_size*sizeof(uint) );
137 0 : vf->tx_desc_frame_sz = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(ushort), tile->iavf.tx_queue_size*sizeof(ushort) );
138 0 : }
139 0 : return (void *)FD_SCRATCH_ALLOC_FINI( scratch, 1UL );
140 0 : }
141 :
142 : static int
143 : fd_iavf_hw_map_regs( fd_iavf_tile_vf_t * vf,
144 0 : ulong queue_id ) {
145 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_map_bar( &vf->vfio ) ) ) return -1;
146 0 : vf->tx_queue.tail = (volatile uint *)(vf->vfio.bar0 + FD_IAVF_TX_TAIL( queue_id ));
147 0 : vf->rx_queue.tail = (volatile uint *)(vf->vfio.bar0 + FD_IAVF_RX_TAIL( queue_id ));
148 0 : return 0;
149 0 : }
150 :
151 : static int
152 : fd_iavf_hw_rx_enqueue( fd_iavf_tile_vf_t * vf,
153 0 : uint recv_cnt ) {
154 0 : fd_iavf_queue_t * rx_queue = &vf->rx_queue;
155 0 : ulong const outstanding = rx_queue->prod-rx_queue->cons;
156 :
157 0 : if( FD_UNLIKELY( outstanding>=rx_queue->depth || recv_cnt>=rx_queue->depth-outstanding ) ) {
158 0 : errno = ENOSPC;
159 0 : return -1;
160 0 : }
161 :
162 0 : for( uint i=0U; i<recv_cnt; i++ ) {
163 0 : uint const chunk = vf->rx_pending_chunk[i];
164 0 : uint const desc_idx = (uint)((rx_queue->prod+i) & (rx_queue->depth-1U));
165 0 : fd_iavf_hw_rx_desc_t * desc = (fd_iavf_hw_rx_desc_t *)rx_queue->ring + desc_idx;
166 :
167 0 : desc->qword[0] = (ulong)chunk<<FD_CHUNK_LG_SZ;
168 0 : desc->qword[1] = 0UL;
169 0 : desc->qword[2] = 0UL;
170 0 : desc->qword[3] = 0UL;
171 :
172 0 : vf->rx_desc_buf_chunk[ desc_idx ] = chunk;
173 0 : }
174 0 : rx_queue->prod += recv_cnt;
175 :
176 0 : fd_iavf_hw_dma_to_device();
177 0 : *rx_queue->tail = (uint)(rx_queue->prod & (rx_queue->depth-1U));
178 0 : FD_COMPILER_MFENCE();
179 0 : rx_queue->posted = rx_queue->prod;
180 0 : return 0;
181 0 : }
182 :
183 : static void
184 : fd_iavf_hw_tx_enqueue( fd_iavf_tile_vf_t * vf,
185 0 : ulong frame_sz ) {
186 0 : fd_iavf_queue_t * tx_queue = &vf->tx_queue;
187 :
188 0 : FD_TEST( tx_queue->prod-tx_queue->cons<tx_queue->depth-1UL );
189 :
190 0 : uint const desc_idx = (uint)(tx_queue->prod & (tx_queue->depth-1U));
191 0 : uint const chunk = vf->tx_desc_buf_chunk[desc_idx];
192 0 : ulong const frame_iova = (ulong)chunk<<FD_CHUNK_LG_SZ;
193 0 : fd_iavf_tx_desc_t * desc = (fd_iavf_tx_desc_t *)tx_queue->ring + desc_idx;
194 0 : desc->buffer_iova = frame_iova;
195 0 : desc->cmd_type_offset_buffer_sz =
196 0 : ((FD_IAVF_TX_DESC_CMD_EOP | FD_IAVF_TX_DESC_CMD_ICRC)<<FD_IAVF_TX_DESC_CMD_SHIFT) |
197 0 : (frame_sz<<FD_IAVF_TX_DESC_BUFFER_SHIFT);
198 0 : vf->tx_desc_frame_sz[desc_idx] = (ushort)frame_sz;
199 0 : tx_queue->prod++;
200 0 : }
201 :
202 : static void
203 0 : fd_iavf_hw_tx_flush( fd_iavf_queue_t * tx_queue ) {
204 0 : if( FD_UNLIKELY( tx_queue->posted==tx_queue->prod ) ) return;
205 :
206 0 : ulong const tx_end = tx_queue->prod;
207 0 : uint const desc_idx = (uint)((tx_end-1UL) & (tx_queue->depth-1U));
208 0 : fd_iavf_tx_desc_t * desc = (fd_iavf_tx_desc_t *)tx_queue->ring + desc_idx;
209 : /* Request completion on the batch's last descriptor */
210 0 : desc->cmd_type_offset_buffer_sz |= FD_IAVF_TX_DESC_CMD_REPORT_STATUS<<FD_IAVF_TX_DESC_CMD_SHIFT;
211 0 : tx_queue->comp_ring[ tx_queue->comp_prod & (tx_queue->depth-1U) ] = tx_end;
212 0 : tx_queue->comp_prod++;
213 :
214 0 : fd_iavf_hw_dma_to_device();
215 0 : *tx_queue->tail = (uint)(tx_end & (tx_queue->depth-1U));
216 0 : FD_COMPILER_MFENCE();
217 0 : tx_queue->posted = tx_end;
218 0 : }
219 :
220 : static ulong
221 : fd_iavf_hw_poll_tx( fd_iavf_tile_vf_t * vf,
222 0 : ulong * comp_bytes ) {
223 0 : fd_iavf_queue_t * tx_queue = &vf->tx_queue;
224 0 : *comp_bytes = 0UL;
225 0 : ulong const comp_cons = tx_queue->comp_cons;
226 0 : if( comp_cons==tx_queue->comp_prod ) return 0UL;
227 :
228 0 : ulong const tx_end = tx_queue->comp_ring[ comp_cons & (tx_queue->depth-1U) ];
229 0 : uint const desc_idx = (uint)((tx_end-1UL) & (tx_queue->depth-1U));
230 0 : fd_iavf_tx_desc_t * desc = (fd_iavf_tx_desc_t *)tx_queue->ring + desc_idx;
231 0 : ulong const cmd = FD_VOLATILE_CONST( desc->cmd_type_offset_buffer_sz );
232 0 : if( FD_LIKELY( (cmd & FD_IAVF_TX_DESC_TYPE_MASK)!=FD_IAVF_TX_DESC_DONE ) ) return 0UL;
233 0 : FD_COMPILER_MFENCE();
234 :
235 0 : ulong const tx_cons = tx_queue->cons;
236 0 : ulong const comp_cnt = tx_end-tx_cons;
237 0 : for( ulong i=tx_cons; i<tx_end; i++ ) {
238 0 : *comp_bytes += vf->tx_desc_frame_sz[ i & (tx_queue->depth-1U) ];
239 0 : }
240 0 : tx_queue->cons = tx_end;
241 0 : tx_queue->comp_cons = comp_cons+1UL;
242 0 : return comp_cnt;
243 0 : }
244 :
245 : static int
246 : fd_iavf_hw_poll_rx( fd_iavf_tile_vf_t * vf,
247 : fd_iavf_tile_rx_comp_t * comp,
248 0 : uint comp_capacity ) {
249 0 : fd_iavf_queue_t * rx_queue = &vf->rx_queue;
250 0 : ulong rx_cons = rx_queue->cons;
251 0 : uint const comp_limit = fd_uint_min( comp_capacity, rx_queue->depth );
252 0 : uint comp_cnt = 0U;
253 0 : while( comp_cnt<comp_limit && rx_cons<rx_queue->posted ) {
254 0 : uint const desc_idx = (uint)(rx_cons & (rx_queue->depth-1U));
255 0 : fd_iavf_hw_rx_desc_t * desc = (fd_iavf_hw_rx_desc_t *)rx_queue->ring + desc_idx;
256 0 : ulong const status = FD_VOLATILE_CONST( desc->qword[1] );
257 :
258 0 : if( !(status & FD_IAVF_RX_DESC_DONE) ) break;
259 :
260 : /* Drop frames that do not fit in one RX buffer. */
261 0 : uint const end_of_packet = (uint)(0!=(status & FD_IAVF_RX_DESC_END_OF_PACKET));
262 0 : uint const discard = rx_queue->discard || !end_of_packet || (0!=(status & FD_IAVF_RX_DESC_MAC_ERR));
263 0 : rx_queue->discard = !end_of_packet;
264 0 : comp[ comp_cnt++ ] = (fd_iavf_tile_rx_comp_t) {
265 0 : .chunk = vf->rx_desc_buf_chunk[ desc_idx ],
266 0 : .discard = discard,
267 0 : .end_of_packet = end_of_packet,
268 0 : .frame_sz = (status>>FD_IAVF_RX_DESC_LENGTH_SHIFT) & FD_IAVF_RX_DESC_LENGTH_MASK
269 0 : };
270 0 : rx_cons++;
271 0 : }
272 0 : if( 0!=comp_cnt ) {
273 0 : fd_iavf_hw_dma_from_device();
274 0 : rx_queue->cons = rx_cons;
275 0 : }
276 0 : return (int)comp_cnt;
277 0 : }
278 :
279 : /* fd_iavf_tile_tx_chunk returns the buffer paired with TX descriptor tx_idx.
280 : The buffer is reusable once tx_queue.cons advances past that descriptor. */
281 : static inline ulong
282 : fd_iavf_tile_tx_chunk( fd_iavf_tile_t const * ctx,
283 0 : ulong tx_idx ) {
284 0 : fd_iavf_tile_vf_t const * vf = &ctx->vfs[ ctx->tx_vf_idx ];
285 0 : return vf->tx_desc_buf_chunk[ tx_idx & (vf->tx_queue.depth-1U) ];
286 0 : }
287 :
288 : static inline void
289 : fd_iavf_tile_rx_recycle( fd_iavf_tile_t * ctx,
290 : fd_iavf_tile_vf_t * vf,
291 0 : ulong chunk ) {
292 0 : vf->rx_pending_chunk[ vf->rx_pending_cnt++ ] = (uint)chunk;
293 : /* Hold partial recycle batches, at most batch_size-1 RX buffers stay
294 : unposted, avoiding a device-ordering barrier and RX tail write per packet. */
295 0 : if( vf->rx_pending_cnt==ctx->batch_size ) {
296 0 : FD_TEST( 0==fd_iavf_hw_rx_enqueue( vf, vf->rx_pending_cnt ) );
297 0 : vf->rx_pending_cnt = 0U;
298 0 : }
299 0 : }
300 :
301 : static ulong
302 : fd_iavf_tile_select_tx_vf( fd_iavf_tile_t * ctx,
303 0 : ulong hash ) {
304 0 : ulong available[ FD_IAVF_MEMBER_MAX ];
305 0 : ulong available_cnt = 0UL;
306 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
307 0 : if( ctx->vfs[vf_idx].vf_info.link_usable ) {
308 0 : available[ available_cnt++ ] = vf_idx;
309 0 : }
310 0 : }
311 0 : return 0!=available_cnt ? available[ hash%available_cnt ] : ULONG_MAX;
312 0 : }
313 :
314 : static inline int
315 : fd_iavf_tile_poll_rx( fd_iavf_tile_t * ctx,
316 0 : fd_stem_context_t * stem ) {
317 0 : int busy = 0;
318 0 : ulong const start_vf_idx = ctx->rx_next_vf;
319 0 : ctx->rx_next_vf = (start_vf_idx+1UL)%ctx->vf_cnt;
320 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
321 0 : ulong const actual_vf_idx = (start_vf_idx+vf_idx)%ctx->vf_cnt;
322 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[ actual_vf_idx ];
323 :
324 0 : fd_iavf_tile_rx_comp_t comp[ FD_IAVF_BATCH_SIZE ];
325 0 : int comp_cnt = fd_iavf_hw_poll_rx( vf, comp, ctx->batch_size );
326 0 : if( 0==comp_cnt ) continue;
327 0 : busy = 1;
328 :
329 0 : int active = vf->vf_info.link_usable;
330 0 : ulong tspub = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
331 :
332 0 : for( uint i=0U; i<(uint)comp_cnt; i++ ) {
333 0 : ulong const chunk = comp[ i ].chunk;
334 0 : if( FD_UNLIKELY( chunk<ctx->net.pkt_buf_chunk0 || chunk>ctx->net.pkt_buf_wmark ) ) {
335 0 : FD_LOG_CRIT(( "VF %lu RX completion chunk %lu out of bounds [%lu,%lu]",
336 0 : actual_vf_idx, chunk, ctx->net.pkt_buf_chunk0, ctx->net.pkt_buf_wmark ));
337 0 : }
338 0 : __builtin_prefetch( fd_chunk_to_laddr_const( ctx->net.pkt_buf_wksp_base, chunk ), 0, 3 );
339 0 : }
340 :
341 0 : for( uint i=0U; i<(uint)comp_cnt; i++ ) {
342 0 : ulong freed_chunk = comp[i].chunk;
343 0 : if( FD_UNLIKELY( !active ) ) ctx->metrics.rx_no_link_cnt += comp[i].end_of_packet;
344 0 : else if( FD_UNLIKELY( comp[i].discard ) ) ctx->net.metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ] += comp[i].end_of_packet;
345 0 : else fd_net_rx_pkt( &ctx->net, stem, comp[i].chunk, 0UL, comp[i].frame_sz, tspub, &freed_chunk, &ctx->metrics.non_fd_pkt_cnt );
346 0 : if( FD_UNLIKELY( 0==freed_chunk ) ) FD_LOG_CRIT(( "invalid RX chunk in mcache" ));
347 0 : fd_iavf_tile_rx_recycle( ctx, vf, freed_chunk );
348 0 : }
349 0 : }
350 0 : return busy;
351 0 : }
352 :
353 : static inline int
354 : fd_iavf_tile_poll_tx( fd_iavf_tile_t * ctx,
355 0 : fd_iavf_tile_vf_t * vf ) {
356 0 : ulong comp_bytes;
357 0 : ulong comp_cnt = fd_iavf_hw_poll_tx( vf, &comp_bytes );
358 0 : ctx->net.metrics.tx_pkt_cnt += comp_cnt;
359 0 : ctx->net.metrics.tx_bytes_total += comp_bytes;
360 0 : return !!comp_cnt;
361 0 : }
362 :
363 : /* fd_iavf_tile_lo_tx_flush drops unsent packets so socket backpressure
364 : cannot stall RX or VF TX. */
365 : static void
366 0 : fd_iavf_tile_lo_tx_flush( fd_iavf_tile_t * ctx ) {
367 0 : if( FD_UNLIKELY( 0==ctx->lo_tx_cnt ) ) return;
368 0 : int const send_cnt = sendmmsg( ctx->lo_tx_sock, ctx->lo_tx_msg, ctx->lo_tx_cnt, MSG_DONTWAIT );
369 0 : uint const sent_cnt = -1==send_cnt ? 0U : (uint)send_cnt;
370 0 : for( uint i=0U; i<sent_cnt; i++ ) ctx->net.metrics.tx_bytes_total += sizeof(fd_eth_hdr_t)+ctx->lo_tx_iov[ i ].iov_len;
371 0 : ctx->net.metrics.tx_pkt_cnt += sent_cnt;
372 0 : ctx->metrics.tx_loopback_fail_cnt += ctx->lo_tx_cnt-sent_cnt;
373 0 : ctx->lo_tx_cnt = 0U;
374 0 : }
375 :
376 : static void
377 : fd_iavf_tile_lo_tx_enqueue( fd_iavf_tile_t * ctx,
378 : fd_ip4_hdr_t const * ip4,
379 0 : ulong ip_sz ) {
380 0 : uint const batch_idx = ctx->lo_tx_cnt;
381 0 : struct mmsghdr * msg = ctx->lo_tx_msg + batch_idx;
382 0 : struct sockaddr_in * sa = ctx->lo_tx_addr + batch_idx;
383 0 : struct iovec * iov = ctx->lo_tx_iov + batch_idx;
384 0 : uchar * buf = ctx->lo_tx_buf[ batch_idx ];
385 :
386 0 : *iov = (struct iovec) {
387 0 : .iov_base = buf,
388 0 : .iov_len = ip_sz,
389 0 : };
390 0 : sa->sin_family = AF_INET;
391 0 : sa->sin_addr.s_addr = ip4->daddr;
392 0 : sa->sin_port = 0; /* Linux raw sockets ignore sin_port */
393 :
394 0 : *msg = (struct mmsghdr) {
395 0 : .msg_hdr = {
396 0 : .msg_name = sa,
397 0 : .msg_namelen = sizeof(struct sockaddr_in),
398 0 : .msg_iov = iov,
399 0 : .msg_iovlen = 1UL
400 0 : }
401 0 : };
402 :
403 0 : fd_memcpy( buf, ip4, ip_sz );
404 0 : ctx->lo_tx_cnt++;
405 0 : if( ctx->lo_tx_cnt==FD_IAVF_BATCH_SIZE ) fd_iavf_tile_lo_tx_flush( ctx );
406 0 : else if( ctx->lo_tx_cnt==1U ) ctx->lo_tx_deadline_ticks = fd_tickcount()+ctx->lo_tx_timeout_ticks;
407 0 : }
408 :
409 : static inline void
410 : before_credit( fd_iavf_tile_t * ctx,
411 : fd_stem_context_t * stem,
412 0 : int * charge_busy ) {
413 0 : fd_net_router_solicit( &ctx->net.router, stem );
414 0 : long now = fd_tickcount();
415 0 : if( 0!=ctx->lo_tx_cnt && now>=ctx->lo_tx_deadline_ticks ) {
416 0 : fd_iavf_tile_lo_tx_flush( ctx );
417 0 : *charge_busy = 1;
418 0 : }
419 0 : for( ulong i=0UL; i<ctx->vf_cnt; i++ ) {
420 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[i];
421 0 : fd_iavf_queue_t * tx_queue = &vf->tx_queue;
422 0 : if( tx_queue->prod!=tx_queue->posted && now>=vf->tx_flush_deadline_ticks ) {
423 0 : fd_iavf_hw_tx_flush( tx_queue );
424 0 : *charge_busy = 1;
425 0 : }
426 0 : if( tx_queue->cons!=tx_queue->posted ) *charge_busy |= fd_iavf_tile_poll_tx( ctx, vf );
427 0 : }
428 0 : }
429 :
430 : static inline void
431 : after_credit( fd_iavf_tile_t * ctx,
432 : fd_stem_context_t * stem,
433 : int * poll_in,
434 0 : int * charge_busy ) {
435 0 : (void)poll_in;
436 0 : *charge_busy |= fd_iavf_tile_poll_rx( ctx, stem );
437 0 : }
438 :
439 : /* before_frag resolves the TX route and checks for space in the TX descriptor ring. */
440 : static inline int
441 : before_frag( fd_iavf_tile_t * ctx,
442 : ulong in_idx,
443 : ulong seq,
444 0 : ulong sig ) {
445 0 : (void)seq;
446 : /* Short circuit TX logic for route updates */
447 0 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) return 0;
448 :
449 0 : if( !fd_net_tx_select( &ctx->net, sig ) ) return 1;
450 :
451 0 : ulong const hash = fd_disco_netmux_sig_hash( sig );
452 0 : fd_net_tx_route_t const * route = &ctx->net.tx_route;
453 0 : ctx->tx_vf_idx = route->use_loopback ? 0UL : fd_iavf_tile_select_tx_vf( ctx, hash );
454 0 : if( FD_UNLIKELY( ctx->tx_vf_idx==ULONG_MAX ) ) {
455 0 : ctx->metrics.tx_no_link_cnt++;
456 0 : return 1;
457 0 : }
458 0 : fd_iavf_queue_t const * tx_queue = &ctx->vfs[ ctx->tx_vf_idx ].tx_queue;
459 0 : if( FD_UNLIKELY( !route->use_loopback && tx_queue->prod-tx_queue->cons>=tx_queue->depth-1UL ) ) {
460 0 : ctx->metrics.tx_no_buffer_cnt++;
461 0 : return 1;
462 0 : }
463 :
464 0 : fd_net_tx_route_result_increment( &ctx->net );
465 0 : return 0;
466 0 : }
467 :
468 : /* during_frag validates and stages the input packet or route update */
469 : static inline void
470 : during_frag( fd_iavf_tile_t * ctx,
471 : ulong in_idx,
472 : ulong seq,
473 : ulong sig,
474 : ulong chunk,
475 : ulong frame_sz,
476 0 : ulong ctl ) {
477 0 : (void)seq; (void)sig; (void)ctl;
478 0 : fd_net_in_frag_validate( &ctx->net, in_idx, chunk, frame_sz );
479 :
480 0 : if( FD_UNLIKELY( fd_net_iproute_msg_stage( &ctx->net, in_idx, chunk, frame_sz ) ) ) return;
481 :
482 0 : ulong dst_chunk = fd_iavf_tile_tx_chunk( ctx, ctx->vfs[ ctx->tx_vf_idx ].tx_queue.prod );
483 0 : uchar * dst = fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, dst_chunk );
484 : /* Speculatively copy frame from in link into buffer */
485 0 : fd_net_tx_pkt_cpy( &ctx->net, dst, chunk, frame_sz, in_idx );
486 0 : }
487 :
488 : /* after_frag enqueues the staged frame or applies the route update only
489 : after stem checks that the input was not overwritten during the copy. */
490 : static void
491 : after_frag( fd_iavf_tile_t * ctx,
492 : ulong in_idx,
493 : ulong seq,
494 : ulong sig,
495 : ulong frame_sz,
496 : ulong tsorig,
497 : ulong tspub,
498 0 : fd_stem_context_t * stem ) {
499 0 : (void)seq; (void)sig; (void)tsorig; (void)tspub;
500 :
501 0 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) {
502 0 : fd_iproute_msg_t const * msg = &ctx->net.iproute_msg;
503 0 : if( msg->op==FD_IPROUTE_OP_FLUSH ) {
504 0 : fd_fib4_clear( ctx->net.router.fib_local );
505 0 : fd_fib4_clear( ctx->net.router.fib_main );
506 0 : return;
507 0 : }
508 :
509 0 : fd_fib4_t * fib;
510 0 : if( msg->table_id==RT_TABLE_LOCAL ) fib = ctx->net.router.fib_local;
511 0 : else if( msg->table_id==RT_TABLE_MAIN ) fib = ctx->net.router.fib_main;
512 0 : else return;
513 :
514 0 : if( msg->op==FD_IPROUTE_OP_UPSERT && FD_UNLIKELY( !fd_fib4_insert( fib, msg->dst_addr, msg->prefix, msg->prio, &msg->hop ) ) ) {
515 0 : FD_LOG_WARNING(( "route update dropped: route table full (increase [tiles.netlink].max_routes or [tiles.netlink].max_peer_routes)" ));
516 0 : fd_stem_publish( stem, ctx->net.router.netlnk_out_idx, FD_NETLINK_ROUTE4_SYNC_SIG, 0UL, 0UL, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
517 0 : } else if( msg->op==FD_IPROUTE_OP_DELETE ) {
518 0 : fd_fib4_remove( fib, msg->dst_addr, msg->prefix, msg->prio );
519 0 : }
520 0 : return;
521 0 : }
522 :
523 0 : ulong chunk = fd_iavf_tile_tx_chunk( ctx, ctx->vfs[ ctx->tx_vf_idx ].tx_queue.prod );
524 0 : uchar * frame = fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, chunk );
525 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[ ctx->tx_vf_idx ];
526 0 : fd_iavf_queue_t * tx_queue = &vf->tx_queue;
527 :
528 0 : if( FD_UNLIKELY( !fd_net_tx_pkt_prep( &ctx->net, frame, &frame_sz, in_idx ) ) ) return;
529 :
530 : /* Packets that failed userland routing are sent on net.sock starting
531 : at the IPv4 header, and the kernel builds the L2 header. The send
532 : is synchronous, so no descriptor is posted and the chunk is reused. */
533 0 : if( FD_UNLIKELY( ctx->net.tx_route.routing_failure ) ) {
534 0 : if( FD_LIKELY( fd_net_tx_sock_send( &ctx->net, frame, frame_sz ) ) ) {
535 0 : ctx->net.metrics.tx_pkt_cnt++;
536 0 : ctx->net.metrics.tx_bytes_total += frame_sz;
537 0 : ctx->net.metrics.tx_gre_cnt += (ulong)ctx->net.tx_route.use_gre;
538 0 : }
539 0 : return;
540 0 : }
541 :
542 0 : if( FD_UNLIKELY( ctx->net.tx_route.use_loopback ) ) {
543 0 : fd_eth_hdr_t const * eth_hdr = (fd_eth_hdr_t const *)frame;
544 0 : fd_ip4_hdr_t const * ip4 = (fd_ip4_hdr_t const *)(eth_hdr+1);
545 0 : ulong const ip_hdr_sz = FD_IP4_GET_LEN( *ip4 );
546 0 : ulong const ip_sz = fd_ushort_bswap( ip4->net_tot_len );
547 0 : if( FD_UNLIKELY( ip4->protocol!=FD_IP4_HDR_PROTOCOL_UDP ||
548 0 : ip4->daddr!=fd_disco_netmux_sig_ip( sig ) ||
549 0 : (fd_ushort_bswap( ip4->net_frag_off ) & ~FD_IP4_HDR_FRAG_OFF_DF) ||
550 0 : ip_sz<ip_hdr_sz+sizeof(fd_udp_hdr_t) ||
551 0 : ip_sz>frame_sz-sizeof(fd_eth_hdr_t) ) ) {
552 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
553 0 : return;
554 0 : }
555 0 : fd_udp_hdr_t const * udp = (fd_udp_hdr_t const *)((uchar const *)ip4+ip_hdr_sz);
556 0 : ulong const udp_sz = fd_ushort_bswap( udp->net_len );
557 0 : if( FD_UNLIKELY( udp_sz<sizeof(fd_udp_hdr_t) || udp_sz>ip_sz-ip_hdr_sz ) ) {
558 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
559 0 : return;
560 0 : }
561 :
562 0 : int owns_udp_dst = 0;
563 0 : if( !ctx->net.bind_address || ctx->net.bind_address==ip4->daddr ) {
564 0 : ushort const dst_port = fd_ushort_bswap( udp->net_dport );
565 0 : for( ulong i=0UL; i<ctx->net.rx_dest_cnt; i++ ) owns_udp_dst |= ctx->net.rx_listen_ports[ i ]==dst_port;
566 0 : }
567 0 : if( !owns_udp_dst ) {
568 0 : fd_iavf_tile_lo_tx_enqueue( ctx, ip4, ip_sz );
569 0 : return;
570 0 : }
571 :
572 0 : ulong freed_chunk;
573 0 : if( fd_net_rx_pkt( &ctx->net, stem, chunk, 0UL, frame_sz, (ulong)fd_frag_meta_ts_comp( fd_tickcount() ), &freed_chunk, NULL ) ) {
574 0 : vf->tx_desc_buf_chunk[ tx_queue->prod & (tx_queue->depth-1U) ] = (uint)freed_chunk;
575 0 : }
576 0 : ctx->net.metrics.tx_pkt_cnt++;
577 0 : ctx->net.metrics.tx_bytes_total += frame_sz;
578 0 : return;
579 0 : }
580 :
581 0 : fd_iavf_hw_tx_enqueue( vf, frame_sz );
582 0 : ctx->net.metrics.tx_gre_cnt += (ulong)ctx->net.tx_route.use_gre;
583 0 : ulong const pending_desc_cnt = tx_queue->prod-tx_queue->posted;
584 0 : if( pending_desc_cnt>=ctx->batch_size || tx_queue->prod-tx_queue->cons>=tx_queue->depth-1UL ) {
585 0 : fd_iavf_hw_tx_flush( tx_queue );
586 0 : } else if( pending_desc_cnt==1UL ) {
587 0 : vf->tx_flush_deadline_ticks = fd_tickcount()+ctx->tx_flush_timeout_ticks;
588 0 : }
589 0 : }
590 :
591 : static inline void
592 0 : metrics_write( fd_iavf_tile_t * ctx ) {
593 :
594 0 : ulong rx_buffer_idle_cnt = 0UL;
595 0 : ulong rx_buffer_busy_cnt = 0UL;
596 0 : ulong tx_buffer_idle_cnt = 0UL;
597 0 : ulong tx_buffer_busy_cnt = 0UL;
598 0 : for( ulong i=0UL; i<ctx->vf_cnt; i++ ) {
599 0 : fd_iavf_queue_t const * rx_queue = &ctx->vfs[i].rx_queue;
600 0 : fd_iavf_queue_t const * tx_queue = &ctx->vfs[i].tx_queue;
601 0 : ulong const tx_capacity = tx_queue->depth-1UL;
602 0 : ulong rx_idle = fd_ulong_min( rx_queue->prod-rx_queue->cons, rx_queue->depth );
603 0 : ulong tx_busy = fd_ulong_min( tx_queue->prod-tx_queue->cons, tx_capacity );
604 0 : rx_buffer_idle_cnt += rx_idle;
605 0 : rx_buffer_busy_cnt += rx_queue->depth-rx_idle;
606 0 : tx_buffer_busy_cnt += tx_busy;
607 0 : tx_buffer_idle_cnt += tx_capacity-tx_busy;
608 0 : }
609 0 : FD_MCNT_SET( IAVF, PKT_RX, ctx->net.metrics.rx_pkt_cnt );
610 0 : FD_MCNT_SET( IAVF, PKT_RX_BYTES, ctx->net.metrics.rx_bytes_total );
611 0 : FD_MCNT_ENUM_COPY( IAVF, PKT_RX_DROPPED, ctx->net.metrics.rx_drop_cnt );
612 0 : FD_MCNT_SET( IAVF, PKT_RX_NO_LINK_DROPPED, ctx->metrics.rx_no_link_cnt );
613 0 : FD_MCNT_SET( IAVF, NON_FD_PKT_RX, ctx->metrics.non_fd_pkt_cnt );
614 0 : FD_MCNT_SET( IAVF, GRE_PKT_RX, ctx->net.metrics.rx_gre_cnt );
615 0 : FD_MCNT_SET( IAVF, VLAN_PKT_RX, ctx->net.metrics.rx_vlan_cnt );
616 0 : FD_MGAUGE_SET( IAVF, RX_BUFFER_USED, rx_buffer_busy_cnt );
617 0 : FD_MGAUGE_SET( IAVF, RX_BUFFER_FREE, rx_buffer_idle_cnt );
618 :
619 0 : FD_MCNT_SET( IAVF, PKT_TX_COMPLETED, ctx->net.metrics.tx_pkt_cnt );
620 0 : FD_MCNT_SET( IAVF, PKT_TX_COMPLETED_BYTES, ctx->net.metrics.tx_bytes_total );
621 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_RING_FULL_IDX ] = ctx->metrics.tx_no_buffer_cnt;
622 0 : FD_MCNT_ENUM_COPY( IAVF, PKT_TX_DROPPED, ctx->net.metrics.tx_drop_cnt );
623 0 : FD_MCNT_ENUM_COPY( IAVF, PKT_TX_ROUTED, ctx->net.metrics.tx_route_result_cnt );
624 0 : FD_MCNT_ENUM_COPY( IAVF, PKT_TX_SLOW_PATH_ROUTED, ctx->net.metrics.tx_slow_path_reason_cnt );
625 0 : FD_MCNT_SET( IAVF, PKT_TX_NO_LINK_DROPPED, ctx->metrics.tx_no_link_cnt );
626 0 : FD_MCNT_SET( IAVF, PKT_TX_LOOPBACK_FAILED, ctx->metrics.tx_loopback_fail_cnt );
627 0 : FD_MCNT_SET( IAVF, GRE_PKT_TX_SUBMITTED, ctx->net.metrics.tx_gre_cnt );
628 0 : FD_MCNT_SET( IAVF, PKT_TX_SLOW_PATH_SUBMITTED, ctx->net.metrics.tx_slow_path_submit_cnt );
629 0 : FD_MGAUGE_SET( IAVF, TX_BUFFER_USED, tx_buffer_busy_cnt );
630 0 : FD_MGAUGE_SET( IAVF, TX_BUFFER_FREE, tx_buffer_idle_cnt );
631 0 : }
632 :
633 : static inline void
634 : fd_iavf_tile_publish_vf_link( fd_iavf_tile_t * ctx,
635 0 : ulong vf_idx ) {
636 0 : ulong const usable = (ulong)ctx->vfs[vf_idx].vf_info.link_usable;
637 0 : __atomic_store_n( &ctx->vf_link_usable[vf_idx], usable, __ATOMIC_RELEASE );
638 0 : }
639 :
640 : static inline void
641 0 : fd_iavf_tile_vf_housekeeping( fd_iavf_tile_t * ctx ) {
642 : /* Other tiles read tile 0's link decisions */
643 0 : if( 0!=ctx->net.kind_id ) {
644 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
645 0 : ctx->vfs[vf_idx].vf_info.link_usable = (int)__atomic_load_n( &ctx->vf_link_usable_shared[vf_idx], __ATOMIC_ACQUIRE );
646 0 : }
647 0 : return;
648 0 : }
649 :
650 : /* There is no need to check VF state on every housekeeping pass */
651 0 : long const now = fd_tickcount();
652 0 : if( FD_LIKELY( now<ctx->vf_housekeeping_deadline_ticks ) ) return;
653 0 : ctx->vf_housekeeping_deadline_ticks = now+ctx->vf_housekeeping_interval_ticks;
654 :
655 : /* Only tile 0 consumes PF events */
656 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
657 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[vf_idx];
658 0 : int changed;
659 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_poll_link( &vf->vfio, &vf->adminq, &vf->vf_info, &changed ) ) ) {
660 0 : FD_LOG_ERR(( "PF member %lu VF link event or reset failure (%i-%s)", vf_idx, errno, fd_io_strerror( errno ) ));
661 0 : }
662 0 : if( changed ) {
663 0 : fd_iavf_tile_publish_vf_link( ctx, vf_idx );
664 0 : FD_LOG_INFO(( "PF member %lu VF link %s", vf_idx, vf->vf_info.link_usable ? "up" : "down" ));
665 0 : }
666 0 : }
667 0 : }
668 :
669 : static inline void
670 0 : during_housekeeping( fd_iavf_tile_t * ctx ) {
671 0 : fd_iavf_tile_vf_housekeeping( ctx );
672 :
673 0 : if( FD_UNLIKELY( fd_netdev_tbl_refresh( &ctx->net.router.netdev_tbl, &ctx->net.router.netdev_shared, &ctx->netdev_seq ) ) ) {
674 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->net.router.netdev_tbl );
675 0 : }
676 0 : }
677 :
678 : static uint
679 0 : fd_iavf_tile_if_ip4_addr( char const * if_name ) {
680 0 : int sock_fd = socket( AF_INET, SOCK_DGRAM, 0 );
681 0 : if( FD_UNLIKELY( -1==sock_fd ) ) {
682 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_DGRAM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
683 0 : }
684 :
685 0 : struct ifreq ifr = { .ifr_addr.sa_family = AF_INET };
686 0 : fd_cstr_ncpy( ifr.ifr_name, if_name, sizeof(ifr.ifr_name) );
687 :
688 0 : if( FD_UNLIKELY( -1==ioctl( sock_fd, SIOCGIFADDR, &ifr ) ) ) {
689 0 : FD_LOG_ERR(( "could not get IP address of interface `%s` (%i-%s)",
690 0 : if_name, errno, fd_io_strerror( errno ) ));
691 0 : }
692 0 : uint ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
693 0 : if( FD_UNLIKELY( -1==close( sock_fd ) ) ) {
694 0 : FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
695 0 : }
696 :
697 0 : return ip4_addr;
698 0 : }
699 :
700 : /* fd_iavf_tile_lo_tx_socket uses IPPROTO_RAW for send only IP_HDRINCL
701 : transmission, preserving the source address and UDP source port. */
702 : static int
703 0 : fd_iavf_tile_lo_tx_socket( void ) {
704 0 : int sock = socket( AF_INET, SOCK_RAW|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_RAW );
705 0 : if( FD_UNLIKELY( -1==sock ) ) {
706 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_RAW,IPPROTO_RAW) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
707 0 : }
708 0 : if( FD_UNLIKELY( -1==setsockopt( sock, SOL_SOCKET, SO_BINDTODEVICE, "lo", sizeof("lo") ) ) ) {
709 0 : FD_LOG_ERR(( "setsockopt(SO_BINDTODEVICE,lo) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
710 0 : }
711 0 : return sock;
712 0 : }
713 :
714 : static ulong
715 0 : scratch_align( void ) {
716 0 : return FD_IAVF_PAGE_SZ;
717 0 : }
718 :
719 : static ulong
720 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
721 0 : ulong queue_sz = fd_iavf_queue_footprint( tile->iavf.tx_queue_size, tile->iavf.rx_queue_size );
722 0 : if( FD_UNLIKELY( 0==queue_sz ) ) return 0UL;
723 0 : ulong layout = FD_LAYOUT_INIT;
724 0 : layout = FD_LAYOUT_APPEND( layout, alignof(fd_iavf_tile_t), sizeof(fd_iavf_tile_t) );
725 0 : for( ulong i=0UL; i<tile->iavf.member_cnt; i++ ) {
726 0 : if( 0==tile->kind_id ) layout = FD_LAYOUT_APPEND( layout, FD_IAVF_PAGE_SZ, fd_iavf_adminq_footprint() );
727 0 : layout = FD_LAYOUT_APPEND( layout, FD_IAVF_PAGE_SZ, queue_sz );
728 0 : layout = FD_LAYOUT_APPEND( layout, alignof(uint), tile->iavf.rx_queue_size*sizeof(uint) );
729 0 : layout = FD_LAYOUT_APPEND( layout, alignof(uint), tile->iavf.tx_queue_size*sizeof(uint) );
730 0 : layout = FD_LAYOUT_APPEND( layout, alignof(ushort), tile->iavf.tx_queue_size*sizeof(ushort) );
731 0 : }
732 0 : layout = FD_LAYOUT_APPEND( layout, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
733 0 : layout = FD_LAYOUT_APPEND( layout, fd_fib4_align(), fd_fib4_footprint( tile->iavf.route_max, tile->iavf.route_peer_max ) );
734 0 : layout = FD_LAYOUT_APPEND( layout, fd_fib4_align(), fd_fib4_footprint( tile->iavf.route_max, tile->iavf.route_peer_max ) );
735 0 : return FD_LAYOUT_FINI( layout, scratch_align() );
736 0 : }
737 :
738 : fd_fib4_t *
739 : fd_iavf_tile_fib4_join( fd_fib4_t * out,
740 : fd_topo_t const * topo,
741 : fd_topo_tile_t const * tile,
742 0 : int main_table ) {
743 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
744 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_iavf_tile_t), sizeof(fd_iavf_tile_t) );
745 0 : ulong queue_sz = fd_iavf_queue_footprint( tile->iavf.tx_queue_size, tile->iavf.rx_queue_size );
746 0 : for( ulong i=0UL; i<tile->iavf.member_cnt; i++ ) {
747 0 : if( 0==tile->kind_id ) (void)FD_SCRATCH_ALLOC_APPEND( scratch, FD_IAVF_PAGE_SZ, fd_iavf_adminq_footprint() );
748 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, FD_IAVF_PAGE_SZ, queue_sz );
749 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->iavf.rx_queue_size*sizeof(uint) );
750 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->iavf.tx_queue_size*sizeof(uint) );
751 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(ushort), tile->iavf.tx_queue_size*sizeof(ushort) );
752 0 : }
753 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
754 0 : void * local_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->iavf.route_max, tile->iavf.route_peer_max ) );
755 0 : void * main_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->iavf.route_max, tile->iavf.route_peer_max ) );
756 0 : return fd_fib4_join( out, main_table ? main_mem : local_mem );
757 0 : }
758 :
759 : /* fd_iavf_tile_join_pkt_bufs uses workspace offsets for packet IOVAs so
760 : descriptors can address buffers directly as chunk<<FD_CHUNK_LG_SZ. */
761 : static void
762 : fd_iavf_tile_join_pkt_bufs( fd_topo_t const * topo,
763 : fd_topo_tile_t const * tile,
764 : fd_iavf_tile_t * ctx,
765 : void ** packet_memory,
766 : ulong * packet_memory_sz,
767 0 : ulong * packet_iova ) {
768 0 : void * const packet_dcache_memory = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
769 0 : void * const packet_dcache = fd_dcache_join( packet_dcache_memory );
770 0 : FD_TEST( NULL!=packet_dcache );
771 0 : ulong const packet_dcache_data_sz = fd_dcache_data_sz( packet_dcache );
772 0 : ulong const frame_sz = FD_NET_MTU;
773 0 : ulong const frame_region_sz = fd_ulong_align_dn( packet_dcache_data_sz, frame_sz );
774 0 : FD_TEST( frame_region_sz>=frame_sz && frame_region_sz<=ULONG_MAX-FD_IAVF_PAGE_SZ &&
775 0 : (ulong)packet_dcache<=ULONG_MAX-frame_region_sz-FD_IAVF_PAGE_SZ );
776 :
777 0 : void * const workspace_base = fd_wksp_containing( packet_dcache_memory );
778 0 : FD_TEST( NULL!=workspace_base );
779 0 : ulong const pkt_buf_chunk0 = ((ulong)packet_dcache-(ulong)workspace_base)>>FD_CHUNK_LG_SZ;
780 0 : ulong const pkt_buf_wmark = pkt_buf_chunk0 + ((frame_region_sz-frame_sz)>>FD_CHUNK_LG_SZ);
781 0 : if( FD_UNLIKELY( 0==pkt_buf_chunk0 || pkt_buf_chunk0>UINT_MAX || pkt_buf_wmark>UINT_MAX || pkt_buf_chunk0>pkt_buf_wmark ) ) {
782 0 : FD_LOG_ERR(( "Calculated invalid packet buffer bounds [%lu,%lu]", pkt_buf_chunk0, pkt_buf_wmark ));
783 0 : }
784 :
785 0 : ctx->net.pkt_buf_wksp_base = workspace_base;
786 0 : ctx->net.pkt_buf_chunk0 = pkt_buf_chunk0;
787 0 : ctx->net.pkt_buf_wmark = pkt_buf_wmark;
788 0 : if( NULL!=packet_memory ) *packet_memory = packet_dcache;
789 : /* VFIO_IOMMU_MAP_DMA requires whole pages. */
790 0 : if( NULL!=packet_memory_sz ) *packet_memory_sz = fd_ulong_align_up( frame_region_sz, FD_IAVF_PAGE_SZ );
791 0 : if( NULL!=packet_iova ) *packet_iova = (ulong)packet_dcache-(ulong)workspace_base;
792 0 : }
793 :
794 : FD_FN_UNUSED static void
795 : fd_iavf_tile_init_pkt_bufs( fd_topo_t const * topo,
796 : fd_topo_tile_t const * tile,
797 0 : fd_iavf_tile_t * ctx ) {
798 0 : ulong frame_chunks = FD_NET_MTU>>FD_CHUNK_LG_SZ;
799 0 : ulong next_chunk = ctx->net.pkt_buf_chunk0;
800 0 : FD_TEST( tile->iavf.rx_queue_size>ctx->batch_size );
801 0 : ulong const rx_fill_cnt = tile->iavf.rx_queue_size-ctx->batch_size;
802 0 : for( ulong i=0UL; i<ctx->vf_cnt; i++ ) {
803 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[i];
804 : /* Post initial RX buffers */
805 0 : for( ulong j=0UL; j<rx_fill_cnt; j++ ) {
806 0 : fd_iavf_tile_rx_recycle( ctx, vf, next_chunk );
807 0 : next_chunk += frame_chunks;
808 0 : }
809 :
810 : /* Reserve one packet buffer per TX descriptor */
811 0 : for( ulong j=0UL; j<tile->iavf.tx_queue_size; j++ ) {
812 0 : vf->tx_desc_buf_chunk[j] = (uint)next_chunk;
813 0 : next_chunk += frame_chunks;
814 0 : }
815 0 : }
816 :
817 : /* Reserve packet buffers for output mcache entries */
818 0 : for( ulong i=0UL; i<tile->out_cnt; i++ ) {
819 0 : fd_topo_link_t const * link = &topo->links[ tile->out_link_id[i] ];
820 0 : fd_frag_meta_t * mcache = fd_mcache_join( fd_topo_obj_laddr( topo, link->mcache_obj_id ) );
821 0 : FD_TEST( NULL!=mcache );
822 0 : ulong depth = fd_mcache_depth( mcache );
823 0 : for( ulong j=0UL; j<depth; j++ ) {
824 0 : mcache[j].chunk = (uint)next_chunk;
825 0 : mcache[j].seq = fd_seq_dec( j, 1UL );
826 0 : next_chunk += frame_chunks;
827 0 : }
828 0 : }
829 0 : if( FD_UNLIKELY( next_chunk-frame_chunks>ctx->net.pkt_buf_wmark ) ) {
830 0 : ulong const required_sz = (next_chunk-ctx->net.pkt_buf_chunk0)<<FD_CHUNK_LG_SZ;
831 0 : ulong const available_sz = ((ctx->net.pkt_buf_wmark-ctx->net.pkt_buf_chunk0)<<FD_CHUNK_LG_SZ)+FD_NET_MTU;
832 0 : FD_LOG_ERR(( "packet dcache needs %lu bytes, available %lu", required_sz, available_sz ));
833 0 : }
834 0 : }
835 :
836 : FD_FN_UNUSED static void
837 : fd_iavf_tile_join_obj_workspace( fd_topo_t * topo,
838 0 : ulong obj_id ) {
839 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->objs[obj_id].wksp_id ];
840 0 : if( NULL==wksp->wksp ) fd_topo_join_workspace( topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE, 0 );
841 0 : }
842 :
843 : #ifndef FD_TILE_TEST
844 : void
845 : fd_topo_install_iavf( fd_topo_t * topo,
846 0 : fd_iavf_fds_t * fds ) {
847 0 : ulong const tile_cnt = fd_topo_tile_name_cnt( topo, "iavf" );
848 0 : if( FD_UNLIKELY( 0==tile_cnt || tile_cnt>FD_IAVF_TILE_MAX ) ) {
849 0 : FD_LOG_ERR(( "IAVF net tile count %lu must be in [1,%lu]", tile_cnt, FD_IAVF_TILE_MAX ));
850 0 : }
851 0 : fd_iavf_tile_t * ctxs [ FD_IAVF_TILE_MAX ];
852 0 : fd_topo_tile_t const * tiles [ FD_IAVF_TILE_MAX ];
853 0 : fd_iavf_vfio_tile_t queues[ FD_IAVF_MEMBER_MAX ][ FD_IAVF_TILE_MAX ];
854 :
855 : /* Prepare tile memory and collect queue pairs by VF */
856 0 : for( ulong tile_idx=0UL; tile_idx<tile_cnt; tile_idx++ ) {
857 0 : ulong const tile_id = fd_topo_find_tile( topo, "iavf", tile_idx );
858 0 : FD_TEST( tile_id!=ULONG_MAX );
859 0 : fd_topo_tile_t const * tile = &topo->tiles[tile_id];
860 0 : tiles[tile_idx] = tile;
861 :
862 0 : fd_iavf_tile_join_obj_workspace( topo, tile->tile_obj_id );
863 0 : fd_iavf_tile_join_obj_workspace( topo, tile->net.umem_dcache_obj_id );
864 0 : for( ulong out_idx=0UL; out_idx<tile->out_cnt; out_idx++ ) {
865 0 : fd_iavf_tile_join_obj_workspace( topo, topo->links[ tile->out_link_id[out_idx] ].mcache_obj_id );
866 0 : }
867 0 : fd_iavf_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
868 0 : ctxs[tile_idx] = ctx;
869 0 : fd_memset( ctx, 0, sizeof(*ctx) );
870 0 : ctx->vf_cnt = tile->iavf.member_cnt;
871 0 : ctx->batch_size = FD_IAVF_BATCH_SIZE;
872 0 : ctx->net.kind_id = tile_idx;
873 0 : ctx->net.tile_cnt = tile_cnt;
874 0 : ctx->vf_link_usable_shared = ctxs[0]->vf_link_usable;
875 0 : fd_iavf_hw_join_queues( ctx, tile );
876 :
877 0 : void * packet_memory;
878 0 : ulong packet_memory_sz;
879 0 : ulong packet_iova;
880 0 : fd_iavf_tile_join_pkt_bufs( topo, tile, ctx, &packet_memory, &packet_memory_sz, &packet_iova );
881 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
882 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[vf_idx];
883 0 : queues[vf_idx][tile_idx] = (fd_iavf_vfio_tile_t) {
884 0 : .tx_queue = &vf->tx_queue,
885 0 : .rx_queue = &vf->rx_queue,
886 0 : .queue_memory = vf->queue_memory,
887 0 : .queue_memory_sz = fd_iavf_queue_footprint( vf->tx_queue.depth, vf->rx_queue.depth ),
888 0 : .packet_memory = packet_memory,
889 0 : .packet_memory_sz = packet_memory_sz,
890 0 : .packet_iova = packet_iova
891 0 : };
892 0 : }
893 0 : fd_net_rx_dests_init( &ctx->net, topo, tile );
894 0 : }
895 :
896 : /* Configure VFs through tile 0's Admin Queues */
897 0 : fd_topo_tile_t const * first_tile = tiles[0];
898 0 : fd_iavf_tile_t * owner = ctxs[0];
899 0 : for( ulong vf_idx=0UL; vf_idx<owner->vf_cnt; vf_idx++ ) {
900 0 : fd_iavf_tile_vf_t * vf = &owner->vfs[vf_idx];
901 0 : char const * pf_if = first_tile->iavf.members[vf_idx];
902 0 : char vf_pci[ FD_IAVF_PCI_ADDR_SZ ];
903 0 : if( FD_UNLIKELY( -1==fd_iavf_get_last_vf( pf_if, NULL, vf_pci, 1 ) ) ) {
904 0 : FD_LOG_ERR(( "VF lookup for PF %s failed (%i-%s)", pf_if, errno, fd_io_strerror( errno ) ));
905 0 : }
906 0 : if( FD_UNLIKELY( -1==fd_iavf_init( &vf->vfio, &vf->adminq, &vf->vf_info,
907 0 : queues[vf_idx], (uint)tile_cnt, vf_pci, FD_NET_MTU, FD_NET_MTU ) ) ) {
908 0 : FD_LOG_ERR(( "VF %s setup failed (%i-%s)", vf_pci, errno, fd_io_strerror( errno ) ));
909 0 : }
910 0 : }
911 :
912 : /* Fill RX descriptors before enabling reception */
913 0 : for( ulong tile_idx=0UL; tile_idx<tile_cnt; tile_idx++ ) {
914 0 : fd_iavf_tile_init_pkt_bufs( topo, tiles[tile_idx], ctxs[tile_idx] );
915 0 : }
916 :
917 : /* Enable queues and distribute VF state to all tiles */
918 0 : if( NULL!=fds ) fds->member_cnt = owner->vf_cnt;
919 0 : for( ulong vf_idx=0UL; vf_idx<owner->vf_cnt; vf_idx++ ) {
920 0 : fd_iavf_tile_vf_t * vf = &owner->vfs[vf_idx];
921 0 : char const * pf_if = first_tile->iavf.members[vf_idx];
922 0 : if( FD_UNLIKELY( -1==fd_iavf_virtchnl_enable_queues( &vf->vfio, &vf->adminq, &vf->vf_info, queues[vf_idx], (uint)tile_cnt ) ) ) {
923 0 : FD_LOG_ERR(( "VF queue enable for PF %s failed (%i-%s)", pf_if, errno, fd_io_strerror( errno ) ));
924 0 : }
925 0 : fd_iavf_tile_publish_vf_link( owner, vf_idx );
926 :
927 : /* Save VFIO descriptors for child launch */
928 0 : if( NULL!=fds ) {
929 0 : fds->container_fd[ vf_idx ] = vf->vfio.container_fd;
930 0 : fds->group_fd [ vf_idx ] = vf->vfio.group_fd;
931 0 : fds->device_fd [ vf_idx ] = vf->vfio.device_fd;
932 0 : }
933 :
934 : /* Each tile remaps BAR0 at launch */
935 0 : if( FD_UNLIKELY( -1==fd_iavf_vfio_unmap_bar( &vf->vfio ) ) ) {
936 0 : FD_LOG_ERR(( "VF BAR unmap for PF %s failed (%i-%s)", pf_if, errno, fd_io_strerror( errno ) ));
937 0 : }
938 :
939 : /* Copy VF resources while retaining each tile's queues */
940 0 : for( ulong tile_idx=0UL; tile_idx<tile_cnt; tile_idx++ ) {
941 0 : fd_iavf_tile_vf_t * tile_vf = &ctxs[tile_idx]->vfs[vf_idx];
942 0 : tile_vf->vfio = vf->vfio;
943 0 : tile_vf->vf_info = vf->vf_info;
944 0 : tile_vf->tx_queue.tail = NULL;
945 0 : tile_vf->rx_queue.tail = NULL;
946 0 : }
947 0 : }
948 0 : for( ulong tile_idx=0UL; tile_idx<tile_cnt; tile_idx++ ) ctxs[tile_idx]->prepared = 1U;
949 0 : }
950 : #endif
951 :
952 : FD_FN_UNUSED static void
953 : privileged_init( fd_topo_t const * topo,
954 0 : fd_topo_tile_t const * tile ) {
955 : /* Prepare VFs if the launcher has not done so */
956 0 : fd_iavf_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
957 0 : if( !ctx->prepared ) {
958 0 : fd_topo_install_iavf( (fd_topo_t *)topo, NULL );
959 0 : }
960 :
961 : /* Join this tile's queue and packet memory */
962 0 : fd_iavf_hw_join_queues( ctx, tile );
963 0 : fd_iavf_tile_join_pkt_bufs( topo, tile, ctx, NULL, NULL, NULL );
964 :
965 : /* Use tile 0's VF link decisions */
966 0 : ulong const owner_tile_id = fd_topo_find_tile( topo, "iavf", 0UL );
967 0 : FD_TEST( owner_tile_id!=ULONG_MAX );
968 0 : fd_iavf_tile_t const * owner = fd_topo_obj_laddr( topo, topo->tiles[ owner_tile_id ].tile_obj_id );
969 0 : ctx->vf_link_usable_shared = owner->vf_link_usable;
970 :
971 : /* Queue IDs match IAVF tile indices */
972 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
973 0 : fd_iavf_tile_vf_t * vf = &ctx->vfs[vf_idx];
974 0 : char const * pf_if = tile->iavf.members[vf_idx];
975 :
976 0 : if( FD_UNLIKELY( -1==fd_iavf_hw_map_regs( vf, tile->kind_id ) ) ) {
977 0 : FD_LOG_ERR(( "VF register mapping for PF %s failed (%i-%s)", pf_if, errno, fd_io_strerror( errno ) ));
978 0 : }
979 0 : }
980 :
981 : /* Routing interface and default source IP */
982 0 : uint const interface_idx = if_nametoindex( tile->iavf.if_name );
983 0 : if( FD_UNLIKELY( 0==interface_idx ) ) {
984 0 : FD_LOG_ERR(( "interface %s lookup failed (%i-%s)",
985 0 : tile->iavf.if_name, errno, fd_io_strerror( errno ) ));
986 0 : }
987 0 : ctx->net.router.if_virt = interface_idx;
988 0 : ctx->net.router.default_address = fd_iavf_tile_if_ip4_addr( tile->iavf.if_name );
989 :
990 : /* Only tile 0 sends loopback traffic */
991 0 : ctx->lo_tx_sock = 0UL==tile->kind_id ? fd_iavf_tile_lo_tx_socket() : -1;
992 0 : if( FD_UNLIKELY( !fd_net_sock_init( &ctx->net.sock ) ) ) {
993 0 : FD_LOG_ERR(( "fd_net_sock_init failed (%i-%s)", errno, fd_io_strerror( errno ) ));
994 0 : }
995 0 : }
996 :
997 : FD_FN_UNUSED static void
998 : unprivileged_init( fd_topo_t const * topo,
999 0 : fd_topo_tile_t const * tile ) {
1000 : /* Tile state and scratch memory */
1001 0 : fd_iavf_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1002 0 : void * after_queues = fd_iavf_hw_join_queues( ctx, tile );
1003 0 : FD_SCRATCH_ALLOC_INIT( scratch, after_queues );
1004 :
1005 0 : ctx->batch_size = FD_IAVF_BATCH_SIZE;
1006 0 : ctx->tx_flush_timeout_ticks = (long)(FD_IAVF_TX_FLUSH_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ));
1007 0 : ctx->lo_tx_timeout_ticks = (long)(FD_IAVF_LO_TX_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ));
1008 0 : ctx->vf_housekeeping_interval_ticks = (long)(FD_IAVF_VF_HOUSEKEEPING_INTERVAL_NS*fd_tempo_tick_per_ns( NULL ));
1009 0 : ctx->vf_housekeeping_deadline_ticks = fd_tickcount();
1010 0 : ctx->lo_tx_cnt = 0U;
1011 0 : ctx->net.kind_id = tile->kind_id;
1012 0 : ctx->net.tile_cnt = fd_topo_tile_name_cnt( topo, tile->name );
1013 :
1014 0 : void * netdev_tbl_local = FD_SCRATCH_ALLOC_APPEND( scratch, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1015 0 : void * fib_local_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->iavf.route_max, tile->iavf.route_peer_max ) );
1016 0 : void * fib_main_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->iavf.route_max, tile->iavf.route_peer_max ) );
1017 :
1018 : /* Input links and buffer bounds */
1019 0 : if( FD_UNLIKELY( tile->in_cnt>FD_NET_IN_MAX ) ) {
1020 0 : FD_LOG_ERR(( "iavf tile in link count %lu exceeds max of %lu", tile->in_cnt, FD_NET_IN_MAX ));
1021 0 : }
1022 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
1023 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
1024 0 : if( 0==strcmp( link->name, "iproute_out" ) ) {
1025 0 : ctx->net.in_kind[ i ] = FD_NET_IN_KIND_IPROUTE;
1026 0 : } else {
1027 0 : ctx->net.in_kind[ i ] = FD_NET_IN_KIND_TX;
1028 0 : if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "iavf tile in link %s MTU %lu, expected %lu", link->name, link->mtu, (ulong)FD_NET_MTU ));
1029 0 : }
1030 :
1031 0 : ctx->net.in_dcache_ctx[ i ].wksp_base = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
1032 0 : ctx->net.in_dcache_ctx[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache );
1033 0 : ctx->net.in_dcache_ctx[ i ].wmark = fd_dcache_compact_wmark( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache, link->mtu );
1034 0 : }
1035 :
1036 : /* Join fib4 tables */
1037 0 : FD_TEST( NULL!=fd_fib4_join( ctx->net.router.fib_local, fd_fib4_new( fib_local_mem, tile->iavf.route_max, tile->iavf.route_peer_max, tile->iavf.route_peer_seed ) ) );
1038 0 : FD_TEST( NULL!=fd_fib4_join( ctx->net.router.fib_main, fd_fib4_new( fib_main_mem, tile->iavf.route_max, tile->iavf.route_peer_max, tile->iavf.route_peer_seed ) ) );
1039 :
1040 : /* Join shared netdev table */
1041 0 : FD_TEST( NULL!=fd_netdev_tbl_join( &ctx->net.router.netdev_shared, fd_topo_obj_laddr( topo, tile->iavf.netdev_tbl_obj_id ) ) );
1042 :
1043 : /* Create and join local netdev table */
1044 0 : FD_TEST( NULL!=fd_netdev_tbl_new( netdev_tbl_local, NETDEV_MAX, BOND_MASTER_MAX ) );
1045 0 : FD_TEST( NULL!=fd_netdev_tbl_join( &ctx->net.router.netdev_tbl, netdev_tbl_local ) );
1046 :
1047 : /* Copy a consistent interface table for routing */
1048 0 : ctx->netdev_seq = fd_netdev_tbl_copy( &ctx->net.router.netdev_tbl, &ctx->net.router.netdev_shared );
1049 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->net.router.netdev_tbl );
1050 0 : ctx->net.bind_address = tile->iavf.net.bind_address;
1051 :
1052 : /* Join shared neighbor table */
1053 0 : ulong neigh4_obj_id = tile->iavf.neigh4_obj_id;
1054 0 : ulong ele_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.ele_max", neigh4_obj_id );
1055 0 : ulong probe_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.probe_max", neigh4_obj_id );
1056 0 : ulong seed = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.seed", neigh4_obj_id );
1057 0 : if( FD_UNLIKELY( (ele_max==ULONG_MAX) | (probe_max==ULONG_MAX) | (seed==ULONG_MAX) ) ) {
1058 0 : FD_LOG_ERR(( "neigh4 hmap properties not set" ));
1059 0 : }
1060 0 : if( FD_UNLIKELY( NULL==fd_neigh4_hmap_join( ctx->net.router.neigh4, fd_topo_obj_laddr( topo, neigh4_obj_id ), ele_max, probe_max, seed ) ) ) {
1061 0 : FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
1062 0 : }
1063 :
1064 : /* Netlink requests for missing neighbors */
1065 0 : ctx->net.router.netlnk_out_idx = fd_topo_find_tile_out_link( topo, tile, "net_netlnk", tile->kind_id );
1066 0 : if( FD_UNLIKELY( ctx->net.router.netlnk_out_idx==ULONG_MAX ) ) FD_LOG_ERR(( "netlink request link not found" ));
1067 0 : ctx->net.router.solicit_ip = 0U;
1068 :
1069 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( scratch, scratch_align() );
1070 0 : if( FD_UNLIKELY( scratch_top>(ulong)ctx+scratch_footprint( tile ) ) ) {
1071 0 : FD_LOG_ERR(( "scratch overflow" ));
1072 0 : }
1073 0 : }
1074 :
1075 : FD_FN_UNUSED static ulong
1076 : populate_allowed_seccomp( fd_topo_t const * topo,
1077 : fd_topo_tile_t const * tile,
1078 : ulong out_cnt,
1079 0 : struct sock_filter * out ) {
1080 0 : fd_iavf_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1081 0 : populate_sock_filter_policy_fd_iavf_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)ctx->lo_tx_sock,
1082 0 : (uint)ctx->net.sock.sock );
1083 0 : return sock_filter_policy_fd_iavf_tile_instr_cnt;
1084 0 : }
1085 :
1086 : FD_FN_UNUSED static ulong
1087 : populate_allowed_fds( fd_topo_t const * topo,
1088 : fd_topo_tile_t const * tile,
1089 : ulong out_fds_cnt,
1090 0 : int * out_fds ) {
1091 0 : fd_iavf_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1092 0 : if( FD_UNLIKELY( out_fds_cnt<5UL+3UL*ctx->vf_cnt ) ) FD_LOG_ERR(( "allowed FD array capacity %lu, expected at least %lu", out_fds_cnt, 5UL+3UL*ctx->vf_cnt ));
1093 0 : ulong out_cnt = 0UL;
1094 0 : out_fds[ out_cnt++ ] = 2;
1095 0 : if( fd_log_private_logfile_fd()!=-1 ) out_fds[ out_cnt++ ] = fd_log_private_logfile_fd();
1096 0 : out_fds[ out_cnt++ ] = ctx->net.sock.sock; /* routing fallback socket */
1097 0 : for( ulong vf_idx=0UL; vf_idx<ctx->vf_cnt; vf_idx++ ) {
1098 0 : fd_iavf_vfio_t const * vfio = &ctx->vfs[vf_idx].vfio;
1099 0 : out_fds[ out_cnt++ ] = vfio->container_fd;
1100 0 : out_fds[ out_cnt++ ] = vfio->group_fd;
1101 0 : out_fds[ out_cnt++ ] = vfio->device_fd;
1102 0 : }
1103 0 : if( ctx->lo_tx_sock>=0 ) out_fds[ out_cnt++ ] = ctx->lo_tx_sock;
1104 0 : return out_cnt;
1105 0 : }
1106 :
1107 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_iavf_tile_t
1108 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_iavf_tile_t)
1109 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
1110 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
1111 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
1112 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
1113 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
1114 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
1115 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
1116 : #define STEM_NEVER_PARK 1
1117 : /* Reserve output credit for one RX batch per VF and one direct loopback
1118 : delivery in the same stem iteration. */
1119 0 : #define STEM_BURST (FD_IAVF_MEMBER_MAX*FD_IAVF_BATCH_SIZE+1UL)
1120 0 : #define STEM_LAZY 270000UL
1121 : #include "../../stem/fd_stem.c"
1122 :
1123 : #ifndef FD_TILE_TEST
1124 : fd_topo_run_tile_t fd_tile_iavf = {
1125 : .name = "iavf",
1126 : .populate_allowed_seccomp = populate_allowed_seccomp,
1127 : .populate_allowed_fds = populate_allowed_fds,
1128 : .scratch_align = scratch_align,
1129 : .scratch_footprint = scratch_footprint,
1130 : .privileged_init = privileged_init,
1131 : .unprivileged_init = unprivileged_init,
1132 : .run = stem_run,
1133 : };
1134 : #endif
|
