Line data Source code
1 : #define _GNU_SOURCE /* sendmmsg */
2 : /* The mlx5 tile translates Ethernet frames between mlx5 work/completion
3 : queues and fd_tango traffic.
4 :
5 : An mlx5 queue pair (QP) contains a send queue (SQ) and a receive queue
6 : (RQ). Work queue entries (WQEs) encode posted work. Completion
7 : queue entries (CQEs) report completed work through completion queues
8 : (CQs). The mlx5 tile uses separate CQs for TX and RX, one each. */
9 :
10 : #include "../fd_net_tile.h"
11 : #include "../fd_net_tile_private.h"
12 : #include "fd_mlx5_private.h"
13 :
14 : #include <errno.h>
15 : #include <fcntl.h>
16 : #include <net/if.h>
17 : #include <stddef.h>
18 : #include <stdlib.h>
19 : #include <time.h>
20 : #include <netinet/in.h>
21 : #include <sys/socket.h>
22 :
23 : #include "../../metrics/fd_metrics.h"
24 : #include "../fd_net_router.h"
25 : #include "../../topo/fd_topo.h"
26 : #include "../../sleep/fd_sleep.h"
27 :
28 : #include "../../../waltz/ip/fd_iproute.h"
29 : #include "../../../util/net/fd_eth.h"
30 : #include "../../../util/net/fd_gre.h"
31 : #include "../../../util/net/fd_ip4.h"
32 : #include "../../../util/net/fd_udp.h"
33 : #include "../../../util/pod/fd_pod_format.h"
34 :
35 : #include <unistd.h>
36 : #include <sys/epoll.h>
37 : #include <sys/ioctl.h>
38 : #include <linux/rtnetlink.h>
39 : #include <rdma/ib_user_verbs.h>
40 :
41 : #include "generated/fd_mlx5_tile_seccomp.h"
42 :
43 :
44 0 : #define FD_MLX5_TX_FLUSH_TIMEOUT_NS (20000L) /* 20us */
45 0 : #define FD_MLX5_LO_TX_TIMEOUT_NS (500000L) /* 500us */
46 :
47 : /* Recent RX packets or TX submissions defer RX CQ rearming and cause
48 : a timed repoll before the tile returns to epoll. */
49 0 : #define FD_MLX5_REPOLL_TIMEOUT_NS ( 200000L) /* 200us */
50 0 : #define FD_MLX5_NS_PER_SEC (1000000000L)
51 :
52 : /* Linux names these UAR offsets MLX5_CQ_DOORBELL and MLX5_BF_OFFSET. */
53 0 : #define FD_MLX5_UAR_CQ_DB_OFFSET (0x20UL)
54 0 : #define FD_MLX5_UAR_SQ_DB_OFFSET (0x800UL)
55 :
56 : /* epoll data of the async fd in the park set (the completion channel
57 : is tagged with its fd, the doorbell with FD_SLEEP_EPOLL_DOORBELL). */
58 0 : #define FD_MLX5_EPOLL_ASYNC (ULONG_MAX-1UL)
59 :
60 : /* mlx5 CQ arm doorbells use the event sequence in bits 28 and 29. */
61 0 : #define FD_MLX5_CQ_ARM_SN_MASK ( 3U)
62 0 : #define FD_MLX5_CQ_ARM_SN_SHIFT (28U)
63 :
64 : /* FD_MLX5_SQ_* are options in a SQ WQE to request certain NIC behaviour.
65 : SEND requests packet transmission. CQ_UPDATE requests a CQE. */
66 0 : #define FD_MLX5_SQ_SEND_PKT (0x0aU) /* SEND */
67 0 : #define FD_MLX5_SQ_REQUEST_CQE (0x08U) /* CQ_UPDATE */
68 :
69 : /* FD_MLX5_CQE_OP_* are mlx5 CQE opcodes used to classify TX and RX completions */
70 0 : #define FD_MLX5_CQE_OP_TX_OK ( 0U) /* MLX5_CQE_REQ */
71 0 : #define FD_MLX5_CQE_OP_RX_OK ( 2U) /* MLX5_CQE_RESP_SEND */
72 0 : #define FD_MLX5_CQE_OP_TX_ERR (13U) /* MLX5_CQE_REQ_ERR */
73 0 : #define FD_MLX5_CQE_OP_RX_ERR (14U) /* MLX5_CQE_RESP_ERR */
74 0 : #define FD_MLX5_CQE_OP_INVALID (15U)
75 0 : #define FD_MLX5_CQE_OWNER_MASK ( 1U) /* MLX5_CQE_OWNER_MASK */
76 :
77 0 : #define FD_MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR (0x01U) /* received packet's length exceeded FD_NET_MTU */
78 :
79 : /* FD_MLX5_ASYNC_EVENT_* are Linux enum ib_event_type values delivered
80 : in ib_uverbs_async_event_desc.event_type. */
81 : #define FD_MLX5_ASYNC_EVENT_CQ_ERR ( 0U)
82 : #define FD_MLX5_ASYNC_EVENT_QP_FATAL ( 1U)
83 : #define FD_MLX5_ASYNC_EVENT_QP_REQ_ERR ( 2U)
84 : #define FD_MLX5_ASYNC_EVENT_QP_ACCESS_ERR ( 3U)
85 0 : #define FD_MLX5_ASYNC_EVENT_DEVICE_FATAL ( 8U)
86 : #define FD_MLX5_ASYNC_EVENT_WQ_FATAL (19U)
87 :
88 : /* The fd_mlx5_hw_* structs below define WQE and CQE formats in the
89 : mlx5 hardware interface. The links show the corresponding definitions
90 : in the Linux mlx5 driver. */
91 :
92 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/qp.h#L205 */
93 : struct __attribute__((packed)) fd_mlx5_hw_wqe_ctrl_seg {
94 : /* mlx5 packs opmod, WQE index, opcode, QP number, and WQE segment
95 : count into two big-endian 32-bit words. */
96 : uint opmod_idx_opcode;
97 : uint qpn_ds;
98 :
99 : uchar signature;
100 : uchar reserved[2];
101 : uchar flags;
102 : uint imm;
103 : };
104 : typedef struct fd_mlx5_hw_wqe_ctrl_seg fd_mlx5_hw_wqe_ctrl_seg_t;
105 :
106 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/qp.h#L265 */
107 : struct __attribute__((packed)) fd_mlx5_hw_wqe_eth_seg {
108 : uchar reserved[12];
109 : ushort inline_hdr_sz;
110 : uchar inline_hdr[2];
111 : };
112 : typedef struct fd_mlx5_hw_wqe_eth_seg fd_mlx5_hw_wqe_eth_seg_t;
113 :
114 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/qp.h#L364 */
115 : struct __attribute__((packed)) fd_mlx5_hw_wqe_data_seg {
116 : uint byte_cnt;
117 : uint lkey;
118 : ulong addr;
119 : };
120 : typedef struct fd_mlx5_hw_wqe_data_seg fd_mlx5_hw_wqe_data_seg_t;
121 :
122 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/device.h#L824 */
123 : struct __attribute__((packed,may_alias)) fd_mlx5_hw_cqe64 {
124 : uchar reserved0[44];
125 : uint byte_cnt;
126 : uchar reserved1[6];
127 : uchar vendor_err;
128 : uchar syndrome;
129 : uint sop_drop_qpn;
130 : ushort wqe_counter;
131 : uchar signature;
132 : uchar op_own;
133 : };
134 : typedef struct fd_mlx5_hw_cqe64 fd_mlx5_hw_cqe64_t;
135 :
136 : struct fd_mlx5_tile_rx_comp {
137 : ulong chunk;
138 : uint byte_len;
139 : uint opcode;
140 : };
141 : typedef struct fd_mlx5_tile_rx_comp fd_mlx5_tile_rx_comp_t;
142 :
143 : /* fd_mlx5_tile_t is private tile state */
144 : struct __attribute__((aligned(64UL))) fd_mlx5_tile {
145 : fd_mlx5_rx_wq_t rx_wq;
146 : uint batch_size; /* used for both SQ/RQ and TX/RX CQ batching */
147 : fd_mlx5_cq_t rx_cq;
148 : uint lo_tx_cnt;
149 : uint rq_pending_cnt;
150 : long sq_flush_deadline_ticks;
151 : fd_mlx5_tx_qp_t tx_qp;
152 : long lo_tx_deadline_ticks;
153 : int rx_comp_channel_fd; /* completion channel of rx_cq, -1 if not efficient mode */
154 : uint has_out_credit;
155 : long repoll_deadline_ticks; /* LONG_MAX when no repoll is scheduled */
156 : fd_net_router_t router; /* TX IP routing */
157 :
158 : long sq_flush_timeout_ticks;
159 : long lo_tx_timeout_ticks;
160 : int lo_tx_sock;
161 : uint lkey;
162 : uint prepared;
163 : fd_uverbs_ctx_t uverbs;
164 : fd_mlx5_cq_t tx_cq;
165 : fd_mlx5_rss_qp_t outer_rss_qp;
166 : fd_mlx5_rss_qp_t gre_rss_qp;
167 :
168 : /* Packet buffer addressing */
169 : uint * sq_wqe_buf_chunk; /* maps SQ WQEs to packet buffers */
170 :
171 : fd_net_tile_t net;
172 :
173 : /* RQ batching */
174 : uint rq_pending_chunk[ FD_MLX5_BATCH_SIZE ];
175 :
176 : /* Local socket TX, used only by mlx5:0. */
177 : struct mmsghdr lo_tx_msg [ FD_MLX5_BATCH_SIZE ];
178 : struct iovec lo_tx_iov [ FD_MLX5_BATCH_SIZE ];
179 : struct sockaddr_in lo_tx_addr[ FD_MLX5_BATCH_SIZE ];
180 : uchar lo_tx_buf [ FD_MLX5_BATCH_SIZE ][ FD_NET_MTU ];
181 :
182 : /* Efficient mode only */
183 : int epoll_fd;
184 : long repoll_timeout_ticks;
185 :
186 : ulong netdev_seq; /* netdev_shared seqlock value of the router.netdev_tbl copy */
187 :
188 : /* Metric tracking */
189 : struct {
190 : ulong tx_no_buffer_cnt;
191 : } metrics;
192 : };
193 : typedef struct fd_mlx5_tile fd_mlx5_tile_t;
194 :
195 : static ulong
196 : fd_mlx5_queue_footprint( uint rx_depth,
197 0 : uint tx_depth ) {
198 0 : ulong layout = FD_LAYOUT_INIT;
199 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_cqe_t) );
200 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_cqe_t) );
201 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_rx_wqe_t) );
202 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_tx_wqe_t) );
203 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, rx_depth*sizeof(uint) );
204 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, tx_depth*sizeof(uint) );
205 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, FD_MLX5_PAGE_SZ );
206 0 : return FD_LAYOUT_FINI( layout, FD_MLX5_PAGE_SZ );
207 0 : }
208 :
209 : static void
210 : fd_mlx5_hw_invalidate_cqes( fd_mlx5_cqe_t * cqes,
211 0 : uint depth ) {
212 0 : for( uint i=0U; i<depth; i++ ) {
213 0 : fd_mlx5_hw_cqe64_t * hw_cqe = (fd_mlx5_hw_cqe64_t *)(cqes+i);
214 0 : hw_cqe->op_own = (uchar)(FD_MLX5_CQE_OP_INVALID<<4);
215 0 : }
216 0 : }
217 :
218 : static fd_mlx5_tile_t *
219 : fd_mlx5_hw_join_queues( fd_mlx5_tile_t * ctx,
220 : void * queue_memory,
221 : uint rx_depth,
222 0 : uint tx_depth ) {
223 0 : ulong const queue_footprint = fd_mlx5_queue_footprint( rx_depth, tx_depth );
224 0 : FD_SCRATCH_ALLOC_INIT( queue, queue_memory );
225 0 : fd_mlx5_cqe_t * rx_cqes = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_cqe_t) );
226 0 : fd_mlx5_cqe_t * tx_cqes = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_cqe_t) );
227 0 : fd_mlx5_rx_wqe_t * rq = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_rx_wqe_t) );
228 0 : fd_mlx5_tx_wqe_t * sq = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_tx_wqe_t) );
229 0 : uint * rq_wqe_buf_chunk = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, rx_depth*sizeof(uint) );
230 0 : uint * sq_wqe_frame_sz = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, tx_depth*sizeof(uint) );
231 0 : uchar * control = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, FD_MLX5_PAGE_SZ );
232 0 : FD_TEST( FD_SCRATCH_ALLOC_FINI( queue, FD_MLX5_PAGE_SZ )==(ulong)queue_memory+queue_footprint );
233 :
234 0 : ctx->rx_cq.entries = rx_cqes;
235 0 : ctx->rx_cq.control = (fd_mlx5_cq_control_t *)control;
236 0 : ctx->rx_cq.depth = rx_depth;
237 :
238 0 : ctx->tx_cq.entries = tx_cqes;
239 0 : ctx->tx_cq.control = (fd_mlx5_cq_control_t *)(control+sizeof(fd_mlx5_cq_control_t));
240 0 : ctx->tx_cq.depth = tx_depth;
241 :
242 0 : ctx->rx_wq.rq = rq;
243 0 : ctx->rx_wq.rx_cq = &ctx->rx_cq;
244 0 : ctx->rx_wq.rx_depth = rx_depth;
245 0 : ctx->rx_wq.rq_wqe_buf_chunk = rq_wqe_buf_chunk;
246 0 : ctx->rx_wq.control = (fd_mlx5_rx_wq_control_t *)(control+2UL*sizeof(fd_mlx5_cq_control_t));
247 :
248 0 : ctx->tx_qp.sq = sq;
249 0 : ctx->tx_qp.tx_cq = &ctx->tx_cq;
250 0 : ctx->tx_qp.tx_depth = tx_depth;
251 0 : ctx->tx_qp.sq_wqe_frame_sz = sq_wqe_frame_sz;
252 0 : ctx->tx_qp.control = (fd_mlx5_qp_control_t *)(control+2UL*sizeof(fd_mlx5_cq_control_t)+sizeof(fd_mlx5_rx_wq_control_t));
253 0 : return ctx;
254 0 : }
255 :
256 : static fd_mlx5_tile_t *
257 : fd_mlx5_hw_init_queues( fd_mlx5_tile_t * ctx,
258 : void * queue_memory,
259 : uint rx_depth,
260 0 : uint tx_depth ) {
261 0 : fd_memset( queue_memory, 0, fd_mlx5_queue_footprint( rx_depth, tx_depth ) );
262 0 : FD_TEST( fd_mlx5_hw_join_queues( ctx, queue_memory, rx_depth, tx_depth ) );
263 0 : fd_mlx5_hw_invalidate_cqes( ctx->rx_cq.entries, rx_depth );
264 0 : fd_mlx5_hw_invalidate_cqes( ctx->tx_cq.entries, tx_depth );
265 0 : return ctx;
266 0 : }
267 :
268 : /* fd_mlx5_tile_tx_chunk returns the buffer paired with SQ WQE sq_idx.
269 : The buffer is reusable once sq_cons advances past that WQE. */
270 : static inline uint
271 : fd_mlx5_tile_tx_chunk( fd_mlx5_tile_t const * ctx,
272 0 : uint sq_idx ) {
273 0 : return ctx->sq_wqe_buf_chunk[ sq_idx & (ctx->tx_qp.tx_depth-1U) ];
274 0 : }
275 :
276 : static inline void
277 0 : fd_mlx5_hw_dma_to_device( void ) {
278 0 : #if FD_HAS_X86
279 0 : FD_COMPILER_MFENCE();
280 : #elif FD_HAS_ARM
281 : __asm__ __volatile__( "dmb oshst" ::: "memory" );
282 : #elif FD_HAS_RISCV
283 : __asm__ __volatile__( "fence ow,ow" ::: "memory" );
284 : #else
285 : FD_HW_MFENCE_ST();
286 : #endif
287 0 : }
288 :
289 : static inline void
290 0 : fd_mlx5_hw_dma_from_device( void ) {
291 0 : #if FD_HAS_X86
292 0 : __asm__ __volatile__( "lfence" ::: "memory" );
293 : #elif FD_HAS_ARM
294 : __asm__ __volatile__( "dmb oshld" ::: "memory" );
295 : #elif FD_HAS_RISCV
296 : __asm__ __volatile__( "fence ir,ir" ::: "memory" );
297 : #else
298 : FD_HW_MFENCE();
299 : #endif
300 0 : }
301 :
302 : static inline int
303 : fd_mlx5_hw_init_tx_wqe( fd_mlx5_tx_wqe_t * wqe,
304 : uint sq_idx,
305 : uint qpn,
306 : void const * frame,
307 : ulong frame_iova,
308 : ulong frame_sz,
309 : uint lkey,
310 0 : ulong inline_hdr_sz ) {
311 0 : if( FD_UNLIKELY( !frame_sz || frame_sz>UINT_MAX || qpn>0xffffffU ||
312 0 : inline_hdr_sz>frame_sz || frame_iova>ULONG_MAX-inline_hdr_sz ) ) return 0;
313 0 : fd_memset( wqe, 0, sizeof(*wqe) );
314 0 : uint const wqe_seg_cnt = inline_hdr_sz ? 4U : 3U;
315 0 : ulong const wqe_data_seg_off = inline_hdr_sz ? 48UL : 32UL;
316 :
317 0 : fd_mlx5_hw_wqe_ctrl_seg_t * wqe_ctrl_seg = (fd_mlx5_hw_wqe_ctrl_seg_t *)wqe;
318 0 : fd_mlx5_hw_wqe_eth_seg_t * wqe_eth_seg = (fd_mlx5_hw_wqe_eth_seg_t *)(wqe->bytes+sizeof(*wqe_ctrl_seg));
319 0 : fd_mlx5_hw_wqe_data_seg_t * wqe_data_seg = (fd_mlx5_hw_wqe_data_seg_t *)(wqe->bytes+wqe_data_seg_off);
320 :
321 0 : wqe_ctrl_seg->opmod_idx_opcode = fd_uint_bswap( ((sq_idx & 0xffffU)<<8) | FD_MLX5_SQ_SEND_PKT );
322 0 : wqe_ctrl_seg->qpn_ds = fd_uint_bswap( (qpn<<8) | wqe_seg_cnt );
323 :
324 0 : wqe_eth_seg->inline_hdr_sz = fd_ushort_bswap( (ushort)inline_hdr_sz );
325 0 : if( inline_hdr_sz ) fd_memcpy( wqe_eth_seg->inline_hdr, frame, inline_hdr_sz );
326 :
327 0 : wqe_data_seg->byte_cnt = fd_uint_bswap( (uint)(frame_sz-inline_hdr_sz) );
328 0 : wqe_data_seg->lkey = fd_uint_bswap( lkey );
329 0 : wqe_data_seg->addr = fd_ulong_bswap( frame_iova+inline_hdr_sz );
330 0 : return 1;
331 0 : }
332 :
333 : static inline void
334 : fd_mlx5_hw_init_rx_wqe( fd_mlx5_rx_wqe_t * wqe,
335 : ulong frame_iova,
336 : ulong frame_sz,
337 0 : uint lkey ) {
338 0 : fd_mlx5_hw_wqe_data_seg_t * wqe_data_seg = (fd_mlx5_hw_wqe_data_seg_t *)wqe;
339 :
340 0 : wqe_data_seg->byte_cnt = fd_uint_bswap( (uint)frame_sz );
341 0 : wqe_data_seg->lkey = fd_uint_bswap( lkey );
342 0 : wqe_data_seg->addr = fd_ulong_bswap( frame_iova );
343 0 : }
344 :
345 : static inline void
346 : fd_mlx5_hw_ring_sq( fd_mlx5_tx_qp_t * tx_qp,
347 0 : fd_mlx5_tx_wqe_t * wqe ) {
348 0 : fd_mlx5_hw_wqe_ctrl_seg_t * wqe_ctrl_seg = (fd_mlx5_hw_wqe_ctrl_seg_t *)wqe;
349 0 : wqe_ctrl_seg->flags = FD_MLX5_SQ_REQUEST_CQE;
350 :
351 0 : fd_mlx5_hw_dma_to_device();
352 0 : FD_VOLATILE( tx_qp->control->sq_prod ) = fd_uint_bswap( tx_qp->sq_prod & 0xffffU );
353 0 : fd_mlx5_hw_dma_to_device();
354 :
355 : /* Ring the SQ through the non-cached UAR. The NIC fetches the complete
356 : WQE from write-back SQ memory. */
357 0 : volatile ulong * doorbell_reg = (volatile ulong *)tx_qp->sq_doorbell;
358 0 : doorbell_reg[0] = FD_LOAD( ulong, wqe->bytes );
359 0 : FD_COMPILER_MFENCE();
360 0 : tx_qp->sq_posted = tx_qp->sq_prod;
361 0 : }
362 :
363 : static inline int
364 : fd_mlx5_hw_sq_enqueue( fd_mlx5_tile_t * ctx,
365 0 : uint frame_sz ) {
366 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
367 0 : uint const outstanding = tx_qp->sq_prod-tx_qp->sq_cons;
368 0 : if( FD_UNLIKELY( outstanding>=tx_qp->tx_depth ) ) {
369 0 : errno = ENOSPC;
370 0 : return -1;
371 0 : }
372 :
373 0 : uint const sq_idx = tx_qp->sq_prod;
374 0 : uint const chunk = fd_mlx5_tile_tx_chunk( ctx, sq_idx );
375 0 : ulong const frame_iova = (ulong)chunk<<FD_CHUNK_LG_SZ;
376 0 : fd_mlx5_tx_wqe_t * wqe = tx_qp->sq+(sq_idx & (tx_qp->tx_depth-1U));
377 0 : if( FD_UNLIKELY( !fd_mlx5_hw_init_tx_wqe( wqe, sq_idx, tx_qp->qpn,
378 0 : fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, chunk ), frame_iova, frame_sz,
379 0 : ctx->lkey, tx_qp->tx_inline_hdr_sz ) ) ) {
380 0 : errno = EINVAL;
381 0 : return -1;
382 0 : }
383 0 : tx_qp->sq_wqe_frame_sz[ sq_idx & (tx_qp->tx_depth-1U) ] = frame_sz;
384 0 : tx_qp->sq_prod++;
385 0 : return 0;
386 0 : }
387 :
388 : static inline int
389 : fd_mlx5_hw_rq_enqueue( fd_mlx5_tile_t * ctx,
390 0 : uint recv_cnt ) {
391 0 : fd_mlx5_rx_wq_t * rx_wq = &ctx->rx_wq;
392 0 : uint const outstanding = rx_wq->rq_prod-rx_wq->rq_cons;
393 0 : if( FD_UNLIKELY( outstanding>rx_wq->rx_depth || recv_cnt>rx_wq->rx_depth-outstanding ) ) {
394 0 : errno = ENOSPC;
395 0 : return -1;
396 0 : }
397 :
398 0 : uint const rq_prod = rx_wq->rq_prod;
399 0 : for( uint i=0U; i<recv_cnt; i++ ) {
400 0 : uint const chunk = ctx->rq_pending_chunk[ i ];
401 0 : uint const rq_idx = rq_prod+i;
402 0 : fd_mlx5_hw_init_rx_wqe( rx_wq->rq+(rq_idx & (rx_wq->rx_depth-1U)),
403 0 : (ulong)chunk<<FD_CHUNK_LG_SZ, FD_NET_MTU, ctx->lkey );
404 0 : rx_wq->rq_wqe_buf_chunk[ rq_idx & (rx_wq->rx_depth-1U) ] = chunk;
405 0 : }
406 0 : rx_wq->rq_prod += recv_cnt;
407 0 : fd_mlx5_hw_dma_to_device();
408 0 : FD_VOLATILE( rx_wq->control->rq_prod ) = fd_uint_bswap( rx_wq->rq_prod & 0xffffU );
409 0 : return 0;
410 0 : }
411 :
412 : static inline int
413 : fd_mlx5_hw_rx_cqe_is_owned( fd_mlx5_cq_t const * rx_cq,
414 : uint cq_cons_idx,
415 0 : uchar op_own ) {
416 0 : return (op_own>>4)!=FD_MLX5_CQE_OP_INVALID &&
417 0 : (op_own & FD_MLX5_CQE_OWNER_MASK)==!!(cq_cons_idx & rx_cq->depth);
418 0 : }
419 :
420 : static inline int
421 : fd_mlx5_hw_poll_rx_cq( fd_mlx5_rx_wq_t * rx_wq,
422 : fd_mlx5_tile_rx_comp_t * comp,
423 0 : uint comp_capacity ) {
424 0 : fd_mlx5_cq_t * rx_cq = rx_wq->rx_cq;
425 0 : uint const comp_limit = fd_uint_min( comp_capacity, rx_cq->depth );
426 0 : uint cq_cons_idx = rx_cq->cons_idx;
427 0 : uint rq_cons_idx = rx_wq->rq_cons;
428 0 : uint const rq_prod_idx = rx_wq->rq_prod;
429 0 : FD_TEST( rq_prod_idx-rq_cons_idx<=rx_wq->rx_depth );
430 :
431 0 : uint comp_cnt = 0U;
432 0 : int poll_failed = 0;
433 0 : while( comp_cnt<comp_limit ) {
434 0 : fd_mlx5_hw_cqe64_t const * rx_cqe = (fd_mlx5_hw_cqe64_t const *)(rx_cq->entries+(cq_cons_idx & (rx_cq->depth-1U)));
435 0 : uchar const op_own = FD_VOLATILE_CONST( rx_cqe->op_own );
436 0 : if( FD_UNLIKELY( !fd_mlx5_hw_rx_cqe_is_owned( rx_cq, cq_cons_idx, op_own ) ) ) break;
437 0 : fd_mlx5_hw_dma_from_device();
438 0 : uint const opcode = (uint)(op_own>>4);
439 :
440 0 : if( FD_UNLIKELY( opcode!=FD_MLX5_CQE_OP_RX_OK && opcode!=FD_MLX5_CQE_OP_RX_ERR ) ) {
441 0 : errno = EINVAL;
442 0 : poll_failed = 1;
443 0 : break;
444 0 : }
445 0 : if( FD_UNLIKELY( opcode==FD_MLX5_CQE_OP_RX_ERR &&
446 0 : rx_cqe->syndrome!=FD_MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR ) ) {
447 0 : FD_LOG_ERR(( "mlx5 RX CQE error (syndrome 0x%02x, vendor syndrome 0x%02x, "
448 0 : "WQE opcode/QPN 0x%08x, WQE counter %u)",
449 0 : (uint)rx_cqe->syndrome, (uint)rx_cqe->vendor_err, fd_uint_bswap( rx_cqe->sop_drop_qpn ),
450 0 : (uint)fd_ushort_bswap( rx_cqe->wqe_counter ) ));
451 0 : }
452 :
453 0 : ushort const wqe_counter = fd_ushort_bswap( rx_cqe->wqe_counter );
454 0 : if( FD_UNLIKELY( rq_cons_idx==rq_prod_idx || wqe_counter!=(ushort)rq_cons_idx ) ) {
455 0 : errno = EPROTO;
456 0 : poll_failed = 1;
457 0 : break;
458 0 : }
459 :
460 0 : comp[ comp_cnt++ ] = (fd_mlx5_tile_rx_comp_t) {
461 0 : .chunk = rx_wq->rq_wqe_buf_chunk[ wqe_counter & (rx_wq->rx_depth-1U) ],
462 0 : .byte_len = fd_uint_bswap( rx_cqe->byte_cnt ),
463 0 : .opcode = opcode
464 0 : };
465 0 : rq_cons_idx++;
466 0 : cq_cons_idx++;
467 0 : }
468 :
469 0 : if( comp_cnt ) {
470 0 : rx_wq->rq_cons = rq_cons_idx;
471 0 : rx_cq->cons_idx = cq_cons_idx;
472 0 : FD_COMPILER_MFENCE();
473 0 : FD_VOLATILE( rx_cq->control->consumer_idx ) = fd_uint_bswap( cq_cons_idx & FD_MLX5_CQ_CONS_IDX_MASK );
474 0 : }
475 0 : return poll_failed ? -1 : (int)comp_cnt;
476 0 : }
477 :
478 : static inline void
479 0 : fd_mlx5_hw_request_rx_notification( fd_mlx5_cq_t * rx_cq ) {
480 0 : uint const notification_request = ((rx_cq->comp_channel_event_seq & FD_MLX5_CQ_ARM_SN_MASK)<<FD_MLX5_CQ_ARM_SN_SHIFT) |
481 0 : (rx_cq->cons_idx & FD_MLX5_CQ_CONS_IDX_MASK);
482 0 : FD_VOLATILE( rx_cq->control->request_notification ) = fd_uint_bswap( notification_request );
483 0 : fd_mlx5_hw_dma_to_device();
484 :
485 0 : uint const doorbell[2] = { fd_uint_bswap( notification_request ), fd_uint_bswap( rx_cq->cqn ) };
486 0 : volatile ulong * doorbell_reg = (volatile ulong *)rx_cq->request_notification_doorbell;
487 0 : doorbell_reg[0] = FD_LOAD( ulong, doorbell );
488 0 : rx_cq->comp_channel_armed = 1U;
489 0 : }
490 :
491 : static inline int
492 : fd_mlx5_hw_poll_tx_cq( fd_mlx5_tx_qp_t * tx_qp,
493 0 : ulong * comp_bytes ) {
494 0 : fd_mlx5_cq_t * tx_cq = tx_qp->tx_cq;
495 0 : uint const cq_cons_idx = tx_cq->cons_idx;
496 0 : uint const cq_entry_idx = cq_cons_idx & (tx_cq->depth-1U);
497 :
498 0 : fd_mlx5_hw_cqe64_t const * tx_cqe = (fd_mlx5_hw_cqe64_t const *)(tx_cq->entries+cq_entry_idx);
499 :
500 0 : uchar const op_own = FD_VOLATILE_CONST( tx_cqe->op_own );
501 0 : uint const opcode = (uint)(op_own>>4);
502 :
503 0 : if( FD_LIKELY( opcode==FD_MLX5_CQE_OP_INVALID ||
504 0 : (op_own & FD_MLX5_CQE_OWNER_MASK)!=!!(cq_cons_idx & tx_cq->depth) ) ) {
505 0 : return 0;
506 0 : }
507 0 : fd_mlx5_hw_dma_from_device();
508 :
509 0 : if( FD_UNLIKELY( opcode!=FD_MLX5_CQE_OP_TX_OK && opcode!=FD_MLX5_CQE_OP_TX_ERR ) ) {
510 0 : errno = EINVAL;
511 0 : return -1;
512 0 : }
513 :
514 0 : if( FD_UNLIKELY( opcode==FD_MLX5_CQE_OP_TX_ERR ) ) {
515 0 : FD_LOG_ERR(( "mlx5 TX CQE error (syndrome 0x%02x, vendor syndrome 0x%02x, "
516 0 : "WQE opcode/QPN 0x%08x, WQE counter %u)",
517 0 : (uint)tx_cqe->syndrome, (uint)tx_cqe->vendor_err, fd_uint_bswap( tx_cqe->sop_drop_qpn ),
518 0 : (uint)fd_ushort_bswap( tx_cqe->wqe_counter ) ));
519 0 : }
520 :
521 0 : uint const sq_cons_idx = tx_qp->sq_cons;
522 0 : uint const wqe_counter = (uint)fd_ushort_bswap( tx_cqe->wqe_counter );
523 0 : uint const outstanding = tx_qp->sq_posted-sq_cons_idx;
524 0 : uint const comp_cnt = (uint)(ushort)(wqe_counter-sq_cons_idx)+1U;
525 :
526 0 : if( FD_UNLIKELY( !outstanding || outstanding>tx_qp->tx_depth || comp_cnt>outstanding ) ) {
527 0 : errno = EPROTO;
528 0 : return -1;
529 0 : }
530 :
531 0 : *comp_bytes = 0UL;
532 0 : for( uint i=0U; i<comp_cnt; i++ ) {
533 0 : *comp_bytes += tx_qp->sq_wqe_frame_sz[ (sq_cons_idx+i) & (tx_qp->tx_depth-1U) ];
534 0 : }
535 0 : tx_qp->sq_cons = sq_cons_idx+comp_cnt;
536 0 : tx_cq->cons_idx = cq_cons_idx+1U;
537 0 : FD_COMPILER_MFENCE();
538 0 : FD_VOLATILE( tx_cq->control->consumer_idx ) = fd_uint_bswap( (cq_cons_idx+1U) & FD_MLX5_CQ_CONS_IDX_MASK );
539 0 : return (int)comp_cnt;
540 0 : }
541 :
542 : static inline void
543 0 : fd_mlx5_tile_sq_flush( fd_mlx5_tile_t * ctx ) {
544 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
545 0 : if( FD_UNLIKELY( tx_qp->sq_posted==tx_qp->sq_prod ) ) return;
546 :
547 0 : uint const last_sq_idx = tx_qp->sq_prod-1U;
548 0 : fd_mlx5_tx_wqe_t * last_wqe = tx_qp->sq+(last_sq_idx & (tx_qp->tx_depth-1U));
549 0 : fd_mlx5_hw_ring_sq( tx_qp, last_wqe );
550 0 : }
551 :
552 : static inline void
553 : fd_mlx5_tile_rx_recycle( fd_mlx5_tile_t * ctx,
554 0 : ulong chunk ) {
555 0 : ctx->rq_pending_chunk[ ctx->rq_pending_cnt++ ] = (uint)chunk;
556 :
557 : /* Hold partial recycle batches: at most batch_size-1 RQ buffers stay
558 : unposted, avoiding a device-ordering barrier and doorbell-record write
559 : per packet. */
560 0 : if( ctx->rq_pending_cnt==ctx->batch_size ) {
561 0 : FD_TEST( !fd_mlx5_hw_rq_enqueue( ctx, ctx->rq_pending_cnt ) );
562 0 : ctx->rq_pending_cnt = 0U;
563 0 : }
564 0 : }
565 :
566 : static inline int
567 : fd_mlx5_tile_poll_rx( fd_mlx5_tile_t * ctx,
568 0 : fd_stem_context_t * stem ) {
569 0 : fd_mlx5_tile_rx_comp_t comp[ FD_MLX5_BATCH_SIZE ];
570 0 : int comp_cnt = fd_mlx5_hw_poll_rx_cq( &ctx->rx_wq, comp, ctx->batch_size );
571 :
572 0 : if( FD_UNLIKELY( comp_cnt<0 ) ) {
573 0 : FD_LOG_ERR(( "direct mlx5 RX CQ poll failed (%i-%s)", errno, fd_io_strerror( errno ) ));
574 0 : }
575 0 : if( FD_UNLIKELY( !comp_cnt ) ) return 0;
576 :
577 0 : ulong const tspub = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
578 :
579 0 : for( uint i=0U; i<(uint)comp_cnt; i++ ) {
580 0 : ulong const chunk = comp[ i ].chunk;
581 0 : if( FD_UNLIKELY( chunk<ctx->net.pkt_buf_chunk0 || chunk>ctx->net.pkt_buf_wmark ) ) {
582 0 : FD_LOG_CRIT(( "RX completion chunk %lu out of bounds [%lu,%lu]", chunk, ctx->net.pkt_buf_chunk0, ctx->net.pkt_buf_wmark ));
583 0 : }
584 0 : __builtin_prefetch( fd_chunk_to_laddr_const( ctx->net.pkt_buf_wksp_base, chunk ), 0, 3 );
585 0 : }
586 :
587 0 : for( uint i=0U; i<(uint)comp_cnt; i++ ) {
588 0 : ulong freed_chunk;
589 0 : if( FD_UNLIKELY( comp[ i ].opcode==FD_MLX5_CQE_OP_RX_ERR ) ) {
590 0 : ctx->net.metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
591 0 : freed_chunk = comp[ i ].chunk;
592 0 : } else {
593 0 : fd_net_rx_pkt( &ctx->net, stem, comp[ i ].chunk, 0UL, comp[ i ].byte_len, tspub, &freed_chunk );
594 0 : }
595 0 : if( FD_UNLIKELY( !freed_chunk ) ) FD_LOG_CRIT(( "invalid chunk in mcache" ));
596 0 : fd_mlx5_tile_rx_recycle( ctx, freed_chunk );
597 0 : }
598 0 : return 1;
599 0 : }
600 :
601 : static inline int
602 0 : fd_mlx5_tile_poll_tx( fd_mlx5_tile_t * ctx ) {
603 0 : ulong tx_pkt_cnt = 0UL;
604 0 : ulong tx_bytes_total = 0UL;
605 :
606 0 : int busy = 0;
607 0 : for( uint poll_cnt=0U; poll_cnt<ctx->batch_size; poll_cnt++ ) {
608 0 : ulong tx_bytes;
609 :
610 0 : int comp_cnt = fd_mlx5_hw_poll_tx_cq( &ctx->tx_qp, &tx_bytes );
611 0 : if( FD_UNLIKELY( comp_cnt<0 ) ) {
612 0 : FD_LOG_ERR(( "direct mlx5 TX CQ poll failed (%i-%s)", errno, fd_io_strerror( errno ) ));
613 0 : }
614 0 : if( FD_UNLIKELY( !comp_cnt ) ) break;
615 :
616 0 : busy = 1;
617 0 : tx_pkt_cnt += (uint)comp_cnt;
618 0 : tx_bytes_total += tx_bytes;
619 0 : }
620 0 : ctx->net.metrics.tx_pkt_cnt += tx_pkt_cnt;
621 0 : ctx->net.metrics.tx_bytes_total += tx_bytes_total;
622 :
623 0 : return busy;
624 0 : }
625 :
626 : /* fd_mlx5_tile_lo_tx_flush makes one nonblocking send attempt per batch,
627 : unsent packets are dropped. */
628 : static void
629 0 : fd_mlx5_tile_lo_tx_flush( fd_mlx5_tile_t * ctx ) {
630 0 : if( FD_UNLIKELY( !ctx->lo_tx_cnt ) ) return;
631 0 : int const send_cnt = sendmmsg( ctx->lo_tx_sock, ctx->lo_tx_msg, ctx->lo_tx_cnt, MSG_DONTWAIT );
632 0 : uint const sent_cnt = send_cnt<0 ? 0U : (uint)send_cnt;
633 0 : for( uint i=0U; i<sent_cnt; i++ ) ctx->net.metrics.tx_bytes_total += sizeof(fd_eth_hdr_t)+ctx->lo_tx_iov[ i ].iov_len;
634 0 : ctx->net.metrics.tx_pkt_cnt += sent_cnt;
635 0 : ctx->lo_tx_cnt = 0U;
636 0 : }
637 :
638 : static void
639 : fd_mlx5_tile_lo_tx_enqueue( fd_mlx5_tile_t * ctx,
640 : fd_ip4_hdr_t const * ip4,
641 0 : ulong ip_sz ) {
642 0 : uint const batch_idx = ctx->lo_tx_cnt;
643 0 : struct mmsghdr * msg = ctx->lo_tx_msg + batch_idx;
644 0 : struct sockaddr_in * sa = ctx->lo_tx_addr + batch_idx;
645 0 : struct iovec * iov = ctx->lo_tx_iov + batch_idx;
646 0 : uchar * buf = ctx->lo_tx_buf[ batch_idx ];
647 :
648 0 : *iov = (struct iovec) {
649 0 : .iov_base = buf,
650 0 : .iov_len = ip_sz,
651 0 : };
652 0 : sa->sin_family = AF_INET;
653 0 : sa->sin_addr.s_addr = ip4->daddr;
654 0 : sa->sin_port = 0; /* ignored */
655 :
656 0 : *msg = (struct mmsghdr) {
657 0 : .msg_hdr = {
658 0 : .msg_name = sa,
659 0 : .msg_namelen = sizeof(struct sockaddr_in),
660 0 : .msg_iov = iov,
661 0 : .msg_iovlen = 1UL
662 0 : }
663 0 : };
664 :
665 0 : fd_memcpy( buf, ip4, ip_sz );
666 0 : ctx->lo_tx_cnt++;
667 0 : if( ctx->lo_tx_cnt==FD_MLX5_BATCH_SIZE ) fd_mlx5_tile_lo_tx_flush( ctx );
668 0 : else if( ctx->lo_tx_cnt==1U ) ctx->lo_tx_deadline_ticks = fd_tickcount()+ctx->lo_tx_timeout_ticks;
669 0 : }
670 :
671 : /* fd_mlx5_tile_drain_rx_comp_channel reads every pending rx_cq completion event.
672 : Each event consumes the CQ's arm, so comp_channel_event_seq advances. */
673 : static void
674 0 : fd_mlx5_tile_drain_rx_comp_channel( fd_mlx5_tile_t * ctx ) {
675 0 : for(;;) {
676 0 : struct ib_uverbs_comp_event_desc comp_event;
677 0 : ssize_t comp_event_read_sz = read( ctx->rx_comp_channel_fd, &comp_event, sizeof(comp_event) );
678 0 : if( FD_UNLIKELY( comp_event_read_sz<0 && errno==EINTR ) ) continue;
679 :
680 0 : if( FD_LIKELY( comp_event_read_sz<0 && (errno==EAGAIN || errno==EWOULDBLOCK) ) ) break;
681 0 : if( FD_UNLIKELY( comp_event_read_sz!=(ssize_t)sizeof(comp_event) ) ) {
682 0 : FD_LOG_ERR(( "mlx5 RX completion event read failed (%i-%s)", errno, fd_io_strerror( errno ) ));
683 0 : }
684 0 : ctx->rx_cq.comp_channel_event_seq++;
685 0 : ctx->rx_cq.comp_channel_armed = 0U;
686 0 : }
687 0 : }
688 :
689 : static inline void
690 : fd_mlx5_tile_repoll_deadline_update( fd_mlx5_tile_t * ctx,
691 : int work_done,
692 0 : long now ) {
693 0 : if( FD_LIKELY( work_done ) ) {
694 0 : ctx->repoll_deadline_ticks = now + ctx->repoll_timeout_ticks;
695 0 : } else if( FD_UNLIKELY( now>=ctx->repoll_deadline_ticks ) ) {
696 0 : ctx->repoll_deadline_ticks = LONG_MAX;
697 0 : }
698 0 : }
699 :
700 : static inline void
701 : before_credit( fd_mlx5_tile_t * ctx,
702 : fd_stem_context_t * stem,
703 0 : int * charge_busy ) {
704 0 : ctx->has_out_credit = 0U;
705 0 : fd_net_router_solicit( &ctx->router, stem );
706 :
707 0 : if( FD_UNLIKELY( ctx->lo_tx_cnt && fd_tickcount()>=ctx->lo_tx_deadline_ticks ) ) {
708 0 : fd_mlx5_tile_lo_tx_flush( ctx );
709 0 : *charge_busy = 1;
710 0 : }
711 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
712 0 : uint const sq_pending_cnt = tx_qp->sq_prod-tx_qp->sq_posted;
713 :
714 0 : if( FD_UNLIKELY( sq_pending_cnt && fd_tickcount()>=ctx->sq_flush_deadline_ticks ) ) {
715 0 : fd_mlx5_tile_sq_flush( ctx );
716 0 : *charge_busy = 1;
717 0 : }
718 0 : if( tx_qp->sq_cons!=tx_qp->sq_posted ) {
719 0 : *charge_busy |= fd_mlx5_tile_poll_tx( ctx );
720 0 : }
721 0 : }
722 :
723 : /* after_credit is called every run loop iteration, provided there is
724 : sufficient downstream credit for forwarding on all output links. */
725 : static inline void
726 : after_credit( fd_mlx5_tile_t * ctx,
727 : fd_stem_context_t * stem,
728 : int * poll_in,
729 0 : int * charge_busy ) {
730 0 : (void)poll_in;
731 0 : ctx->has_out_credit = 1U;
732 0 : int rx_busy = fd_mlx5_tile_poll_rx( ctx, stem );
733 0 : if( FD_UNLIKELY( ctx->rx_comp_channel_fd>=0 ) ) {
734 0 : fd_mlx5_tile_repoll_deadline_update( ctx, rx_busy, fd_tickcount() );
735 0 : }
736 0 : *charge_busy |= rx_busy;
737 0 : }
738 :
739 : static void
740 0 : fd_mlx5_tile_async_drain( fd_mlx5_tile_t * ctx ) {
741 0 : int const async_event_fd = ctx->uverbs.async_fd;
742 0 : for(;;) {
743 0 : struct ib_uverbs_async_event_desc async_event;
744 0 : ssize_t async_event_read_sz = read( async_event_fd, &async_event, sizeof(async_event) );
745 0 : if( FD_UNLIKELY( async_event_read_sz<0 && errno==EINTR ) ) continue;
746 :
747 0 : if( FD_LIKELY( async_event_read_sz<0 && (errno==EAGAIN || errno==EWOULDBLOCK) ) ) break;
748 0 : if( FD_UNLIKELY( async_event_read_sz!=(ssize_t)sizeof(async_event) ) ) {
749 0 : FD_LOG_ERR(( "mlx5 async event read failed (%i-%s)", errno, fd_io_strerror( errno ) ));
750 0 : }
751 :
752 0 : uint const async_event_type = async_event.event_type;
753 0 : ulong const async_event_element = (ulong)async_event.element;
754 0 : if( FD_UNLIKELY( async_event_type==FD_MLX5_ASYNC_EVENT_CQ_ERR ||
755 0 : async_event_type==FD_MLX5_ASYNC_EVENT_QP_FATAL ||
756 0 : async_event_type==FD_MLX5_ASYNC_EVENT_QP_REQ_ERR ||
757 0 : async_event_type==FD_MLX5_ASYNC_EVENT_QP_ACCESS_ERR ||
758 0 : async_event_type==FD_MLX5_ASYNC_EVENT_DEVICE_FATAL ||
759 0 : async_event_type==FD_MLX5_ASYNC_EVENT_WQ_FATAL ) ) {
760 0 : FD_LOG_ERR(( "fatal mlx5 async event %u on element %lu", async_event_type, async_event_element ));
761 0 : }
762 0 : FD_LOG_INFO(( "mlx5 async event %u on element %lu", async_event_type, async_event_element ));
763 0 : }
764 0 : }
765 :
766 : static inline int
767 0 : prevent_park( fd_mlx5_tile_t * ctx ) {
768 0 : if( FD_UNLIKELY( ctx->tx_qp.sq_prod!=ctx->tx_qp.sq_posted ) ) fd_mlx5_tile_sq_flush( ctx );
769 0 : if( FD_UNLIKELY( ctx->lo_tx_cnt ) ) fd_mlx5_tile_lo_tx_flush( ctx );
770 0 : if( FD_UNLIKELY( !ctx->has_out_credit ) ) return 0;
771 0 : fd_mlx5_cq_t * rx_cq = &ctx->rx_cq;
772 0 : if( FD_UNLIKELY( !rx_cq->comp_channel_armed &&
773 0 : ctx->repoll_deadline_ticks==LONG_MAX ) ) {
774 0 : fd_mlx5_hw_request_rx_notification( rx_cq );
775 0 : }
776 0 : uint const cq_cons_idx = rx_cq->cons_idx;
777 0 : fd_mlx5_hw_cqe64_t const * rx_cqe = (fd_mlx5_hw_cqe64_t const *)(rx_cq->entries+(cq_cons_idx & (rx_cq->depth-1U)));
778 0 : uchar const op_own = FD_VOLATILE_CONST( rx_cqe->op_own );
779 0 : return fd_mlx5_hw_rx_cqe_is_owned( rx_cq, cq_cons_idx, op_own );
780 0 : }
781 :
782 : static inline int
783 : park_wait( fd_mlx5_tile_t * ctx,
784 : ulong * word,
785 : long deadline,
786 0 : double tick_per_ns ) {
787 0 : if( FD_UNLIKELY( ctx->has_out_credit && ctx->repoll_deadline_ticks!=LONG_MAX ) ) {
788 0 : struct timespec timeout;
789 0 : clock_gettime( CLOCK_MONOTONIC, &timeout );
790 0 : long const remaining_ns = (long)((double)(ctx->repoll_deadline_ticks-fd_tickcount())/tick_per_ns);
791 0 : if( FD_UNLIKELY( remaining_ns<=0L ) ) return FD_SLEEP_UNPARK_DEADLINE;
792 0 : timeout.tv_sec += remaining_ns/FD_MLX5_NS_PER_SEC;
793 0 : timeout.tv_nsec += remaining_ns%FD_MLX5_NS_PER_SEC;
794 0 : if( timeout.tv_nsec>=FD_MLX5_NS_PER_SEC ) {
795 0 : timeout.tv_sec++;
796 0 : timeout.tv_nsec -= FD_MLX5_NS_PER_SEC;
797 0 : }
798 0 : for(;;) {
799 0 : int const err = clock_nanosleep( CLOCK_MONOTONIC, TIMER_ABSTIME, &timeout, NULL );
800 0 : if( FD_LIKELY( !err ) ) return FD_SLEEP_UNPARK_DEADLINE;
801 0 : if( FD_UNLIKELY( err!=EINTR ) ) {
802 0 : FD_LOG_ERR(( "clock_nanosleep() failed (%i-%s)", err, fd_io_strerror( err ) ));
803 0 : }
804 0 : }
805 0 : }
806 :
807 0 : struct epoll_event evs[ 3 ];
808 0 : int ev_cnt;
809 0 : int cause = fd_sleep_park_wait_epoll( ctx->epoll_fd, word, evs, 3, &ev_cnt, deadline, tick_per_ns );
810 0 : for( int i=0; i<ev_cnt; i++ ) {
811 0 : if( evs[ i ].data.u64==(ulong)ctx->rx_comp_channel_fd ) fd_mlx5_tile_drain_rx_comp_channel( ctx );
812 0 : else if( evs[ i ].data.u64==FD_MLX5_EPOLL_ASYNC ) fd_mlx5_tile_async_drain( ctx );
813 0 : }
814 0 : return cause;
815 0 : }
816 :
817 : static inline long
818 0 : next_deadline( fd_mlx5_tile_t * ctx ) {
819 0 : if( !ctx->has_out_credit ) return LONG_MAX; /* If no out credit then don't repoll */
820 :
821 0 : return ctx->repoll_deadline_ticks; /* LONG_MAX if no repoll */
822 0 : }
823 :
824 : /* before_frag resolves the TX route and checks SQ capacity */
825 : static inline int
826 : before_frag( fd_mlx5_tile_t * ctx,
827 : ulong in_idx,
828 : ulong seq,
829 0 : ulong sig ) {
830 0 : (void)seq;
831 0 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) return 0;
832 :
833 : /* Resolve the TX route for outgoing packets */
834 0 : ulong dst_proto = fd_disco_netmux_sig_proto( sig );
835 0 : if( FD_UNLIKELY( dst_proto!=DST_PROTO_OUTGOING ) ) return 1;
836 :
837 0 : ulong const hash = fd_disco_netmux_sig_hash( sig );
838 0 : ulong target_idx = hash % ctx->net.tile_cnt;
839 0 : ulong const kind_id = ctx->net.kind_id;
840 0 : if( kind_id!=0UL && kind_id!=target_idx ) return 1;
841 :
842 0 : uint dst_ip = fd_disco_netmux_sig_ip( sig );
843 0 : fd_net_tx_route_t * route = &ctx->net.tx_route;
844 0 : if( FD_UNLIKELY( !fd_net_tx_route( &ctx->router, &ctx->net, dst_ip, route ) ) ) return 1;
845 0 : if( FD_UNLIKELY( route->use_gre ) ) {
846 0 : fd_net_tx_route_t outer_route;
847 0 : uint const inner_src_ip = route->src_ip;
848 0 : uint const outer_src_ip = route->gre_outer_src_ip;
849 0 : uint const outer_dst_ip = route->gre_outer_dst_ip;
850 :
851 0 : if( FD_UNLIKELY( !inner_src_ip || !outer_dst_ip ) ) {
852 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
853 0 : return 1;
854 0 : }
855 : /* fd_net_tx_route counts its own drop reason */
856 0 : if( FD_UNLIKELY( !fd_net_tx_route( &ctx->router, &ctx->net, outer_dst_ip, &outer_route ) ) ) return 1;
857 0 : if( FD_UNLIKELY( outer_route.use_gre || outer_route.use_loopback ) ) {
858 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_GRE_NO_ROUTE_IDX ]++;
859 0 : return 1;
860 0 : }
861 0 : *route = outer_route;
862 0 : route->src_ip = inner_src_ip;
863 0 : route->gre_outer_src_ip = fd_uint_if( !outer_src_ip, outer_route.src_ip, outer_src_ip );
864 0 : route->gre_outer_dst_ip = outer_dst_ip;
865 0 : route->use_gre = 1U;
866 0 : }
867 :
868 0 : if( route->use_loopback ) target_idx = 0UL;
869 0 : if( kind_id!=target_idx ) return 1;
870 :
871 : /* Drop the packet if no SQ WQE is available. */
872 0 : fd_mlx5_tx_qp_t const * tx_qp = &ctx->tx_qp;
873 0 : uint const outstanding = tx_qp->sq_prod-tx_qp->sq_cons;
874 0 : if( FD_UNLIKELY( outstanding>=tx_qp->tx_depth ) ) {
875 0 : ctx->metrics.tx_no_buffer_cnt++;
876 0 : return 1;
877 0 : }
878 :
879 0 : return 0; /* continue */
880 0 : }
881 :
882 : /* during_frag validates and stages the input packet or route update */
883 : static inline void
884 : during_frag( fd_mlx5_tile_t * ctx,
885 : ulong in_idx,
886 : ulong seq,
887 : ulong sig,
888 : ulong chunk,
889 : ulong frame_sz,
890 0 : ulong ctl ) {
891 0 : (void)seq; (void)sig; (void)ctl;
892 0 : fd_net_in_frag_validate( &ctx->net, in_idx, chunk, frame_sz );
893 :
894 0 : if( FD_UNLIKELY( fd_net_iproute_msg_stage( &ctx->net, in_idx, chunk, frame_sz ) ) ) return;
895 :
896 0 : ulong dst_chunk = fd_mlx5_tile_tx_chunk( ctx, ctx->tx_qp.sq_prod );
897 0 : uchar * dst = fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, dst_chunk );
898 : /* Speculatively copy frame from in link into buffer */
899 0 : fd_net_tx_pkt_cpy( &ctx->net, dst, chunk, frame_sz, in_idx );
900 0 : }
901 :
902 : /* after_frag applies a route update or completes and submits the staged packet */
903 : static void
904 : after_frag( fd_mlx5_tile_t * ctx,
905 : ulong in_idx,
906 : ulong seq,
907 : ulong sig,
908 : ulong frame_sz,
909 : ulong tsorig,
910 : ulong tspub,
911 0 : fd_stem_context_t * stem ) {
912 0 : (void)seq; (void)sig; (void)tsorig; (void)tspub;
913 :
914 0 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) {
915 0 : fd_iproute_msg_t const * msg = &ctx->net.iproute_msg;
916 0 : if( msg->op==FD_IPROUTE_OP_FLUSH ) {
917 0 : fd_fib4_clear( ctx->router.fib_local );
918 0 : fd_fib4_clear( ctx->router.fib_main );
919 0 : return;
920 0 : }
921 :
922 0 : fd_fib4_t * fib;
923 0 : if( msg->table_id==RT_TABLE_LOCAL ) fib = ctx->router.fib_local;
924 0 : else if( msg->table_id==RT_TABLE_MAIN ) fib = ctx->router.fib_main;
925 0 : else return;
926 :
927 0 : if( msg->op==FD_IPROUTE_OP_UPSERT && FD_UNLIKELY( !fd_fib4_insert( fib, msg->dst_addr, msg->prefix, msg->prio, &msg->hop ) ) ) {
928 0 : FD_LOG_WARNING(( "route update dropped: route table full (increase [net.max_routes] or [net.max_peer_routes])" ));
929 0 : fd_stem_publish( stem, ctx->router.netlnk_out_idx, FD_NETLINK_ROUTE4_SYNC_SIG, 0UL, 0UL, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
930 0 : } else if( msg->op==FD_IPROUTE_OP_DELETE ) {
931 0 : fd_fib4_remove( fib, msg->dst_addr, msg->prefix, msg->prio );
932 0 : }
933 0 : return;
934 0 : }
935 :
936 0 : ulong chunk = fd_mlx5_tile_tx_chunk( ctx, ctx->tx_qp.sq_prod );
937 0 : uchar * frame = fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, chunk );
938 :
939 0 : if( FD_UNLIKELY( !fd_net_tx_pkt_prep( &ctx->net, frame, &frame_sz, in_idx ) ) ) return;
940 :
941 0 : if( FD_UNLIKELY( ctx->net.tx_route.use_loopback ) ) {
942 0 : fd_eth_hdr_t const * eth_hdr = (fd_eth_hdr_t const *)frame;
943 0 : fd_ip4_hdr_t const * ip4 = (fd_ip4_hdr_t const *)(eth_hdr+1);
944 0 : ulong const ip_hdr_sz = FD_IP4_GET_LEN( *ip4 );
945 0 : ulong const ip_sz = fd_ushort_bswap( ip4->net_tot_len );
946 0 : if( FD_UNLIKELY( ip4->protocol!=FD_IP4_HDR_PROTOCOL_UDP ||
947 0 : ip4->daddr!=fd_disco_netmux_sig_ip( sig ) ||
948 0 : (fd_ushort_bswap( ip4->net_frag_off ) & ~FD_IP4_HDR_FRAG_OFF_DF) ||
949 0 : ip_sz<ip_hdr_sz+sizeof(fd_udp_hdr_t) ||
950 0 : ip_sz>frame_sz-sizeof(fd_eth_hdr_t) ) ) {
951 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
952 0 : return;
953 0 : }
954 0 : fd_udp_hdr_t const * udp = (fd_udp_hdr_t const *)((uchar const *)ip4+ip_hdr_sz);
955 0 : ulong const udp_sz = fd_ushort_bswap( udp->net_len );
956 0 : if( FD_UNLIKELY( udp_sz<sizeof(fd_udp_hdr_t) || udp_sz>ip_sz-ip_hdr_sz ) ) {
957 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
958 0 : return;
959 0 : }
960 :
961 0 : int owned = 0;
962 0 : if( !ctx->router.bind_address || ctx->router.bind_address==ip4->daddr ) {
963 0 : ushort const dst_port = fd_ushort_bswap( udp->net_dport );
964 0 : for( ulong i=0UL; i<ctx->net.dst_port_cnt; i++ ) owned |= ctx->net.dst_ports[ i ]==dst_port;
965 0 : }
966 0 : if( !owned ) {
967 0 : fd_mlx5_tile_lo_tx_enqueue( ctx, ip4, ip_sz );
968 0 : return;
969 0 : }
970 :
971 0 : ulong freed_chunk;
972 0 : if( fd_net_rx_pkt( &ctx->net, stem, chunk, 0UL, frame_sz, (ulong)fd_frag_meta_ts_comp( fd_tickcount() ), &freed_chunk ) ) {
973 0 : ctx->sq_wqe_buf_chunk[ ctx->tx_qp.sq_prod & (ctx->tx_qp.tx_depth-1U) ] = (uint)freed_chunk;
974 0 : }
975 0 : ctx->net.metrics.tx_pkt_cnt++;
976 0 : ctx->net.metrics.tx_bytes_total += frame_sz;
977 0 : return;
978 0 : }
979 :
980 0 : FD_TEST( !fd_mlx5_hw_sq_enqueue( ctx, (uint)frame_sz ) );
981 :
982 0 : ctx->net.metrics.tx_gre_cnt += (ulong)ctx->net.tx_route.use_gre;
983 :
984 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
985 0 : uint const sq_pending_cnt = tx_qp->sq_prod-tx_qp->sq_posted;
986 :
987 0 : if( sq_pending_cnt>=ctx->batch_size || tx_qp->sq_prod-tx_qp->sq_cons>=tx_qp->tx_depth ) {
988 0 : fd_mlx5_tile_sq_flush( ctx );
989 0 : } else if( sq_pending_cnt==1U ) {
990 0 : ctx->sq_flush_deadline_ticks = fd_tickcount()+ctx->sq_flush_timeout_ticks;
991 0 : }
992 0 : if( FD_UNLIKELY( ctx->rx_comp_channel_fd>=0 ) ) {
993 0 : fd_mlx5_tile_repoll_deadline_update( ctx, 1, fd_tickcount() );
994 0 : }
995 0 : }
996 :
997 : static inline void
998 0 : metrics_write( fd_mlx5_tile_t * ctx ) {
999 0 : fd_mlx5_rx_wq_t const * rx_wq = &ctx->rx_wq;
1000 0 : fd_mlx5_tx_qp_t const * tx_qp = &ctx->tx_qp;
1001 0 : ulong const rx_buffer_idle_cnt = fd_ulong_min( (ulong)(rx_wq->rq_prod-rx_wq->rq_cons), rx_wq->rx_depth );
1002 0 : ulong const rx_buffer_busy_cnt = rx_wq->rx_depth-rx_buffer_idle_cnt;
1003 0 : ulong const tx_buffer_busy_cnt = fd_ulong_min( (ulong)(tx_qp->sq_prod-tx_qp->sq_cons), tx_qp->tx_depth );
1004 0 : ulong const tx_buffer_idle_cnt = tx_qp->tx_depth-tx_buffer_busy_cnt;
1005 :
1006 0 : FD_MCNT_SET( MLX5, PKT_RX, ctx->net.metrics.rx_pkt_cnt );
1007 0 : FD_MCNT_SET( MLX5, PKT_RX_BYTES, ctx->net.metrics.rx_bytes_total );
1008 0 : FD_MCNT_ENUM_COPY( MLX5, PKT_RX_DROPPED, ctx->net.metrics.rx_drop_cnt );
1009 0 : FD_MCNT_SET( MLX5, GRE_PKT_RX, ctx->net.metrics.rx_gre_cnt );
1010 0 : FD_MGAUGE_SET( MLX5, RX_BUFFER_USED, rx_buffer_busy_cnt );
1011 0 : FD_MGAUGE_SET( MLX5, RX_BUFFER_FREE, rx_buffer_idle_cnt );
1012 :
1013 0 : FD_MCNT_SET( MLX5, PKT_TX_COMPLETED, ctx->net.metrics.tx_pkt_cnt );
1014 0 : FD_MCNT_SET( MLX5, PKT_TX_COMPLETED_BYTES, ctx->net.metrics.tx_bytes_total );
1015 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_NO_NEIGHBOR_IDX ] = ctx->router.metrics.tx_neigh_fail_cnt;
1016 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_RING_FULL_IDX ] = ctx->metrics.tx_no_buffer_cnt;
1017 0 : FD_MCNT_ENUM_COPY( MLX5, PKT_TX_DROPPED, ctx->net.metrics.tx_drop_cnt );
1018 0 : FD_MCNT_SET( MLX5, GRE_PKT_TX_SUBMITTED, ctx->net.metrics.tx_gre_cnt );
1019 0 : FD_MGAUGE_SET( MLX5, TX_BUFFER_USED, tx_buffer_busy_cnt );
1020 0 : FD_MGAUGE_SET( MLX5, TX_BUFFER_FREE, tx_buffer_idle_cnt );
1021 0 : }
1022 :
1023 : static inline void
1024 0 : during_housekeeping( fd_mlx5_tile_t * ctx ) {
1025 0 : fd_mlx5_tile_async_drain( ctx );
1026 :
1027 0 : if( FD_UNLIKELY( fd_netdev_tbl_refresh( &ctx->router.netdev_tbl, &ctx->router.netdev_shared, &ctx->netdev_seq ) ) ) {
1028 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->router.netdev_tbl );
1029 0 : }
1030 0 : }
1031 :
1032 : static uint
1033 0 : fd_mlx5_tile_if_ip4_addr( char const * if_name ) {
1034 0 : int sock_fd = socket( AF_INET, SOCK_DGRAM, 0 );
1035 0 : if( FD_UNLIKELY( sock_fd<0 ) ) {
1036 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_DGRAM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1037 0 : }
1038 :
1039 0 : struct ifreq ifr = { .ifr_addr.sa_family = AF_INET };
1040 0 : fd_cstr_ncpy( ifr.ifr_name, if_name, sizeof(ifr.ifr_name) );
1041 :
1042 0 : if( FD_UNLIKELY( ioctl( sock_fd, SIOCGIFADDR, &ifr ) ) ) {
1043 0 : FD_LOG_ERR(( "could not get IP address of interface `%s` (%i-%s)",
1044 0 : if_name, errno, fd_io_strerror( errno ) ));
1045 0 : }
1046 0 : uint ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
1047 0 : if( FD_UNLIKELY( close( sock_fd ) ) ) {
1048 0 : FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1049 0 : }
1050 :
1051 0 : return ip4_addr;
1052 0 : }
1053 :
1054 : /* fd_mlx5_tile_lo_tx_socket uses IPPROTO_RAW for send only IP_HDRINCL
1055 : transmission, preserving the source address and UDP source port. */
1056 : static int
1057 0 : fd_mlx5_tile_lo_tx_socket( void ) {
1058 0 : int sock = socket( AF_INET, SOCK_RAW|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_RAW );
1059 0 : if( FD_UNLIKELY( sock<0 ) ) {
1060 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_RAW,IPPROTO_RAW) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1061 0 : }
1062 0 : if( FD_UNLIKELY( 0!=setsockopt( sock, SOL_SOCKET, SO_BINDTODEVICE, "lo", sizeof("lo") ) ) ) {
1063 0 : FD_LOG_ERR(( "setsockopt(SO_BINDTODEVICE,lo) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1064 0 : }
1065 0 : return sock;
1066 0 : }
1067 :
1068 : static ulong
1069 0 : scratch_align( void ) {
1070 0 : ulong a = alignof( fd_mlx5_tile_t );
1071 0 : a = fd_ulong_max( a, FD_MLX5_PAGE_SZ );
1072 0 : a = fd_ulong_max( a, fd_netdev_tbl_align() );
1073 0 : a = fd_ulong_max( a, fd_fib4_align() );
1074 0 : return a;
1075 0 : }
1076 :
1077 : static ulong
1078 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
1079 0 : ulong layout = FD_LAYOUT_INIT;
1080 0 : ulong const queue_footprint = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size,
1081 0 : tile->mlx5.tx_queue_size );
1082 0 : if( FD_UNLIKELY( !queue_footprint ) ) return 0UL;
1083 :
1084 0 : layout = FD_LAYOUT_APPEND( layout, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1085 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, queue_footprint );
1086 0 : layout = FD_LAYOUT_APPEND( layout, alignof(uint), tile->mlx5.tx_queue_size*sizeof(uint) );
1087 0 : layout = FD_LAYOUT_APPEND( layout, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1088 0 : layout = FD_LAYOUT_APPEND( layout, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1089 0 : layout = FD_LAYOUT_APPEND( layout, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1090 :
1091 0 : return FD_LAYOUT_FINI( layout, scratch_align() );
1092 0 : }
1093 :
1094 : fd_fib4_t *
1095 : fd_mlx5_tile_fib4_join( fd_fib4_t * out,
1096 : fd_topo_t const * topo,
1097 : fd_topo_tile_t const * tile,
1098 0 : int main_table ) {
1099 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1100 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1101 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size ) );
1102 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->mlx5.tx_queue_size*sizeof(uint) );
1103 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1104 0 : void * local_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1105 0 : void * main_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1106 0 : return fd_fib4_join( out, main_table ? main_mem : local_mem );
1107 0 : }
1108 :
1109 : static void
1110 : fd_mlx5_tile_packet_memory( fd_topo_t const * topo,
1111 : fd_topo_tile_t const * tile,
1112 : fd_mlx5_tile_t * ctx,
1113 : void ** packet_memory,
1114 : ulong * packet_memory_sz,
1115 0 : ulong * packet_iova ) {
1116 0 : void * const packet_dcache_memory = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
1117 0 : void * const packet_dcache = fd_dcache_join( packet_dcache_memory );
1118 0 : if( FD_UNLIKELY( !packet_dcache ) ) FD_LOG_ERR(( "Failed to join packet dcache" ));
1119 0 : ulong const packet_dcache_data_sz = fd_dcache_data_sz( packet_dcache );
1120 0 : ulong const frame_sz = FD_NET_MTU;
1121 0 : ulong const frame_region_sz = fd_ulong_align_dn( packet_dcache_data_sz, frame_sz );
1122 :
1123 0 : void * const workspace_base = fd_wksp_containing( packet_dcache_memory );
1124 0 : if( FD_UNLIKELY( !workspace_base ) ) FD_LOG_ERR(( "Packet dcache is not in a workspace" ));
1125 0 : ulong const pkt_buf_chunk0 = ((ulong)packet_dcache-(ulong)workspace_base)>>FD_CHUNK_LG_SZ;
1126 0 : ulong const pkt_buf_wmark = pkt_buf_chunk0 + ((frame_region_sz-frame_sz)>>FD_CHUNK_LG_SZ);
1127 0 : if( FD_UNLIKELY( pkt_buf_chunk0>UINT_MAX || pkt_buf_wmark>UINT_MAX || pkt_buf_chunk0>pkt_buf_wmark ) ) {
1128 0 : FD_LOG_ERR(( "Calculated invalid packet buffer bounds [%lu,%lu]", pkt_buf_chunk0, pkt_buf_wmark ));
1129 0 : }
1130 :
1131 0 : ctx->net.pkt_buf_wksp_base = workspace_base;
1132 0 : ctx->net.pkt_buf_chunk0 = pkt_buf_chunk0;
1133 0 : ctx->net.pkt_buf_wmark = pkt_buf_wmark;
1134 0 : if( packet_memory ) *packet_memory = packet_dcache;
1135 0 : if( packet_memory_sz ) *packet_memory_sz = frame_region_sz;
1136 0 : if( packet_iova ) *packet_iova = (ulong)packet_dcache-(ulong)workspace_base;
1137 0 : }
1138 :
1139 : FD_FN_UNUSED static void
1140 : fd_mlx5_tile_join_obj_workspace( fd_topo_t * topo,
1141 0 : ulong obj_id ) {
1142 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->objs[ obj_id ].wksp_id ];
1143 0 : if( FD_LIKELY( !wksp->wksp ) ) {
1144 0 : fd_topo_join_workspace( topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE, 0 );
1145 0 : }
1146 0 : }
1147 :
1148 : #ifndef FD_TILE_TEST
1149 : void
1150 : fd_topo_install_mlx5( fd_topo_t * topo,
1151 0 : fd_mlx5_fds_t * fds ) {
1152 0 : ulong const tile_cnt = fd_topo_tile_name_cnt( topo, "mlx5" );
1153 0 : if( FD_UNLIKELY( 0==tile_cnt || tile_cnt>FD_MLX5_TILE_MAX ) ) {
1154 0 : FD_LOG_ERR(( "mlx5 tile count %lu must be in [1,%lu]", tile_cnt, FD_MLX5_TILE_MAX ));
1155 0 : }
1156 :
1157 0 : fd_mlx5_tile_t * ctxs [ FD_MLX5_TILE_MAX ];
1158 0 : fd_mlx5_uverbs_tile_t queues[ FD_MLX5_TILE_MAX ];
1159 0 : fd_topo_tile_t const * first_tile = NULL;
1160 0 : char rdma_device_name[ FD_MLX5_RDMA_NAME_MAX ];
1161 0 : uint rdma_port_num = 0U;
1162 :
1163 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1164 0 : ulong const tile_id = fd_topo_find_tile( topo, "mlx5", i );
1165 0 : FD_TEST( tile_id!=ULONG_MAX );
1166 0 : fd_topo_tile_t const * tile = topo->tiles+tile_id;
1167 0 : fd_mlx5_tile_join_obj_workspace( topo, tile->tile_obj_id );
1168 0 : fd_mlx5_tile_join_obj_workspace( topo, tile->net.umem_dcache_obj_id );
1169 :
1170 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1171 0 : fd_mlx5_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1172 0 : ulong const queue_memory_sz = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size );
1173 0 : void * queue_memory = FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, queue_memory_sz );
1174 0 : ctxs[ i ] = ctx;
1175 0 : fd_memset( ctx, 0, sizeof(*ctx) );
1176 0 : ctx->rx_comp_channel_fd = -1;
1177 0 : ctx->epoll_fd = -1;
1178 0 : FD_TEST( fd_mlx5_hw_init_queues( ctx, queue_memory,
1179 0 : tile->mlx5.rx_queue_size,
1180 0 : tile->mlx5.tx_queue_size ) );
1181 :
1182 0 : void * packet_memory;
1183 0 : ulong packet_memory_sz;
1184 0 : ulong packet_iova;
1185 0 : fd_mlx5_tile_packet_memory( topo, tile, ctx, &packet_memory, &packet_memory_sz, &packet_iova );
1186 0 : queues[ i ] = (fd_mlx5_uverbs_tile_t) {
1187 0 : .rx_cq = &ctx->rx_cq,
1188 0 : .tx_cq = &ctx->tx_cq,
1189 0 : .rx_wq = &ctx->rx_wq,
1190 0 : .tx_qp = &ctx->tx_qp,
1191 0 : .lkey = &ctx->lkey,
1192 0 : .rx_comp_channel_fd = tile->sleep_eventfd ? &ctx->rx_comp_channel_fd : NULL,
1193 0 : .packet_memory = packet_memory,
1194 0 : .packet_memory_sz = packet_memory_sz,
1195 0 : .packet_iova = packet_iova,
1196 0 : };
1197 0 : fd_net_rx_dst_ports_init( &ctx->net, topo, tile );
1198 :
1199 0 : if( !i ) {
1200 0 : first_tile = tile;
1201 0 : if( FD_UNLIKELY( !fd_mlx5_rdma_dev_find( rdma_device_name, &rdma_port_num, tile->mlx5.if_name ) ) ) {
1202 0 : if( errno==EEXIST ) FD_LOG_ERR(( "multiple RDMA ports match interface `%s`", tile->mlx5.if_name ));
1203 0 : if( errno==ENOENT ) FD_LOG_ERR(( "RDMA device port for interface `%s` not found", tile->mlx5.if_name ));
1204 0 : FD_LOG_ERR(( "finding RDMA device port for interface `%s` failed (%i-%s)",
1205 0 : tile->mlx5.if_name, errno, fd_io_strerror( errno ) ));
1206 0 : }
1207 0 : }
1208 0 : }
1209 :
1210 0 : fd_mlx5_tile_t * first = ctxs[ 0 ];
1211 0 : if( FD_UNLIKELY( !fd_mlx5_uverbs_avail() ) ) {
1212 0 : FD_LOG_ERR(( "failed to run mlx5 tile: uverbs interface unavailable. "
1213 0 : "Run `sudo modprobe ib_uverbs` and retry. "
1214 0 : "If it still fails, please set [net.provider] to `xdp`" ));
1215 0 : }
1216 0 : FD_LOG_INFO(( "Opening direct mlx5 device `%s` port %u for %lu tiles",
1217 0 : rdma_device_name, rdma_port_num, tile_cnt ));
1218 0 : if( FD_UNLIKELY( !fd_uverbs_init( &first->uverbs, queues, tile_cnt,
1219 0 : &first->outer_rss_qp, &first->gre_rss_qp,
1220 0 : rdma_device_name, rdma_port_num ) ) ) {
1221 0 : FD_LOG_ERR(( "direct mlx5 setup failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1222 0 : }
1223 :
1224 0 : int const async_event_fd = first->uverbs.async_fd;
1225 0 : int const async_event_flags = fcntl( async_event_fd, F_GETFL );
1226 0 : if( FD_UNLIKELY( async_event_flags<0 ||
1227 0 : fcntl( async_event_fd, F_SETFL, async_event_flags|O_NONBLOCK )<0 ) ) {
1228 0 : FD_LOG_ERR(( "making mlx5 async fd non-blocking failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1229 0 : }
1230 :
1231 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1232 0 : ctxs[ i ]->uverbs = first->uverbs;
1233 0 : ctxs[ i ]->outer_rss_qp = first->outer_rss_qp;
1234 0 : ctxs[ i ]->gre_rss_qp = first->gre_rss_qp;
1235 0 : ctxs[ i ]->prepared = 1U;
1236 :
1237 0 : int const rx_comp_channel_fd = ctxs[ i ]->rx_comp_channel_fd;
1238 0 : if( rx_comp_channel_fd<0 ) continue;
1239 0 : int const rx_comp_channel_fd_flags = fcntl( rx_comp_channel_fd, F_GETFL );
1240 0 : if( FD_UNLIKELY( rx_comp_channel_fd_flags<0 ||
1241 0 : fcntl( rx_comp_channel_fd, F_SETFL, rx_comp_channel_fd_flags|O_NONBLOCK )<0 ) ) {
1242 0 : FD_LOG_ERR(( "making mlx5 RX completion channel non-blocking failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1243 0 : }
1244 0 : }
1245 :
1246 0 : for( ulong flow_idx=0UL; flow_idx<first->net.dst_port_cnt; flow_idx++ ) {
1247 0 : if( FD_UNLIKELY( fd_uverbs_create_udp_flow( &first->uverbs, &first->outer_rss_qp,
1248 0 : first_tile->mlx5.net.bind_address,
1249 0 : first->net.dst_ports[ flow_idx ] ) ) ) {
1250 0 : FD_LOG_ERR(( "fd_uverbs_create_udp_flow failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1251 0 : }
1252 0 : if( FD_UNLIKELY( fd_uverbs_create_gre_udp_flow( &first->uverbs, &first->gre_rss_qp,
1253 0 : first_tile->mlx5.net.bind_address,
1254 0 : first->net.dst_ports[ flow_idx ] ) ) ) {
1255 0 : FD_LOG_ERR(( "fd_uverbs_create_gre_udp_flow failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1256 0 : }
1257 0 : }
1258 0 : FD_LOG_INFO(( "Installed %lu direct mlx5 flow rules", 2UL*first->net.dst_port_cnt ));
1259 :
1260 0 : if( fds ) {
1261 0 : fds->cmd_fd = first->uverbs.cmd_fd;
1262 0 : fds->async_fd = first->uverbs.async_fd;
1263 0 : fds->rx_comp_channel_fd_cnt = first->rx_comp_channel_fd>=0 ? tile_cnt : 0UL;
1264 0 : for( ulong i=0UL; i<fds->rx_comp_channel_fd_cnt; i++ )
1265 0 : fds->rx_comp_channel_fd[ i ] = ctxs[ i ]->rx_comp_channel_fd;
1266 0 : }
1267 0 : }
1268 : #endif
1269 :
1270 : FD_FN_UNUSED static void
1271 : privileged_init( fd_topo_t const * topo,
1272 0 : fd_topo_tile_t const * tile ) {
1273 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1274 0 : fd_mlx5_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1275 0 : ulong const queue_memory_sz = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size );
1276 0 : void * queue_memory = FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, queue_memory_sz );
1277 0 : if( FD_UNLIKELY( !ctx->prepared ) ) {
1278 0 : if( FD_UNLIKELY( fd_topo_tile_name_cnt( topo, "mlx5" )!=1UL ) ) {
1279 0 : FD_LOG_ERR(( "multi-tile mlx5 setup must run before tile launch" ));
1280 0 : }
1281 0 : fd_topo_install_mlx5( (fd_topo_t *)topo, NULL );
1282 0 : }
1283 0 : FD_TEST( fd_mlx5_hw_join_queues( ctx, queue_memory,
1284 0 : tile->mlx5.rx_queue_size,
1285 0 : tile->mlx5.tx_queue_size ) );
1286 0 : volatile uchar * uar = fd_uverbs_map_uar( &ctx->uverbs, ctx->tx_qp.uar_mmap_offset );
1287 0 : if( FD_UNLIKELY( !uar ) ) {
1288 0 : FD_LOG_ERR(( "mapping mlx5 UAR failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1289 0 : }
1290 0 : ctx->tx_qp.sq_doorbell = uar+FD_MLX5_UAR_SQ_DB_OFFSET;
1291 0 : ctx->rx_cq.request_notification_doorbell = uar+FD_MLX5_UAR_CQ_DB_OFFSET;
1292 :
1293 0 : ctx->batch_size = tile->mlx5.batch_size;
1294 0 : ctx->sq_wqe_buf_chunk = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint),
1295 0 : tile->mlx5.tx_queue_size*sizeof(uint) );
1296 :
1297 0 : fd_mlx5_tile_packet_memory( topo, tile, ctx, NULL, NULL, NULL );
1298 :
1299 : /* Resolve the netdev. */
1300 0 : uint const interface_idx = if_nametoindex( tile->mlx5.if_name );
1301 0 : if( FD_UNLIKELY( !interface_idx ) ) {
1302 0 : FD_LOG_ERR(( "if_nametoindex(%s) failed (%i-%s)",
1303 0 : tile->mlx5.if_name, errno, fd_io_strerror( errno ) ));
1304 0 : }
1305 0 : ctx->router.if_virt = interface_idx;
1306 0 : ctx->router.default_address = fd_mlx5_tile_if_ip4_addr( tile->mlx5.if_name );
1307 0 : ctx->lo_tx_sock = tile->kind_id==0UL ? fd_mlx5_tile_lo_tx_socket() : -1;
1308 :
1309 0 : if( FD_UNLIKELY( tile->sleep_eventfd ) ) {
1310 0 : if( FD_UNLIKELY( ctx->rx_comp_channel_fd<0 ) ) {
1311 0 : FD_LOG_ERR(( "mlx5:%lu has no RX completion channel", tile->kind_id ));
1312 0 : }
1313 0 : ctx->epoll_fd = epoll_create1( EPOLL_CLOEXEC );
1314 0 : if( FD_UNLIKELY( ctx->epoll_fd<0 ) ) {
1315 0 : FD_LOG_ERR(( "epoll_create1() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1316 0 : }
1317 0 : struct epoll_event doorbell_ev = { .events = EPOLLIN|EPOLLET, .data.u64 = FD_SLEEP_EPOLL_DOORBELL };
1318 0 : if( FD_UNLIKELY( -1==epoll_ctl( ctx->epoll_fd, EPOLL_CTL_ADD, FD_SLEEP_EVENTFD( tile->id ), &doorbell_ev ) ) ) {
1319 0 : FD_LOG_ERR(( "epoll_ctl(ADD,sleep_eventfd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1320 0 : }
1321 0 : struct epoll_event comp_ev = { .events = EPOLLIN, .data.u64 = (ulong)ctx->rx_comp_channel_fd };
1322 0 : if( FD_UNLIKELY( -1==epoll_ctl( ctx->epoll_fd, EPOLL_CTL_ADD, ctx->rx_comp_channel_fd, &comp_ev ) ) ) {
1323 0 : FD_LOG_ERR(( "epoll_ctl(ADD,rx_comp_channel_fd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1324 0 : }
1325 0 : struct epoll_event async_ev = { .events = EPOLLIN, .data.u64 = FD_MLX5_EPOLL_ASYNC };
1326 0 : if( FD_UNLIKELY( -1==epoll_ctl( ctx->epoll_fd, EPOLL_CTL_ADD, ctx->uverbs.async_fd, &async_ev ) ) ) {
1327 0 : FD_LOG_ERR(( "epoll_ctl(ADD,async_fd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1328 0 : }
1329 0 : }
1330 0 : }
1331 :
1332 : FD_FN_UNUSED static void
1333 : unprivileged_init( fd_topo_t const * topo,
1334 0 : fd_topo_tile_t const * tile ) {
1335 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1336 0 : fd_mlx5_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1337 0 : ulong queue_footprint = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size );
1338 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, queue_footprint );
1339 :
1340 0 : ctx->sq_wqe_buf_chunk = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->mlx5.tx_queue_size*sizeof(uint) );
1341 0 : ctx->batch_size = tile->mlx5.batch_size;
1342 0 : ctx->sq_flush_timeout_ticks = (long)( FD_MLX5_TX_FLUSH_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ) );
1343 0 : ctx->lo_tx_timeout_ticks = (long)( FD_MLX5_LO_TX_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ) );
1344 :
1345 0 : ctx->repoll_timeout_ticks = (long)( FD_MLX5_REPOLL_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ) );
1346 0 : ctx->repoll_deadline_ticks = LONG_MAX;
1347 :
1348 0 : ctx->lo_tx_cnt = 0U;
1349 0 : ctx->net.kind_id = tile->kind_id;
1350 0 : ctx->net.tile_cnt = fd_topo_tile_name_cnt( topo, tile->name );
1351 :
1352 0 : void * netdev_tbl_local = FD_SCRATCH_ALLOC_APPEND( scratch, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1353 0 : void * fib_local_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1354 0 : void * fib_main_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1355 :
1356 : /* chunk 0 is used as a sentinel value, so ensure actual chunk indices
1357 : do not use that value. */
1358 0 : FD_TEST( ctx->net.pkt_buf_chunk0>0UL );
1359 :
1360 0 : ctx->rq_pending_cnt = 0U;
1361 :
1362 : /* Post RQ WQEs */
1363 0 : ulong frame_chunks = FD_NET_MTU>>FD_CHUNK_LG_SZ;
1364 0 : ulong next_chunk = ctx->net.pkt_buf_chunk0;
1365 :
1366 0 : FD_TEST( tile->mlx5.rx_queue_size>ctx->batch_size );
1367 :
1368 0 : ulong const rx_fill_cnt = tile->mlx5.rx_queue_size-ctx->batch_size;
1369 0 : for( ulong i=0UL; i<rx_fill_cnt; i++ ) {
1370 0 : fd_mlx5_tile_rx_recycle( ctx, next_chunk );
1371 0 : next_chunk += frame_chunks;
1372 0 : }
1373 :
1374 : /* Assign chunks to RX mcaches */
1375 0 : for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
1376 0 : fd_frag_meta_t * mcache = topo->links[ tile->out_link_id[ i ] ].mcache;
1377 0 : ulong const depth = fd_mcache_depth( mcache );
1378 0 : for( ulong j=0UL; j<depth; j++ ) {
1379 0 : mcache[ j ].chunk = (uint)next_chunk;
1380 0 : mcache[ j ].seq = fd_seq_dec( j, 1UL ); /* mark seq as invalid */
1381 0 : next_chunk += frame_chunks;
1382 0 : }
1383 0 : }
1384 : /* Assign one TX buffer to each SQ WQE */
1385 0 : for( ulong i=0UL; i<tile->mlx5.tx_queue_size; i++ ) {
1386 0 : ctx->sq_wqe_buf_chunk[ i ] = (uint)next_chunk;
1387 0 : next_chunk += frame_chunks;
1388 0 : }
1389 : /* Init TX */
1390 0 : if( FD_UNLIKELY( tile->in_cnt>FD_NET_IN_MAX ) ) {
1391 0 : FD_LOG_ERR(( "mlx5 tile in link count %lu exceeds max of %lu", tile->in_cnt, FD_NET_IN_MAX ));
1392 0 : }
1393 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
1394 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
1395 0 : if( !strcmp( link->name, "iproute_out" ) ) {
1396 0 : ctx->net.in_kind[ i ] = FD_NET_IN_KIND_IPROUTE;
1397 0 : } else {
1398 0 : ctx->net.in_kind[ i ] = FD_NET_IN_KIND_TX;
1399 0 : if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "mlx5 tile in link does not have a normal MTU" ));
1400 0 : }
1401 :
1402 0 : ctx->net.in_dcache_ctx[ i ].wksp_base = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
1403 0 : ctx->net.in_dcache_ctx[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache );
1404 0 : ctx->net.in_dcache_ctx[ i ].wmark = fd_dcache_compact_wmark( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache, link->mtu );
1405 0 : }
1406 :
1407 : /* Join netbase objects */
1408 0 : FD_TEST( fd_fib4_join( ctx->router.fib_local, fd_fib4_new( fib_local_mem, tile->mlx5.route_max, tile->mlx5.route_peer_max, tile->mlx5.route_peer_seed ) ) );
1409 0 : FD_TEST( fd_fib4_join( ctx->router.fib_main, fd_fib4_new( fib_main_mem, tile->mlx5.route_max, tile->mlx5.route_peer_max, tile->mlx5.route_peer_seed ) ) );
1410 0 : FD_TEST( fd_netdev_tbl_join( &ctx->router.netdev_shared, fd_topo_obj_laddr( topo, tile->mlx5.netdev_tbl_obj_id ) ) );
1411 0 : FD_TEST( fd_netdev_tbl_new( netdev_tbl_local, NETDEV_MAX, BOND_MASTER_MAX ) );
1412 0 : FD_TEST( fd_netdev_tbl_join( &ctx->router.netdev_tbl, netdev_tbl_local ) );
1413 :
1414 0 : ctx->netdev_seq = fd_netdev_tbl_copy( &ctx->router.netdev_tbl, &ctx->router.netdev_shared );
1415 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->router.netdev_tbl );
1416 0 : ctx->router.bind_address = tile->mlx5.net.bind_address;
1417 0 : ctx->net.bind_address = tile->mlx5.net.bind_address;
1418 :
1419 0 : ulong neigh4_obj_id = tile->mlx5.neigh4_obj_id;
1420 0 : ulong ele_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.ele_max", neigh4_obj_id );
1421 0 : ulong probe_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.probe_max", neigh4_obj_id );
1422 0 : ulong seed = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.seed", neigh4_obj_id );
1423 0 : if( FD_UNLIKELY( (ele_max==ULONG_MAX) | (probe_max==ULONG_MAX) | (seed==ULONG_MAX) ) ) {
1424 0 : FD_LOG_ERR(( "neigh4 hmap properties not set" ));
1425 0 : }
1426 0 : if( FD_UNLIKELY( !fd_neigh4_hmap_join( ctx->router.neigh4, fd_topo_obj_laddr( topo, neigh4_obj_id ), ele_max, probe_max, seed ) ) ) {
1427 0 : FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
1428 0 : }
1429 :
1430 0 : ctx->router.netlnk_out_idx = fd_topo_find_tile_out_link( topo, tile, "net_netlnk", tile->kind_id );
1431 0 : if( FD_UNLIKELY( ctx->router.netlnk_out_idx==ULONG_MAX ) ) FD_LOG_ERR(( "netlink request link not found" ));
1432 0 : ctx->router.solicit_ip = 0U;
1433 :
1434 : /* Check if all chunks are in bound */
1435 0 : if( FD_UNLIKELY( next_chunk>ctx->net.pkt_buf_wmark ) ) {
1436 0 : FD_LOG_ERR(( "dcache is too small (topology bug)" ));
1437 0 : }
1438 :
1439 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( scratch, scratch_align() );
1440 0 : if( FD_UNLIKELY( scratch_top>(ulong)ctx+scratch_footprint( tile ) ) ) {
1441 0 : FD_LOG_ERR(( "scratch overflow" ));
1442 0 : }
1443 0 : }
1444 :
1445 : FD_FN_UNUSED static ulong
1446 : populate_allowed_seccomp( fd_topo_t const * topo,
1447 : fd_topo_tile_t const * tile,
1448 : ulong out_cnt,
1449 0 : struct sock_filter * out ) {
1450 0 : fd_mlx5_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1451 0 : populate_sock_filter_policy_fd_mlx5_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(),
1452 0 : (uint)ctx->uverbs.async_fd, (uint)ctx->lo_tx_sock,
1453 0 : (uint)ctx->rx_comp_channel_fd, (uint)ctx->epoll_fd );
1454 0 : return sock_filter_policy_fd_mlx5_tile_instr_cnt;
1455 0 : }
1456 :
1457 : FD_FN_UNUSED static ulong
1458 : populate_allowed_fds( fd_topo_t const * topo,
1459 : fd_topo_tile_t const * tile,
1460 : ulong out_fds_cnt,
1461 0 : int * out_fds ) {
1462 0 : fd_mlx5_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1463 0 : if( FD_UNLIKELY( out_fds_cnt<8UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
1464 0 : ulong out_cnt = 0UL;
1465 0 : out_fds[ out_cnt++ ] = 2;
1466 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) ) out_fds[ out_cnt++ ] = fd_log_private_logfile_fd();
1467 0 : out_fds[ out_cnt++ ] = ctx->uverbs.cmd_fd;
1468 0 : out_fds[ out_cnt++ ] = ctx->uverbs.async_fd;
1469 0 : if( ctx->lo_tx_sock>=0 ) out_fds[ out_cnt++ ] = ctx->lo_tx_sock;
1470 0 : if( FD_UNLIKELY( tile->sleep_eventfd ) ) {
1471 0 : out_fds[ out_cnt++ ] = ctx->rx_comp_channel_fd;
1472 0 : out_fds[ out_cnt++ ] = ctx->epoll_fd;
1473 0 : out_fds[ out_cnt++ ] = FD_SLEEP_EVENTFD( tile->id );
1474 0 : }
1475 0 : return out_cnt;
1476 0 : }
1477 :
1478 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_mlx5_tile_t
1479 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_mlx5_tile_t)
1480 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
1481 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
1482 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
1483 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
1484 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
1485 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
1486 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
1487 : #define STEM_CALLBACK_PREVENT_PARK prevent_park
1488 0 : #define STEM_CALLBACK_PARK_WAIT park_wait
1489 0 : #define STEM_CALLBACK_NEXT_DEADLINE next_deadline
1490 0 : #define STEM_BURST (FD_MLX5_BATCH_SIZE+1) /* +1 accounts for possible LB TX packet */
1491 0 : #define STEM_LAZY 270000UL /* 270us */
1492 : #include "../../stem/fd_stem.c"
1493 :
1494 : #ifndef FD_TILE_TEST
1495 : fd_topo_run_tile_t fd_tile_mlx5 = {
1496 : .name = "mlx5",
1497 : .populate_allowed_seccomp = populate_allowed_seccomp,
1498 : .populate_allowed_fds = populate_allowed_fds,
1499 : .scratch_align = scratch_align,
1500 : .scratch_footprint = scratch_footprint,
1501 : .privileged_init = privileged_init,
1502 : .unprivileged_init = unprivileged_init,
1503 : .run = stem_run,
1504 : };
1505 : #endif
|
