Line data Source code
1 : #define _GNU_SOURCE /* sendmmsg */
2 : /* The mlx5 tile translates Ethernet frames between mlx5 work/completion
3 : queues and fd_tango traffic.
4 :
5 : An mlx5 queue pair (QP) contains a send queue (SQ) and a receive queue
6 : (RQ). Work queue entries (WQEs) encode posted work. Completion
7 : queue entries (CQEs) report completed work through completion queues
8 : (CQs). The mlx5 tile uses separate CQs for TX and RX, one each. */
9 :
10 : #include "../fd_net_tile.h"
11 : #include "../fd_net_tile_private.h"
12 : #include "fd_mlx5_private.h"
13 :
14 : #include <errno.h>
15 : #include <fcntl.h>
16 : #include <net/if.h>
17 : #include <stddef.h>
18 : #include <stdlib.h>
19 : #include <time.h>
20 : #include <netinet/in.h>
21 : #include <sys/socket.h>
22 :
23 : #include "../../metrics/fd_metrics.h"
24 : #include "../../topo/fd_topo.h"
25 : #include "../../sleep/fd_sleep.h"
26 :
27 : #include "../../../waltz/ip/fd_iproute.h"
28 : #include "../../../util/net/fd_eth.h"
29 : #include "../../../util/net/fd_gre.h"
30 : #include "../../../util/net/fd_ip4.h"
31 : #include "../../../util/net/fd_udp.h"
32 : #include "../../../util/pod/fd_pod_format.h"
33 :
34 : #include <unistd.h>
35 : #include <sys/epoll.h>
36 : #include <sys/ioctl.h>
37 : #include <linux/rtnetlink.h>
38 : #include <rdma/ib_user_verbs.h>
39 :
40 : #define FD_SOCKADDR_IN_SZ sizeof(struct sockaddr_in)
41 : #include "generated/fd_mlx5_tile_seccomp.h"
42 :
43 :
44 0 : #define FD_MLX5_TX_FLUSH_TIMEOUT_NS (20000L) /* 20us */
45 0 : #define FD_MLX5_LO_TX_TIMEOUT_NS (500000L) /* 500us */
46 :
47 : /* Recent RX packets or TX submissions defer RX CQ rearming and cause
48 : a timed repoll before the tile returns to epoll. */
49 0 : #define FD_MLX5_REPOLL_TIMEOUT_NS ( 200000L) /* 200us */
50 0 : #define FD_MLX5_NS_PER_SEC (1000000000L)
51 :
52 : /* Linux names these UAR offsets MLX5_CQ_DOORBELL and MLX5_BF_OFFSET. */
53 0 : #define FD_MLX5_UAR_CQ_DB_OFFSET (0x20UL)
54 0 : #define FD_MLX5_UAR_SQ_DB_OFFSET (0x800UL)
55 :
56 : /* epoll data of the async fd in the park set (the completion channel
57 : is tagged with its fd, the doorbell with FD_SLEEP_EPOLL_DOORBELL). */
58 0 : #define FD_MLX5_EPOLL_ASYNC (ULONG_MAX-1UL)
59 :
60 : /* mlx5 CQ arm doorbells use the event sequence in bits 28 and 29. */
61 0 : #define FD_MLX5_CQ_ARM_SN_MASK ( 3U)
62 0 : #define FD_MLX5_CQ_ARM_SN_SHIFT (28U)
63 :
64 : /* FD_MLX5_SQ_* are options in a SQ WQE to request certain NIC behaviour.
65 : SEND requests packet transmission. CQ_UPDATE requests a CQE. */
66 0 : #define FD_MLX5_SQ_SEND_PKT (0x0aU) /* SEND */
67 0 : #define FD_MLX5_SQ_REQUEST_CQE (0x08U) /* CQ_UPDATE */
68 :
69 : /* FD_MLX5_CQE_OP_* are mlx5 CQE opcodes used to classify TX and RX completions */
70 0 : #define FD_MLX5_CQE_OP_TX_OK ( 0U) /* MLX5_CQE_REQ */
71 0 : #define FD_MLX5_CQE_OP_RX_OK ( 2U) /* MLX5_CQE_RESP_SEND */
72 0 : #define FD_MLX5_CQE_OP_TX_ERR (13U) /* MLX5_CQE_REQ_ERR */
73 0 : #define FD_MLX5_CQE_OP_RX_ERR (14U) /* MLX5_CQE_RESP_ERR */
74 0 : #define FD_MLX5_CQE_OP_INVALID (15U)
75 0 : #define FD_MLX5_CQE_OWNER_MASK ( 1U) /* MLX5_CQE_OWNER_MASK */
76 :
77 0 : #define FD_MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR (0x01U) /* received packet's length exceeded FD_NET_MTU */
78 :
79 : /* FD_MLX5_ASYNC_EVENT_* are Linux enum ib_event_type values delivered
80 : in ib_uverbs_async_event_desc.event_type. */
81 : #define FD_MLX5_ASYNC_EVENT_CQ_ERR ( 0U)
82 : #define FD_MLX5_ASYNC_EVENT_QP_FATAL ( 1U)
83 : #define FD_MLX5_ASYNC_EVENT_QP_REQ_ERR ( 2U)
84 : #define FD_MLX5_ASYNC_EVENT_QP_ACCESS_ERR ( 3U)
85 0 : #define FD_MLX5_ASYNC_EVENT_DEVICE_FATAL ( 8U)
86 : #define FD_MLX5_ASYNC_EVENT_WQ_FATAL (19U)
87 :
88 : /* The fd_mlx5_hw_* structs below define WQE and CQE formats in the
89 : mlx5 hardware interface. The links show the corresponding definitions
90 : in the Linux mlx5 driver. */
91 :
92 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/qp.h#L205 */
93 : struct __attribute__((packed)) fd_mlx5_hw_wqe_ctrl_seg {
94 : /* mlx5 packs opmod, WQE index, opcode, QP number, and WQE segment
95 : count into two big-endian 32-bit words. */
96 : uint opmod_idx_opcode;
97 : uint qpn_ds;
98 :
99 : uchar signature;
100 : uchar reserved[2];
101 : uchar flags;
102 : uint imm;
103 : };
104 : typedef struct fd_mlx5_hw_wqe_ctrl_seg fd_mlx5_hw_wqe_ctrl_seg_t;
105 :
106 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/qp.h#L265 */
107 : struct __attribute__((packed)) fd_mlx5_hw_wqe_eth_seg {
108 : uchar reserved[12];
109 : ushort inline_hdr_sz;
110 : uchar inline_hdr[2];
111 : };
112 : typedef struct fd_mlx5_hw_wqe_eth_seg fd_mlx5_hw_wqe_eth_seg_t;
113 :
114 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/qp.h#L364 */
115 : struct __attribute__((packed)) fd_mlx5_hw_wqe_data_seg {
116 : uint byte_cnt;
117 : uint lkey;
118 : ulong addr;
119 : };
120 : typedef struct fd_mlx5_hw_wqe_data_seg fd_mlx5_hw_wqe_data_seg_t;
121 :
122 : /* https://elixir.bootlin.com/linux/v7.1.8/source/include/linux/mlx5/device.h#L824 */
123 : struct __attribute__((packed,may_alias)) fd_mlx5_hw_cqe64 {
124 : uchar reserved0[44];
125 : uint byte_cnt;
126 : uchar reserved1[6];
127 : uchar vendor_err;
128 : uchar syndrome;
129 : uint sop_drop_qpn;
130 : ushort wqe_counter;
131 : uchar signature;
132 : uchar op_own;
133 : };
134 : typedef struct fd_mlx5_hw_cqe64 fd_mlx5_hw_cqe64_t;
135 :
136 : struct fd_mlx5_tile_rx_comp {
137 : ulong chunk;
138 : uint byte_len;
139 : uint opcode;
140 : };
141 : typedef struct fd_mlx5_tile_rx_comp fd_mlx5_tile_rx_comp_t;
142 :
143 : /* fd_mlx5_tile_t is private tile state */
144 : struct __attribute__((aligned(64UL))) fd_mlx5_tile {
145 : fd_mlx5_rx_wq_t rx_wq;
146 : uint batch_size; /* used for both SQ/RQ and TX/RX CQ batching */
147 : fd_mlx5_cq_t rx_cq;
148 : uint lo_tx_cnt;
149 : uint rq_pending_cnt;
150 : long sq_flush_deadline_ticks;
151 : fd_mlx5_tx_qp_t tx_qp;
152 : long lo_tx_deadline_ticks;
153 : int rx_comp_channel_fd; /* completion channel of rx_cq, -1 if not efficient mode */
154 : uint has_out_credit;
155 : long repoll_deadline_ticks; /* LONG_MAX when no repoll is scheduled */
156 :
157 : long sq_flush_timeout_ticks;
158 : long lo_tx_timeout_ticks;
159 : int lo_tx_sock;
160 : uint lkey;
161 : uint prepared;
162 : fd_uverbs_ctx_t uverbs;
163 : fd_mlx5_cq_t tx_cq;
164 : fd_mlx5_rss_qp_t outer_rss_qp;
165 : fd_mlx5_rss_qp_t gre_rss_qp;
166 :
167 : /* Packet buffer addressing */
168 : uint * sq_wqe_buf_chunk; /* maps SQ WQEs to packet buffers */
169 :
170 : fd_net_tile_t net;
171 :
172 : /* RQ batching */
173 : uint rq_pending_chunk[ FD_MLX5_BATCH_SIZE ];
174 :
175 : /* Local socket TX, used only by mlx5:0. */
176 : struct mmsghdr lo_tx_msg [ FD_MLX5_BATCH_SIZE ];
177 : struct iovec lo_tx_iov [ FD_MLX5_BATCH_SIZE ];
178 : struct sockaddr_in lo_tx_addr[ FD_MLX5_BATCH_SIZE ];
179 : uchar lo_tx_buf [ FD_MLX5_BATCH_SIZE ][ FD_NET_MTU ];
180 :
181 : /* Efficient mode only */
182 : int epoll_fd;
183 : long repoll_timeout_ticks;
184 :
185 : ulong netdev_seq; /* netdev_shared seqlock value of the net.router.netdev_tbl copy */
186 :
187 : /* Metric tracking */
188 : struct {
189 : ulong tx_no_buffer_cnt;
190 : } metrics;
191 : };
192 : typedef struct fd_mlx5_tile fd_mlx5_tile_t;
193 :
194 : static ulong
195 : fd_mlx5_queue_footprint( uint rx_depth,
196 0 : uint tx_depth ) {
197 0 : ulong layout = FD_LAYOUT_INIT;
198 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_cqe_t) );
199 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_cqe_t) );
200 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_rx_wqe_t) );
201 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_tx_wqe_t) );
202 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, rx_depth*sizeof(uint) );
203 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, tx_depth*sizeof(uint) );
204 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, FD_MLX5_PAGE_SZ );
205 0 : return FD_LAYOUT_FINI( layout, FD_MLX5_PAGE_SZ );
206 0 : }
207 :
208 : static void
209 : fd_mlx5_hw_invalidate_cqes( fd_mlx5_cqe_t * cqes,
210 0 : uint depth ) {
211 0 : for( uint i=0U; i<depth; i++ ) {
212 0 : fd_mlx5_hw_cqe64_t * hw_cqe = (fd_mlx5_hw_cqe64_t *)(cqes+i);
213 0 : hw_cqe->op_own = (uchar)(FD_MLX5_CQE_OP_INVALID<<4);
214 0 : }
215 0 : }
216 :
217 : static fd_mlx5_tile_t *
218 : fd_mlx5_hw_join_queues( fd_mlx5_tile_t * ctx,
219 : void * queue_memory,
220 : uint rx_depth,
221 0 : uint tx_depth ) {
222 0 : ulong const queue_footprint = fd_mlx5_queue_footprint( rx_depth, tx_depth );
223 0 : FD_SCRATCH_ALLOC_INIT( queue, queue_memory );
224 0 : fd_mlx5_cqe_t * rx_cqes = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_cqe_t) );
225 0 : fd_mlx5_cqe_t * tx_cqes = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_cqe_t) );
226 0 : fd_mlx5_rx_wqe_t * rq = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, rx_depth*sizeof(fd_mlx5_rx_wqe_t) );
227 0 : fd_mlx5_tx_wqe_t * sq = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, tx_depth*sizeof(fd_mlx5_tx_wqe_t) );
228 0 : uint * rq_wqe_buf_chunk = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, rx_depth*sizeof(uint) );
229 0 : uint * sq_wqe_frame_sz = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, tx_depth*sizeof(uint) );
230 0 : uchar * control = FD_SCRATCH_ALLOC_APPEND( queue, FD_MLX5_PAGE_SZ, FD_MLX5_PAGE_SZ );
231 0 : FD_TEST( FD_SCRATCH_ALLOC_FINI( queue, FD_MLX5_PAGE_SZ )==(ulong)queue_memory+queue_footprint );
232 :
233 0 : ctx->rx_cq.entries = rx_cqes;
234 0 : ctx->rx_cq.control = (fd_mlx5_cq_control_t *)control;
235 0 : ctx->rx_cq.depth = rx_depth;
236 :
237 0 : ctx->tx_cq.entries = tx_cqes;
238 0 : ctx->tx_cq.control = (fd_mlx5_cq_control_t *)(control+sizeof(fd_mlx5_cq_control_t));
239 0 : ctx->tx_cq.depth = tx_depth;
240 :
241 0 : ctx->rx_wq.rq = rq;
242 0 : ctx->rx_wq.rx_cq = &ctx->rx_cq;
243 0 : ctx->rx_wq.rx_depth = rx_depth;
244 0 : ctx->rx_wq.rq_wqe_buf_chunk = rq_wqe_buf_chunk;
245 0 : ctx->rx_wq.control = (fd_mlx5_rx_wq_control_t *)(control+2UL*sizeof(fd_mlx5_cq_control_t));
246 :
247 0 : ctx->tx_qp.sq = sq;
248 0 : ctx->tx_qp.tx_cq = &ctx->tx_cq;
249 0 : ctx->tx_qp.tx_depth = tx_depth;
250 0 : ctx->tx_qp.sq_wqe_frame_sz = sq_wqe_frame_sz;
251 0 : ctx->tx_qp.control = (fd_mlx5_qp_control_t *)(control+2UL*sizeof(fd_mlx5_cq_control_t)+sizeof(fd_mlx5_rx_wq_control_t));
252 0 : return ctx;
253 0 : }
254 :
255 : static fd_mlx5_tile_t *
256 : fd_mlx5_hw_init_queues( fd_mlx5_tile_t * ctx,
257 : void * queue_memory,
258 : uint rx_depth,
259 0 : uint tx_depth ) {
260 0 : fd_memset( queue_memory, 0, fd_mlx5_queue_footprint( rx_depth, tx_depth ) );
261 0 : FD_TEST( fd_mlx5_hw_join_queues( ctx, queue_memory, rx_depth, tx_depth ) );
262 0 : fd_mlx5_hw_invalidate_cqes( ctx->rx_cq.entries, rx_depth );
263 0 : fd_mlx5_hw_invalidate_cqes( ctx->tx_cq.entries, tx_depth );
264 0 : return ctx;
265 0 : }
266 :
267 : /* fd_mlx5_tile_tx_chunk returns the buffer paired with SQ WQE sq_idx.
268 : The buffer is reusable once sq_cons advances past that WQE. */
269 : static inline uint
270 : fd_mlx5_tile_tx_chunk( fd_mlx5_tile_t const * ctx,
271 0 : uint sq_idx ) {
272 0 : return ctx->sq_wqe_buf_chunk[ sq_idx & (ctx->tx_qp.tx_depth-1U) ];
273 0 : }
274 :
275 : static inline void
276 0 : fd_mlx5_hw_dma_to_device( void ) {
277 0 : #if FD_HAS_X86
278 0 : FD_COMPILER_MFENCE();
279 : #elif FD_HAS_ARM
280 : __asm__ __volatile__( "dmb oshst" ::: "memory" );
281 : #elif FD_HAS_RISCV
282 : __asm__ __volatile__( "fence ow,ow" ::: "memory" );
283 : #else
284 : FD_HW_MFENCE_ST();
285 : #endif
286 0 : }
287 :
288 : static inline void
289 0 : fd_mlx5_hw_dma_from_device( void ) {
290 0 : #if FD_HAS_X86
291 0 : __asm__ __volatile__( "lfence" ::: "memory" );
292 : #elif FD_HAS_ARM
293 : __asm__ __volatile__( "dmb oshld" ::: "memory" );
294 : #elif FD_HAS_RISCV
295 : __asm__ __volatile__( "fence ir,ir" ::: "memory" );
296 : #else
297 : FD_HW_MFENCE();
298 : #endif
299 0 : }
300 :
301 : static inline int
302 : fd_mlx5_hw_init_tx_wqe( fd_mlx5_tx_wqe_t * wqe,
303 : uint sq_idx,
304 : uint qpn,
305 : void const * frame,
306 : ulong frame_iova,
307 : ulong frame_sz,
308 : uint lkey,
309 0 : ulong inline_hdr_sz ) {
310 0 : if( FD_UNLIKELY( !frame_sz || frame_sz>UINT_MAX || qpn>0xffffffU ||
311 0 : inline_hdr_sz>frame_sz || frame_iova>ULONG_MAX-inline_hdr_sz ) ) return 0;
312 0 : fd_memset( wqe, 0, sizeof(*wqe) );
313 0 : uint const wqe_seg_cnt = inline_hdr_sz ? 4U : 3U;
314 0 : ulong const wqe_data_seg_off = inline_hdr_sz ? 48UL : 32UL;
315 :
316 0 : fd_mlx5_hw_wqe_ctrl_seg_t * wqe_ctrl_seg = (fd_mlx5_hw_wqe_ctrl_seg_t *)wqe;
317 0 : fd_mlx5_hw_wqe_eth_seg_t * wqe_eth_seg = (fd_mlx5_hw_wqe_eth_seg_t *)(wqe->bytes+sizeof(*wqe_ctrl_seg));
318 0 : fd_mlx5_hw_wqe_data_seg_t * wqe_data_seg = (fd_mlx5_hw_wqe_data_seg_t *)(wqe->bytes+wqe_data_seg_off);
319 :
320 0 : wqe_ctrl_seg->opmod_idx_opcode = fd_uint_bswap( ((sq_idx & 0xffffU)<<8) | FD_MLX5_SQ_SEND_PKT );
321 0 : wqe_ctrl_seg->qpn_ds = fd_uint_bswap( (qpn<<8) | wqe_seg_cnt );
322 :
323 0 : wqe_eth_seg->inline_hdr_sz = fd_ushort_bswap( (ushort)inline_hdr_sz );
324 0 : if( inline_hdr_sz ) fd_memcpy( wqe_eth_seg->inline_hdr, frame, inline_hdr_sz );
325 :
326 0 : wqe_data_seg->byte_cnt = fd_uint_bswap( (uint)(frame_sz-inline_hdr_sz) );
327 0 : wqe_data_seg->lkey = fd_uint_bswap( lkey );
328 0 : wqe_data_seg->addr = fd_ulong_bswap( frame_iova+inline_hdr_sz );
329 0 : return 1;
330 0 : }
331 :
332 : static inline void
333 : fd_mlx5_hw_init_rx_wqe( fd_mlx5_rx_wqe_t * wqe,
334 : ulong frame_iova,
335 : ulong frame_sz,
336 0 : uint lkey ) {
337 0 : fd_mlx5_hw_wqe_data_seg_t * wqe_data_seg = (fd_mlx5_hw_wqe_data_seg_t *)wqe;
338 :
339 0 : wqe_data_seg->byte_cnt = fd_uint_bswap( (uint)frame_sz );
340 0 : wqe_data_seg->lkey = fd_uint_bswap( lkey );
341 0 : wqe_data_seg->addr = fd_ulong_bswap( frame_iova );
342 0 : }
343 :
344 : static inline void
345 : fd_mlx5_hw_ring_sq( fd_mlx5_tx_qp_t * tx_qp,
346 0 : fd_mlx5_tx_wqe_t * wqe ) {
347 0 : fd_mlx5_hw_wqe_ctrl_seg_t * wqe_ctrl_seg = (fd_mlx5_hw_wqe_ctrl_seg_t *)wqe;
348 0 : wqe_ctrl_seg->flags = FD_MLX5_SQ_REQUEST_CQE;
349 :
350 0 : fd_mlx5_hw_dma_to_device();
351 0 : FD_VOLATILE( tx_qp->control->sq_prod ) = fd_uint_bswap( tx_qp->sq_prod & 0xffffU );
352 0 : fd_mlx5_hw_dma_to_device();
353 :
354 : /* Ring the SQ through the non-cached UAR. The NIC fetches the complete
355 : WQE from write-back SQ memory. */
356 0 : volatile ulong * doorbell_reg = (volatile ulong *)tx_qp->sq_doorbell;
357 0 : doorbell_reg[0] = FD_LOAD( ulong, wqe->bytes );
358 0 : FD_COMPILER_MFENCE();
359 0 : tx_qp->sq_posted = tx_qp->sq_prod;
360 0 : }
361 :
362 : static inline int
363 : fd_mlx5_hw_sq_enqueue( fd_mlx5_tile_t * ctx,
364 0 : uint frame_sz ) {
365 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
366 0 : uint const outstanding = tx_qp->sq_prod-tx_qp->sq_cons;
367 0 : if( FD_UNLIKELY( outstanding>=tx_qp->tx_depth ) ) {
368 0 : errno = ENOSPC;
369 0 : return -1;
370 0 : }
371 :
372 0 : uint const sq_idx = tx_qp->sq_prod;
373 0 : uint const chunk = fd_mlx5_tile_tx_chunk( ctx, sq_idx );
374 0 : ulong const frame_iova = (ulong)chunk<<FD_CHUNK_LG_SZ;
375 0 : fd_mlx5_tx_wqe_t * wqe = tx_qp->sq+(sq_idx & (tx_qp->tx_depth-1U));
376 0 : if( FD_UNLIKELY( !fd_mlx5_hw_init_tx_wqe( wqe, sq_idx, tx_qp->qpn,
377 0 : fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, chunk ), frame_iova, frame_sz,
378 0 : ctx->lkey, tx_qp->tx_inline_hdr_sz ) ) ) {
379 0 : errno = EINVAL;
380 0 : return -1;
381 0 : }
382 0 : tx_qp->sq_wqe_frame_sz[ sq_idx & (tx_qp->tx_depth-1U) ] = frame_sz;
383 0 : tx_qp->sq_prod++;
384 0 : return 0;
385 0 : }
386 :
387 : static inline int
388 : fd_mlx5_hw_rq_enqueue( fd_mlx5_tile_t * ctx,
389 0 : uint recv_cnt ) {
390 0 : fd_mlx5_rx_wq_t * rx_wq = &ctx->rx_wq;
391 0 : uint const outstanding = rx_wq->rq_prod-rx_wq->rq_cons;
392 0 : if( FD_UNLIKELY( outstanding>rx_wq->rx_depth || recv_cnt>rx_wq->rx_depth-outstanding ) ) {
393 0 : errno = ENOSPC;
394 0 : return -1;
395 0 : }
396 :
397 0 : uint const rq_prod = rx_wq->rq_prod;
398 0 : for( uint i=0U; i<recv_cnt; i++ ) {
399 0 : uint const chunk = ctx->rq_pending_chunk[ i ];
400 0 : uint const rq_idx = rq_prod+i;
401 0 : fd_mlx5_hw_init_rx_wqe( rx_wq->rq+(rq_idx & (rx_wq->rx_depth-1U)),
402 0 : (ulong)chunk<<FD_CHUNK_LG_SZ, FD_NET_MTU, ctx->lkey );
403 0 : rx_wq->rq_wqe_buf_chunk[ rq_idx & (rx_wq->rx_depth-1U) ] = chunk;
404 0 : }
405 0 : rx_wq->rq_prod += recv_cnt;
406 0 : fd_mlx5_hw_dma_to_device();
407 0 : FD_VOLATILE( rx_wq->control->rq_prod ) = fd_uint_bswap( rx_wq->rq_prod & 0xffffU );
408 0 : return 0;
409 0 : }
410 :
411 : static inline int
412 : fd_mlx5_hw_rx_cqe_is_owned( fd_mlx5_cq_t const * rx_cq,
413 : uint cq_cons_idx,
414 0 : uchar op_own ) {
415 0 : return (op_own>>4)!=FD_MLX5_CQE_OP_INVALID &&
416 0 : (op_own & FD_MLX5_CQE_OWNER_MASK)==!!(cq_cons_idx & rx_cq->depth);
417 0 : }
418 :
419 : static inline int
420 : fd_mlx5_hw_poll_rx_cq( fd_mlx5_rx_wq_t * rx_wq,
421 : fd_mlx5_tile_rx_comp_t * comp,
422 0 : uint comp_capacity ) {
423 0 : fd_mlx5_cq_t * rx_cq = rx_wq->rx_cq;
424 0 : uint const comp_limit = fd_uint_min( comp_capacity, rx_cq->depth );
425 0 : uint cq_cons_idx = rx_cq->cons_idx;
426 0 : uint rq_cons_idx = rx_wq->rq_cons;
427 0 : uint const rq_prod_idx = rx_wq->rq_prod;
428 0 : FD_TEST( rq_prod_idx-rq_cons_idx<=rx_wq->rx_depth );
429 :
430 0 : uint comp_cnt = 0U;
431 0 : int poll_failed = 0;
432 0 : while( comp_cnt<comp_limit ) {
433 0 : fd_mlx5_hw_cqe64_t const * rx_cqe = (fd_mlx5_hw_cqe64_t const *)(rx_cq->entries+(cq_cons_idx & (rx_cq->depth-1U)));
434 0 : uchar const op_own = FD_VOLATILE_CONST( rx_cqe->op_own );
435 0 : if( FD_UNLIKELY( !fd_mlx5_hw_rx_cqe_is_owned( rx_cq, cq_cons_idx, op_own ) ) ) break;
436 0 : fd_mlx5_hw_dma_from_device();
437 0 : uint const opcode = (uint)(op_own>>4);
438 :
439 0 : if( FD_UNLIKELY( opcode!=FD_MLX5_CQE_OP_RX_OK && opcode!=FD_MLX5_CQE_OP_RX_ERR ) ) {
440 0 : errno = EINVAL;
441 0 : poll_failed = 1;
442 0 : break;
443 0 : }
444 0 : if( FD_UNLIKELY( opcode==FD_MLX5_CQE_OP_RX_ERR &&
445 0 : rx_cqe->syndrome!=FD_MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR ) ) {
446 0 : FD_LOG_ERR(( "mlx5 RX CQE error (syndrome 0x%02x, vendor syndrome 0x%02x, "
447 0 : "WQE opcode/QPN 0x%08x, WQE counter %u)",
448 0 : (uint)rx_cqe->syndrome, (uint)rx_cqe->vendor_err, fd_uint_bswap( rx_cqe->sop_drop_qpn ),
449 0 : (uint)fd_ushort_bswap( rx_cqe->wqe_counter ) ));
450 0 : }
451 :
452 0 : ushort const wqe_counter = fd_ushort_bswap( rx_cqe->wqe_counter );
453 0 : if( FD_UNLIKELY( rq_cons_idx==rq_prod_idx || wqe_counter!=(ushort)rq_cons_idx ) ) {
454 0 : errno = EPROTO;
455 0 : poll_failed = 1;
456 0 : break;
457 0 : }
458 :
459 0 : comp[ comp_cnt++ ] = (fd_mlx5_tile_rx_comp_t) {
460 0 : .chunk = rx_wq->rq_wqe_buf_chunk[ wqe_counter & (rx_wq->rx_depth-1U) ],
461 0 : .byte_len = fd_uint_bswap( rx_cqe->byte_cnt ),
462 0 : .opcode = opcode
463 0 : };
464 0 : rq_cons_idx++;
465 0 : cq_cons_idx++;
466 0 : }
467 :
468 0 : if( comp_cnt ) {
469 0 : rx_wq->rq_cons = rq_cons_idx;
470 0 : rx_cq->cons_idx = cq_cons_idx;
471 0 : FD_COMPILER_MFENCE();
472 0 : FD_VOLATILE( rx_cq->control->consumer_idx ) = fd_uint_bswap( cq_cons_idx & FD_MLX5_CQ_CONS_IDX_MASK );
473 0 : }
474 0 : return poll_failed ? -1 : (int)comp_cnt;
475 0 : }
476 :
477 : static inline void
478 0 : fd_mlx5_hw_request_rx_notification( fd_mlx5_cq_t * rx_cq ) {
479 0 : uint const notification_request = ((rx_cq->comp_channel_event_seq & FD_MLX5_CQ_ARM_SN_MASK)<<FD_MLX5_CQ_ARM_SN_SHIFT) |
480 0 : (rx_cq->cons_idx & FD_MLX5_CQ_CONS_IDX_MASK);
481 0 : FD_VOLATILE( rx_cq->control->request_notification ) = fd_uint_bswap( notification_request );
482 0 : fd_mlx5_hw_dma_to_device();
483 :
484 0 : uint const doorbell[2] = { fd_uint_bswap( notification_request ), fd_uint_bswap( rx_cq->cqn ) };
485 0 : volatile ulong * doorbell_reg = (volatile ulong *)rx_cq->request_notification_doorbell;
486 0 : doorbell_reg[0] = FD_LOAD( ulong, doorbell );
487 0 : rx_cq->comp_channel_armed = 1U;
488 0 : }
489 :
490 : static inline int
491 : fd_mlx5_hw_poll_tx_cq( fd_mlx5_tx_qp_t * tx_qp,
492 0 : ulong * comp_bytes ) {
493 0 : fd_mlx5_cq_t * tx_cq = tx_qp->tx_cq;
494 0 : uint const cq_cons_idx = tx_cq->cons_idx;
495 0 : uint const cq_entry_idx = cq_cons_idx & (tx_cq->depth-1U);
496 :
497 0 : fd_mlx5_hw_cqe64_t const * tx_cqe = (fd_mlx5_hw_cqe64_t const *)(tx_cq->entries+cq_entry_idx);
498 :
499 0 : uchar const op_own = FD_VOLATILE_CONST( tx_cqe->op_own );
500 0 : uint const opcode = (uint)(op_own>>4);
501 :
502 0 : if( FD_LIKELY( opcode==FD_MLX5_CQE_OP_INVALID ||
503 0 : (op_own & FD_MLX5_CQE_OWNER_MASK)!=!!(cq_cons_idx & tx_cq->depth) ) ) {
504 0 : return 0;
505 0 : }
506 0 : fd_mlx5_hw_dma_from_device();
507 :
508 0 : if( FD_UNLIKELY( opcode!=FD_MLX5_CQE_OP_TX_OK && opcode!=FD_MLX5_CQE_OP_TX_ERR ) ) {
509 0 : errno = EINVAL;
510 0 : return -1;
511 0 : }
512 :
513 0 : if( FD_UNLIKELY( opcode==FD_MLX5_CQE_OP_TX_ERR ) ) {
514 0 : FD_LOG_ERR(( "mlx5 TX CQE error (syndrome 0x%02x, vendor syndrome 0x%02x, "
515 0 : "WQE opcode/QPN 0x%08x, WQE counter %u)",
516 0 : (uint)tx_cqe->syndrome, (uint)tx_cqe->vendor_err, fd_uint_bswap( tx_cqe->sop_drop_qpn ),
517 0 : (uint)fd_ushort_bswap( tx_cqe->wqe_counter ) ));
518 0 : }
519 :
520 0 : uint const sq_cons_idx = tx_qp->sq_cons;
521 0 : uint const wqe_counter = (uint)fd_ushort_bswap( tx_cqe->wqe_counter );
522 0 : uint const outstanding = tx_qp->sq_posted-sq_cons_idx;
523 0 : uint const comp_cnt = (uint)(ushort)(wqe_counter-sq_cons_idx)+1U;
524 :
525 0 : if( FD_UNLIKELY( !outstanding || outstanding>tx_qp->tx_depth || comp_cnt>outstanding ) ) {
526 0 : errno = EPROTO;
527 0 : return -1;
528 0 : }
529 :
530 0 : *comp_bytes = 0UL;
531 0 : for( uint i=0U; i<comp_cnt; i++ ) {
532 0 : *comp_bytes += tx_qp->sq_wqe_frame_sz[ (sq_cons_idx+i) & (tx_qp->tx_depth-1U) ];
533 0 : }
534 0 : tx_qp->sq_cons = sq_cons_idx+comp_cnt;
535 0 : tx_cq->cons_idx = cq_cons_idx+1U;
536 0 : FD_COMPILER_MFENCE();
537 0 : FD_VOLATILE( tx_cq->control->consumer_idx ) = fd_uint_bswap( (cq_cons_idx+1U) & FD_MLX5_CQ_CONS_IDX_MASK );
538 0 : return (int)comp_cnt;
539 0 : }
540 :
541 : static inline void
542 0 : fd_mlx5_tile_sq_flush( fd_mlx5_tile_t * ctx ) {
543 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
544 0 : if( FD_UNLIKELY( tx_qp->sq_posted==tx_qp->sq_prod ) ) return;
545 :
546 0 : uint const last_sq_idx = tx_qp->sq_prod-1U;
547 0 : fd_mlx5_tx_wqe_t * last_wqe = tx_qp->sq+(last_sq_idx & (tx_qp->tx_depth-1U));
548 0 : fd_mlx5_hw_ring_sq( tx_qp, last_wqe );
549 0 : }
550 :
551 : static inline void
552 : fd_mlx5_tile_rx_recycle( fd_mlx5_tile_t * ctx,
553 0 : ulong chunk ) {
554 0 : ctx->rq_pending_chunk[ ctx->rq_pending_cnt++ ] = (uint)chunk;
555 :
556 : /* Hold partial recycle batches: at most batch_size-1 RQ buffers stay
557 : unposted, avoiding a device-ordering barrier and doorbell-record write
558 : per packet. */
559 0 : if( ctx->rq_pending_cnt==ctx->batch_size ) {
560 0 : FD_TEST( !fd_mlx5_hw_rq_enqueue( ctx, ctx->rq_pending_cnt ) );
561 0 : ctx->rq_pending_cnt = 0U;
562 0 : }
563 0 : }
564 :
565 : static inline int
566 : fd_mlx5_tile_poll_rx( fd_mlx5_tile_t * ctx,
567 0 : fd_stem_context_t * stem ) {
568 0 : fd_mlx5_tile_rx_comp_t comp[ FD_MLX5_BATCH_SIZE ];
569 0 : int comp_cnt = fd_mlx5_hw_poll_rx_cq( &ctx->rx_wq, comp, ctx->batch_size );
570 :
571 0 : if( FD_UNLIKELY( comp_cnt<0 ) ) {
572 0 : FD_LOG_ERR(( "direct mlx5 RX CQ poll failed (%i-%s)", errno, fd_io_strerror( errno ) ));
573 0 : }
574 0 : if( FD_UNLIKELY( !comp_cnt ) ) return 0;
575 :
576 0 : ulong const tspub = (ulong)fd_frag_meta_ts_comp( fd_tickcount() );
577 :
578 0 : for( uint i=0U; i<(uint)comp_cnt; i++ ) {
579 0 : ulong const chunk = comp[ i ].chunk;
580 0 : if( FD_UNLIKELY( chunk<ctx->net.pkt_buf_chunk0 || chunk>ctx->net.pkt_buf_wmark ) ) {
581 0 : FD_LOG_CRIT(( "RX completion chunk %lu out of bounds [%lu,%lu]", chunk, ctx->net.pkt_buf_chunk0, ctx->net.pkt_buf_wmark ));
582 0 : }
583 0 : __builtin_prefetch( fd_chunk_to_laddr_const( ctx->net.pkt_buf_wksp_base, chunk ), 0, 3 );
584 0 : }
585 :
586 0 : for( uint i=0U; i<(uint)comp_cnt; i++ ) {
587 0 : ulong freed_chunk;
588 0 : if( FD_UNLIKELY( comp[ i ].opcode==FD_MLX5_CQE_OP_RX_ERR ) ) {
589 0 : ctx->net.metrics.rx_drop_cnt[ FD_METRICS_ENUM_PKT_RX_DROP_REASON_V_MALFORMED_IDX ]++;
590 0 : freed_chunk = comp[ i ].chunk;
591 0 : } else {
592 0 : fd_net_rx_pkt( &ctx->net, stem, comp[ i ].chunk, 0UL, comp[ i ].byte_len, tspub, &freed_chunk, NULL );
593 0 : }
594 0 : if( FD_UNLIKELY( !freed_chunk ) ) FD_LOG_CRIT(( "invalid chunk in mcache" ));
595 0 : fd_mlx5_tile_rx_recycle( ctx, freed_chunk );
596 0 : }
597 0 : return 1;
598 0 : }
599 :
600 : static inline int
601 0 : fd_mlx5_tile_poll_tx( fd_mlx5_tile_t * ctx ) {
602 0 : ulong tx_pkt_cnt = 0UL;
603 0 : ulong tx_bytes_total = 0UL;
604 :
605 0 : int busy = 0;
606 0 : for( uint poll_cnt=0U; poll_cnt<ctx->batch_size; poll_cnt++ ) {
607 0 : ulong tx_bytes;
608 :
609 0 : int comp_cnt = fd_mlx5_hw_poll_tx_cq( &ctx->tx_qp, &tx_bytes );
610 0 : if( FD_UNLIKELY( comp_cnt<0 ) ) {
611 0 : FD_LOG_ERR(( "direct mlx5 TX CQ poll failed (%i-%s)", errno, fd_io_strerror( errno ) ));
612 0 : }
613 0 : if( FD_UNLIKELY( !comp_cnt ) ) break;
614 :
615 0 : busy = 1;
616 0 : tx_pkt_cnt += (uint)comp_cnt;
617 0 : tx_bytes_total += tx_bytes;
618 0 : }
619 0 : ctx->net.metrics.tx_pkt_cnt += tx_pkt_cnt;
620 0 : ctx->net.metrics.tx_bytes_total += tx_bytes_total;
621 :
622 0 : return busy;
623 0 : }
624 :
625 : /* fd_mlx5_tile_lo_tx_flush makes one nonblocking send attempt per batch,
626 : unsent packets are dropped. */
627 : static void
628 0 : fd_mlx5_tile_lo_tx_flush( fd_mlx5_tile_t * ctx ) {
629 0 : if( FD_UNLIKELY( !ctx->lo_tx_cnt ) ) return;
630 0 : int const send_cnt = sendmmsg( ctx->lo_tx_sock, ctx->lo_tx_msg, ctx->lo_tx_cnt, MSG_DONTWAIT );
631 0 : uint const sent_cnt = send_cnt<0 ? 0U : (uint)send_cnt;
632 0 : for( uint i=0U; i<sent_cnt; i++ ) ctx->net.metrics.tx_bytes_total += sizeof(fd_eth_hdr_t)+ctx->lo_tx_iov[ i ].iov_len;
633 0 : ctx->net.metrics.tx_pkt_cnt += sent_cnt;
634 0 : ctx->lo_tx_cnt = 0U;
635 0 : }
636 :
637 : static void
638 : fd_mlx5_tile_lo_tx_enqueue( fd_mlx5_tile_t * ctx,
639 : fd_ip4_hdr_t const * ip4,
640 0 : ulong ip_sz ) {
641 0 : uint const batch_idx = ctx->lo_tx_cnt;
642 0 : struct mmsghdr * msg = ctx->lo_tx_msg + batch_idx;
643 0 : struct sockaddr_in * sa = ctx->lo_tx_addr + batch_idx;
644 0 : struct iovec * iov = ctx->lo_tx_iov + batch_idx;
645 0 : uchar * buf = ctx->lo_tx_buf[ batch_idx ];
646 :
647 0 : *iov = (struct iovec) {
648 0 : .iov_base = buf,
649 0 : .iov_len = ip_sz,
650 0 : };
651 0 : sa->sin_family = AF_INET;
652 0 : sa->sin_addr.s_addr = ip4->daddr;
653 0 : sa->sin_port = 0; /* ignored */
654 :
655 0 : *msg = (struct mmsghdr) {
656 0 : .msg_hdr = {
657 0 : .msg_name = sa,
658 0 : .msg_namelen = sizeof(struct sockaddr_in),
659 0 : .msg_iov = iov,
660 0 : .msg_iovlen = 1UL
661 0 : }
662 0 : };
663 :
664 0 : fd_memcpy( buf, ip4, ip_sz );
665 0 : ctx->lo_tx_cnt++;
666 0 : if( ctx->lo_tx_cnt==FD_MLX5_BATCH_SIZE ) fd_mlx5_tile_lo_tx_flush( ctx );
667 0 : else if( ctx->lo_tx_cnt==1U ) ctx->lo_tx_deadline_ticks = fd_tickcount()+ctx->lo_tx_timeout_ticks;
668 0 : }
669 :
670 : /* fd_mlx5_tile_drain_rx_comp_channel reads every pending rx_cq completion event.
671 : Each event consumes the CQ's arm, so comp_channel_event_seq advances. */
672 : static void
673 0 : fd_mlx5_tile_drain_rx_comp_channel( fd_mlx5_tile_t * ctx ) {
674 0 : for(;;) {
675 0 : struct ib_uverbs_comp_event_desc comp_event;
676 0 : ssize_t comp_event_read_sz = read( ctx->rx_comp_channel_fd, &comp_event, sizeof(comp_event) );
677 0 : if( FD_UNLIKELY( comp_event_read_sz<0 && errno==EINTR ) ) continue;
678 :
679 0 : if( FD_LIKELY( comp_event_read_sz<0 && (errno==EAGAIN || errno==EWOULDBLOCK) ) ) break;
680 0 : if( FD_UNLIKELY( comp_event_read_sz!=(ssize_t)sizeof(comp_event) ) ) {
681 0 : FD_LOG_ERR(( "mlx5 RX completion event read failed (%i-%s)", errno, fd_io_strerror( errno ) ));
682 0 : }
683 0 : ctx->rx_cq.comp_channel_event_seq++;
684 0 : ctx->rx_cq.comp_channel_armed = 0U;
685 0 : }
686 0 : }
687 :
688 : static inline void
689 : fd_mlx5_tile_repoll_deadline_update( fd_mlx5_tile_t * ctx,
690 : int work_done,
691 0 : long now ) {
692 0 : if( FD_LIKELY( work_done ) ) {
693 0 : ctx->repoll_deadline_ticks = now + ctx->repoll_timeout_ticks;
694 0 : } else if( FD_UNLIKELY( now>=ctx->repoll_deadline_ticks ) ) {
695 0 : ctx->repoll_deadline_ticks = LONG_MAX;
696 0 : }
697 0 : }
698 :
699 : static inline void
700 : before_credit( fd_mlx5_tile_t * ctx,
701 : fd_stem_context_t * stem,
702 0 : int * charge_busy ) {
703 0 : ctx->has_out_credit = 0U;
704 0 : fd_net_router_solicit( &ctx->net.router, stem );
705 :
706 0 : if( FD_UNLIKELY( ctx->lo_tx_cnt && fd_tickcount()>=ctx->lo_tx_deadline_ticks ) ) {
707 0 : fd_mlx5_tile_lo_tx_flush( ctx );
708 0 : *charge_busy = 1;
709 0 : }
710 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
711 0 : uint const sq_pending_cnt = tx_qp->sq_prod-tx_qp->sq_posted;
712 :
713 0 : if( FD_UNLIKELY( sq_pending_cnt && fd_tickcount()>=ctx->sq_flush_deadline_ticks ) ) {
714 0 : fd_mlx5_tile_sq_flush( ctx );
715 0 : *charge_busy = 1;
716 0 : }
717 0 : if( tx_qp->sq_cons!=tx_qp->sq_posted ) {
718 0 : *charge_busy |= fd_mlx5_tile_poll_tx( ctx );
719 0 : }
720 0 : }
721 :
722 : /* after_credit is called every run loop iteration, provided there is
723 : sufficient downstream credit for forwarding on all output links. */
724 : static inline void
725 : after_credit( fd_mlx5_tile_t * ctx,
726 : fd_stem_context_t * stem,
727 : int * poll_in,
728 0 : int * charge_busy ) {
729 0 : (void)poll_in;
730 0 : ctx->has_out_credit = 1U;
731 0 : int rx_busy = fd_mlx5_tile_poll_rx( ctx, stem );
732 0 : if( FD_UNLIKELY( ctx->rx_comp_channel_fd>=0 ) ) {
733 0 : fd_mlx5_tile_repoll_deadline_update( ctx, rx_busy, fd_tickcount() );
734 0 : }
735 0 : *charge_busy |= rx_busy;
736 0 : }
737 :
738 : static void
739 0 : fd_mlx5_tile_async_drain( fd_mlx5_tile_t * ctx ) {
740 0 : int const async_event_fd = ctx->uverbs.async_fd;
741 0 : for(;;) {
742 0 : struct ib_uverbs_async_event_desc async_event;
743 0 : ssize_t async_event_read_sz = read( async_event_fd, &async_event, sizeof(async_event) );
744 0 : if( FD_UNLIKELY( async_event_read_sz<0 && errno==EINTR ) ) continue;
745 :
746 0 : if( FD_LIKELY( async_event_read_sz<0 && (errno==EAGAIN || errno==EWOULDBLOCK) ) ) break;
747 0 : if( FD_UNLIKELY( async_event_read_sz!=(ssize_t)sizeof(async_event) ) ) {
748 0 : FD_LOG_ERR(( "mlx5 async event read failed (%i-%s)", errno, fd_io_strerror( errno ) ));
749 0 : }
750 :
751 0 : uint const async_event_type = async_event.event_type;
752 0 : ulong const async_event_element = (ulong)async_event.element;
753 0 : if( FD_UNLIKELY( async_event_type==FD_MLX5_ASYNC_EVENT_CQ_ERR ||
754 0 : async_event_type==FD_MLX5_ASYNC_EVENT_QP_FATAL ||
755 0 : async_event_type==FD_MLX5_ASYNC_EVENT_QP_REQ_ERR ||
756 0 : async_event_type==FD_MLX5_ASYNC_EVENT_QP_ACCESS_ERR ||
757 0 : async_event_type==FD_MLX5_ASYNC_EVENT_DEVICE_FATAL ||
758 0 : async_event_type==FD_MLX5_ASYNC_EVENT_WQ_FATAL ) ) {
759 0 : FD_LOG_ERR(( "fatal mlx5 async event %u on element %lu", async_event_type, async_event_element ));
760 0 : }
761 0 : FD_LOG_INFO(( "mlx5 async event %u on element %lu", async_event_type, async_event_element ));
762 0 : }
763 0 : }
764 :
765 : static inline int
766 0 : prevent_park( fd_mlx5_tile_t * ctx ) {
767 0 : if( FD_UNLIKELY( ctx->tx_qp.sq_prod!=ctx->tx_qp.sq_posted ) ) fd_mlx5_tile_sq_flush( ctx );
768 0 : if( FD_UNLIKELY( ctx->lo_tx_cnt ) ) fd_mlx5_tile_lo_tx_flush( ctx );
769 0 : if( FD_UNLIKELY( !ctx->has_out_credit ) ) return 0;
770 0 : fd_mlx5_cq_t * rx_cq = &ctx->rx_cq;
771 0 : if( FD_UNLIKELY( !rx_cq->comp_channel_armed &&
772 0 : ctx->repoll_deadline_ticks==LONG_MAX ) ) {
773 0 : fd_mlx5_hw_request_rx_notification( rx_cq );
774 0 : }
775 0 : uint const cq_cons_idx = rx_cq->cons_idx;
776 0 : fd_mlx5_hw_cqe64_t const * rx_cqe = (fd_mlx5_hw_cqe64_t const *)(rx_cq->entries+(cq_cons_idx & (rx_cq->depth-1U)));
777 0 : uchar const op_own = FD_VOLATILE_CONST( rx_cqe->op_own );
778 0 : return fd_mlx5_hw_rx_cqe_is_owned( rx_cq, cq_cons_idx, op_own );
779 0 : }
780 :
781 : static inline int
782 : park_wait( fd_mlx5_tile_t * ctx,
783 : ulong * word,
784 : long deadline,
785 0 : double tick_per_ns ) {
786 0 : if( FD_UNLIKELY( ctx->has_out_credit && ctx->repoll_deadline_ticks!=LONG_MAX ) ) {
787 0 : struct timespec timeout;
788 0 : clock_gettime( CLOCK_MONOTONIC, &timeout );
789 0 : long const remaining_ns = (long)((double)(ctx->repoll_deadline_ticks-fd_tickcount())/tick_per_ns);
790 0 : if( FD_UNLIKELY( remaining_ns<=0L ) ) return FD_SLEEP_UNPARK_DEADLINE;
791 0 : timeout.tv_sec += remaining_ns/FD_MLX5_NS_PER_SEC;
792 0 : timeout.tv_nsec += remaining_ns%FD_MLX5_NS_PER_SEC;
793 0 : if( timeout.tv_nsec>=FD_MLX5_NS_PER_SEC ) {
794 0 : timeout.tv_sec++;
795 0 : timeout.tv_nsec -= FD_MLX5_NS_PER_SEC;
796 0 : }
797 0 : for(;;) {
798 0 : int const err = clock_nanosleep( CLOCK_MONOTONIC, TIMER_ABSTIME, &timeout, NULL );
799 0 : if( FD_LIKELY( !err ) ) return FD_SLEEP_UNPARK_DEADLINE;
800 0 : if( FD_UNLIKELY( err!=EINTR ) ) {
801 0 : FD_LOG_ERR(( "clock_nanosleep() failed (%i-%s)", err, fd_io_strerror( err ) ));
802 0 : }
803 0 : }
804 0 : }
805 :
806 0 : struct epoll_event evs[ 3 ];
807 0 : int ev_cnt;
808 0 : int cause = fd_sleep_park_wait_epoll( ctx->epoll_fd, word, evs, 3, &ev_cnt, deadline, tick_per_ns );
809 0 : for( int i=0; i<ev_cnt; i++ ) {
810 0 : if( evs[ i ].data.u64==(ulong)ctx->rx_comp_channel_fd ) fd_mlx5_tile_drain_rx_comp_channel( ctx );
811 0 : else if( evs[ i ].data.u64==FD_MLX5_EPOLL_ASYNC ) fd_mlx5_tile_async_drain( ctx );
812 0 : }
813 0 : return cause;
814 0 : }
815 :
816 : static inline long
817 0 : next_deadline( fd_mlx5_tile_t * ctx ) {
818 0 : if( !ctx->has_out_credit ) return LONG_MAX; /* If no out credit then don't repoll */
819 :
820 0 : return ctx->repoll_deadline_ticks; /* LONG_MAX if no repoll */
821 0 : }
822 :
823 : /* before_frag resolves the TX route and checks SQ capacity */
824 : static inline int
825 : before_frag( fd_mlx5_tile_t * ctx,
826 : ulong in_idx,
827 : ulong seq,
828 0 : ulong sig ) {
829 0 : (void)seq;
830 : /* Short circuit TX logic for route updates */
831 0 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) return 0;
832 :
833 0 : if( !fd_net_tx_select( &ctx->net, sig ) ) return 1;
834 :
835 : /* Drop the packet if no SQ WQE is available. */
836 0 : fd_mlx5_tx_qp_t const * tx_qp = &ctx->tx_qp;
837 0 : uint const outstanding = tx_qp->sq_prod-tx_qp->sq_cons;
838 0 : if( FD_UNLIKELY( outstanding>=tx_qp->tx_depth ) ) {
839 0 : ctx->metrics.tx_no_buffer_cnt++;
840 0 : return 1;
841 0 : }
842 :
843 0 : fd_net_tx_route_result_increment( &ctx->net );
844 0 : return 0; /* continue */
845 0 : }
846 :
847 : /* during_frag validates and stages the input packet or route update */
848 : static inline void
849 : during_frag( fd_mlx5_tile_t * ctx,
850 : ulong in_idx,
851 : ulong seq,
852 : ulong sig,
853 : ulong chunk,
854 : ulong frame_sz,
855 0 : ulong ctl ) {
856 0 : (void)seq; (void)sig; (void)ctl;
857 0 : fd_net_in_frag_validate( &ctx->net, in_idx, chunk, frame_sz );
858 :
859 0 : if( FD_UNLIKELY( fd_net_iproute_msg_stage( &ctx->net, in_idx, chunk, frame_sz ) ) ) return;
860 :
861 0 : ulong dst_chunk = fd_mlx5_tile_tx_chunk( ctx, ctx->tx_qp.sq_prod );
862 0 : uchar * dst = fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, dst_chunk );
863 : /* Speculatively copy frame from in link into buffer */
864 0 : fd_net_tx_pkt_cpy( &ctx->net, dst, chunk, frame_sz, in_idx );
865 0 : }
866 :
867 : /* after_frag applies a route update or completes and submits the staged packet */
868 : static void
869 : after_frag( fd_mlx5_tile_t * ctx,
870 : ulong in_idx,
871 : ulong seq,
872 : ulong sig,
873 : ulong frame_sz,
874 : ulong tsorig,
875 : ulong tspub,
876 0 : fd_stem_context_t * stem ) {
877 0 : (void)seq; (void)sig; (void)tsorig; (void)tspub;
878 :
879 0 : if( FD_UNLIKELY( ctx->net.in_kind[ in_idx ]==FD_NET_IN_KIND_IPROUTE ) ) {
880 0 : fd_iproute_msg_t const * msg = &ctx->net.iproute_msg;
881 0 : if( msg->op==FD_IPROUTE_OP_FLUSH ) {
882 0 : fd_fib4_clear( ctx->net.router.fib_local );
883 0 : fd_fib4_clear( ctx->net.router.fib_main );
884 0 : return;
885 0 : }
886 :
887 0 : fd_fib4_t * fib;
888 0 : if( msg->table_id==RT_TABLE_LOCAL ) fib = ctx->net.router.fib_local;
889 0 : else if( msg->table_id==RT_TABLE_MAIN ) fib = ctx->net.router.fib_main;
890 0 : else return;
891 :
892 0 : if( msg->op==FD_IPROUTE_OP_UPSERT && FD_UNLIKELY( !fd_fib4_insert( fib, msg->dst_addr, msg->prefix, msg->prio, &msg->hop ) ) ) {
893 0 : FD_LOG_WARNING(( "route update dropped: route table full (increase [tiles.netlink].max_routes or [tiles.netlink].max_peer_routes)" ));
894 0 : fd_stem_publish( stem, ctx->net.router.netlnk_out_idx, FD_NETLINK_ROUTE4_SYNC_SIG, 0UL, 0UL, 0UL, 0UL, fd_frag_meta_ts_comp( fd_tickcount() ) );
895 0 : } else if( msg->op==FD_IPROUTE_OP_DELETE ) {
896 0 : fd_fib4_remove( fib, msg->dst_addr, msg->prefix, msg->prio );
897 0 : }
898 0 : return;
899 0 : }
900 :
901 0 : ulong chunk = fd_mlx5_tile_tx_chunk( ctx, ctx->tx_qp.sq_prod );
902 0 : uchar * frame = fd_chunk_to_laddr( ctx->net.pkt_buf_wksp_base, chunk );
903 :
904 0 : if( FD_UNLIKELY( !fd_net_tx_pkt_prep( &ctx->net, frame, &frame_sz, in_idx ) ) ) return;
905 :
906 : /* Packets that failed userland routing are sent on net.sock starting
907 : at the IPv4 header, and the kernel builds the L2 header. The send
908 : is synchronous, so no WQE is posted and the chunk is reused. */
909 0 : if( FD_UNLIKELY( ctx->net.tx_route.routing_failure ) ) {
910 0 : if( FD_LIKELY( fd_net_tx_sock_send( &ctx->net, frame, frame_sz ) ) ) {
911 0 : ctx->net.metrics.tx_pkt_cnt++;
912 0 : ctx->net.metrics.tx_bytes_total += frame_sz;
913 0 : ctx->net.metrics.tx_gre_cnt += (ulong)ctx->net.tx_route.use_gre;
914 0 : }
915 0 : return;
916 0 : }
917 :
918 0 : if( FD_UNLIKELY( ctx->net.tx_route.use_loopback ) ) {
919 0 : fd_eth_hdr_t const * eth_hdr = (fd_eth_hdr_t const *)frame;
920 0 : fd_ip4_hdr_t const * ip4 = (fd_ip4_hdr_t const *)(eth_hdr+1);
921 0 : ulong const ip_hdr_sz = FD_IP4_GET_LEN( *ip4 );
922 0 : ulong const ip_sz = fd_ushort_bswap( ip4->net_tot_len );
923 0 : if( FD_UNLIKELY( ip4->protocol!=FD_IP4_HDR_PROTOCOL_UDP ||
924 0 : ip4->daddr!=fd_disco_netmux_sig_ip( sig ) ||
925 0 : (fd_ushort_bswap( ip4->net_frag_off ) & ~FD_IP4_HDR_FRAG_OFF_DF) ||
926 0 : ip_sz<ip_hdr_sz+sizeof(fd_udp_hdr_t) ||
927 0 : ip_sz>frame_sz-sizeof(fd_eth_hdr_t) ) ) {
928 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
929 0 : return;
930 0 : }
931 0 : fd_udp_hdr_t const * udp = (fd_udp_hdr_t const *)((uchar const *)ip4+ip_hdr_sz);
932 0 : ulong const udp_sz = fd_ushort_bswap( udp->net_len );
933 0 : if( FD_UNLIKELY( udp_sz<sizeof(fd_udp_hdr_t) || udp_sz>ip_sz-ip_hdr_sz ) ) {
934 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_INVALID_IDX ]++;
935 0 : return;
936 0 : }
937 :
938 0 : int owned = 0;
939 0 : if( !ctx->net.bind_address || ctx->net.bind_address==ip4->daddr ) {
940 0 : ushort const dst_port = fd_ushort_bswap( udp->net_dport );
941 0 : for( ulong i=0UL; i<ctx->net.rx_dest_cnt; i++ ) owned |= ctx->net.rx_listen_ports[ i ]==dst_port;
942 0 : }
943 0 : if( !owned ) {
944 0 : fd_mlx5_tile_lo_tx_enqueue( ctx, ip4, ip_sz );
945 0 : return;
946 0 : }
947 :
948 0 : ulong freed_chunk;
949 0 : if( fd_net_rx_pkt( &ctx->net, stem, chunk, 0UL, frame_sz, (ulong)fd_frag_meta_ts_comp( fd_tickcount() ), &freed_chunk, NULL ) ) {
950 0 : ctx->sq_wqe_buf_chunk[ ctx->tx_qp.sq_prod & (ctx->tx_qp.tx_depth-1U) ] = (uint)freed_chunk;
951 0 : }
952 0 : ctx->net.metrics.tx_pkt_cnt++;
953 0 : ctx->net.metrics.tx_bytes_total += frame_sz;
954 0 : return;
955 0 : }
956 :
957 0 : FD_TEST( !fd_mlx5_hw_sq_enqueue( ctx, (uint)frame_sz ) );
958 :
959 0 : ctx->net.metrics.tx_gre_cnt += (ulong)ctx->net.tx_route.use_gre;
960 :
961 0 : fd_mlx5_tx_qp_t * tx_qp = &ctx->tx_qp;
962 0 : uint const sq_pending_cnt = tx_qp->sq_prod-tx_qp->sq_posted;
963 :
964 0 : if( sq_pending_cnt>=ctx->batch_size || tx_qp->sq_prod-tx_qp->sq_cons>=tx_qp->tx_depth ) {
965 0 : fd_mlx5_tile_sq_flush( ctx );
966 0 : } else if( sq_pending_cnt==1U ) {
967 0 : ctx->sq_flush_deadline_ticks = fd_tickcount()+ctx->sq_flush_timeout_ticks;
968 0 : }
969 0 : if( FD_UNLIKELY( ctx->rx_comp_channel_fd>=0 ) ) {
970 0 : fd_mlx5_tile_repoll_deadline_update( ctx, 1, fd_tickcount() );
971 0 : }
972 0 : }
973 :
974 : static inline void
975 0 : metrics_write( fd_mlx5_tile_t * ctx ) {
976 0 : fd_mlx5_rx_wq_t const * rx_wq = &ctx->rx_wq;
977 0 : fd_mlx5_tx_qp_t const * tx_qp = &ctx->tx_qp;
978 0 : ulong const rx_buffer_idle_cnt = fd_ulong_min( (ulong)(rx_wq->rq_prod-rx_wq->rq_cons), rx_wq->rx_depth );
979 0 : ulong const rx_buffer_busy_cnt = rx_wq->rx_depth-rx_buffer_idle_cnt;
980 0 : ulong const tx_buffer_busy_cnt = fd_ulong_min( (ulong)(tx_qp->sq_prod-tx_qp->sq_cons), tx_qp->tx_depth );
981 0 : ulong const tx_buffer_idle_cnt = tx_qp->tx_depth-tx_buffer_busy_cnt;
982 :
983 0 : FD_MCNT_SET( MLX5, PKT_RX, ctx->net.metrics.rx_pkt_cnt );
984 0 : FD_MCNT_SET( MLX5, PKT_RX_BYTES, ctx->net.metrics.rx_bytes_total );
985 0 : FD_MCNT_ENUM_COPY( MLX5, PKT_RX_DROPPED, ctx->net.metrics.rx_drop_cnt );
986 0 : FD_MCNT_SET( MLX5, GRE_PKT_RX, ctx->net.metrics.rx_gre_cnt );
987 0 : FD_MCNT_SET( MLX5, VLAN_PKT_RX, ctx->net.metrics.rx_vlan_cnt );
988 0 : FD_MGAUGE_SET( MLX5, RX_BUFFER_USED, rx_buffer_busy_cnt );
989 0 : FD_MGAUGE_SET( MLX5, RX_BUFFER_FREE, rx_buffer_idle_cnt );
990 :
991 0 : FD_MCNT_SET( MLX5, PKT_TX_COMPLETED, ctx->net.metrics.tx_pkt_cnt );
992 0 : FD_MCNT_SET( MLX5, PKT_TX_COMPLETED_BYTES, ctx->net.metrics.tx_bytes_total );
993 0 : ctx->net.metrics.tx_drop_cnt[ FD_METRICS_ENUM_PKT_TX_DROP_REASON_V_RING_FULL_IDX ] = ctx->metrics.tx_no_buffer_cnt;
994 0 : FD_MCNT_ENUM_COPY( MLX5, PKT_TX_DROPPED, ctx->net.metrics.tx_drop_cnt );
995 0 : FD_MCNT_ENUM_COPY( MLX5, PKT_TX_ROUTED, ctx->net.metrics.tx_route_result_cnt );
996 0 : FD_MCNT_ENUM_COPY( MLX5, PKT_TX_SLOW_PATH_ROUTED, ctx->net.metrics.tx_slow_path_reason_cnt );
997 0 : FD_MCNT_SET( MLX5, GRE_PKT_TX_SUBMITTED, ctx->net.metrics.tx_gre_cnt );
998 0 : FD_MCNT_SET( MLX5, PKT_TX_SLOW_PATH_SUBMITTED, ctx->net.metrics.tx_slow_path_submit_cnt );
999 0 : FD_MGAUGE_SET( MLX5, TX_BUFFER_USED, tx_buffer_busy_cnt );
1000 0 : FD_MGAUGE_SET( MLX5, TX_BUFFER_FREE, tx_buffer_idle_cnt );
1001 0 : }
1002 :
1003 : static inline void
1004 0 : during_housekeeping( fd_mlx5_tile_t * ctx ) {
1005 0 : fd_mlx5_tile_async_drain( ctx );
1006 :
1007 0 : if( FD_UNLIKELY( fd_netdev_tbl_refresh( &ctx->net.router.netdev_tbl, &ctx->net.router.netdev_shared, &ctx->netdev_seq ) ) ) {
1008 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->net.router.netdev_tbl );
1009 0 : }
1010 0 : }
1011 :
1012 : static uint
1013 0 : fd_mlx5_tile_if_ip4_addr( char const * if_name ) {
1014 0 : int sock_fd = socket( AF_INET, SOCK_DGRAM, 0 );
1015 0 : if( FD_UNLIKELY( sock_fd<0 ) ) {
1016 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_DGRAM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1017 0 : }
1018 :
1019 0 : struct ifreq ifr = { .ifr_addr.sa_family = AF_INET };
1020 0 : fd_cstr_ncpy( ifr.ifr_name, if_name, sizeof(ifr.ifr_name) );
1021 :
1022 0 : if( FD_UNLIKELY( ioctl( sock_fd, SIOCGIFADDR, &ifr ) ) ) {
1023 0 : FD_LOG_ERR(( "could not get IP address of interface `%s` (%i-%s)",
1024 0 : if_name, errno, fd_io_strerror( errno ) ));
1025 0 : }
1026 0 : uint ip4_addr = ((struct sockaddr_in *)fd_type_pun( &ifr.ifr_addr ))->sin_addr.s_addr;
1027 0 : if( FD_UNLIKELY( close( sock_fd ) ) ) {
1028 0 : FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1029 0 : }
1030 :
1031 0 : return ip4_addr;
1032 0 : }
1033 :
1034 : /* fd_mlx5_tile_lo_tx_socket uses IPPROTO_RAW for send only IP_HDRINCL
1035 : transmission, preserving the source address and UDP source port. */
1036 : static int
1037 0 : fd_mlx5_tile_lo_tx_socket( void ) {
1038 0 : int sock = socket( AF_INET, SOCK_RAW|SOCK_CLOEXEC|SOCK_NONBLOCK, IPPROTO_RAW );
1039 0 : if( FD_UNLIKELY( sock<0 ) ) {
1040 0 : FD_LOG_ERR(( "socket(AF_INET,SOCK_RAW,IPPROTO_RAW) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1041 0 : }
1042 0 : if( FD_UNLIKELY( 0!=setsockopt( sock, SOL_SOCKET, SO_BINDTODEVICE, "lo", sizeof("lo") ) ) ) {
1043 0 : FD_LOG_ERR(( "setsockopt(SO_BINDTODEVICE,lo) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1044 0 : }
1045 0 : return sock;
1046 0 : }
1047 :
1048 : static ulong
1049 0 : scratch_align( void ) {
1050 0 : ulong a = alignof( fd_mlx5_tile_t );
1051 0 : a = fd_ulong_max( a, FD_MLX5_PAGE_SZ );
1052 0 : a = fd_ulong_max( a, fd_netdev_tbl_align() );
1053 0 : a = fd_ulong_max( a, fd_fib4_align() );
1054 0 : return a;
1055 0 : }
1056 :
1057 : static ulong
1058 0 : scratch_footprint( fd_topo_tile_t const * tile ) {
1059 0 : ulong layout = FD_LAYOUT_INIT;
1060 0 : ulong const queue_footprint = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size,
1061 0 : tile->mlx5.tx_queue_size );
1062 0 : if( FD_UNLIKELY( !queue_footprint ) ) return 0UL;
1063 :
1064 0 : layout = FD_LAYOUT_APPEND( layout, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1065 0 : layout = FD_LAYOUT_APPEND( layout, FD_MLX5_PAGE_SZ, queue_footprint );
1066 0 : layout = FD_LAYOUT_APPEND( layout, alignof(uint), tile->mlx5.tx_queue_size*sizeof(uint) );
1067 0 : layout = FD_LAYOUT_APPEND( layout, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1068 0 : layout = FD_LAYOUT_APPEND( layout, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1069 0 : layout = FD_LAYOUT_APPEND( layout, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1070 :
1071 0 : return FD_LAYOUT_FINI( layout, scratch_align() );
1072 0 : }
1073 :
1074 : fd_fib4_t *
1075 : fd_mlx5_tile_fib4_join( fd_fib4_t * out,
1076 : fd_topo_t const * topo,
1077 : fd_topo_tile_t const * tile,
1078 0 : int main_table ) {
1079 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1080 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1081 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size ) );
1082 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->mlx5.tx_queue_size*sizeof(uint) );
1083 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1084 0 : void * local_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1085 0 : void * main_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1086 0 : return fd_fib4_join( out, main_table ? main_mem : local_mem );
1087 0 : }
1088 :
1089 : static void
1090 : fd_mlx5_tile_packet_memory( fd_topo_t const * topo,
1091 : fd_topo_tile_t const * tile,
1092 : fd_mlx5_tile_t * ctx,
1093 : void ** packet_memory,
1094 : ulong * packet_memory_sz,
1095 0 : ulong * packet_iova ) {
1096 0 : void * const packet_dcache_memory = fd_topo_obj_laddr( topo, tile->net.umem_dcache_obj_id );
1097 0 : void * const packet_dcache = fd_dcache_join( packet_dcache_memory );
1098 0 : if( FD_UNLIKELY( !packet_dcache ) ) FD_LOG_ERR(( "Failed to join packet dcache" ));
1099 0 : ulong const packet_dcache_data_sz = fd_dcache_data_sz( packet_dcache );
1100 0 : ulong const frame_sz = FD_NET_MTU;
1101 0 : ulong const frame_region_sz = fd_ulong_align_dn( packet_dcache_data_sz, frame_sz );
1102 :
1103 0 : void * const workspace_base = fd_wksp_containing( packet_dcache_memory );
1104 0 : if( FD_UNLIKELY( !workspace_base ) ) FD_LOG_ERR(( "Packet dcache is not in a workspace" ));
1105 0 : ulong const pkt_buf_chunk0 = ((ulong)packet_dcache-(ulong)workspace_base)>>FD_CHUNK_LG_SZ;
1106 0 : ulong const pkt_buf_wmark = pkt_buf_chunk0 + ((frame_region_sz-frame_sz)>>FD_CHUNK_LG_SZ);
1107 0 : if( FD_UNLIKELY( pkt_buf_chunk0>UINT_MAX || pkt_buf_wmark>UINT_MAX || pkt_buf_chunk0>pkt_buf_wmark ) ) {
1108 0 : FD_LOG_ERR(( "Calculated invalid packet buffer bounds [%lu,%lu]", pkt_buf_chunk0, pkt_buf_wmark ));
1109 0 : }
1110 :
1111 0 : ctx->net.pkt_buf_wksp_base = workspace_base;
1112 0 : ctx->net.pkt_buf_chunk0 = pkt_buf_chunk0;
1113 0 : ctx->net.pkt_buf_wmark = pkt_buf_wmark;
1114 0 : if( packet_memory ) *packet_memory = packet_dcache;
1115 0 : if( packet_memory_sz ) *packet_memory_sz = frame_region_sz;
1116 0 : if( packet_iova ) *packet_iova = (ulong)packet_dcache-(ulong)workspace_base;
1117 0 : }
1118 :
1119 : FD_FN_UNUSED static void
1120 : fd_mlx5_tile_join_obj_workspace( fd_topo_t * topo,
1121 0 : ulong obj_id ) {
1122 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->objs[ obj_id ].wksp_id ];
1123 0 : if( FD_LIKELY( !wksp->wksp ) ) {
1124 0 : fd_topo_join_workspace( topo, wksp, FD_SHMEM_JOIN_MODE_READ_WRITE, 0 );
1125 0 : }
1126 0 : }
1127 :
1128 : #ifndef FD_TILE_TEST
1129 : void
1130 : fd_topo_install_mlx5( fd_topo_t * topo,
1131 0 : fd_mlx5_fds_t * fds ) {
1132 0 : ulong const tile_cnt = fd_topo_tile_name_cnt( topo, "mlx5" );
1133 0 : if( FD_UNLIKELY( 0==tile_cnt || tile_cnt>FD_MLX5_TILE_MAX ) ) {
1134 0 : FD_LOG_ERR(( "mlx5 tile count %lu must be in [1,%lu]", tile_cnt, FD_MLX5_TILE_MAX ));
1135 0 : }
1136 :
1137 0 : fd_mlx5_tile_t * ctxs [ FD_MLX5_TILE_MAX ];
1138 0 : fd_mlx5_uverbs_tile_t queues[ FD_MLX5_TILE_MAX ];
1139 0 : fd_topo_tile_t const * first_tile = NULL;
1140 0 : char rdma_device_name[ FD_MLX5_RDMA_NAME_MAX ];
1141 0 : uint rdma_port_num = 0U;
1142 :
1143 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1144 0 : ulong const tile_id = fd_topo_find_tile( topo, "mlx5", i );
1145 0 : FD_TEST( tile_id!=ULONG_MAX );
1146 0 : fd_topo_tile_t const * tile = topo->tiles+tile_id;
1147 0 : fd_mlx5_tile_join_obj_workspace( topo, tile->tile_obj_id );
1148 0 : fd_mlx5_tile_join_obj_workspace( topo, tile->net.umem_dcache_obj_id );
1149 :
1150 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1151 0 : fd_mlx5_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1152 0 : ulong const queue_memory_sz = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size );
1153 0 : void * queue_memory = FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, queue_memory_sz );
1154 0 : ctxs[ i ] = ctx;
1155 0 : fd_memset( ctx, 0, sizeof(*ctx) );
1156 0 : ctx->rx_comp_channel_fd = -1;
1157 0 : ctx->epoll_fd = -1;
1158 0 : FD_TEST( fd_mlx5_hw_init_queues( ctx, queue_memory,
1159 0 : tile->mlx5.rx_queue_size,
1160 0 : tile->mlx5.tx_queue_size ) );
1161 :
1162 0 : void * packet_memory;
1163 0 : ulong packet_memory_sz;
1164 0 : ulong packet_iova;
1165 0 : fd_mlx5_tile_packet_memory( topo, tile, ctx, &packet_memory, &packet_memory_sz, &packet_iova );
1166 0 : queues[ i ] = (fd_mlx5_uverbs_tile_t) {
1167 0 : .rx_cq = &ctx->rx_cq,
1168 0 : .tx_cq = &ctx->tx_cq,
1169 0 : .rx_wq = &ctx->rx_wq,
1170 0 : .tx_qp = &ctx->tx_qp,
1171 0 : .lkey = &ctx->lkey,
1172 0 : .rx_comp_channel_fd = tile->sleep_eventfd ? &ctx->rx_comp_channel_fd : NULL,
1173 0 : .packet_memory = packet_memory,
1174 0 : .packet_memory_sz = packet_memory_sz,
1175 0 : .packet_iova = packet_iova,
1176 0 : };
1177 0 : fd_net_rx_dests_init( &ctx->net, topo, tile );
1178 :
1179 0 : if( !i ) {
1180 0 : first_tile = tile;
1181 0 : if( FD_UNLIKELY( !fd_mlx5_rdma_dev_find( rdma_device_name, &rdma_port_num, tile->mlx5.if_name ) ) ) {
1182 0 : if( errno==EEXIST ) FD_LOG_ERR(( "multiple RDMA ports match interface `%s`", tile->mlx5.if_name ));
1183 0 : if( errno==ENOENT ) FD_LOG_ERR(( "RDMA device port for interface `%s` not found", tile->mlx5.if_name ));
1184 0 : FD_LOG_ERR(( "finding RDMA device port for interface `%s` failed (%i-%s)",
1185 0 : tile->mlx5.if_name, errno, fd_io_strerror( errno ) ));
1186 0 : }
1187 0 : }
1188 0 : }
1189 :
1190 0 : fd_mlx5_tile_t * first = ctxs[ 0 ];
1191 0 : if( FD_UNLIKELY( !fd_mlx5_uverbs_avail() ) ) {
1192 0 : FD_LOG_ERR(( "failed to run mlx5 tile: uverbs interface unavailable. "
1193 0 : "Run `sudo modprobe ib_uverbs` and retry. "
1194 0 : "If it still fails, please set [net.provider] to `xdp`" ));
1195 0 : }
1196 0 : FD_LOG_INFO(( "Opening direct mlx5 device `%s` port %u for %lu tiles",
1197 0 : rdma_device_name, rdma_port_num, tile_cnt ));
1198 0 : if( FD_UNLIKELY( !fd_uverbs_init( &first->uverbs, queues, tile_cnt,
1199 0 : &first->outer_rss_qp, &first->gre_rss_qp,
1200 0 : rdma_device_name, rdma_port_num ) ) ) {
1201 0 : FD_LOG_ERR(( "direct mlx5 setup failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1202 0 : }
1203 :
1204 0 : int const async_event_fd = first->uverbs.async_fd;
1205 0 : int const async_event_flags = fcntl( async_event_fd, F_GETFL );
1206 0 : if( FD_UNLIKELY( async_event_flags<0 ||
1207 0 : fcntl( async_event_fd, F_SETFL, async_event_flags|O_NONBLOCK )<0 ) ) {
1208 0 : FD_LOG_ERR(( "making mlx5 async fd non-blocking failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1209 0 : }
1210 :
1211 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1212 0 : ctxs[ i ]->uverbs = first->uverbs;
1213 0 : ctxs[ i ]->outer_rss_qp = first->outer_rss_qp;
1214 0 : ctxs[ i ]->gre_rss_qp = first->gre_rss_qp;
1215 0 : ctxs[ i ]->prepared = 1U;
1216 :
1217 0 : int const rx_comp_channel_fd = ctxs[ i ]->rx_comp_channel_fd;
1218 0 : if( rx_comp_channel_fd<0 ) continue;
1219 0 : int const rx_comp_channel_fd_flags = fcntl( rx_comp_channel_fd, F_GETFL );
1220 0 : if( FD_UNLIKELY( rx_comp_channel_fd_flags<0 ||
1221 0 : fcntl( rx_comp_channel_fd, F_SETFL, rx_comp_channel_fd_flags|O_NONBLOCK )<0 ) ) {
1222 0 : FD_LOG_ERR(( "making mlx5 RX completion channel non-blocking failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1223 0 : }
1224 0 : }
1225 :
1226 0 : for( ulong flow_idx=0UL; flow_idx<first->net.rx_dest_cnt; flow_idx++ ) {
1227 0 : if( FD_UNLIKELY( fd_uverbs_create_udp_flow( &first->uverbs, &first->outer_rss_qp,
1228 0 : first_tile->mlx5.net.bind_address,
1229 0 : first->net.rx_listen_ports[ flow_idx ] ) ) ) {
1230 0 : FD_LOG_ERR(( "fd_uverbs_create_udp_flow failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1231 0 : }
1232 0 : if( FD_UNLIKELY( fd_uverbs_create_gre_udp_flow( &first->uverbs, &first->gre_rss_qp,
1233 0 : first_tile->mlx5.net.bind_address,
1234 0 : first->net.rx_listen_ports[ flow_idx ] ) ) ) {
1235 0 : FD_LOG_ERR(( "fd_uverbs_create_gre_udp_flow failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1236 0 : }
1237 0 : }
1238 0 : FD_LOG_INFO(( "Installed %lu direct mlx5 flow rules", 2UL*first->net.rx_dest_cnt ));
1239 :
1240 0 : if( fds ) {
1241 0 : fds->cmd_fd = first->uverbs.cmd_fd;
1242 0 : fds->async_fd = first->uverbs.async_fd;
1243 0 : fds->rx_comp_channel_fd_cnt = first->rx_comp_channel_fd>=0 ? tile_cnt : 0UL;
1244 0 : for( ulong i=0UL; i<fds->rx_comp_channel_fd_cnt; i++ )
1245 0 : fds->rx_comp_channel_fd[ i ] = ctxs[ i ]->rx_comp_channel_fd;
1246 0 : }
1247 0 : }
1248 : #endif
1249 :
1250 : FD_FN_UNUSED static void
1251 : privileged_init( fd_topo_t const * topo,
1252 0 : fd_topo_tile_t const * tile ) {
1253 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1254 0 : fd_mlx5_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1255 0 : ulong const queue_memory_sz = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size );
1256 0 : void * queue_memory = FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, queue_memory_sz );
1257 0 : if( FD_UNLIKELY( !ctx->prepared ) ) {
1258 0 : if( FD_UNLIKELY( fd_topo_tile_name_cnt( topo, "mlx5" )!=1UL ) ) {
1259 0 : FD_LOG_ERR(( "multi-tile mlx5 setup must run before tile launch" ));
1260 0 : }
1261 0 : fd_topo_install_mlx5( (fd_topo_t *)topo, NULL );
1262 0 : }
1263 0 : FD_TEST( fd_mlx5_hw_join_queues( ctx, queue_memory,
1264 0 : tile->mlx5.rx_queue_size,
1265 0 : tile->mlx5.tx_queue_size ) );
1266 0 : volatile uchar * uar = fd_uverbs_map_uar( &ctx->uverbs, ctx->tx_qp.uar_mmap_offset );
1267 0 : if( FD_UNLIKELY( !uar ) ) {
1268 0 : FD_LOG_ERR(( "mapping mlx5 UAR failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1269 0 : }
1270 0 : ctx->tx_qp.sq_doorbell = uar+FD_MLX5_UAR_SQ_DB_OFFSET;
1271 0 : ctx->rx_cq.request_notification_doorbell = uar+FD_MLX5_UAR_CQ_DB_OFFSET;
1272 :
1273 0 : ctx->batch_size = tile->mlx5.batch_size;
1274 0 : ctx->sq_wqe_buf_chunk = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint),
1275 0 : tile->mlx5.tx_queue_size*sizeof(uint) );
1276 :
1277 0 : fd_mlx5_tile_packet_memory( topo, tile, ctx, NULL, NULL, NULL );
1278 :
1279 : /* Resolve the netdev. */
1280 0 : uint const interface_idx = if_nametoindex( tile->mlx5.if_name );
1281 0 : if( FD_UNLIKELY( !interface_idx ) ) {
1282 0 : FD_LOG_ERR(( "if_nametoindex(%s) failed (%i-%s)",
1283 0 : tile->mlx5.if_name, errno, fd_io_strerror( errno ) ));
1284 0 : }
1285 0 : ctx->net.router.if_virt = interface_idx;
1286 0 : ctx->net.router.default_address = fd_mlx5_tile_if_ip4_addr( tile->mlx5.if_name );
1287 0 : ctx->lo_tx_sock = tile->kind_id==0UL ? fd_mlx5_tile_lo_tx_socket() : -1;
1288 0 : if( FD_UNLIKELY( !fd_net_sock_init( &ctx->net.sock ) ) ) {
1289 0 : FD_LOG_ERR(( "fd_net_sock_init failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1290 0 : }
1291 :
1292 0 : if( FD_UNLIKELY( tile->sleep_eventfd ) ) {
1293 0 : if( FD_UNLIKELY( ctx->rx_comp_channel_fd<0 ) ) {
1294 0 : FD_LOG_ERR(( "mlx5:%lu has no RX completion channel", tile->kind_id ));
1295 0 : }
1296 0 : ctx->epoll_fd = epoll_create1( EPOLL_CLOEXEC );
1297 0 : if( FD_UNLIKELY( ctx->epoll_fd<0 ) ) {
1298 0 : FD_LOG_ERR(( "epoll_create1() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1299 0 : }
1300 0 : struct epoll_event doorbell_ev = { .events = EPOLLIN|EPOLLET, .data.u64 = FD_SLEEP_EPOLL_DOORBELL };
1301 0 : if( FD_UNLIKELY( -1==epoll_ctl( ctx->epoll_fd, EPOLL_CTL_ADD, FD_SLEEP_EVENTFD( tile->id ), &doorbell_ev ) ) ) {
1302 0 : FD_LOG_ERR(( "epoll_ctl(ADD,sleep_eventfd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1303 0 : }
1304 0 : struct epoll_event comp_ev = { .events = EPOLLIN, .data.u64 = (ulong)ctx->rx_comp_channel_fd };
1305 0 : if( FD_UNLIKELY( -1==epoll_ctl( ctx->epoll_fd, EPOLL_CTL_ADD, ctx->rx_comp_channel_fd, &comp_ev ) ) ) {
1306 0 : FD_LOG_ERR(( "epoll_ctl(ADD,rx_comp_channel_fd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1307 0 : }
1308 0 : struct epoll_event async_ev = { .events = EPOLLIN, .data.u64 = FD_MLX5_EPOLL_ASYNC };
1309 0 : if( FD_UNLIKELY( -1==epoll_ctl( ctx->epoll_fd, EPOLL_CTL_ADD, ctx->uverbs.async_fd, &async_ev ) ) ) {
1310 0 : FD_LOG_ERR(( "epoll_ctl(ADD,async_fd) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
1311 0 : }
1312 0 : }
1313 0 : }
1314 :
1315 : FD_FN_UNUSED static void
1316 : unprivileged_init( fd_topo_t const * topo,
1317 0 : fd_topo_tile_t const * tile ) {
1318 0 : FD_SCRATCH_ALLOC_INIT( scratch, fd_topo_obj_laddr( topo, tile->tile_obj_id ) );
1319 0 : fd_mlx5_tile_t * ctx = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(fd_mlx5_tile_t), sizeof(fd_mlx5_tile_t) );
1320 0 : ulong queue_footprint = fd_mlx5_queue_footprint( tile->mlx5.rx_queue_size, tile->mlx5.tx_queue_size );
1321 0 : (void)FD_SCRATCH_ALLOC_APPEND( scratch, FD_MLX5_PAGE_SZ, queue_footprint );
1322 :
1323 0 : ctx->sq_wqe_buf_chunk = FD_SCRATCH_ALLOC_APPEND( scratch, alignof(uint), tile->mlx5.tx_queue_size*sizeof(uint) );
1324 0 : ctx->batch_size = tile->mlx5.batch_size;
1325 0 : ctx->sq_flush_timeout_ticks = (long)( FD_MLX5_TX_FLUSH_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ) );
1326 0 : ctx->lo_tx_timeout_ticks = (long)( FD_MLX5_LO_TX_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ) );
1327 :
1328 0 : ctx->repoll_timeout_ticks = (long)( FD_MLX5_REPOLL_TIMEOUT_NS*fd_tempo_tick_per_ns( NULL ) );
1329 0 : ctx->repoll_deadline_ticks = LONG_MAX;
1330 :
1331 0 : ctx->lo_tx_cnt = 0U;
1332 0 : ctx->net.kind_id = tile->kind_id;
1333 0 : ctx->net.tile_cnt = fd_topo_tile_name_cnt( topo, tile->name );
1334 :
1335 0 : void * netdev_tbl_local = FD_SCRATCH_ALLOC_APPEND( scratch, fd_netdev_tbl_align(), fd_netdev_tbl_footprint( NETDEV_MAX, BOND_MASTER_MAX ) );
1336 0 : void * fib_local_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1337 0 : void * fib_main_mem = FD_SCRATCH_ALLOC_APPEND( scratch, fd_fib4_align(), fd_fib4_footprint( tile->mlx5.route_max, tile->mlx5.route_peer_max ) );
1338 :
1339 : /* chunk 0 is used as a sentinel value, so ensure actual chunk indices
1340 : do not use that value. */
1341 0 : FD_TEST( ctx->net.pkt_buf_chunk0>0UL );
1342 :
1343 0 : ctx->rq_pending_cnt = 0U;
1344 :
1345 : /* Post RQ WQEs */
1346 0 : ulong frame_chunks = FD_NET_MTU>>FD_CHUNK_LG_SZ;
1347 0 : ulong next_chunk = ctx->net.pkt_buf_chunk0;
1348 :
1349 0 : FD_TEST( tile->mlx5.rx_queue_size>ctx->batch_size );
1350 :
1351 0 : ulong const rx_fill_cnt = tile->mlx5.rx_queue_size-ctx->batch_size;
1352 0 : for( ulong i=0UL; i<rx_fill_cnt; i++ ) {
1353 0 : fd_mlx5_tile_rx_recycle( ctx, next_chunk );
1354 0 : next_chunk += frame_chunks;
1355 0 : }
1356 :
1357 : /* Assign chunks to RX mcaches */
1358 0 : for( ulong i=0UL; i<(tile->out_cnt); i++ ) {
1359 0 : fd_frag_meta_t * mcache = topo->links[ tile->out_link_id[ i ] ].mcache;
1360 0 : ulong const depth = fd_mcache_depth( mcache );
1361 0 : for( ulong j=0UL; j<depth; j++ ) {
1362 0 : mcache[ j ].chunk = (uint)next_chunk;
1363 0 : mcache[ j ].seq = fd_seq_dec( j, 1UL ); /* mark seq as invalid */
1364 0 : next_chunk += frame_chunks;
1365 0 : }
1366 0 : }
1367 : /* Assign one TX buffer to each SQ WQE */
1368 0 : for( ulong i=0UL; i<tile->mlx5.tx_queue_size; i++ ) {
1369 0 : ctx->sq_wqe_buf_chunk[ i ] = (uint)next_chunk;
1370 0 : next_chunk += frame_chunks;
1371 0 : }
1372 : /* Init TX */
1373 0 : if( FD_UNLIKELY( tile->in_cnt>FD_NET_IN_MAX ) ) {
1374 0 : FD_LOG_ERR(( "mlx5 tile in link count %lu exceeds max of %lu", tile->in_cnt, FD_NET_IN_MAX ));
1375 0 : }
1376 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
1377 0 : fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ];
1378 0 : if( !strcmp( link->name, "iproute_out" ) ) {
1379 0 : ctx->net.in_kind[ i ] = FD_NET_IN_KIND_IPROUTE;
1380 0 : } else {
1381 0 : ctx->net.in_kind[ i ] = FD_NET_IN_KIND_TX;
1382 0 : if( FD_UNLIKELY( link->mtu!=FD_NET_MTU ) ) FD_LOG_ERR(( "mlx5 tile in link does not have a normal MTU" ));
1383 0 : }
1384 :
1385 0 : ctx->net.in_dcache_ctx[ i ].wksp_base = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp;
1386 0 : ctx->net.in_dcache_ctx[ i ].chunk0 = fd_dcache_compact_chunk0( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache );
1387 0 : ctx->net.in_dcache_ctx[ i ].wmark = fd_dcache_compact_wmark( ctx->net.in_dcache_ctx[ i ].wksp_base, link->dcache, link->mtu );
1388 0 : }
1389 :
1390 : /* Join netbase objects */
1391 0 : FD_TEST( fd_fib4_join( ctx->net.router.fib_local, fd_fib4_new( fib_local_mem, tile->mlx5.route_max, tile->mlx5.route_peer_max, tile->mlx5.route_peer_seed ) ) );
1392 0 : FD_TEST( fd_fib4_join( ctx->net.router.fib_main, fd_fib4_new( fib_main_mem, tile->mlx5.route_max, tile->mlx5.route_peer_max, tile->mlx5.route_peer_seed ) ) );
1393 0 : FD_TEST( fd_netdev_tbl_join( &ctx->net.router.netdev_shared, fd_topo_obj_laddr( topo, tile->mlx5.netdev_tbl_obj_id ) ) );
1394 0 : FD_TEST( fd_netdev_tbl_new( netdev_tbl_local, NETDEV_MAX, BOND_MASTER_MAX ) );
1395 0 : FD_TEST( fd_netdev_tbl_join( &ctx->net.router.netdev_tbl, netdev_tbl_local ) );
1396 :
1397 0 : ctx->netdev_seq = fd_netdev_tbl_copy( &ctx->net.router.netdev_tbl, &ctx->net.router.netdev_shared );
1398 0 : fd_net_gre_tunnels_refresh( &ctx->net, &ctx->net.router.netdev_tbl );
1399 0 : ctx->net.bind_address = tile->mlx5.net.bind_address;
1400 :
1401 0 : ulong neigh4_obj_id = tile->mlx5.neigh4_obj_id;
1402 0 : ulong ele_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.ele_max", neigh4_obj_id );
1403 0 : ulong probe_max = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.probe_max", neigh4_obj_id );
1404 0 : ulong seed = fd_pod_queryf_ulong( topo->props, ULONG_MAX, "obj.%lu.seed", neigh4_obj_id );
1405 0 : if( FD_UNLIKELY( (ele_max==ULONG_MAX) | (probe_max==ULONG_MAX) | (seed==ULONG_MAX) ) ) {
1406 0 : FD_LOG_ERR(( "neigh4 hmap properties not set" ));
1407 0 : }
1408 0 : if( FD_UNLIKELY( !fd_neigh4_hmap_join( ctx->net.router.neigh4, fd_topo_obj_laddr( topo, neigh4_obj_id ), ele_max, probe_max, seed ) ) ) {
1409 0 : FD_LOG_ERR(( "fd_neigh4_hmap_join failed" ));
1410 0 : }
1411 :
1412 0 : ctx->net.router.netlnk_out_idx = fd_topo_find_tile_out_link( topo, tile, "net_netlnk", tile->kind_id );
1413 0 : if( FD_UNLIKELY( ctx->net.router.netlnk_out_idx==ULONG_MAX ) ) FD_LOG_ERR(( "netlink request link not found" ));
1414 0 : ctx->net.router.solicit_ip = 0U;
1415 :
1416 : /* Check if all chunks are in bound */
1417 0 : if( FD_UNLIKELY( next_chunk>ctx->net.pkt_buf_wmark ) ) {
1418 0 : FD_LOG_ERR(( "dcache is too small (topology bug)" ));
1419 0 : }
1420 :
1421 0 : ulong scratch_top = FD_SCRATCH_ALLOC_FINI( scratch, scratch_align() );
1422 0 : if( FD_UNLIKELY( scratch_top>(ulong)ctx+scratch_footprint( tile ) ) ) {
1423 0 : FD_LOG_ERR(( "scratch overflow" ));
1424 0 : }
1425 0 : }
1426 :
1427 : FD_FN_UNUSED static ulong
1428 : populate_allowed_seccomp( fd_topo_t const * topo,
1429 : fd_topo_tile_t const * tile,
1430 : ulong out_cnt,
1431 0 : struct sock_filter * out ) {
1432 0 : fd_mlx5_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1433 0 : populate_sock_filter_policy_fd_mlx5_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(),
1434 0 : (uint)ctx->uverbs.async_fd, (uint)ctx->lo_tx_sock,
1435 0 : (uint)ctx->net.sock.sock, (uint)ctx->rx_comp_channel_fd,
1436 0 : (uint)ctx->epoll_fd );
1437 0 : return sock_filter_policy_fd_mlx5_tile_instr_cnt;
1438 0 : }
1439 :
1440 : FD_FN_UNUSED static ulong
1441 : populate_allowed_fds( fd_topo_t const * topo,
1442 : fd_topo_tile_t const * tile,
1443 : ulong out_fds_cnt,
1444 0 : int * out_fds ) {
1445 0 : fd_mlx5_tile_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id );
1446 0 : if( FD_UNLIKELY( out_fds_cnt<9UL ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));
1447 0 : ulong out_cnt = 0UL;
1448 0 : out_fds[ out_cnt++ ] = 2;
1449 0 : if( FD_LIKELY( fd_log_private_logfile_fd()!=-1 ) ) out_fds[ out_cnt++ ] = fd_log_private_logfile_fd();
1450 0 : out_fds[ out_cnt++ ] = ctx->uverbs.cmd_fd;
1451 0 : out_fds[ out_cnt++ ] = ctx->uverbs.async_fd;
1452 0 : out_fds[ out_cnt++ ] = ctx->net.sock.sock; /* routing fallback socket */
1453 0 : if( ctx->lo_tx_sock>=0 ) out_fds[ out_cnt++ ] = ctx->lo_tx_sock;
1454 0 : if( FD_UNLIKELY( tile->sleep_eventfd ) ) {
1455 0 : out_fds[ out_cnt++ ] = ctx->rx_comp_channel_fd;
1456 0 : out_fds[ out_cnt++ ] = ctx->epoll_fd;
1457 0 : out_fds[ out_cnt++ ] = FD_SLEEP_EVENTFD( tile->id );
1458 0 : }
1459 0 : return out_cnt;
1460 0 : }
1461 :
1462 0 : #define STEM_CALLBACK_CONTEXT_TYPE fd_mlx5_tile_t
1463 0 : #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_mlx5_tile_t)
1464 0 : #define STEM_CALLBACK_BEFORE_CREDIT before_credit
1465 0 : #define STEM_CALLBACK_AFTER_CREDIT after_credit
1466 0 : #define STEM_CALLBACK_BEFORE_FRAG before_frag
1467 0 : #define STEM_CALLBACK_DURING_FRAG during_frag
1468 0 : #define STEM_CALLBACK_AFTER_FRAG after_frag
1469 0 : #define STEM_CALLBACK_METRICS_WRITE metrics_write
1470 0 : #define STEM_CALLBACK_DURING_HOUSEKEEPING during_housekeeping
1471 : #define STEM_CALLBACK_PREVENT_PARK prevent_park
1472 0 : #define STEM_CALLBACK_PARK_WAIT park_wait
1473 0 : #define STEM_CALLBACK_NEXT_DEADLINE next_deadline
1474 0 : #define STEM_BURST (FD_MLX5_BATCH_SIZE+1) /* +1 accounts for possible LB TX packet */
1475 0 : #define STEM_LAZY 270000UL /* 270us */
1476 : #include "../../stem/fd_stem.c"
1477 :
1478 : #ifndef FD_TILE_TEST
1479 : fd_topo_run_tile_t fd_tile_mlx5 = {
1480 : .name = "mlx5",
1481 : .populate_allowed_seccomp = populate_allowed_seccomp,
1482 : .populate_allowed_fds = populate_allowed_fds,
1483 : .scratch_align = scratch_align,
1484 : .scratch_footprint = scratch_footprint,
1485 : .privileged_init = privileged_init,
1486 : .unprivileged_init = unprivileged_init,
1487 : .run = stem_run,
1488 : };
1489 : #endif
|
