Line data Source code
1 : #define _GNU_SOURCE
2 : #include "../fd_net_tile.h"
3 : #include "fd_mlx5_private.h"
4 : #include "../../../util/log/fd_log.h"
5 : #include "../../../util/net/fd_eth.h"
6 : #include "../../../util/net/fd_ip4.h"
7 : #include "../../../util/rng/fd_rng.h"
8 : #include "../fd_linux_bond.h"
9 :
10 : #include <ctype.h>
11 : #include <dirent.h>
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <net/if.h>
16 : #include <stddef.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <sys/ioctl.h>
20 : #include <sys/mman.h>
21 : #include <sys/stat.h>
22 : #include <sys/syscall.h>
23 : #include <unistd.h>
24 :
25 : #include <rdma/ib_user_ioctl_cmds.h>
26 : #include <rdma/ib_user_ioctl_verbs.h>
27 : #include <rdma/ib_user_verbs.h>
28 : #include <rdma/mlx5-abi.h>
29 : #include <rdma/mlx5_user_ioctl_cmds.h>
30 : #include <rdma/mlx5_user_ioctl_verbs.h>
31 : #include <rdma/rdma_user_ioctl_cmds.h>
32 :
33 : /* Provide uverbs and mlx5 UAPI definitions missing from older Linux headers. */
34 : #ifndef IB_UVERBS_ACCESS_LOCAL_WRITE
35 0 : #define IB_UVERBS_ACCESS_LOCAL_WRITE (1U)
36 : #endif
37 : #ifndef IB_UVERBS_QPT_RAW_PACKET
38 0 : #define IB_UVERBS_QPT_RAW_PACKET (8U)
39 : #endif
40 : #ifndef IB_UVERBS_CREATE_QP_MASK_IND_TABLE
41 0 : #define IB_UVERBS_CREATE_QP_MASK_IND_TABLE (1U)
42 : #endif
43 : #ifndef IB_UVERBS_WQT_RQ
44 0 : #define IB_UVERBS_WQT_RQ (0U)
45 : #endif
46 : #ifndef MLX5_LIB_CAP_DYN_UAR
47 0 : #define MLX5_LIB_CAP_DYN_UAR (2UL)
48 : #endif
49 : #ifndef MLX5_IB_OBJECT_UAR
50 0 : #define MLX5_IB_OBJECT_UAR (4104U)
51 : #endif
52 : #ifndef MLX5_IB_METHOD_UAR_OBJ_ALLOC
53 0 : #define MLX5_IB_METHOD_UAR_OBJ_ALLOC (4096U)
54 : #endif
55 : #ifndef MLX5_IB_METHOD_UAR_OBJ_DESTROY
56 0 : #define MLX5_IB_METHOD_UAR_OBJ_DESTROY (4097U)
57 : #endif
58 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE
59 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE (4096U)
60 : #endif
61 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE
62 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE (4097U)
63 : #endif
64 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET
65 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET (4098U)
66 : #endif
67 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH
68 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH (4099U)
69 : #endif
70 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID
71 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID (4100U)
72 : #endif
73 : #ifndef MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE
74 0 : #define MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE (4096U)
75 : #endif
76 : #ifndef MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC
77 0 : #define MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC (1U)
78 : #endif
79 : #ifndef MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX
80 0 : #define MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX (2U)
81 : #endif
82 : #ifndef MLX5_QP_FLAG_UAR_PAGE_INDEX
83 0 : #define MLX5_QP_FLAG_UAR_PAGE_INDEX (1024U)
84 : #endif
85 :
86 0 : #define FD_MLX5_ETH_INLINE_HDR_SZ (18UL) /* MLX5_ETH_L2_INLINE_HEADER_SIZE */
87 : #define FD_MLX5_LINK_LAYER_ETHERNET (2U) /* IB_LINK_LAYER_ETHERNET */
88 0 : #define FD_UVERBS_NAME_MAX (32UL)
89 :
90 0 : #define FD_MLX5_QPS_INIT (1U) /* IB_QPS_INIT */
91 : #define FD_MLX5_QPS_RTR (2U) /* IB_QPS_RTR */
92 : #define FD_MLX5_QPS_RTS (3U) /* IB_QPS_RTS */
93 :
94 0 : #define FD_MLX5_QP_ATTR_STATE (1U) /* IB_QP_STATE */
95 0 : #define FD_MLX5_QP_ATTR_PORT (1U<<5) /* IB_QP_PORT */
96 :
97 0 : #define FD_MLX5_WQS_RESET (0U) /* IB_WQS_RESET */
98 0 : #define FD_MLX5_WQS_RDY (1U) /* IB_WQS_RDY */
99 :
100 0 : #define FD_MLX5_WQ_ATTR_STATE (1U) /* IB_WQ_STATE */
101 0 : #define FD_MLX5_WQ_ATTR_CUR_STATE (1U<<1) /* IB_WQ_CUR_STATE */
102 0 : #define FD_MLX5_WQ_ATTR_FLAGS (1U<<2) /* IB_WQ_FLAGS */
103 :
104 0 : #define FD_MLX5_WQ_FLAGS_CVLAN_STRIPPING (1U) /* IB_WQ_FLAGS_CVLAN_STRIPPING */
105 :
106 0 : #define FD_MLX5_RX_HASH_FUNC_TOEPLITZ (1U) /* MLX5_RX_HASH_FUNC_TOEPLITZ */
107 0 : #define FD_MLX5_RX_HASH_KEY_SZ (40U) /* MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) */
108 0 : #define FD_MLX5_RX_HASH_SRC_IPV4 (1UL<<0) /* MLX5_RX_HASH_SRC_IPV4 */
109 0 : #define FD_MLX5_RX_HASH_DST_IPV4 (1UL<<1) /* MLX5_RX_HASH_DST_IPV4 */
110 0 : #define FD_MLX5_RX_HASH_SRC_PORT_UDP (1UL<<6) /* MLX5_RX_HASH_SRC_PORT_UDP */
111 0 : #define FD_MLX5_RX_HASH_DST_PORT_UDP (1UL<<7) /* MLX5_RX_HASH_DST_PORT_UDP */
112 0 : #define FD_MLX5_RX_HASH_INNER (1UL<<31) /* MLX5_RX_HASH_INNER */
113 0 : #define FD_MLX5_RX_HASH_IPV4_UDP ( FD_MLX5_RX_HASH_SRC_IPV4 | FD_MLX5_RX_HASH_DST_IPV4 | \
114 0 : FD_MLX5_RX_HASH_SRC_PORT_UDP | FD_MLX5_RX_HASH_DST_PORT_UDP )
115 : #define FD_MLX5_QP_TUNNEL_OFFLOADS (1U<<2) /* MLX5_QP_FLAG_TUNNEL_OFFLOADS */
116 : #define FD_MLX5_TUNNEL_OFFLOADS_GRE (1U<<1) /* MLX5_IB_TUNNELED_OFFLOADS_GRE */
117 0 : #define FD_MLX5_INDIRECTION_COUNT (1UL<<8) /* Spreads RSS more equally across non power of 2 tile counts */
118 :
119 : static int
120 : fd_mlx5_rdma_port_contains_if( char const * rdma_name,
121 : uint port,
122 0 : char const * interface_name ) {
123 0 : char netdev_dir_path[ PATH_MAX ];
124 0 : int path_sz = snprintf( netdev_dir_path, sizeof(netdev_dir_path),
125 0 : "/sys/class/infiniband/%s/ports/%u/gid_attrs/ndevs", rdma_name, port );
126 0 : if( FD_UNLIKELY( path_sz<=0 || (ulong)path_sz>=sizeof(netdev_dir_path) ) ) {
127 0 : errno = ENAMETOOLONG;
128 0 : return -1;
129 0 : }
130 :
131 0 : DIR * netdev_dir = opendir( netdev_dir_path );
132 0 : if( FD_UNLIKELY( !netdev_dir ) ) return errno==ENOENT ? 0 : -1;
133 :
134 0 : int result = 0;
135 0 : for(;;) {
136 0 : errno = 0;
137 0 : struct dirent * netdev_entry = readdir( netdev_dir );
138 0 : if( FD_UNLIKELY( !netdev_entry ) ) {
139 0 : if( errno ) result = -1;
140 0 : break;
141 0 : }
142 0 : if( netdev_entry->d_name[0]=='.' ) continue;
143 :
144 0 : char netdev_name_path[ PATH_MAX ];
145 0 : int netdev_name_path_sz = snprintf( netdev_name_path, sizeof(netdev_name_path),
146 0 : "%s/%s", netdev_dir_path, netdev_entry->d_name );
147 0 : if( FD_UNLIKELY( netdev_name_path_sz<=0 || (ulong)netdev_name_path_sz>=sizeof(netdev_name_path) ) ) {
148 0 : errno = ENAMETOOLONG;
149 0 : result = -1;
150 0 : break;
151 0 : }
152 :
153 0 : int netdev_fd = open( netdev_name_path, O_RDONLY );
154 0 : if( FD_UNLIKELY( netdev_fd<0 ) ) {
155 0 : if( errno==ENOENT ) continue;
156 0 : result = -1;
157 0 : break;
158 0 : }
159 :
160 0 : char netdev_name[ IF_NAMESIZE+1 ];
161 0 : ssize_t read_sz = read( netdev_fd, netdev_name, IF_NAMESIZE );
162 0 : int const read_err = read_sz<0 ? errno : 0;
163 0 : int const close_err = close( netdev_fd ) ? errno : 0;
164 0 : if( FD_UNLIKELY( (read_err && read_err!=EINVAL) || close_err ) ) {
165 0 : errno = read_err && read_err!=EINVAL ? read_err : close_err;
166 0 : result = -1;
167 0 : break;
168 0 : }
169 : /* Empty Linux RDMA GID attribute entries return EINVAL. */
170 0 : if( FD_UNLIKELY( read_err==EINVAL ) ) continue;
171 :
172 0 : if( FD_UNLIKELY( !read_sz ) ) continue;
173 0 : while( read_sz>0 && (netdev_name[read_sz-1]=='\n' || netdev_name[read_sz-1]=='\r') ) read_sz--;
174 0 : netdev_name[read_sz] = '\0';
175 0 : if( !strcmp( netdev_name, interface_name ) ) {
176 0 : result = 1;
177 0 : break;
178 0 : }
179 0 : }
180 :
181 0 : int err = result<0 ? errno : 0;
182 0 : if( FD_UNLIKELY( closedir( netdev_dir ) && !err ) ) err = errno;
183 0 : if( FD_UNLIKELY( err ) ) {
184 0 : errno = err;
185 0 : return -1;
186 0 : }
187 0 : return result;
188 0 : }
189 :
190 : static int
191 : fd_mlx5_rdma_port_matches_if( char const * rdma_name,
192 : uint port,
193 0 : char const * interface_name ) {
194 0 : int match = fd_mlx5_rdma_port_contains_if( rdma_name, port, interface_name );
195 0 : if( FD_UNLIKELY( match<0 ) ) return -1;
196 0 : if( match ) return 1;
197 0 : if( !fd_bonding_is_master( interface_name ) ) return 0;
198 :
199 0 : fd_bonding_slave_iter_t iter_mem[1];
200 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_mem, interface_name );
201 0 : while( !fd_bonding_slave_iter_done( iter ) ) {
202 0 : char const * slave_interface_name = fd_bonding_slave_iter_ele( iter );
203 0 : match = fd_mlx5_rdma_port_contains_if( rdma_name, port, slave_interface_name );
204 0 : if( FD_UNLIKELY( match<0 ) ) return -1;
205 0 : if( match ) return 1;
206 0 : fd_bonding_slave_iter_next( iter );
207 0 : }
208 :
209 0 : return 0;
210 0 : }
211 :
212 : int
213 : fd_mlx5_rdma_dev_find( char rdma_name[ FD_MLX5_RDMA_NAME_MAX ],
214 : uint * rdma_port,
215 0 : char const * interface_name ) {
216 0 : rdma_name[0] = '\0';
217 0 : *rdma_port = 0U;
218 :
219 0 : DIR * infiniband_dir = opendir( "/sys/class/infiniband" );
220 0 : if( FD_UNLIKELY( !infiniband_dir ) ) return 0;
221 :
222 0 : ulong match_cnt = 0UL;
223 0 : int err = 0;
224 0 : for(;;) {
225 0 : errno = 0;
226 0 : struct dirent * device_entry = readdir( infiniband_dir );
227 0 : if( FD_UNLIKELY( !device_entry ) ) {
228 0 : err = errno;
229 0 : break;
230 0 : }
231 0 : if( device_entry->d_name[0]=='.' ) continue;
232 :
233 0 : char device_ports_path[ PATH_MAX ];
234 0 : int const device_ports_path_sz = snprintf( device_ports_path, sizeof(device_ports_path),
235 0 : "/sys/class/infiniband/%s/ports", device_entry->d_name );
236 0 : if( FD_UNLIKELY( device_ports_path_sz<=0 || (ulong)device_ports_path_sz>=sizeof(device_ports_path) ) ) {
237 0 : err = ENAMETOOLONG;
238 0 : break;
239 0 : }
240 :
241 0 : DIR * ports_dir = opendir( device_ports_path );
242 0 : if( FD_UNLIKELY( !ports_dir ) ) {
243 0 : if( errno==ENOENT ) continue;
244 0 : err = errno;
245 0 : break;
246 0 : }
247 :
248 0 : for(;;) {
249 0 : errno = 0;
250 0 : struct dirent * port_entry = readdir( ports_dir );
251 0 : if( FD_UNLIKELY( !port_entry ) ) {
252 0 : err = errno;
253 0 : break;
254 0 : }
255 0 : char * port_end;
256 0 : ulong const port_num = strtoul( port_entry->d_name, &port_end, 10 );
257 0 : if( !port_num || *port_end || port_num>(ulong)UCHAR_MAX ) continue;
258 0 : int const match = fd_mlx5_rdma_port_matches_if( device_entry->d_name, (uint)port_num, interface_name );
259 0 : if( FD_UNLIKELY( match<0 ) ) {
260 0 : err = errno;
261 0 : break;
262 0 : }
263 0 : if( !match ) continue;
264 :
265 0 : match_cnt++;
266 0 : if( match_cnt==1UL ) {
267 0 : fd_cstr_ncpy( rdma_name, device_entry->d_name, FD_MLX5_RDMA_NAME_MAX );
268 0 : *rdma_port = (uint)port_num;
269 0 : }
270 0 : }
271 0 : if( FD_UNLIKELY( closedir( ports_dir ) && !err ) ) err = errno;
272 0 : if( FD_UNLIKELY( err ) ) break;
273 0 : }
274 :
275 0 : if( FD_UNLIKELY( closedir( infiniband_dir ) && !err ) ) err = errno;
276 0 : if( FD_UNLIKELY( err ) ) {
277 0 : rdma_name[0] = '\0';
278 0 : *rdma_port = 0U;
279 0 : errno = err;
280 0 : return 0;
281 0 : }
282 0 : if( FD_UNLIKELY( match_cnt!=1UL ) ) {
283 0 : errno = match_cnt ? EEXIST : ENOENT;
284 0 : rdma_name[0] = '\0';
285 0 : *rdma_port = 0U;
286 0 : return 0;
287 0 : }
288 0 : return 1;
289 0 : }
290 :
291 : int
292 0 : fd_mlx5_uverbs_avail( void ) {
293 0 : struct stat class_stat;
294 0 : return !stat( "/sys/class/infiniband_verbs", &class_stat ) && S_ISDIR( class_stat.st_mode );
295 0 : }
296 :
297 : struct fd_mlx5_pd {
298 : fd_uverbs_ctx_t * ctx; /* uverbs context */
299 : uint handle; /* protection domain handle */
300 : };
301 : typedef struct fd_mlx5_pd fd_mlx5_pd_t;
302 :
303 : struct fd_mlx5_caps {
304 : ulong max_mr_size;
305 : uint max_send_wqe; /* max_send_wqebb, SQ WQE capacity */
306 : uint max_recv_wr;
307 : uint max_cqe;
308 : uchar eth_min_inline_hdr_sz;
309 : };
310 : typedef struct fd_mlx5_caps fd_mlx5_caps_t;
311 :
312 : /* fd_uverbs_* types define Linux uverbs requests and responses */
313 : struct fd_uverbs_ex_hdr {
314 : struct ib_uverbs_cmd_hdr cmd;
315 : struct ib_uverbs_ex_cmd_hdr ex;
316 : };
317 : typedef struct fd_uverbs_ex_hdr fd_uverbs_ex_hdr_t;
318 : FD_STATIC_ASSERT( sizeof(fd_uverbs_ex_hdr_t)==24UL, uverbs_ex_hdr_sz );
319 :
320 : struct fd_uverbs_get_context_req {
321 : struct ib_uverbs_cmd_hdr hdr;
322 : ulong response;
323 : struct mlx5_ib_alloc_ucontext_req_v2 mlx5;
324 : };
325 : typedef struct fd_uverbs_get_context_req fd_uverbs_get_context_req_t;
326 : FD_STATIC_ASSERT( sizeof(fd_uverbs_get_context_req_t)==48UL, uverbs_get_context_req_sz );
327 :
328 : struct fd_uverbs_get_context_resp {
329 : uint async_fd;
330 : uint num_comp_vectors;
331 : struct mlx5_ib_alloc_ucontext_resp mlx5;
332 : };
333 : typedef struct fd_uverbs_get_context_resp fd_uverbs_get_context_resp_t;
334 : FD_STATIC_ASSERT( sizeof(fd_uverbs_get_context_resp_t)==80UL, uverbs_get_context_resp_sz );
335 :
336 : struct fd_uverbs_query_device_req {
337 : fd_uverbs_ex_hdr_t hdr;
338 : struct ib_uverbs_ex_query_device core;
339 : };
340 : typedef struct fd_uverbs_query_device_req fd_uverbs_query_device_req_t;
341 :
342 : struct fd_uverbs_query_device_resp {
343 : struct ib_uverbs_ex_query_device_resp core;
344 : struct mlx5_ib_query_device_resp mlx5;
345 : };
346 : typedef struct fd_uverbs_query_device_resp fd_uverbs_query_device_resp_t;
347 :
348 : struct fd_uverbs_query_port_req {
349 : struct ib_uverbs_cmd_hdr hdr;
350 : ulong response;
351 : uchar port_num;
352 : uchar reserved[ 7 ];
353 : };
354 : typedef struct fd_uverbs_query_port_req fd_uverbs_query_port_req_t;
355 : FD_STATIC_ASSERT( sizeof(fd_uverbs_query_port_req_t)==24UL, uverbs_query_port_req_sz );
356 :
357 : struct fd_uverbs_alloc_pd_req {
358 : struct ib_uverbs_cmd_hdr hdr;
359 : ulong response;
360 : };
361 : typedef struct fd_uverbs_alloc_pd_req fd_uverbs_alloc_pd_req_t;
362 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_pd_req_t)==16UL, uverbs_alloc_pd_req_sz );
363 :
364 : struct fd_uverbs_alloc_pd_resp {
365 : uint pd_handle;
366 : struct mlx5_ib_alloc_pd_resp mlx5;
367 : };
368 : typedef struct fd_uverbs_alloc_pd_resp fd_uverbs_alloc_pd_resp_t;
369 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_pd_resp_t)==8UL, uverbs_alloc_pd_resp_sz );
370 :
371 : struct fd_uverbs_reg_mr_req {
372 : struct ib_uverbs_cmd_hdr hdr;
373 : ulong response;
374 : ulong start;
375 : ulong length;
376 : ulong hca_va;
377 : uint pd_handle;
378 : uint access_flags;
379 : };
380 : typedef struct fd_uverbs_reg_mr_req fd_uverbs_reg_mr_req_t;
381 : FD_STATIC_ASSERT( sizeof(fd_uverbs_reg_mr_req_t)==48UL, uverbs_reg_mr_req_sz );
382 :
383 : struct fd_uverbs_reg_mr_resp {
384 : uint mr_handle;
385 : uint lkey;
386 : uint rkey;
387 : };
388 : typedef struct fd_uverbs_reg_mr_resp fd_uverbs_reg_mr_resp_t;
389 : FD_STATIC_ASSERT( sizeof(fd_uverbs_reg_mr_resp_t)==12UL, uverbs_reg_mr_resp_sz );
390 :
391 : struct fd_uverbs_ioctl_hdr {
392 : ushort length;
393 : ushort object_id;
394 : ushort method_id;
395 : ushort num_attrs;
396 : ulong reserved1;
397 : uint driver_id;
398 : uint reserved2;
399 : };
400 : typedef struct fd_uverbs_ioctl_hdr fd_uverbs_ioctl_hdr_t;
401 : FD_STATIC_ASSERT( sizeof(fd_uverbs_ioctl_hdr_t)==24UL, uverbs_ioctl_hdr_sz );
402 :
403 : struct fd_uverbs_alloc_uar_req {
404 : fd_uverbs_ioctl_hdr_t hdr;
405 : struct ib_uverbs_attr attrs[ 5 ];
406 : };
407 : typedef struct fd_uverbs_alloc_uar_req fd_uverbs_alloc_uar_req_t;
408 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_uar_req_t)==104UL, uverbs_alloc_uar_req_sz );
409 :
410 : struct fd_uverbs_destroy_uar_req {
411 : fd_uverbs_ioctl_hdr_t hdr;
412 : struct ib_uverbs_attr attrs[ 1 ];
413 : };
414 : typedef struct fd_uverbs_destroy_uar_req fd_uverbs_destroy_uar_req_t;
415 : FD_STATIC_ASSERT( sizeof(fd_uverbs_destroy_uar_req_t)==40UL, uverbs_destroy_uar_req_sz );
416 :
417 : struct fd_uverbs_create_comp_channel_req {
418 : struct ib_uverbs_cmd_hdr hdr;
419 : ulong response;
420 : };
421 : typedef struct fd_uverbs_create_comp_channel_req fd_uverbs_create_comp_channel_req_t;
422 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_comp_channel_req_t)==16UL, uverbs_create_comp_channel_req_sz );
423 :
424 : struct fd_uverbs_create_cq_req {
425 : struct ib_uverbs_cmd_hdr hdr;
426 : ulong response;
427 : ulong user_handle;
428 : uint cqe;
429 : uint comp_vector;
430 : int comp_channel;
431 : uint reserved;
432 : /* mlx5_ib_create_cq gained uar_page_index in Linux 5.7 */
433 : union {
434 : struct mlx5_ib_create_cq fields;
435 : struct {
436 : uchar prefix[ 24 ];
437 : ushort uar_page_index;
438 : } uar;
439 : } mlx5;
440 : };
441 : typedef struct fd_uverbs_create_cq_req fd_uverbs_create_cq_req_t;
442 :
443 : struct fd_uverbs_create_cq_resp {
444 : uint cq_handle;
445 : uint cqe;
446 : struct mlx5_ib_create_cq_resp mlx5;
447 : };
448 : typedef struct fd_uverbs_create_cq_resp fd_uverbs_create_cq_resp_t;
449 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_cq_resp_t)==16UL, uverbs_create_cq_resp_sz );
450 :
451 : struct fd_uverbs_destroy_cq_req {
452 : struct ib_uverbs_cmd_hdr hdr;
453 : ulong response;
454 : uint cq_handle;
455 : uint reserved;
456 : };
457 : typedef struct fd_uverbs_destroy_cq_req fd_uverbs_destroy_cq_req_t;
458 : FD_STATIC_ASSERT( sizeof(fd_uverbs_destroy_cq_req_t)==24UL, uverbs_destroy_cq_req_sz );
459 :
460 : struct fd_uverbs_create_tx_qp_req {
461 : struct ib_uverbs_cmd_hdr hdr;
462 : ulong response;
463 : ulong user_handle;
464 : uint pd_handle;
465 : uint send_cq_handle;
466 : uint recv_cq_handle;
467 : uint srq_handle;
468 : uint max_send_wr;
469 : uint max_recv_wr;
470 : uint max_send_sge;
471 : uint max_recv_sge;
472 : uint max_inline_data;
473 : uchar sq_sig_all;
474 : uchar qp_type;
475 : uchar is_srq;
476 : uchar reserved;
477 : struct mlx5_ib_create_qp mlx5;
478 : };
479 : typedef struct fd_uverbs_create_tx_qp_req fd_uverbs_create_tx_qp_req_t;
480 :
481 : struct fd_uverbs_create_tx_qp_resp {
482 : struct ib_uverbs_create_qp_resp core;
483 : struct mlx5_ib_create_qp_resp mlx5;
484 : };
485 : typedef struct fd_uverbs_create_tx_qp_resp fd_uverbs_create_tx_qp_resp_t;
486 :
487 : struct fd_uverbs_modify_qp_req {
488 : struct ib_uverbs_cmd_hdr hdr;
489 : struct ib_uverbs_modify_qp core;
490 : };
491 : typedef struct fd_uverbs_modify_qp_req fd_uverbs_modify_qp_req_t;
492 : FD_STATIC_ASSERT( sizeof(fd_uverbs_modify_qp_req_t)==120UL, uverbs_modify_qp_req_sz );
493 :
494 : struct fd_uverbs_create_rx_wq_req {
495 : fd_uverbs_ex_hdr_t hdr;
496 : struct ib_uverbs_ex_create_wq core;
497 : struct mlx5_ib_create_wq mlx5;
498 : };
499 : typedef struct fd_uverbs_create_rx_wq_req fd_uverbs_create_rx_wq_req_t;
500 :
501 : struct fd_uverbs_create_rx_wq_resp {
502 : struct ib_uverbs_ex_create_wq_resp core;
503 : struct mlx5_ib_create_wq_resp mlx5;
504 : };
505 : typedef struct fd_uverbs_create_rx_wq_resp fd_uverbs_create_rx_wq_resp_t;
506 :
507 : struct fd_uverbs_modify_rx_wq_req {
508 : fd_uverbs_ex_hdr_t hdr;
509 : struct ib_uverbs_ex_modify_wq core;
510 : struct mlx5_ib_modify_wq mlx5;
511 : };
512 : typedef struct fd_uverbs_modify_rx_wq_req fd_uverbs_modify_rx_wq_req_t;
513 :
514 : struct fd_uverbs_create_rwq_ind_table_req {
515 : fd_uverbs_ex_hdr_t hdr;
516 : uint comp_mask;
517 : uint log_ind_tbl_size;
518 : uint wq_handles[];
519 : };
520 : typedef struct fd_uverbs_create_rwq_ind_table_req fd_uverbs_create_rwq_ind_table_req_t;
521 :
522 : struct fd_uverbs_create_rss_qp_req {
523 : fd_uverbs_ex_hdr_t hdr;
524 : struct ib_uverbs_ex_create_qp core;
525 : struct mlx5_ib_create_qp_rss mlx5;
526 : };
527 : typedef struct fd_uverbs_create_rss_qp_req fd_uverbs_create_rss_qp_req_t;
528 :
529 : struct fd_uverbs_create_rss_qp_resp {
530 : struct ib_uverbs_ex_create_qp_resp core;
531 : struct mlx5_ib_create_qp_resp mlx5;
532 : };
533 : typedef struct fd_uverbs_create_rss_qp_resp fd_uverbs_create_rss_qp_resp_t;
534 :
535 : struct fd_uverbs_create_udp_flow_req {
536 : fd_uverbs_ex_hdr_t hdr;
537 : uint comp_mask;
538 : uint qp_handle;
539 : uint type;
540 : ushort size;
541 : ushort priority;
542 : uchar num_of_specs;
543 : uchar reserved[2];
544 : uchar port;
545 : uint flags;
546 : struct ib_uverbs_flow_spec_eth eth;
547 : struct ib_uverbs_flow_spec_ipv4 ipv4;
548 : struct ib_uverbs_flow_spec_tcp_udp udp;
549 : uint mlx5_ncounters_data;
550 : uint mlx5_reserved;
551 : };
552 : typedef struct fd_uverbs_create_udp_flow_req fd_uverbs_create_udp_flow_req_t;
553 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_udp_flow_req_t)==144UL, uverbs_create_udp_flow_req_sz );
554 :
555 : struct fd_uverbs_create_gre_flow_req {
556 : fd_uverbs_ex_hdr_t hdr;
557 : uint comp_mask;
558 : uint qp_handle;
559 : uint type;
560 : ushort size;
561 : ushort priority;
562 : uchar num_of_specs;
563 : uchar reserved[2];
564 : uchar port;
565 : uint flags;
566 : struct ib_uverbs_flow_spec_eth eth;
567 : struct ib_uverbs_flow_spec_ipv4 outer_ipv4;
568 : struct ib_uverbs_flow_spec_gre gre;
569 : struct ib_uverbs_flow_spec_ipv4 inner_ipv4;
570 : struct ib_uverbs_flow_spec_tcp_udp inner_udp;
571 : uint mlx5_ncounters_data;
572 : uint mlx5_reserved;
573 : };
574 : typedef struct fd_uverbs_create_gre_flow_req fd_uverbs_create_gre_flow_req_t;
575 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_gre_flow_req_t)==200UL, uverbs_create_gre_flow_req_sz );
576 :
577 : #define FD_RDMA_PATH_MAX (256UL)
578 :
579 0 : #define FD_MLX5_FLOW_SPEC_ETH (0x20U) /* IB_FLOW_SPEC_ETH */
580 0 : #define FD_MLX5_FLOW_SPEC_IPV4 (0x30U) /* IB_FLOW_SPEC_IPV4 */
581 0 : #define FD_MLX5_FLOW_SPEC_UDP (0x41U) /* IB_FLOW_SPEC_UDP */
582 0 : #define FD_MLX5_FLOW_SPEC_GRE (0x51U) /* IB_FLOW_SPEC_GRE */
583 0 : #define FD_MLX5_FLOW_SPEC_INNER (0x100U) /* IB_FLOW_SPEC_INNER */
584 :
585 : /* fd_rdma_* helpers discover and inspect the selected Linux RDMA device */
586 : static int
587 : fd_rdma_read_text( char const * path,
588 : char * buf,
589 0 : ulong buf_sz ) {
590 0 : FD_TEST( buf_sz>1UL );
591 :
592 0 : int text_fd = open( path, O_RDONLY | O_CLOEXEC );
593 0 : if( FD_UNLIKELY( text_fd<0 ) ) return -1;
594 :
595 0 : long read_sz = read( text_fd, buf, buf_sz );
596 0 : int err = 0;
597 0 : if( FD_UNLIKELY( read_sz<0L ) ) err = errno;
598 0 : else if( FD_UNLIKELY( (ulong)read_sz==buf_sz ) ) err = EOVERFLOW;
599 0 : if( FD_UNLIKELY( close( text_fd ) && !err ) ) err = errno;
600 0 : if( FD_UNLIKELY( err ) ) {
601 0 : errno = err;
602 0 : return -1;
603 0 : }
604 :
605 0 : ulong text_sz = (ulong)read_sz;
606 0 : while( text_sz && fd_isspace( (int)(uchar)buf[ text_sz-1UL ] ) ) text_sz--;
607 0 : if( FD_UNLIKELY( !text_sz ) ) {
608 0 : errno = EPROTO;
609 0 : return -1;
610 0 : }
611 0 : buf[ text_sz ] = '\0';
612 0 : return 0;
613 0 : }
614 :
615 : static int
616 : fd_rdma_read_uint( char const * path,
617 0 : uint * value ) {
618 0 : char buf[ 32 ];
619 0 : if( FD_UNLIKELY( fd_rdma_read_text( path, buf, sizeof(buf) ) ) ) return -1;
620 :
621 0 : char * end;
622 0 : ulong parsed_value = strtoul( buf, &end, 10 );
623 0 : if( FD_UNLIKELY( *end || parsed_value>UINT_MAX ) ) {
624 0 : errno = EPROTO;
625 0 : return -1;
626 0 : }
627 0 : *value = (uint)parsed_value;
628 0 : return 0;
629 0 : }
630 :
631 : static int
632 : fd_rdma_name_valid( char const * name,
633 0 : ulong name_max ) {
634 0 : if( FD_UNLIKELY( !name || !name[0] ) ) return 0;
635 0 : if( FD_UNLIKELY( name[0]=='.' && (!name[1] || (name[1]=='.' && !name[2])) ) ) return 0;
636 0 : for( ulong i=0UL; i<name_max; i++ ) {
637 0 : uchar c = (uchar)name[ i ];
638 0 : if( !c ) return 1;
639 0 : if( !((c>=(uchar)'a' && c<=(uchar)'z') ||
640 0 : (c>=(uchar)'A' && c<=(uchar)'Z') ||
641 0 : (c>=(uchar)'0' && c<=(uchar)'9') ||
642 0 : c==(uchar)'_' || c==(uchar)'-' || c==(uchar)'.') ) return 0;
643 0 : }
644 0 : return 0;
645 0 : }
646 :
647 : /* fd_uverbs_* helpers build and submit Linux uverbs commands */
648 : static int
649 0 : fd_uverbs_name_valid( char const * name ) {
650 0 : ulong name_sz = strnlen( name, FD_UVERBS_NAME_MAX );
651 0 : if( name_sz==FD_UVERBS_NAME_MAX || name_sz==6UL || strncmp( name, "uverbs", 6UL ) ) return 0;
652 0 : for( char const * digit_cur=name+6; *digit_cur; digit_cur++ )
653 0 : if( *digit_cur<'0' || *digit_cur>'9' ) return 0;
654 0 : return 1;
655 0 : }
656 :
657 : static int
658 : fd_uverbs_resolve( char uverbs_name[ FD_UVERBS_NAME_MAX ],
659 0 : char const * rdma_name ) {
660 0 : if( FD_UNLIKELY( !fd_rdma_name_valid( rdma_name, FD_MLX5_RDMA_NAME_MAX ) ) ) {
661 0 : errno = EINVAL;
662 0 : return -1;
663 0 : }
664 :
665 0 : DIR * uverbs_dir = opendir( "/sys/class/infiniband_verbs" );
666 0 : if( FD_UNLIKELY( !uverbs_dir ) ) return -1;
667 :
668 0 : char matched_uverbs_name[ FD_UVERBS_NAME_MAX ];
669 0 : int err;
670 0 : for(;;) {
671 0 : errno = 0;
672 0 : struct dirent * uverbs_entry = readdir( uverbs_dir );
673 0 : if( !uverbs_entry ) {
674 0 : err = errno ? errno : ENODEV;
675 0 : break;
676 0 : }
677 0 : if( !fd_uverbs_name_valid( uverbs_entry->d_name ) ) continue;
678 :
679 0 : char path[ FD_RDMA_PATH_MAX ];
680 0 : char entry_rdma_name[ FD_MLX5_RDMA_NAME_MAX ];
681 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
682 0 : "/sys/class/infiniband_verbs/%s/ibdev", uverbs_entry->d_name ) );
683 0 : if( FD_UNLIKELY( fd_rdma_read_text( path, entry_rdma_name, sizeof(entry_rdma_name) ) ) ) {
684 0 : err = errno;
685 0 : break;
686 0 : }
687 0 : if( strcmp( entry_rdma_name, rdma_name ) ) continue;
688 :
689 0 : ulong name_sz = strlen( uverbs_entry->d_name );
690 0 : fd_memcpy( matched_uverbs_name, uverbs_entry->d_name, name_sz+1UL );
691 0 : err = 0;
692 0 : break;
693 0 : }
694 0 : closedir( uverbs_dir );
695 0 : if( FD_UNLIKELY( err ) ) {
696 0 : errno = err;
697 0 : return -1;
698 0 : }
699 :
700 0 : char path[ FD_RDMA_PATH_MAX ];
701 0 : uint core_abi;
702 0 : uint provider_abi;
703 0 : if( FD_UNLIKELY( fd_rdma_read_uint( "/sys/class/infiniband_verbs/abi_version", &core_abi ) ) ) return -1;
704 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
705 0 : "/sys/class/infiniband_verbs/%s/abi_version", matched_uverbs_name ) );
706 0 : if( FD_UNLIKELY( fd_rdma_read_uint( path, &provider_abi ) ) ) return -1;
707 0 : if( FD_UNLIKELY( core_abi!=IB_USER_VERBS_ABI_VERSION ||
708 0 : provider_abi!=MLX5_IB_UVERBS_ABI_VERSION ) ) {
709 0 : FD_LOG_WARNING(( "mlx5 uverbs ABI mismatch (core %u, expected %u, provider %u, expected %u)",
710 0 : core_abi, (uint)IB_USER_VERBS_ABI_VERSION,
711 0 : provider_abi, (uint)MLX5_IB_UVERBS_ABI_VERSION ));
712 0 : errno = EPROTONOSUPPORT;
713 0 : return -1;
714 0 : }
715 :
716 0 : fd_memcpy( uverbs_name, matched_uverbs_name, strlen(matched_uverbs_name)+1UL );
717 0 : return 0;
718 0 : }
719 :
720 : static int
721 : fd_uverbs_init_cmd_hdr( struct ib_uverbs_cmd_hdr * hdr,
722 : uint command,
723 : ulong request_sz,
724 0 : ulong response_sz ) {
725 0 : if( FD_UNLIKELY( !hdr || command>IB_USER_VERBS_CMD_COMMAND_MASK || request_sz<sizeof(*hdr) ) ) return -1;
726 :
727 0 : hdr->command = command;
728 0 : hdr->in_words = (ushort)(request_sz / 4UL);
729 0 : hdr->out_words = (ushort)(response_sz / 4UL);
730 0 : return 0;
731 0 : }
732 :
733 : static int
734 : fd_uverbs_init_ex_hdr( fd_uverbs_ex_hdr_t * hdr,
735 : uint command,
736 : ulong core_request_sz,
737 : ulong request_sz,
738 : void * response,
739 : ulong core_response_sz,
740 0 : ulong response_sz ) {
741 0 : ulong const hdr_sz = sizeof(*hdr);
742 0 : if( FD_UNLIKELY( !hdr || command>IB_USER_VERBS_CMD_COMMAND_MASK ||
743 0 : core_request_sz<hdr_sz || request_sz<core_request_sz ||
744 0 : response_sz<core_response_sz || (response_sz && !response) ) ) return -1;
745 :
746 0 : hdr->cmd.command = IB_USER_VERBS_CMD_FLAG_EXTENDED | command;
747 0 : hdr->cmd.in_words = (ushort)((core_request_sz-hdr_sz) / 8UL);
748 0 : hdr->cmd.out_words = (ushort)(core_response_sz / 8UL);
749 0 : hdr->ex.provider_in_words = (ushort)((request_sz-core_request_sz) / 8UL);
750 0 : hdr->ex.provider_out_words = (ushort)((response_sz-core_response_sz) / 8UL);
751 0 : hdr->ex.response = (ulong)response;
752 0 : hdr->ex.cmd_hdr_reserved = 0U;
753 0 : return 0;
754 0 : }
755 :
756 : static int
757 : fd_uverbs_write_cmd( int cmd_fd,
758 : void const * req,
759 0 : ulong req_sz ) {
760 0 : ssize_t write_sz = write( cmd_fd, req, req_sz );
761 0 : if( FD_UNLIKELY( write_sz<0 ) ) return -1;
762 0 : if( FD_UNLIKELY( (ulong)write_sz!=req_sz ) ) {
763 0 : errno = EPROTO;
764 0 : return -1;
765 0 : }
766 0 : return 0;
767 0 : }
768 :
769 : static int
770 : fd_uverbs_get_context( fd_uverbs_ctx_t * ctx,
771 0 : fd_mlx5_caps_t * caps ) {
772 0 : fd_uverbs_get_context_req_t req [1];
773 0 : fd_uverbs_get_context_resp_t resp[1];
774 0 : fd_memset( req, 0, sizeof(req ) );
775 0 : fd_memset( resp, 0, sizeof(resp) );
776 :
777 0 : req->response = (ulong)resp;
778 0 : req->mlx5.total_num_bfregs = 16U;
779 0 : req->mlx5.num_low_latency_bfregs = 4U;
780 0 : req->mlx5.max_cqe_version = 1U;
781 0 : req->mlx5.lib_caps = MLX5_LIB_CAP_4K_UAR | MLX5_LIB_CAP_DYN_UAR;
782 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_GET_CONTEXT,
783 0 : sizeof(req), sizeof(resp) ) );
784 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
785 0 : if( FD_UNLIKELY( resp->async_fd>(uint)INT_MAX ) ) {
786 0 : errno = EPROTO;
787 0 : return -1;
788 0 : }
789 :
790 0 : ctx->async_fd = (int)resp->async_fd;
791 0 : int fd_flags = fcntl( ctx->async_fd, F_GETFD );
792 0 : if( FD_UNLIKELY( fd_flags<0 || fcntl( ctx->async_fd, F_SETFD, fd_flags | FD_CLOEXEC ) ) ) return -1;
793 :
794 0 : ulong const uar_resp_sz = offsetof( struct mlx5_ib_alloc_ucontext_resp, num_uars_per_page )+
795 0 : sizeof(resp->mlx5.num_uars_per_page);
796 0 : if( FD_UNLIKELY( resp->mlx5.response_length<uar_resp_sz ||
797 0 : resp->mlx5.cqe_version>1U ||
798 0 : resp->mlx5.max_sq_desc_sz<sizeof(fd_mlx5_tx_wqe_t) ||
799 0 : resp->mlx5.max_rq_desc_sz<sizeof(fd_mlx5_rx_wqe_t) ||
800 0 : resp->mlx5.log_uar_size!=(uint)fd_ulong_find_lsb( FD_MLX5_PAGE_SZ ) ||
801 0 : resp->mlx5.num_uars_per_page!=1U ||
802 0 : resp->mlx5.tot_bfregs ) ) {
803 0 : FD_LOG_WARNING(( "unsupported mlx5 context capabilities (response length %u, CQE version %u, "
804 0 : "max SQ descriptor %u, max RQ descriptor %u, log UAR size %u, UARs per page %u, "
805 0 : "legacy BF registers %u)",
806 0 : resp->mlx5.response_length, (uint)resp->mlx5.cqe_version,
807 0 : (uint)resp->mlx5.max_sq_desc_sz, (uint)resp->mlx5.max_rq_desc_sz,
808 0 : resp->mlx5.log_uar_size, resp->mlx5.num_uars_per_page, resp->mlx5.tot_bfregs ));
809 0 : errno = EPROTONOSUPPORT;
810 0 : return -1;
811 0 : }
812 :
813 0 : caps->max_send_wqe = resp->mlx5.max_send_wqebb;
814 0 : caps->max_recv_wr = resp->mlx5.max_recv_wr;
815 0 : switch( resp->mlx5.eth_min_inline ) {
816 0 : case MLX5_USER_INLINE_MODE_NONE: caps->eth_min_inline_hdr_sz = 0U; break;
817 0 : case MLX5_USER_INLINE_MODE_L2: caps->eth_min_inline_hdr_sz = FD_MLX5_ETH_INLINE_HDR_SZ; break;
818 0 : default:
819 0 : FD_LOG_WARNING(( "unsupported mlx5 minimum inline mode %u", (uint)resp->mlx5.eth_min_inline ));
820 0 : errno = EPROTONOSUPPORT;
821 0 : return -1;
822 0 : }
823 0 : return 0;
824 0 : }
825 :
826 : static int
827 : fd_uverbs_query_device( fd_uverbs_ctx_t * ctx,
828 0 : fd_mlx5_caps_t * caps ) {
829 0 : fd_uverbs_query_device_req_t req [1];
830 0 : fd_uverbs_query_device_resp_t resp[1];
831 0 : fd_memset( req, 0, sizeof(req ) );
832 0 : fd_memset( resp, 0, sizeof(resp) );
833 :
834 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
835 0 : sizeof(req), sizeof(req), resp,
836 0 : sizeof(resp->core), sizeof(resp) ) );
837 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
838 :
839 0 : struct ib_uverbs_query_device_resp const * core = &resp->core.base;
840 0 : caps->max_mr_size = core->max_mr_size;
841 0 : caps->max_cqe = core->max_cqe;
842 0 : if( FD_UNLIKELY( resp->core.response_length<sizeof(*core) ||
843 0 : !caps->max_mr_size ||
844 0 : !core->max_qp_wr ||
845 0 : !core->max_sge ||
846 0 : !caps->max_cqe ||
847 0 : core->phys_port_cnt<ctx->port_num ) ) {
848 0 : FD_LOG_WARNING(( "unsupported mlx5 device capabilities (max MR size %lu, max QP WRs %u, max SGEs %u, "
849 0 : "max CQEs %u, physical ports %u, requested port %u)",
850 0 : caps->max_mr_size, core->max_qp_wr, core->max_sge, caps->max_cqe,
851 0 : (uint)core->phys_port_cnt, ctx->port_num ));
852 0 : errno = EPROTONOSUPPORT;
853 0 : return -1;
854 0 : }
855 :
856 0 : ulong const rss_caps_sz = offsetof( struct ib_uverbs_ex_query_device_resp, rss_caps )+
857 0 : sizeof(resp->core.rss_caps);
858 0 : if( FD_UNLIKELY( resp->core.response_length<rss_caps_sz ) ) {
859 0 : FD_LOG_WARNING(( "mlx5 device query did not return RSS capabilities (response length %u, required %lu)",
860 0 : resp->core.response_length, rss_caps_sz ));
861 0 : errno = EPROTONOSUPPORT;
862 0 : return -1;
863 0 : }
864 0 : if( FD_UNLIKELY( resp->core.rss_caps.max_rwq_indirection_table_size<FD_MLX5_INDIRECTION_COUNT ) ) {
865 0 : FD_LOG_WARNING(( "mlx5 RSS indirection table requires %lu entries, device supports only %u",
866 0 : FD_MLX5_INDIRECTION_COUNT, resp->core.rss_caps.max_rwq_indirection_table_size ));
867 0 : errno = EPROTONOSUPPORT;
868 0 : return -1;
869 0 : }
870 :
871 0 : ulong const tunnel_caps_sz = offsetof( struct mlx5_ib_query_device_resp, tunnel_offloads_caps )+
872 0 : sizeof(resp->mlx5.tunnel_offloads_caps);
873 0 : ulong const required_hash_fields = FD_MLX5_RX_HASH_IPV4_UDP | FD_MLX5_RX_HASH_INNER;
874 :
875 0 : if( FD_UNLIKELY( resp->mlx5.response_length<tunnel_caps_sz ||
876 0 : !(resp->mlx5.rss_caps.rx_hash_function & FD_MLX5_RX_HASH_FUNC_TOEPLITZ) ||
877 0 : (resp->mlx5.rss_caps.rx_hash_fields_mask & required_hash_fields)!=required_hash_fields ||
878 0 : !(resp->mlx5.tunnel_offloads_caps & FD_MLX5_TUNNEL_OFFLOADS_GRE) ) ) {
879 0 : FD_LOG_WARNING(( "unsupported mlx5 GRE RSS capabilities (response length %u, hash functions %#x, "
880 0 : "hash fields %#lx, tunnel offloads %#x)",
881 0 : resp->mlx5.response_length, (uint)resp->mlx5.rss_caps.rx_hash_function,
882 0 : (ulong)resp->mlx5.rss_caps.rx_hash_fields_mask, resp->mlx5.tunnel_offloads_caps ));
883 0 : errno = EPROTONOSUPPORT;
884 0 : return -1;
885 0 : }
886 0 : return 0;
887 0 : }
888 :
889 : static int
890 0 : fd_uverbs_query_port( fd_uverbs_ctx_t * ctx ) {
891 0 : fd_uverbs_query_port_req_t req [1];
892 0 : struct ib_uverbs_query_port_resp resp[1];
893 0 : fd_memset( req, 0, sizeof(req ) );
894 0 : fd_memset( resp, 0, sizeof(resp) );
895 :
896 0 : req->response = (ulong)resp;
897 0 : req->port_num = (uchar)ctx->port_num;
898 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_QUERY_PORT,
899 0 : sizeof(req), sizeof(resp) ) );
900 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
901 :
902 0 : if( FD_UNLIKELY( resp->link_layer!=FD_MLX5_LINK_LAYER_ETHERNET ) ) {
903 0 : FD_LOG_WARNING(( "unsupported mlx5 link layer %u, expected Ethernet (%u)",
904 0 : (uint)resp->link_layer, FD_MLX5_LINK_LAYER_ETHERNET ));
905 0 : errno = EPROTONOSUPPORT;
906 0 : return -1;
907 0 : }
908 0 : return 0;
909 0 : }
910 :
911 : static fd_uverbs_ctx_t *
912 : fd_uverbs_open_context( fd_uverbs_ctx_t * ctx,
913 : fd_mlx5_caps_t * caps,
914 : char const * rdma_name,
915 0 : uint port_num ) {
916 0 : fd_memset( ctx, 0, sizeof(*ctx) );
917 0 : fd_memset( caps, 0, sizeof(*caps) );
918 0 : ctx->cmd_fd = -1;
919 0 : ctx->async_fd = -1;
920 0 : if( FD_UNLIKELY( !port_num || port_num>(uint)UCHAR_MAX ) ) {
921 0 : errno = EINVAL;
922 0 : return NULL;
923 0 : }
924 0 : ctx->port_num = port_num;
925 :
926 0 : char uverbs_name[ FD_UVERBS_NAME_MAX ];
927 0 : if( FD_UNLIKELY( fd_uverbs_resolve( uverbs_name, rdma_name ) ) ) return NULL;
928 :
929 0 : char path[ FD_RDMA_PATH_MAX ];
930 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/dev/infiniband/%s", uverbs_name ) );
931 0 : ctx->cmd_fd = open( path, O_RDWR | O_CLOEXEC );
932 0 : if( FD_UNLIKELY( ctx->cmd_fd<0 ) ) return NULL;
933 :
934 0 : struct stat cmd_fd_stat;
935 0 : if( FD_UNLIKELY( fstat( ctx->cmd_fd, &cmd_fd_stat ) ) ) return NULL;
936 0 : if( FD_UNLIKELY( !S_ISCHR( cmd_fd_stat.st_mode ) ) ) {
937 0 : errno = ENODEV;
938 0 : return NULL;
939 0 : }
940 :
941 0 : if( FD_UNLIKELY( fd_uverbs_get_context( ctx, caps ) ||
942 0 : fd_uverbs_query_device( ctx, caps ) ||
943 0 : fd_uverbs_query_port( ctx ) ) ) return NULL;
944 0 : return ctx;
945 0 : }
946 :
947 : static fd_mlx5_pd_t *
948 : fd_uverbs_alloc_pd( fd_mlx5_pd_t * pd,
949 0 : fd_uverbs_ctx_t * ctx ) {
950 0 : if( FD_UNLIKELY( !pd ) ) {
951 0 : errno = EINVAL;
952 0 : return NULL;
953 0 : }
954 0 : fd_memset( pd, 0, sizeof(*pd) );
955 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 ) ) {
956 0 : errno = EINVAL;
957 0 : return NULL;
958 0 : }
959 :
960 0 : fd_uverbs_alloc_pd_req_t req [1];
961 0 : fd_uverbs_alloc_pd_resp_t resp[1];
962 0 : fd_memset( req, 0, sizeof(req ) );
963 0 : fd_memset( resp, 0, sizeof(resp) );
964 0 : req->response = (ulong)resp;
965 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_ALLOC_PD,
966 0 : sizeof(req), sizeof(resp) ) );
967 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
968 :
969 0 : pd->ctx = ctx;
970 0 : pd->handle = resp->pd_handle;
971 0 : return pd;
972 0 : }
973 :
974 : static uint *
975 : fd_uverbs_register_mr( uint * lkey,
976 : fd_mlx5_pd_t * pd,
977 : void * memory,
978 : ulong memory_sz,
979 : ulong iova,
980 0 : ulong max_mr_size ) {
981 0 : ulong memory_addr = (ulong)memory;
982 0 : if( FD_UNLIKELY( !memory || !memory_sz || memory_sz>ULONG_MAX-memory_addr || memory_sz>ULONG_MAX-iova ||
983 0 : memory_sz>max_mr_size ||
984 0 : (memory_addr & (FD_MLX5_PAGE_SZ-1UL))!=(iova & (FD_MLX5_PAGE_SZ-1UL)) ) ) {
985 0 : errno = EINVAL;
986 0 : return NULL;
987 0 : }
988 :
989 0 : fd_uverbs_reg_mr_req_t req [1];
990 0 : fd_uverbs_reg_mr_resp_t resp[1];
991 0 : fd_memset( req, 0, sizeof(req ) );
992 0 : fd_memset( resp, 0, sizeof(resp) );
993 0 : req->response = (ulong)resp;
994 0 : req->start = memory_addr;
995 0 : req->length = memory_sz;
996 0 : req->hca_va = iova;
997 0 : req->pd_handle = pd->handle;
998 0 : req->access_flags = IB_UVERBS_ACCESS_LOCAL_WRITE;
999 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_REG_MR,
1000 0 : sizeof(req), sizeof(resp) ) );
1001 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( pd->ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1002 :
1003 0 : *lkey = resp->lkey;
1004 0 : return lkey;
1005 0 : }
1006 :
1007 : static void
1008 : fd_uverbs_init_ioctl_attr( struct ib_uverbs_attr * attr,
1009 : ushort attr_id,
1010 : ushort len,
1011 0 : ulong data ) {
1012 0 : fd_memset( attr, 0, sizeof(*attr) );
1013 0 : attr->attr_id = attr_id;
1014 0 : attr->len = len;
1015 0 : attr->flags = UVERBS_ATTR_F_MANDATORY;
1016 0 : attr->data = data;
1017 0 : }
1018 :
1019 : static int
1020 : fd_uverbs_init_alloc_uar_req( fd_uverbs_alloc_uar_req_t * req,
1021 : ulong * mmap_offset,
1022 : uint * mmap_sz,
1023 0 : uint * page_id ) {
1024 0 : if( FD_UNLIKELY( !req || !mmap_offset || !mmap_sz || !page_id ) ) return -1;
1025 0 : fd_memset( req, 0, sizeof(*req) );
1026 0 : req->hdr.length = (ushort)sizeof(*req);
1027 0 : req->hdr.object_id = MLX5_IB_OBJECT_UAR;
1028 0 : req->hdr.method_id = MLX5_IB_METHOD_UAR_OBJ_ALLOC;
1029 0 : req->hdr.num_attrs = 5U;
1030 0 : req->hdr.driver_id = RDMA_DRIVER_MLX5;
1031 0 : fd_uverbs_init_ioctl_attr( req->attrs+0, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE, 0U, 0UL );
1032 0 : fd_uverbs_init_ioctl_attr( req->attrs+1, MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE, 8U, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC );
1033 0 : fd_uverbs_init_ioctl_attr( req->attrs+2, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET, 8U, (ulong)mmap_offset );
1034 0 : fd_uverbs_init_ioctl_attr( req->attrs+3, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH, 4U, (ulong)mmap_sz );
1035 0 : fd_uverbs_init_ioctl_attr( req->attrs+4, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID, 4U, (ulong)page_id );
1036 0 : return 0;
1037 0 : }
1038 :
1039 : static int
1040 : fd_uverbs_init_destroy_uar_req( fd_uverbs_destroy_uar_req_t * req,
1041 0 : uint handle ) {
1042 0 : if( FD_UNLIKELY( !req ) ) return -1;
1043 0 : fd_memset( req, 0, sizeof(*req) );
1044 0 : req->hdr.length = (ushort)sizeof(*req);
1045 0 : req->hdr.object_id = MLX5_IB_OBJECT_UAR;
1046 0 : req->hdr.method_id = MLX5_IB_METHOD_UAR_OBJ_DESTROY;
1047 0 : req->hdr.num_attrs = 1U;
1048 0 : req->hdr.driver_id = RDMA_DRIVER_MLX5;
1049 0 : fd_uverbs_init_ioctl_attr( req->attrs, MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE, 0U, handle );
1050 0 : return 0;
1051 0 : }
1052 :
1053 : static int
1054 : fd_uverbs_destroy_uar( fd_uverbs_ctx_t * ctx,
1055 0 : uint handle ) {
1056 0 : fd_uverbs_destroy_uar_req_t req[1];
1057 0 : if( FD_UNLIKELY( fd_uverbs_init_destroy_uar_req( req, handle ) ) ) {
1058 0 : errno = EINVAL;
1059 0 : return -1;
1060 0 : }
1061 0 : return (int)syscall( SYS_ioctl, ctx->cmd_fd, RDMA_VERBS_IOCTL, &req->hdr );
1062 0 : }
1063 :
1064 : static ulong *
1065 : fd_uverbs_alloc_uar( fd_uverbs_ctx_t * ctx,
1066 : uint * page_id,
1067 0 : ulong * mmap_offset_out ) {
1068 0 : ulong mmap_offset = 0UL;
1069 0 : uint mmap_sz = 0U;
1070 0 : uint uar_page_id = 0U;
1071 0 : fd_uverbs_alloc_uar_req_t req[1];
1072 0 : if( FD_UNLIKELY( fd_uverbs_init_alloc_uar_req( req, &mmap_offset, &mmap_sz, &uar_page_id ) ) ) {
1073 0 : errno = EINVAL;
1074 0 : return NULL;
1075 0 : }
1076 0 : if( FD_UNLIKELY( syscall( SYS_ioctl, ctx->cmd_fd, RDMA_VERBS_IOCTL, &req->hdr ) ) ) return NULL;
1077 :
1078 0 : ulong handle_raw = req->attrs[0].data;
1079 0 : if( FD_UNLIKELY( handle_raw>UINT_MAX || mmap_sz!=FD_MLX5_PAGE_SZ ||
1080 0 : (mmap_offset & (FD_MLX5_PAGE_SZ-1UL)) || mmap_offset>(ulong)LONG_MAX ) ) {
1081 0 : FD_LOG_WARNING(( "invalid mlx5 UAR response (handle %lu, mapping size %u, mapping offset %#lx)",
1082 0 : handle_raw, mmap_sz, mmap_offset ));
1083 0 : int err = EPROTO;
1084 0 : if( handle_raw<=UINT_MAX ) fd_uverbs_destroy_uar( ctx, (uint)handle_raw );
1085 0 : errno = err;
1086 0 : return NULL;
1087 0 : }
1088 :
1089 0 : *page_id = uar_page_id;
1090 0 : *mmap_offset_out = mmap_offset;
1091 0 : return mmap_offset_out;
1092 0 : }
1093 :
1094 : volatile uchar *
1095 : fd_uverbs_map_uar( fd_uverbs_ctx_t * ctx,
1096 0 : ulong mmap_offset ) {
1097 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 ||
1098 0 : (mmap_offset & (FD_MLX5_PAGE_SZ-1UL)) || mmap_offset>(ulong)LONG_MAX ) ) {
1099 0 : errno = EINVAL;
1100 0 : return NULL;
1101 0 : }
1102 0 : void * uar_mapping = mmap( NULL, FD_MLX5_PAGE_SZ, PROT_WRITE, MAP_SHARED,
1103 0 : ctx->cmd_fd, (off_t)mmap_offset );
1104 0 : if( FD_UNLIKELY( uar_mapping==MAP_FAILED ) ) return NULL;
1105 0 : return (volatile uchar *)uar_mapping;
1106 0 : }
1107 :
1108 : static int
1109 0 : fd_uverbs_create_comp_channel( fd_uverbs_ctx_t * ctx ) {
1110 0 : fd_uverbs_create_comp_channel_req_t req [1];
1111 0 : struct ib_uverbs_create_comp_channel_resp resp[1];
1112 0 : fd_memset( req, 0, sizeof(req ) );
1113 0 : fd_memset( resp, 0, sizeof(resp) );
1114 :
1115 0 : req->response = (ulong)resp;
1116 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
1117 0 : sizeof(req), sizeof(resp) ) );
1118 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
1119 0 : return (int)resp->fd;
1120 0 : }
1121 :
1122 : static uint *
1123 : fd_uverbs_create_cq( uint * handle,
1124 : fd_uverbs_ctx_t * ctx,
1125 : fd_mlx5_cq_t * cq,
1126 : int comp_channel_fd,
1127 : uint page_id,
1128 0 : uint max_cqe ) {
1129 0 : if( FD_UNLIKELY( !handle ) ) {
1130 0 : errno = EINVAL;
1131 0 : return NULL;
1132 0 : }
1133 0 : *handle = 0U;
1134 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 || !cq || !cq->entries || !cq->control ||
1135 0 : !fd_uint_is_pow2( cq->depth ) || cq->depth-1U>max_cqe || page_id>(uint)USHRT_MAX ) ) {
1136 0 : errno = EINVAL;
1137 0 : return NULL;
1138 0 : }
1139 :
1140 0 : fd_uverbs_create_cq_req_t req [1];
1141 0 : fd_uverbs_create_cq_resp_t resp[1];
1142 0 : fd_memset( req, 0, sizeof(req ) );
1143 0 : fd_memset( resp, 0, sizeof(resp) );
1144 :
1145 0 : req->response = (ulong)resp;
1146 0 : req->user_handle = (ulong)cq;
1147 0 : req->cqe = cq->depth-1U;
1148 0 : req->comp_channel = comp_channel_fd;
1149 0 : req->mlx5.fields.buf_addr = (ulong)cq->entries;
1150 0 : req->mlx5.fields.db_addr = (ulong)cq->control;
1151 0 : req->mlx5.fields.cqe_size = sizeof(fd_mlx5_cqe_t);
1152 0 : req->mlx5.fields.flags = MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX;
1153 0 : req->mlx5.uar.uar_page_index = (ushort)page_id;
1154 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_CQ,
1155 0 : sizeof(req), sizeof(resp) ) );
1156 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1157 0 : if( FD_UNLIKELY( resp->cqe!=cq->depth-1U ) ) {
1158 0 : fd_uverbs_destroy_cq_req_t destroy[1];
1159 0 : struct ib_uverbs_destroy_cq_resp destroy_resp[1];
1160 0 : fd_memset( destroy, 0, sizeof(destroy) );
1161 0 : fd_memset( destroy_resp, 0, sizeof(destroy_resp) );
1162 :
1163 0 : destroy->response = (ulong)destroy_resp;
1164 0 : destroy->cq_handle = resp->cq_handle;
1165 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &destroy->hdr, IB_USER_VERBS_CMD_DESTROY_CQ,
1166 0 : sizeof(destroy), sizeof(destroy_resp) ) );
1167 0 : fd_uverbs_write_cmd( ctx->cmd_fd, destroy, sizeof(destroy) );
1168 0 : FD_LOG_WARNING(( "invalid mlx5 CQ response (CQEs %u, requested %u)", resp->cqe, cq->depth-1U ));
1169 0 : errno = EPROTO;
1170 0 : return NULL;
1171 0 : }
1172 :
1173 0 : cq->cqn = resp->mlx5.cqn;
1174 0 : *handle = resp->cq_handle;
1175 0 : return handle;
1176 0 : }
1177 :
1178 : static int
1179 : fd_uverbs_modify_qp( fd_uverbs_ctx_t * uverbs,
1180 : fd_mlx5_tx_qp_t * tx_qp,
1181 0 : uint state ) {
1182 0 : fd_uverbs_modify_qp_req_t req[1];
1183 0 : fd_memset( req, 0, sizeof(req) );
1184 0 : req->core.qp_handle = tx_qp->handle;
1185 0 : req->core.attr_mask = FD_MLX5_QP_ATTR_STATE;
1186 0 : req->core.qp_state = (uchar)state;
1187 0 : if( state==FD_MLX5_QPS_INIT ) {
1188 0 : req->core.attr_mask |= FD_MLX5_QP_ATTR_PORT;
1189 0 : req->core.port_num = (uchar)uverbs->port_num;
1190 0 : }
1191 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_MODIFY_QP,
1192 0 : sizeof(req), 0UL ) );
1193 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1194 0 : return 0;
1195 0 : }
1196 :
1197 : static fd_mlx5_tx_qp_t *
1198 : fd_uverbs_create_tx_qp( fd_uverbs_ctx_t * uverbs,
1199 : fd_mlx5_tx_qp_t * tx_qp,
1200 : fd_mlx5_pd_t * pd,
1201 : uint tx_cq_handle,
1202 0 : uint uar_page_id ) {
1203 0 : if( FD_UNLIKELY( !uverbs || !tx_qp || !tx_qp->tx_cq || !tx_qp->sq ||
1204 0 : !tx_qp->control || !pd || pd->ctx!=uverbs ) ) {
1205 0 : errno = EINVAL;
1206 0 : return NULL;
1207 0 : }
1208 :
1209 0 : fd_uverbs_create_tx_qp_req_t req [1];
1210 0 : fd_uverbs_create_tx_qp_resp_t resp[1];
1211 0 : fd_memset( req, 0, sizeof(req ) );
1212 0 : fd_memset( resp, 0, sizeof(resp) );
1213 :
1214 0 : req->response = (ulong)resp;
1215 0 : req->user_handle = (ulong)tx_qp;
1216 0 : req->pd_handle = pd->handle;
1217 0 : req->send_cq_handle = tx_cq_handle;
1218 : /* Linux requires both CQ handles even when max_recv_wr is zero. */
1219 0 : req->recv_cq_handle = tx_cq_handle;
1220 0 : req->max_send_wr = tx_qp->tx_depth;
1221 0 : req->max_send_sge = 1U;
1222 0 : req->qp_type = IB_UVERBS_QPT_RAW_PACKET;
1223 0 : req->mlx5.db_addr = (ulong)tx_qp->control;
1224 0 : req->mlx5.sq_wqe_count = tx_qp->tx_depth;
1225 0 : req->mlx5.flags = MLX5_QP_FLAG_UAR_PAGE_INDEX;
1226 0 : req->mlx5.uidx = 0U;
1227 0 : req->mlx5.bfreg_index = uar_page_id;
1228 0 : req->mlx5.sq_buf_addr = (ulong)tx_qp->sq;
1229 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_QP,
1230 0 : sizeof(req), sizeof(resp) ) );
1231 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1232 :
1233 0 : if( FD_UNLIKELY( resp->core.max_send_wr<tx_qp->tx_depth ||
1234 0 : resp->core.max_recv_wr ||
1235 0 : resp->core.max_send_sge<1U ||
1236 0 : resp->core.max_recv_sge ||
1237 0 : resp->core.qpn>0xffffffU ) ) {
1238 0 : FD_LOG_WARNING(( "invalid mlx5 QP response (max send WRs %u, max receive WRs %u, max send SGEs %u, "
1239 0 : "max receive SGEs %u, QPN %u)",
1240 0 : resp->core.max_send_wr, resp->core.max_recv_wr, resp->core.max_send_sge,
1241 0 : resp->core.max_recv_sge, resp->core.qpn ));
1242 0 : errno = EPROTO;
1243 0 : return NULL;
1244 0 : }
1245 :
1246 0 : tx_qp->handle = resp->core.qp_handle;
1247 0 : tx_qp->qpn = resp->core.qpn;
1248 0 : return tx_qp;
1249 0 : }
1250 :
1251 : static fd_mlx5_tx_qp_t *
1252 : fd_uverbs_start_tx_qp( fd_uverbs_ctx_t * uverbs,
1253 0 : fd_mlx5_tx_qp_t * tx_qp ) {
1254 0 : if( FD_UNLIKELY( fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_INIT ) ||
1255 0 : fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_RTR ) ||
1256 0 : fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_RTS ) ) ) return NULL;
1257 0 : return tx_qp;
1258 0 : }
1259 :
1260 : static fd_mlx5_rx_wq_t *
1261 : fd_uverbs_create_rx_wq( fd_uverbs_ctx_t * uverbs,
1262 : fd_mlx5_rx_wq_t * rx_wq,
1263 : fd_mlx5_pd_t * pd,
1264 0 : uint rx_cq_handle ) {
1265 0 : fd_uverbs_create_rx_wq_req_t req [1];
1266 0 : fd_uverbs_create_rx_wq_resp_t resp[1];
1267 0 : fd_memset( req, 0, sizeof(req ) );
1268 0 : fd_memset( resp, 0, sizeof(resp) );
1269 :
1270 0 : req->core.wq_type = IB_UVERBS_WQT_RQ;
1271 0 : req->core.user_handle = (ulong)rx_wq;
1272 0 : req->core.pd_handle = pd->handle;
1273 0 : req->core.cq_handle = rx_cq_handle;
1274 0 : req->core.max_wr = rx_wq->rx_depth;
1275 0 : req->core.max_sge = 1U;
1276 0 : req->mlx5.buf_addr = (ulong)rx_wq->rq;
1277 0 : req->mlx5.db_addr = (ulong)rx_wq->control;
1278 0 : req->mlx5.rq_wqe_count = rx_wq->rx_depth;
1279 0 : req->mlx5.rq_wqe_shift = (uint)fd_ulong_find_lsb( sizeof(fd_mlx5_rx_wqe_t) );
1280 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_rx_wq_req_t, mlx5 );
1281 0 : ulong const core_resp_sz = offsetof( fd_uverbs_create_rx_wq_resp_t, mlx5 );
1282 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_WQ,
1283 0 : core_req_sz, sizeof(req), resp,
1284 0 : core_resp_sz, sizeof(resp) ) );
1285 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1286 0 : if( FD_UNLIKELY( resp->core.max_wr<rx_wq->rx_depth ||
1287 0 : resp->core.max_sge<1U ||
1288 0 : resp->core.wqn>0xffffffU ) ) {
1289 0 : FD_LOG_WARNING(( "invalid mlx5 receive WQ response (max WRs %u, max SGEs %u, WQN %u)",
1290 0 : resp->core.max_wr, resp->core.max_sge, resp->core.wqn ));
1291 0 : errno = EPROTO;
1292 0 : return NULL;
1293 0 : }
1294 :
1295 0 : rx_wq->handle = resp->core.wq_handle;
1296 0 : rx_wq->wqn = resp->core.wqn;
1297 0 : return rx_wq;
1298 0 : }
1299 :
1300 : static fd_mlx5_rx_wq_t *
1301 : fd_uverbs_start_rx_wq( fd_uverbs_ctx_t * uverbs,
1302 0 : fd_mlx5_rx_wq_t * rx_wq ) {
1303 0 : fd_uverbs_modify_rx_wq_req_t req[1];
1304 0 : fd_memset( req, 0, sizeof(req) );
1305 0 : req->core.attr_mask = FD_MLX5_WQ_ATTR_STATE | FD_MLX5_WQ_ATTR_CUR_STATE;
1306 0 : req->core.wq_handle = rx_wq->handle;
1307 0 : req->core.wq_state = FD_MLX5_WQS_RDY;
1308 0 : req->core.curr_wq_state = FD_MLX5_WQS_RESET;
1309 0 : ulong const core_req_sz = offsetof( fd_uverbs_modify_rx_wq_req_t, mlx5 );
1310 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_MODIFY_WQ,
1311 0 : core_req_sz, sizeof(req), NULL, 0UL, 0UL ) );
1312 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1313 :
1314 : /* Explicitly disable CVLAN stripping. Some kernel configurations will
1315 : still strip it anyway so we explicitly disable it so all providers
1316 : follow the same software path. */
1317 0 : fd_memset( req, 0, sizeof(req) );
1318 0 : req->core.attr_mask = FD_MLX5_WQ_ATTR_STATE | FD_MLX5_WQ_ATTR_CUR_STATE | FD_MLX5_WQ_ATTR_FLAGS;
1319 0 : req->core.wq_handle = rx_wq->handle;
1320 0 : req->core.wq_state = FD_MLX5_WQS_RDY;
1321 0 : req->core.curr_wq_state = FD_MLX5_WQS_RDY;
1322 0 : req->core.flags = 0U;
1323 0 : req->core.flags_mask = FD_MLX5_WQ_FLAGS_CVLAN_STRIPPING;
1324 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_MODIFY_WQ,
1325 0 : core_req_sz, sizeof(req), NULL, 0UL, 0UL ) );
1326 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) {
1327 0 : FD_LOG_WARNING(( "failed to disable mlx5 CVLAN stripping on WQ %u (%i-%s)",
1328 0 : rx_wq->wqn, errno, fd_io_strerror( errno ) ));
1329 0 : }
1330 0 : return rx_wq;
1331 0 : }
1332 :
1333 : static uint *
1334 : fd_uverbs_create_rwq_ind_table( uint * handle,
1335 : fd_uverbs_ctx_t * uverbs,
1336 : fd_mlx5_uverbs_tile_t const * tiles,
1337 0 : ulong tile_cnt ) {
1338 0 : if( FD_UNLIKELY( 0==tile_cnt || tile_cnt>FD_MLX5_TILE_MAX ) ) {
1339 0 : errno = EINVAL;
1340 0 : return NULL;
1341 0 : }
1342 :
1343 0 : ulong const req_sz = fd_ulong_align_up( sizeof(fd_uverbs_create_rwq_ind_table_req_t)+FD_MLX5_INDIRECTION_COUNT*sizeof(uint), 8UL );
1344 0 : fd_uverbs_create_rwq_ind_table_req_t * req = aligned_alloc( 8UL, req_sz );
1345 0 : if( FD_UNLIKELY( !req ) ) return NULL;
1346 0 : struct ib_uverbs_ex_create_rwq_ind_table_resp resp[1];
1347 0 : fd_memset( req, 0, req_sz );
1348 0 : fd_memset( resp, 0, sizeof(resp) );
1349 0 : req->log_ind_tbl_size = (uint)fd_ulong_find_lsb( FD_MLX5_INDIRECTION_COUNT );
1350 0 : for( ulong i=0UL; i<FD_MLX5_INDIRECTION_COUNT; i++ ) req->wq_handles[ i ] = tiles[ i%tile_cnt ].rx_wq->handle;
1351 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
1352 0 : req_sz, req_sz, resp,
1353 0 : sizeof(resp), sizeof(resp) ) );
1354 0 : int err = fd_uverbs_write_cmd( uverbs->cmd_fd, req, req_sz );
1355 0 : int saved_errno = errno;
1356 0 : free( req );
1357 0 : if( FD_UNLIKELY( err ) ) {
1358 0 : errno = saved_errno;
1359 0 : return NULL;
1360 0 : }
1361 0 : *handle = resp->ind_tbl_handle;
1362 0 : return handle;
1363 0 : }
1364 :
1365 : static fd_mlx5_rss_qp_t *
1366 : fd_uverbs_create_rss_qp( fd_uverbs_ctx_t * uverbs,
1367 : fd_mlx5_rss_qp_t * rss_qp,
1368 : fd_mlx5_pd_t * pd,
1369 : uint rwq_ind_table_handle,
1370 : ulong hash_fields,
1371 0 : uint flags ) {
1372 0 : fd_uverbs_create_rss_qp_req_t req [1];
1373 0 : fd_uverbs_create_rss_qp_resp_t resp[1];
1374 0 : fd_memset( req, 0, sizeof(req ) );
1375 0 : fd_memset( resp, 0, sizeof(resp) );
1376 0 : req->core.user_handle = (ulong)rss_qp;
1377 0 : req->core.pd_handle = pd->handle;
1378 0 : req->core.qp_type = IB_UVERBS_QPT_RAW_PACKET;
1379 0 : req->core.comp_mask = IB_UVERBS_CREATE_QP_MASK_IND_TABLE;
1380 0 : req->core.rwq_ind_tbl_handle = rwq_ind_table_handle;
1381 0 : req->mlx5.rx_hash_fields_mask = hash_fields;
1382 0 : req->mlx5.rx_hash_function = FD_MLX5_RX_HASH_FUNC_TOEPLITZ;
1383 0 : req->mlx5.rx_key_len = FD_MLX5_RX_HASH_KEY_SZ;
1384 0 : req->mlx5.flags = flags;
1385 :
1386 0 : if( FD_UNLIKELY( !fd_rng_secure( req->mlx5.rx_hash_key, FD_MLX5_RX_HASH_KEY_SZ ) ) ) return NULL;
1387 :
1388 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_rss_qp_req_t, mlx5 );
1389 0 : ulong const core_resp_sz = offsetof( fd_uverbs_create_rss_qp_resp_t, mlx5 );
1390 :
1391 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_QP,
1392 0 : core_req_sz, sizeof(req), resp,
1393 0 : core_resp_sz, sizeof(resp) ) );
1394 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1395 0 : rss_qp->handle = resp->core.base.qp_handle;
1396 0 : return rss_qp;
1397 0 : }
1398 :
1399 : static int
1400 : fd_uverbs_init_udp_flow_req( fd_uverbs_create_udp_flow_req_t * req,
1401 : struct ib_uverbs_create_flow_resp * resp,
1402 : uint qp_handle,
1403 : uint port_num,
1404 : uint dst_ip,
1405 0 : ushort dst_port ) {
1406 0 : if( FD_UNLIKELY( !req || !resp || !port_num || port_num>UCHAR_MAX || !dst_port ) ) return -1;
1407 :
1408 0 : fd_memset( req, 0, sizeof(*req ) );
1409 0 : fd_memset( resp, 0, sizeof(*resp) );
1410 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_udp_flow_req_t, mlx5_ncounters_data );
1411 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_FLOW,
1412 0 : core_req_sz, sizeof(*req), resp,
1413 0 : sizeof(*resp), sizeof(*resp) ) );
1414 :
1415 0 : req->qp_handle = qp_handle;
1416 0 : req->size = sizeof(req->eth)+sizeof(req->ipv4)+sizeof(req->udp);
1417 0 : req->num_of_specs = 3U;
1418 0 : req->port = (uchar)port_num;
1419 0 : req->eth.type = FD_MLX5_FLOW_SPEC_ETH;
1420 0 : req->eth.size = sizeof(req->eth);
1421 0 : req->eth.val.ether_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP );
1422 0 : req->eth.mask.ether_type = USHORT_MAX;
1423 0 : req->ipv4.type = FD_MLX5_FLOW_SPEC_IPV4;
1424 0 : req->ipv4.size = sizeof(req->ipv4);
1425 0 : req->ipv4.val.dst_ip = dst_ip;
1426 0 : req->ipv4.mask.dst_ip = dst_ip ? UINT_MAX : 0U;
1427 0 : req->udp.type = FD_MLX5_FLOW_SPEC_UDP;
1428 0 : req->udp.size = sizeof(req->udp);
1429 0 : req->udp.val.dst_port = fd_ushort_bswap( dst_port );
1430 0 : req->udp.mask.dst_port = USHORT_MAX;
1431 0 : return 0;
1432 0 : }
1433 :
1434 : int
1435 : fd_uverbs_create_udp_flow( fd_uverbs_ctx_t * uverbs,
1436 : fd_mlx5_rss_qp_t const * rss_qp,
1437 : uint dst_ip,
1438 0 : ushort dst_port ) {
1439 0 : if( FD_UNLIKELY( !uverbs || uverbs->cmd_fd<0 || !rss_qp || !dst_port ) ) {
1440 0 : errno = EINVAL;
1441 0 : return -1;
1442 0 : }
1443 :
1444 0 : fd_uverbs_create_udp_flow_req_t req [1];
1445 0 : struct ib_uverbs_create_flow_resp resp[1];
1446 0 : if( FD_UNLIKELY( fd_uverbs_init_udp_flow_req( req, resp, rss_qp->handle,
1447 0 : uverbs->port_num, dst_ip,
1448 0 : dst_port ) ) ) {
1449 0 : errno = EINVAL;
1450 0 : return -1;
1451 0 : }
1452 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1453 0 : return 0;
1454 0 : }
1455 :
1456 : static int
1457 : fd_uverbs_init_gre_flow_req( fd_uverbs_create_gre_flow_req_t * req,
1458 : struct ib_uverbs_create_flow_resp * resp,
1459 : uint qp_handle,
1460 : uint port_num,
1461 : uint inner_dst_ip,
1462 0 : ushort inner_dst_port ) {
1463 0 : if( FD_UNLIKELY( !req || !resp || !port_num || port_num>UCHAR_MAX || !inner_dst_port ) ) return -1;
1464 :
1465 0 : fd_memset( req, 0, sizeof(*req ) );
1466 0 : fd_memset( resp, 0, sizeof(*resp) );
1467 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_gre_flow_req_t, mlx5_ncounters_data );
1468 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_FLOW,
1469 0 : core_req_sz, sizeof(*req), resp,
1470 0 : sizeof(*resp), sizeof(*resp) ) );
1471 :
1472 0 : req->qp_handle = qp_handle;
1473 0 : req->size = sizeof(req->eth)+sizeof(req->outer_ipv4)+sizeof(req->gre)+
1474 0 : sizeof(req->inner_ipv4)+sizeof(req->inner_udp);
1475 0 : req->num_of_specs = 5U;
1476 0 : req->port = (uchar)port_num;
1477 0 : req->eth = (struct ib_uverbs_flow_spec_eth) {
1478 0 : .type = FD_MLX5_FLOW_SPEC_ETH,
1479 0 : .size = sizeof(req->eth),
1480 0 : .val = { .ether_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) },
1481 0 : .mask = { .ether_type = USHORT_MAX },
1482 0 : };
1483 0 : req->outer_ipv4 = (struct ib_uverbs_flow_spec_ipv4) {
1484 0 : .type = FD_MLX5_FLOW_SPEC_IPV4,
1485 0 : .size = sizeof(req->outer_ipv4),
1486 0 : .val = { .proto = FD_IP4_HDR_PROTOCOL_GRE },
1487 0 : .mask = { .proto = UCHAR_MAX },
1488 0 : };
1489 0 : req->gre = (struct ib_uverbs_flow_spec_gre) {
1490 0 : .type = FD_MLX5_FLOW_SPEC_GRE,
1491 0 : .size = sizeof(req->gre),
1492 0 : .val = { .protocol = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) },
1493 0 : .mask = { .protocol = USHORT_MAX },
1494 0 : };
1495 0 : req->inner_ipv4 = (struct ib_uverbs_flow_spec_ipv4) {
1496 0 : .type = FD_MLX5_FLOW_SPEC_INNER | FD_MLX5_FLOW_SPEC_IPV4,
1497 0 : .size = sizeof(req->inner_ipv4),
1498 0 : .val = { .dst_ip = inner_dst_ip, .proto = FD_IP4_HDR_PROTOCOL_UDP },
1499 0 : .mask = { .dst_ip = inner_dst_ip ? UINT_MAX : 0U, .proto = UCHAR_MAX },
1500 0 : };
1501 0 : req->inner_udp = (struct ib_uverbs_flow_spec_tcp_udp) {
1502 0 : .type = FD_MLX5_FLOW_SPEC_INNER | FD_MLX5_FLOW_SPEC_UDP,
1503 0 : .size = sizeof(req->inner_udp),
1504 0 : .val = { .dst_port = fd_ushort_bswap( inner_dst_port ) },
1505 0 : .mask = { .dst_port = USHORT_MAX },
1506 0 : };
1507 0 : return 0;
1508 0 : }
1509 :
1510 : int
1511 : fd_uverbs_create_gre_udp_flow( fd_uverbs_ctx_t * uverbs,
1512 : fd_mlx5_rss_qp_t const * rss_qp,
1513 : uint inner_dst_ip,
1514 0 : ushort inner_dst_port ) {
1515 0 : if( FD_UNLIKELY( !uverbs || uverbs->cmd_fd<0 || !rss_qp || !inner_dst_port ) ) {
1516 0 : errno = EINVAL;
1517 0 : return -1;
1518 0 : }
1519 :
1520 0 : fd_uverbs_create_gre_flow_req_t req [1];
1521 0 : struct ib_uverbs_create_flow_resp resp[1];
1522 0 : if( FD_UNLIKELY( fd_uverbs_init_gre_flow_req( req, resp, rss_qp->handle,
1523 0 : uverbs->port_num, inner_dst_ip,
1524 0 : inner_dst_port ) ) ) {
1525 0 : errno = EINVAL;
1526 0 : return -1;
1527 0 : }
1528 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1529 0 : return 0;
1530 0 : }
1531 :
1532 : fd_mlx5_rss_qp_t *
1533 : fd_uverbs_init( fd_uverbs_ctx_t * uverbs,
1534 : fd_mlx5_uverbs_tile_t * tiles,
1535 : ulong tile_cnt,
1536 : fd_mlx5_rss_qp_t * outer_rss_qp,
1537 : fd_mlx5_rss_qp_t * gre_rss_qp,
1538 : char const * rdma_name,
1539 0 : uint port_num ) {
1540 0 : if( FD_UNLIKELY( !uverbs || !tiles || 0==tile_cnt || tile_cnt>FD_MLX5_TILE_MAX || !outer_rss_qp || !gre_rss_qp ) ) {
1541 0 : errno = EINVAL;
1542 0 : return NULL;
1543 0 : }
1544 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1545 0 : fd_mlx5_uverbs_tile_t const * tile = tiles+i;
1546 0 : if( FD_UNLIKELY( !tile->rx_cq || !tile->tx_cq || !tile->rx_wq || !tile->tx_qp || !tile->lkey ||
1547 0 : !tile->packet_memory || !tile->packet_memory_sz || !tile->rx_wq->rq ||
1548 0 : !tile->rx_wq->control || tile->rx_wq->rx_cq!=tile->rx_cq || tile->tx_qp->tx_cq!=tile->tx_cq ||
1549 0 : tile->rx_wq->rx_depth!=tile->rx_cq->depth || tile->tx_qp->tx_depth!=tile->tx_cq->depth ) ) {
1550 0 : errno = EINVAL;
1551 0 : return NULL;
1552 0 : }
1553 0 : }
1554 :
1555 0 : fd_mlx5_caps_t caps[1];
1556 0 : fd_mlx5_pd_t pd[1];
1557 0 : uint rwq_ind_table_handle;
1558 0 : if( FD_UNLIKELY( !fd_uverbs_open_context( uverbs, caps, rdma_name, port_num ) ) ) return NULL;
1559 0 : if( FD_UNLIKELY( !fd_uverbs_alloc_pd( pd, uverbs ) ) ) return NULL;
1560 :
1561 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1562 0 : fd_mlx5_uverbs_tile_t * tile = tiles+i;
1563 0 : fd_mlx5_rx_wq_t * rx_wq = tile->rx_wq;
1564 0 : fd_mlx5_tx_qp_t * tx_qp = tile->tx_qp;
1565 0 : if( FD_UNLIKELY( !fd_uint_is_pow2( rx_wq->rx_depth ) ||
1566 0 : !fd_uint_is_pow2( tx_qp->tx_depth ) ||
1567 0 : rx_wq->rx_depth>caps->max_recv_wr ||
1568 0 : tx_qp->tx_depth>caps->max_send_wqe ||
1569 0 : rx_wq->rx_depth-1U>caps->max_cqe ||
1570 0 : tx_qp->tx_depth-1U>caps->max_cqe ) ) {
1571 0 : errno = EINVAL;
1572 0 : return NULL;
1573 0 : }
1574 :
1575 0 : int rx_comp_channel_fd = -1;
1576 0 : if( tile->rx_comp_channel_fd ) {
1577 0 : rx_comp_channel_fd = fd_uverbs_create_comp_channel( uverbs );
1578 0 : if( FD_UNLIKELY( rx_comp_channel_fd<0 ) ) return NULL;
1579 0 : *tile->rx_comp_channel_fd = rx_comp_channel_fd;
1580 0 : }
1581 :
1582 0 : uint uar_page_id;
1583 0 : uint rx_cq_handle;
1584 0 : uint tx_cq_handle;
1585 0 : if( FD_UNLIKELY( !fd_uverbs_alloc_uar( uverbs, &uar_page_id, &tx_qp->uar_mmap_offset ) ||
1586 0 : !fd_uverbs_create_cq( &rx_cq_handle, uverbs, tile->rx_cq, rx_comp_channel_fd, uar_page_id, caps->max_cqe ) ||
1587 0 : !fd_uverbs_create_cq( &tx_cq_handle, uverbs, tile->tx_cq, -1, uar_page_id, caps->max_cqe ) ||
1588 0 : !fd_uverbs_register_mr( tile->lkey, pd, tile->packet_memory, tile->packet_memory_sz, tile->packet_iova, caps->max_mr_size ) ||
1589 0 : !fd_uverbs_create_tx_qp( uverbs, tx_qp, pd, tx_cq_handle, uar_page_id ) ||
1590 0 : !fd_uverbs_start_tx_qp( uverbs, tx_qp ) ||
1591 0 : !fd_uverbs_create_rx_wq( uverbs, rx_wq, pd, rx_cq_handle ) ||
1592 0 : !fd_uverbs_start_rx_wq( uverbs, rx_wq ) ) ) {
1593 0 : return NULL;
1594 0 : }
1595 0 : tx_qp->tx_inline_hdr_sz = caps->eth_min_inline_hdr_sz;
1596 0 : }
1597 :
1598 0 : if( FD_UNLIKELY( !fd_uverbs_create_rwq_ind_table( &rwq_ind_table_handle, uverbs, tiles, tile_cnt ) ||
1599 0 : !fd_uverbs_create_rss_qp( uverbs, outer_rss_qp, pd, rwq_ind_table_handle,
1600 0 : FD_MLX5_RX_HASH_IPV4_UDP, 0U ) ||
1601 0 : !fd_uverbs_create_rss_qp( uverbs, gre_rss_qp, pd, rwq_ind_table_handle,
1602 0 : FD_MLX5_RX_HASH_IPV4_UDP | FD_MLX5_RX_HASH_INNER,
1603 0 : FD_MLX5_QP_TUNNEL_OFFLOADS ) ) ) return NULL;
1604 0 : return outer_rss_qp;
1605 0 : }
|
