Line data Source code
1 : #define _GNU_SOURCE
2 : #include "../fd_net_tile.h"
3 : #include "fd_mlx5_private.h"
4 : #include "../../../util/log/fd_log.h"
5 : #include "../../../util/net/fd_eth.h"
6 : #include "../../../util/net/fd_ip4.h"
7 : #include "../../../util/rng/fd_rng.h"
8 : #include "../fd_linux_bond.h"
9 :
10 : #include <ctype.h>
11 : #include <dirent.h>
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <net/if.h>
16 : #include <stddef.h>
17 : #include <stdio.h>
18 : #include <stdlib.h>
19 : #include <sys/ioctl.h>
20 : #include <sys/mman.h>
21 : #include <sys/stat.h>
22 : #include <sys/syscall.h>
23 : #include <sys/wait.h>
24 : #include <unistd.h>
25 :
26 : #include <rdma/ib_user_ioctl_cmds.h>
27 : #include <rdma/ib_user_ioctl_verbs.h>
28 : #include <rdma/ib_user_verbs.h>
29 : #include <rdma/mlx5-abi.h>
30 : #include <rdma/mlx5_user_ioctl_cmds.h>
31 : #include <rdma/mlx5_user_ioctl_verbs.h>
32 : #include <rdma/rdma_user_ioctl_cmds.h>
33 :
34 : /* Provide uverbs and mlx5 UAPI definitions missing from older Linux headers. */
35 : #ifndef IB_UVERBS_ACCESS_LOCAL_WRITE
36 0 : #define IB_UVERBS_ACCESS_LOCAL_WRITE (1U)
37 : #endif
38 : #ifndef IB_UVERBS_QPT_RAW_PACKET
39 0 : #define IB_UVERBS_QPT_RAW_PACKET (8U)
40 : #endif
41 : #ifndef IB_UVERBS_CREATE_QP_MASK_IND_TABLE
42 0 : #define IB_UVERBS_CREATE_QP_MASK_IND_TABLE (1U)
43 : #endif
44 : #ifndef IB_UVERBS_WQT_RQ
45 0 : #define IB_UVERBS_WQT_RQ (0U)
46 : #endif
47 : #ifndef MLX5_LIB_CAP_DYN_UAR
48 0 : #define MLX5_LIB_CAP_DYN_UAR (2UL)
49 : #endif
50 : #ifndef MLX5_IB_OBJECT_UAR
51 0 : #define MLX5_IB_OBJECT_UAR (4104U)
52 : #endif
53 : #ifndef MLX5_IB_METHOD_UAR_OBJ_ALLOC
54 0 : #define MLX5_IB_METHOD_UAR_OBJ_ALLOC (4096U)
55 : #endif
56 : #ifndef MLX5_IB_METHOD_UAR_OBJ_DESTROY
57 0 : #define MLX5_IB_METHOD_UAR_OBJ_DESTROY (4097U)
58 : #endif
59 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE
60 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE (4096U)
61 : #endif
62 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE
63 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE (4097U)
64 : #endif
65 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET
66 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET (4098U)
67 : #endif
68 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH
69 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH (4099U)
70 : #endif
71 : #ifndef MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID
72 0 : #define MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID (4100U)
73 : #endif
74 : #ifndef MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE
75 0 : #define MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE (4096U)
76 : #endif
77 : #ifndef MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC
78 0 : #define MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC (1U)
79 : #endif
80 : #ifndef MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX
81 0 : #define MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX (2U)
82 : #endif
83 : #ifndef MLX5_QP_FLAG_UAR_PAGE_INDEX
84 0 : #define MLX5_QP_FLAG_UAR_PAGE_INDEX (1024U)
85 : #endif
86 :
87 0 : #define FD_MLX5_ETH_INLINE_HDR_SZ (18UL) /* MLX5_ETH_L2_INLINE_HEADER_SIZE */
88 : #define FD_MLX5_LINK_LAYER_ETHERNET (2U) /* IB_LINK_LAYER_ETHERNET */
89 0 : #define FD_UVERBS_NAME_MAX (32UL)
90 :
91 0 : #define FD_MLX5_QPS_INIT (1U) /* IB_QPS_INIT */
92 : #define FD_MLX5_QPS_RTR (2U) /* IB_QPS_RTR */
93 : #define FD_MLX5_QPS_RTS (3U) /* IB_QPS_RTS */
94 :
95 0 : #define FD_MLX5_QP_ATTR_STATE (1U) /* IB_QP_STATE */
96 0 : #define FD_MLX5_QP_ATTR_PORT (1U<<5) /* IB_QP_PORT */
97 :
98 0 : #define FD_MLX5_WQS_RESET (0U) /* IB_WQS_RESET */
99 0 : #define FD_MLX5_WQS_RDY (1U) /* IB_WQS_RDY */
100 :
101 0 : #define FD_MLX5_WQ_ATTR_STATE (1U) /* IB_WQ_STATE */
102 0 : #define FD_MLX5_WQ_ATTR_CUR_STATE (1U<<1) /* IB_WQ_CUR_STATE */
103 :
104 0 : #define FD_MLX5_RX_HASH_FUNC_TOEPLITZ (1U) /* MLX5_RX_HASH_FUNC_TOEPLITZ */
105 0 : #define FD_MLX5_RX_HASH_KEY_SZ (40U) /* MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) */
106 0 : #define FD_MLX5_RX_HASH_SRC_IPV4 (1UL<<0) /* MLX5_RX_HASH_SRC_IPV4 */
107 0 : #define FD_MLX5_RX_HASH_DST_IPV4 (1UL<<1) /* MLX5_RX_HASH_DST_IPV4 */
108 0 : #define FD_MLX5_RX_HASH_SRC_PORT_UDP (1UL<<6) /* MLX5_RX_HASH_SRC_PORT_UDP */
109 0 : #define FD_MLX5_RX_HASH_DST_PORT_UDP (1UL<<7) /* MLX5_RX_HASH_DST_PORT_UDP */
110 0 : #define FD_MLX5_RX_HASH_INNER (1UL<<31) /* MLX5_RX_HASH_INNER */
111 0 : #define FD_MLX5_RX_HASH_IPV4_UDP ( FD_MLX5_RX_HASH_SRC_IPV4 | FD_MLX5_RX_HASH_DST_IPV4 | \
112 0 : FD_MLX5_RX_HASH_SRC_PORT_UDP | FD_MLX5_RX_HASH_DST_PORT_UDP )
113 : #define FD_MLX5_QP_TUNNEL_OFFLOADS (1U<<2) /* MLX5_QP_FLAG_TUNNEL_OFFLOADS */
114 : #define FD_MLX5_TUNNEL_OFFLOADS_GRE (1U<<1) /* MLX5_IB_TUNNELED_OFFLOADS_GRE */
115 0 : #define FD_MLX5_INDIRECTION_COUNT (1UL<<8) /* Spreads RSS more equally across non power of 2 tile counts */
116 :
117 : static int
118 : fd_mlx5_rdma_port_contains_if( char const * rdma_name,
119 : uint port,
120 0 : char const * interface_name ) {
121 0 : char netdev_dir_path[ PATH_MAX ];
122 0 : int path_sz = snprintf( netdev_dir_path, sizeof(netdev_dir_path),
123 0 : "/sys/class/infiniband/%s/ports/%u/gid_attrs/ndevs", rdma_name, port );
124 0 : if( FD_UNLIKELY( path_sz<=0 || (ulong)path_sz>=sizeof(netdev_dir_path) ) ) {
125 0 : errno = ENAMETOOLONG;
126 0 : return -1;
127 0 : }
128 :
129 0 : DIR * netdev_dir = opendir( netdev_dir_path );
130 0 : if( FD_UNLIKELY( !netdev_dir ) ) return errno==ENOENT ? 0 : -1;
131 :
132 0 : int result = 0;
133 0 : for(;;) {
134 0 : errno = 0;
135 0 : struct dirent * netdev_entry = readdir( netdev_dir );
136 0 : if( FD_UNLIKELY( !netdev_entry ) ) {
137 0 : if( errno ) result = -1;
138 0 : break;
139 0 : }
140 0 : if( netdev_entry->d_name[0]=='.' ) continue;
141 :
142 0 : char netdev_name_path[ PATH_MAX ];
143 0 : int netdev_name_path_sz = snprintf( netdev_name_path, sizeof(netdev_name_path),
144 0 : "%s/%s", netdev_dir_path, netdev_entry->d_name );
145 0 : if( FD_UNLIKELY( netdev_name_path_sz<=0 || (ulong)netdev_name_path_sz>=sizeof(netdev_name_path) ) ) {
146 0 : errno = ENAMETOOLONG;
147 0 : result = -1;
148 0 : break;
149 0 : }
150 :
151 0 : int netdev_fd = open( netdev_name_path, O_RDONLY );
152 0 : if( FD_UNLIKELY( netdev_fd<0 ) ) {
153 0 : if( errno==ENOENT ) continue;
154 0 : result = -1;
155 0 : break;
156 0 : }
157 :
158 0 : char netdev_name[ IF_NAMESIZE+1 ];
159 0 : ssize_t read_sz = read( netdev_fd, netdev_name, IF_NAMESIZE );
160 0 : int const read_err = read_sz<0 ? errno : 0;
161 0 : int const close_err = close( netdev_fd ) ? errno : 0;
162 0 : if( FD_UNLIKELY( (read_err && read_err!=EINVAL) || close_err ) ) {
163 0 : errno = read_err && read_err!=EINVAL ? read_err : close_err;
164 0 : result = -1;
165 0 : break;
166 0 : }
167 : /* Empty Linux RDMA GID attribute entries return EINVAL. */
168 0 : if( FD_UNLIKELY( read_err==EINVAL ) ) continue;
169 :
170 0 : if( FD_UNLIKELY( !read_sz ) ) continue;
171 0 : while( read_sz>0 && (netdev_name[read_sz-1]=='\n' || netdev_name[read_sz-1]=='\r') ) read_sz--;
172 0 : netdev_name[read_sz] = '\0';
173 0 : if( !strcmp( netdev_name, interface_name ) ) {
174 0 : result = 1;
175 0 : break;
176 0 : }
177 0 : }
178 :
179 0 : int err = result<0 ? errno : 0;
180 0 : if( FD_UNLIKELY( closedir( netdev_dir ) && !err ) ) err = errno;
181 0 : if( FD_UNLIKELY( err ) ) {
182 0 : errno = err;
183 0 : return -1;
184 0 : }
185 0 : return result;
186 0 : }
187 :
188 : static int
189 : fd_mlx5_rdma_port_matches_if( char const * rdma_name,
190 : uint port,
191 0 : char const * interface_name ) {
192 0 : int match = fd_mlx5_rdma_port_contains_if( rdma_name, port, interface_name );
193 0 : if( FD_UNLIKELY( match<0 ) ) return -1;
194 0 : if( match ) return 1;
195 0 : if( !fd_bonding_is_master( interface_name ) ) return 0;
196 :
197 0 : fd_bonding_slave_iter_t iter_mem[1];
198 0 : fd_bonding_slave_iter_t * iter = fd_bonding_slave_iter_init( iter_mem, interface_name );
199 0 : while( !fd_bonding_slave_iter_done( iter ) ) {
200 0 : char const * slave_interface_name = fd_bonding_slave_iter_ele( iter );
201 0 : match = fd_mlx5_rdma_port_contains_if( rdma_name, port, slave_interface_name );
202 0 : if( FD_UNLIKELY( match<0 ) ) return -1;
203 0 : if( match ) return 1;
204 0 : fd_bonding_slave_iter_next( iter );
205 0 : }
206 :
207 0 : return 0;
208 0 : }
209 :
210 : int
211 : fd_mlx5_rdma_dev_find( char rdma_name[ FD_MLX5_RDMA_NAME_MAX ],
212 : uint * rdma_port,
213 0 : char const * interface_name ) {
214 0 : rdma_name[0] = '\0';
215 0 : *rdma_port = 0U;
216 :
217 0 : DIR * infiniband_dir = opendir( "/sys/class/infiniband" );
218 0 : if( FD_UNLIKELY( !infiniband_dir ) ) return 0;
219 :
220 0 : ulong match_cnt = 0UL;
221 0 : int err = 0;
222 0 : for(;;) {
223 0 : errno = 0;
224 0 : struct dirent * device_entry = readdir( infiniband_dir );
225 0 : if( FD_UNLIKELY( !device_entry ) ) {
226 0 : err = errno;
227 0 : break;
228 0 : }
229 0 : if( device_entry->d_name[0]=='.' ) continue;
230 :
231 0 : char device_ports_path[ PATH_MAX ];
232 0 : int const device_ports_path_sz = snprintf( device_ports_path, sizeof(device_ports_path),
233 0 : "/sys/class/infiniband/%s/ports", device_entry->d_name );
234 0 : if( FD_UNLIKELY( device_ports_path_sz<=0 || (ulong)device_ports_path_sz>=sizeof(device_ports_path) ) ) {
235 0 : err = ENAMETOOLONG;
236 0 : break;
237 0 : }
238 :
239 0 : DIR * ports_dir = opendir( device_ports_path );
240 0 : if( FD_UNLIKELY( !ports_dir ) ) {
241 0 : if( errno==ENOENT ) continue;
242 0 : err = errno;
243 0 : break;
244 0 : }
245 :
246 0 : for(;;) {
247 0 : errno = 0;
248 0 : struct dirent * port_entry = readdir( ports_dir );
249 0 : if( FD_UNLIKELY( !port_entry ) ) {
250 0 : err = errno;
251 0 : break;
252 0 : }
253 0 : char * port_end;
254 0 : ulong const port_num = strtoul( port_entry->d_name, &port_end, 10 );
255 0 : if( !port_num || *port_end || port_num>(ulong)UCHAR_MAX ) continue;
256 0 : int const match = fd_mlx5_rdma_port_matches_if( device_entry->d_name, (uint)port_num, interface_name );
257 0 : if( FD_UNLIKELY( match<0 ) ) {
258 0 : err = errno;
259 0 : break;
260 0 : }
261 0 : if( !match ) continue;
262 :
263 0 : match_cnt++;
264 0 : if( match_cnt==1UL ) {
265 0 : fd_cstr_ncpy( rdma_name, device_entry->d_name, FD_MLX5_RDMA_NAME_MAX );
266 0 : *rdma_port = (uint)port_num;
267 0 : }
268 0 : }
269 0 : if( FD_UNLIKELY( closedir( ports_dir ) && !err ) ) err = errno;
270 0 : if( FD_UNLIKELY( err ) ) break;
271 0 : }
272 :
273 0 : if( FD_UNLIKELY( closedir( infiniband_dir ) && !err ) ) err = errno;
274 0 : if( FD_UNLIKELY( err ) ) {
275 0 : rdma_name[0] = '\0';
276 0 : *rdma_port = 0U;
277 0 : errno = err;
278 0 : return 0;
279 0 : }
280 0 : if( FD_UNLIKELY( match_cnt!=1UL ) ) {
281 0 : errno = match_cnt ? EEXIST : ENOENT;
282 0 : rdma_name[0] = '\0';
283 0 : *rdma_port = 0U;
284 0 : return 0;
285 0 : }
286 0 : return 1;
287 0 : }
288 :
289 : int
290 0 : fd_mlx5_uverbs_avail( void ) {
291 0 : struct stat class_stat;
292 0 : return !stat( "/sys/class/infiniband_verbs", &class_stat ) && S_ISDIR( class_stat.st_mode );
293 0 : }
294 :
295 : int
296 0 : fd_mlx5_uverbs_modprobe( int is_dry_run ) {
297 0 : pid_t pid = fork();
298 0 : if( FD_UNLIKELY( pid<0 ) ) {
299 0 : FD_LOG_WARNING(( "fork() for modprobe failed (%i-%s)", errno, fd_io_strerror( errno ) ));
300 0 : return -1;
301 0 : }
302 0 : if( !pid ) {
303 0 : int null_fd = open( "/dev/null", O_RDWR );
304 0 : if( FD_UNLIKELY( null_fd<0 ) ) {
305 0 : FD_LOG_WARNING(( "open(/dev/null) for modprobe failed (%i-%s)", errno, fd_io_strerror( errno ) ));
306 0 : _exit( 1 );
307 0 : }
308 0 : if( FD_UNLIKELY( dup2( null_fd, STDIN_FILENO )<0 ||
309 0 : ( is_dry_run && dup2( null_fd, STDOUT_FILENO )<0 ) ) ) {
310 0 : FD_LOG_WARNING(( "dup2() for modprobe failed (%i-%s)", errno, fd_io_strerror( errno ) ));
311 0 : _exit( 1 );
312 0 : }
313 0 : if( null_fd!=STDIN_FILENO &&
314 0 : !( is_dry_run && null_fd==STDOUT_FILENO ) ) close( null_fd );
315 :
316 0 : char * argv[] = { "modprobe", "--quiet", "ib_uverbs", NULL, NULL };
317 0 : if( is_dry_run ) {
318 0 : argv[2] = "--dry-run";
319 0 : argv[3] = "ib_uverbs";
320 0 : }
321 0 : char * const envp[] = { NULL };
322 0 : execve( "/sbin/modprobe", argv, envp );
323 0 : FD_LOG_WARNING(( "execve(/sbin/modprobe) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
324 0 : _exit( 1 );
325 0 : }
326 :
327 0 : int status;
328 0 : while( FD_UNLIKELY( waitpid( pid, &status, 0 )<0 ) ) {
329 0 : if( errno==EINTR ) continue;
330 0 : FD_LOG_WARNING(( "waitpid() for modprobe failed (%i-%s)", errno, fd_io_strerror( errno ) ));
331 0 : return -1;
332 0 : }
333 0 : return WIFEXITED( status ) && !WEXITSTATUS( status ) ? 0 : -1;
334 0 : }
335 :
336 : struct fd_mlx5_pd {
337 : fd_uverbs_ctx_t * ctx; /* uverbs context */
338 : uint handle; /* protection domain handle */
339 : };
340 : typedef struct fd_mlx5_pd fd_mlx5_pd_t;
341 :
342 : struct fd_mlx5_caps {
343 : ulong max_mr_size;
344 : uint max_send_wqe; /* max_send_wqebb, SQ WQE capacity */
345 : uint max_recv_wr;
346 : uint max_cqe;
347 : uchar eth_min_inline_hdr_sz;
348 : };
349 : typedef struct fd_mlx5_caps fd_mlx5_caps_t;
350 :
351 : /* fd_uverbs_* types define Linux uverbs requests and responses */
352 : struct fd_uverbs_ex_hdr {
353 : struct ib_uverbs_cmd_hdr cmd;
354 : struct ib_uverbs_ex_cmd_hdr ex;
355 : };
356 : typedef struct fd_uverbs_ex_hdr fd_uverbs_ex_hdr_t;
357 : FD_STATIC_ASSERT( sizeof(fd_uverbs_ex_hdr_t)==24UL, uverbs_ex_hdr_sz );
358 :
359 : struct fd_uverbs_get_context_req {
360 : struct ib_uverbs_cmd_hdr hdr;
361 : ulong response;
362 : struct mlx5_ib_alloc_ucontext_req_v2 mlx5;
363 : };
364 : typedef struct fd_uverbs_get_context_req fd_uverbs_get_context_req_t;
365 : FD_STATIC_ASSERT( sizeof(fd_uverbs_get_context_req_t)==48UL, uverbs_get_context_req_sz );
366 :
367 : struct fd_uverbs_get_context_resp {
368 : uint async_fd;
369 : uint num_comp_vectors;
370 : struct mlx5_ib_alloc_ucontext_resp mlx5;
371 : };
372 : typedef struct fd_uverbs_get_context_resp fd_uverbs_get_context_resp_t;
373 : FD_STATIC_ASSERT( sizeof(fd_uverbs_get_context_resp_t)==80UL, uverbs_get_context_resp_sz );
374 :
375 : struct fd_uverbs_query_device_req {
376 : fd_uverbs_ex_hdr_t hdr;
377 : struct ib_uverbs_ex_query_device core;
378 : };
379 : typedef struct fd_uverbs_query_device_req fd_uverbs_query_device_req_t;
380 :
381 : struct fd_uverbs_query_device_resp {
382 : struct ib_uverbs_ex_query_device_resp core;
383 : struct mlx5_ib_query_device_resp mlx5;
384 : };
385 : typedef struct fd_uverbs_query_device_resp fd_uverbs_query_device_resp_t;
386 :
387 : struct fd_uverbs_query_port_req {
388 : struct ib_uverbs_cmd_hdr hdr;
389 : ulong response;
390 : uchar port_num;
391 : uchar reserved[ 7 ];
392 : };
393 : typedef struct fd_uverbs_query_port_req fd_uverbs_query_port_req_t;
394 : FD_STATIC_ASSERT( sizeof(fd_uverbs_query_port_req_t)==24UL, uverbs_query_port_req_sz );
395 :
396 : struct fd_uverbs_alloc_pd_req {
397 : struct ib_uverbs_cmd_hdr hdr;
398 : ulong response;
399 : };
400 : typedef struct fd_uverbs_alloc_pd_req fd_uverbs_alloc_pd_req_t;
401 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_pd_req_t)==16UL, uverbs_alloc_pd_req_sz );
402 :
403 : struct fd_uverbs_alloc_pd_resp {
404 : uint pd_handle;
405 : struct mlx5_ib_alloc_pd_resp mlx5;
406 : };
407 : typedef struct fd_uverbs_alloc_pd_resp fd_uverbs_alloc_pd_resp_t;
408 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_pd_resp_t)==8UL, uverbs_alloc_pd_resp_sz );
409 :
410 : struct fd_uverbs_reg_mr_req {
411 : struct ib_uverbs_cmd_hdr hdr;
412 : ulong response;
413 : ulong start;
414 : ulong length;
415 : ulong hca_va;
416 : uint pd_handle;
417 : uint access_flags;
418 : };
419 : typedef struct fd_uverbs_reg_mr_req fd_uverbs_reg_mr_req_t;
420 : FD_STATIC_ASSERT( sizeof(fd_uverbs_reg_mr_req_t)==48UL, uverbs_reg_mr_req_sz );
421 :
422 : struct fd_uverbs_reg_mr_resp {
423 : uint mr_handle;
424 : uint lkey;
425 : uint rkey;
426 : };
427 : typedef struct fd_uverbs_reg_mr_resp fd_uverbs_reg_mr_resp_t;
428 : FD_STATIC_ASSERT( sizeof(fd_uverbs_reg_mr_resp_t)==12UL, uverbs_reg_mr_resp_sz );
429 :
430 : struct fd_uverbs_ioctl_hdr {
431 : ushort length;
432 : ushort object_id;
433 : ushort method_id;
434 : ushort num_attrs;
435 : ulong reserved1;
436 : uint driver_id;
437 : uint reserved2;
438 : };
439 : typedef struct fd_uverbs_ioctl_hdr fd_uverbs_ioctl_hdr_t;
440 : FD_STATIC_ASSERT( sizeof(fd_uverbs_ioctl_hdr_t)==24UL, uverbs_ioctl_hdr_sz );
441 :
442 : struct fd_uverbs_alloc_uar_req {
443 : fd_uverbs_ioctl_hdr_t hdr;
444 : struct ib_uverbs_attr attrs[ 5 ];
445 : };
446 : typedef struct fd_uverbs_alloc_uar_req fd_uverbs_alloc_uar_req_t;
447 : FD_STATIC_ASSERT( sizeof(fd_uverbs_alloc_uar_req_t)==104UL, uverbs_alloc_uar_req_sz );
448 :
449 : struct fd_uverbs_destroy_uar_req {
450 : fd_uverbs_ioctl_hdr_t hdr;
451 : struct ib_uverbs_attr attrs[ 1 ];
452 : };
453 : typedef struct fd_uverbs_destroy_uar_req fd_uverbs_destroy_uar_req_t;
454 : FD_STATIC_ASSERT( sizeof(fd_uverbs_destroy_uar_req_t)==40UL, uverbs_destroy_uar_req_sz );
455 :
456 : struct fd_uverbs_create_comp_channel_req {
457 : struct ib_uverbs_cmd_hdr hdr;
458 : ulong response;
459 : };
460 : typedef struct fd_uverbs_create_comp_channel_req fd_uverbs_create_comp_channel_req_t;
461 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_comp_channel_req_t)==16UL, uverbs_create_comp_channel_req_sz );
462 :
463 : struct fd_uverbs_create_cq_req {
464 : struct ib_uverbs_cmd_hdr hdr;
465 : ulong response;
466 : ulong user_handle;
467 : uint cqe;
468 : uint comp_vector;
469 : int comp_channel;
470 : uint reserved;
471 : /* mlx5_ib_create_cq gained uar_page_index in Linux 5.7 */
472 : union {
473 : struct mlx5_ib_create_cq fields;
474 : struct {
475 : uchar prefix[ 24 ];
476 : ushort uar_page_index;
477 : } uar;
478 : } mlx5;
479 : };
480 : typedef struct fd_uverbs_create_cq_req fd_uverbs_create_cq_req_t;
481 :
482 : struct fd_uverbs_create_cq_resp {
483 : uint cq_handle;
484 : uint cqe;
485 : struct mlx5_ib_create_cq_resp mlx5;
486 : };
487 : typedef struct fd_uverbs_create_cq_resp fd_uverbs_create_cq_resp_t;
488 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_cq_resp_t)==16UL, uverbs_create_cq_resp_sz );
489 :
490 : struct fd_uverbs_destroy_cq_req {
491 : struct ib_uverbs_cmd_hdr hdr;
492 : ulong response;
493 : uint cq_handle;
494 : uint reserved;
495 : };
496 : typedef struct fd_uverbs_destroy_cq_req fd_uverbs_destroy_cq_req_t;
497 : FD_STATIC_ASSERT( sizeof(fd_uverbs_destroy_cq_req_t)==24UL, uverbs_destroy_cq_req_sz );
498 :
499 : struct fd_uverbs_create_tx_qp_req {
500 : struct ib_uverbs_cmd_hdr hdr;
501 : ulong response;
502 : ulong user_handle;
503 : uint pd_handle;
504 : uint send_cq_handle;
505 : uint recv_cq_handle;
506 : uint srq_handle;
507 : uint max_send_wr;
508 : uint max_recv_wr;
509 : uint max_send_sge;
510 : uint max_recv_sge;
511 : uint max_inline_data;
512 : uchar sq_sig_all;
513 : uchar qp_type;
514 : uchar is_srq;
515 : uchar reserved;
516 : struct mlx5_ib_create_qp mlx5;
517 : };
518 : typedef struct fd_uverbs_create_tx_qp_req fd_uverbs_create_tx_qp_req_t;
519 :
520 : struct fd_uverbs_create_tx_qp_resp {
521 : struct ib_uverbs_create_qp_resp core;
522 : struct mlx5_ib_create_qp_resp mlx5;
523 : };
524 : typedef struct fd_uverbs_create_tx_qp_resp fd_uverbs_create_tx_qp_resp_t;
525 :
526 : struct fd_uverbs_modify_qp_req {
527 : struct ib_uverbs_cmd_hdr hdr;
528 : struct ib_uverbs_modify_qp core;
529 : };
530 : typedef struct fd_uverbs_modify_qp_req fd_uverbs_modify_qp_req_t;
531 : FD_STATIC_ASSERT( sizeof(fd_uverbs_modify_qp_req_t)==120UL, uverbs_modify_qp_req_sz );
532 :
533 : struct fd_uverbs_create_rx_wq_req {
534 : fd_uverbs_ex_hdr_t hdr;
535 : struct ib_uverbs_ex_create_wq core;
536 : struct mlx5_ib_create_wq mlx5;
537 : };
538 : typedef struct fd_uverbs_create_rx_wq_req fd_uverbs_create_rx_wq_req_t;
539 :
540 : struct fd_uverbs_create_rx_wq_resp {
541 : struct ib_uverbs_ex_create_wq_resp core;
542 : struct mlx5_ib_create_wq_resp mlx5;
543 : };
544 : typedef struct fd_uverbs_create_rx_wq_resp fd_uverbs_create_rx_wq_resp_t;
545 :
546 : struct fd_uverbs_modify_rx_wq_req {
547 : fd_uverbs_ex_hdr_t hdr;
548 : struct ib_uverbs_ex_modify_wq core;
549 : struct mlx5_ib_modify_wq mlx5;
550 : };
551 : typedef struct fd_uverbs_modify_rx_wq_req fd_uverbs_modify_rx_wq_req_t;
552 :
553 : struct fd_uverbs_create_rwq_ind_table_req {
554 : fd_uverbs_ex_hdr_t hdr;
555 : uint comp_mask;
556 : uint log_ind_tbl_size;
557 : uint wq_handles[];
558 : };
559 : typedef struct fd_uverbs_create_rwq_ind_table_req fd_uverbs_create_rwq_ind_table_req_t;
560 :
561 : struct fd_uverbs_create_rss_qp_req {
562 : fd_uverbs_ex_hdr_t hdr;
563 : struct ib_uverbs_ex_create_qp core;
564 : struct mlx5_ib_create_qp_rss mlx5;
565 : };
566 : typedef struct fd_uverbs_create_rss_qp_req fd_uverbs_create_rss_qp_req_t;
567 :
568 : struct fd_uverbs_create_rss_qp_resp {
569 : struct ib_uverbs_ex_create_qp_resp core;
570 : struct mlx5_ib_create_qp_resp mlx5;
571 : };
572 : typedef struct fd_uverbs_create_rss_qp_resp fd_uverbs_create_rss_qp_resp_t;
573 :
574 : struct fd_uverbs_create_udp_flow_req {
575 : fd_uverbs_ex_hdr_t hdr;
576 : uint comp_mask;
577 : uint qp_handle;
578 : uint type;
579 : ushort size;
580 : ushort priority;
581 : uchar num_of_specs;
582 : uchar reserved[2];
583 : uchar port;
584 : uint flags;
585 : struct ib_uverbs_flow_spec_eth eth;
586 : struct ib_uverbs_flow_spec_ipv4 ipv4;
587 : struct ib_uverbs_flow_spec_tcp_udp udp;
588 : uint mlx5_ncounters_data;
589 : uint mlx5_reserved;
590 : };
591 : typedef struct fd_uverbs_create_udp_flow_req fd_uverbs_create_udp_flow_req_t;
592 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_udp_flow_req_t)==144UL, uverbs_create_udp_flow_req_sz );
593 :
594 : struct fd_uverbs_create_gre_flow_req {
595 : fd_uverbs_ex_hdr_t hdr;
596 : uint comp_mask;
597 : uint qp_handle;
598 : uint type;
599 : ushort size;
600 : ushort priority;
601 : uchar num_of_specs;
602 : uchar reserved[2];
603 : uchar port;
604 : uint flags;
605 : struct ib_uverbs_flow_spec_eth eth;
606 : struct ib_uverbs_flow_spec_ipv4 outer_ipv4;
607 : struct ib_uverbs_flow_spec_gre gre;
608 : struct ib_uverbs_flow_spec_ipv4 inner_ipv4;
609 : struct ib_uverbs_flow_spec_tcp_udp inner_udp;
610 : uint mlx5_ncounters_data;
611 : uint mlx5_reserved;
612 : };
613 : typedef struct fd_uverbs_create_gre_flow_req fd_uverbs_create_gre_flow_req_t;
614 : FD_STATIC_ASSERT( sizeof(fd_uverbs_create_gre_flow_req_t)==200UL, uverbs_create_gre_flow_req_sz );
615 :
616 : #define FD_RDMA_PATH_MAX (256UL)
617 :
618 0 : #define FD_MLX5_FLOW_SPEC_ETH (0x20U) /* IB_FLOW_SPEC_ETH */
619 0 : #define FD_MLX5_FLOW_SPEC_IPV4 (0x30U) /* IB_FLOW_SPEC_IPV4 */
620 0 : #define FD_MLX5_FLOW_SPEC_UDP (0x41U) /* IB_FLOW_SPEC_UDP */
621 0 : #define FD_MLX5_FLOW_SPEC_GRE (0x51U) /* IB_FLOW_SPEC_GRE */
622 0 : #define FD_MLX5_FLOW_SPEC_INNER (0x100U) /* IB_FLOW_SPEC_INNER */
623 :
624 : /* fd_rdma_* helpers discover and inspect the selected Linux RDMA device */
625 : static int
626 : fd_rdma_read_text( char const * path,
627 : char * buf,
628 0 : ulong buf_sz ) {
629 0 : FD_TEST( buf_sz>1UL );
630 :
631 0 : int text_fd = open( path, O_RDONLY | O_CLOEXEC );
632 0 : if( FD_UNLIKELY( text_fd<0 ) ) return -1;
633 :
634 0 : long read_sz = read( text_fd, buf, buf_sz );
635 0 : int err = 0;
636 0 : if( FD_UNLIKELY( read_sz<0L ) ) err = errno;
637 0 : else if( FD_UNLIKELY( (ulong)read_sz==buf_sz ) ) err = EOVERFLOW;
638 0 : if( FD_UNLIKELY( close( text_fd ) && !err ) ) err = errno;
639 0 : if( FD_UNLIKELY( err ) ) {
640 0 : errno = err;
641 0 : return -1;
642 0 : }
643 :
644 0 : ulong text_sz = (ulong)read_sz;
645 0 : while( text_sz && fd_isspace( (int)(uchar)buf[ text_sz-1UL ] ) ) text_sz--;
646 0 : if( FD_UNLIKELY( !text_sz ) ) {
647 0 : errno = EPROTO;
648 0 : return -1;
649 0 : }
650 0 : buf[ text_sz ] = '\0';
651 0 : return 0;
652 0 : }
653 :
654 : static int
655 : fd_rdma_read_uint( char const * path,
656 0 : uint * value ) {
657 0 : char buf[ 32 ];
658 0 : if( FD_UNLIKELY( fd_rdma_read_text( path, buf, sizeof(buf) ) ) ) return -1;
659 :
660 0 : char * end;
661 0 : ulong parsed_value = strtoul( buf, &end, 10 );
662 0 : if( FD_UNLIKELY( *end || parsed_value>UINT_MAX ) ) {
663 0 : errno = EPROTO;
664 0 : return -1;
665 0 : }
666 0 : *value = (uint)parsed_value;
667 0 : return 0;
668 0 : }
669 :
670 : static int
671 : fd_rdma_name_valid( char const * name,
672 0 : ulong name_max ) {
673 0 : if( FD_UNLIKELY( !name || !name[0] ) ) return 0;
674 0 : if( FD_UNLIKELY( name[0]=='.' && (!name[1] || (name[1]=='.' && !name[2])) ) ) return 0;
675 0 : for( ulong i=0UL; i<name_max; i++ ) {
676 0 : uchar c = (uchar)name[ i ];
677 0 : if( !c ) return 1;
678 0 : if( !((c>=(uchar)'a' && c<=(uchar)'z') ||
679 0 : (c>=(uchar)'A' && c<=(uchar)'Z') ||
680 0 : (c>=(uchar)'0' && c<=(uchar)'9') ||
681 0 : c==(uchar)'_' || c==(uchar)'-' || c==(uchar)'.') ) return 0;
682 0 : }
683 0 : return 0;
684 0 : }
685 :
686 : /* fd_uverbs_* helpers build and submit Linux uverbs commands */
687 : static int
688 0 : fd_uverbs_name_valid( char const * name ) {
689 0 : ulong name_sz = strnlen( name, FD_UVERBS_NAME_MAX );
690 0 : if( name_sz==FD_UVERBS_NAME_MAX || name_sz==6UL || strncmp( name, "uverbs", 6UL ) ) return 0;
691 0 : for( char const * digit_cur=name+6; *digit_cur; digit_cur++ )
692 0 : if( *digit_cur<'0' || *digit_cur>'9' ) return 0;
693 0 : return 1;
694 0 : }
695 :
696 : static int
697 : fd_uverbs_resolve( char uverbs_name[ FD_UVERBS_NAME_MAX ],
698 0 : char const * rdma_name ) {
699 0 : if( FD_UNLIKELY( !fd_rdma_name_valid( rdma_name, FD_MLX5_RDMA_NAME_MAX ) ) ) {
700 0 : errno = EINVAL;
701 0 : return -1;
702 0 : }
703 :
704 0 : DIR * uverbs_dir = opendir( "/sys/class/infiniband_verbs" );
705 0 : if( FD_UNLIKELY( !uverbs_dir ) ) return -1;
706 :
707 0 : char matched_uverbs_name[ FD_UVERBS_NAME_MAX ];
708 0 : int err;
709 0 : for(;;) {
710 0 : errno = 0;
711 0 : struct dirent * uverbs_entry = readdir( uverbs_dir );
712 0 : if( !uverbs_entry ) {
713 0 : err = errno ? errno : ENODEV;
714 0 : break;
715 0 : }
716 0 : if( !fd_uverbs_name_valid( uverbs_entry->d_name ) ) continue;
717 :
718 0 : char path[ FD_RDMA_PATH_MAX ];
719 0 : char entry_rdma_name[ FD_MLX5_RDMA_NAME_MAX ];
720 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
721 0 : "/sys/class/infiniband_verbs/%s/ibdev", uverbs_entry->d_name ) );
722 0 : if( FD_UNLIKELY( fd_rdma_read_text( path, entry_rdma_name, sizeof(entry_rdma_name) ) ) ) {
723 0 : err = errno;
724 0 : break;
725 0 : }
726 0 : if( strcmp( entry_rdma_name, rdma_name ) ) continue;
727 :
728 0 : ulong name_sz = strlen( uverbs_entry->d_name );
729 0 : fd_memcpy( matched_uverbs_name, uverbs_entry->d_name, name_sz+1UL );
730 0 : err = 0;
731 0 : break;
732 0 : }
733 0 : closedir( uverbs_dir );
734 0 : if( FD_UNLIKELY( err ) ) {
735 0 : errno = err;
736 0 : return -1;
737 0 : }
738 :
739 0 : char path[ FD_RDMA_PATH_MAX ];
740 0 : uint core_abi;
741 0 : uint provider_abi;
742 0 : if( FD_UNLIKELY( fd_rdma_read_uint( "/sys/class/infiniband_verbs/abi_version", &core_abi ) ) ) return -1;
743 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL,
744 0 : "/sys/class/infiniband_verbs/%s/abi_version", matched_uverbs_name ) );
745 0 : if( FD_UNLIKELY( fd_rdma_read_uint( path, &provider_abi ) ) ) return -1;
746 0 : if( FD_UNLIKELY( core_abi!=IB_USER_VERBS_ABI_VERSION ||
747 0 : provider_abi!=MLX5_IB_UVERBS_ABI_VERSION ) ) {
748 0 : FD_LOG_WARNING(( "mlx5 uverbs ABI mismatch (core %u, expected %u, provider %u, expected %u)",
749 0 : core_abi, (uint)IB_USER_VERBS_ABI_VERSION,
750 0 : provider_abi, (uint)MLX5_IB_UVERBS_ABI_VERSION ));
751 0 : errno = EPROTONOSUPPORT;
752 0 : return -1;
753 0 : }
754 :
755 0 : fd_memcpy( uverbs_name, matched_uverbs_name, strlen(matched_uverbs_name)+1UL );
756 0 : return 0;
757 0 : }
758 :
759 : static int
760 : fd_uverbs_init_cmd_hdr( struct ib_uverbs_cmd_hdr * hdr,
761 : uint command,
762 : ulong request_sz,
763 0 : ulong response_sz ) {
764 0 : if( FD_UNLIKELY( !hdr || command>IB_USER_VERBS_CMD_COMMAND_MASK || request_sz<sizeof(*hdr) ) ) return -1;
765 :
766 0 : hdr->command = command;
767 0 : hdr->in_words = (ushort)(request_sz / 4UL);
768 0 : hdr->out_words = (ushort)(response_sz / 4UL);
769 0 : return 0;
770 0 : }
771 :
772 : static int
773 : fd_uverbs_init_ex_hdr( fd_uverbs_ex_hdr_t * hdr,
774 : uint command,
775 : ulong core_request_sz,
776 : ulong request_sz,
777 : void * response,
778 : ulong core_response_sz,
779 0 : ulong response_sz ) {
780 0 : ulong const hdr_sz = sizeof(*hdr);
781 0 : if( FD_UNLIKELY( !hdr || command>IB_USER_VERBS_CMD_COMMAND_MASK ||
782 0 : core_request_sz<hdr_sz || request_sz<core_request_sz ||
783 0 : response_sz<core_response_sz || (response_sz && !response) ) ) return -1;
784 :
785 0 : hdr->cmd.command = IB_USER_VERBS_CMD_FLAG_EXTENDED | command;
786 0 : hdr->cmd.in_words = (ushort)((core_request_sz-hdr_sz) / 8UL);
787 0 : hdr->cmd.out_words = (ushort)(core_response_sz / 8UL);
788 0 : hdr->ex.provider_in_words = (ushort)((request_sz-core_request_sz) / 8UL);
789 0 : hdr->ex.provider_out_words = (ushort)((response_sz-core_response_sz) / 8UL);
790 0 : hdr->ex.response = (ulong)response;
791 0 : hdr->ex.cmd_hdr_reserved = 0U;
792 0 : return 0;
793 0 : }
794 :
795 : static int
796 : fd_uverbs_write_cmd( int cmd_fd,
797 : void const * req,
798 0 : ulong req_sz ) {
799 0 : ssize_t write_sz = write( cmd_fd, req, req_sz );
800 0 : if( FD_UNLIKELY( write_sz<0 ) ) return -1;
801 0 : if( FD_UNLIKELY( (ulong)write_sz!=req_sz ) ) {
802 0 : errno = EPROTO;
803 0 : return -1;
804 0 : }
805 0 : return 0;
806 0 : }
807 :
808 : static int
809 : fd_uverbs_get_context( fd_uverbs_ctx_t * ctx,
810 0 : fd_mlx5_caps_t * caps ) {
811 0 : fd_uverbs_get_context_req_t req [1];
812 0 : fd_uverbs_get_context_resp_t resp[1];
813 0 : fd_memset( req, 0, sizeof(req ) );
814 0 : fd_memset( resp, 0, sizeof(resp) );
815 :
816 0 : req->response = (ulong)resp;
817 0 : req->mlx5.total_num_bfregs = 16U;
818 0 : req->mlx5.num_low_latency_bfregs = 4U;
819 0 : req->mlx5.max_cqe_version = 1U;
820 0 : req->mlx5.lib_caps = MLX5_LIB_CAP_4K_UAR | MLX5_LIB_CAP_DYN_UAR;
821 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_GET_CONTEXT,
822 0 : sizeof(req), sizeof(resp) ) );
823 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
824 0 : if( FD_UNLIKELY( resp->async_fd>(uint)INT_MAX ) ) {
825 0 : errno = EPROTO;
826 0 : return -1;
827 0 : }
828 :
829 0 : ctx->async_fd = (int)resp->async_fd;
830 0 : int fd_flags = fcntl( ctx->async_fd, F_GETFD );
831 0 : if( FD_UNLIKELY( fd_flags<0 || fcntl( ctx->async_fd, F_SETFD, fd_flags | FD_CLOEXEC ) ) ) return -1;
832 :
833 0 : ulong const uar_resp_sz = offsetof( struct mlx5_ib_alloc_ucontext_resp, num_uars_per_page )+
834 0 : sizeof(resp->mlx5.num_uars_per_page);
835 0 : if( FD_UNLIKELY( resp->mlx5.response_length<uar_resp_sz ||
836 0 : resp->mlx5.cqe_version>1U ||
837 0 : resp->mlx5.max_sq_desc_sz<sizeof(fd_mlx5_tx_wqe_t) ||
838 0 : resp->mlx5.max_rq_desc_sz<sizeof(fd_mlx5_rx_wqe_t) ||
839 0 : resp->mlx5.log_uar_size!=(uint)fd_ulong_find_lsb( FD_MLX5_PAGE_SZ ) ||
840 0 : resp->mlx5.num_uars_per_page!=1U ||
841 0 : resp->mlx5.tot_bfregs ) ) {
842 0 : FD_LOG_WARNING(( "unsupported mlx5 context capabilities (response length %u, CQE version %u, "
843 0 : "max SQ descriptor %u, max RQ descriptor %u, log UAR size %u, UARs per page %u, "
844 0 : "legacy BF registers %u)",
845 0 : resp->mlx5.response_length, (uint)resp->mlx5.cqe_version,
846 0 : (uint)resp->mlx5.max_sq_desc_sz, (uint)resp->mlx5.max_rq_desc_sz,
847 0 : resp->mlx5.log_uar_size, resp->mlx5.num_uars_per_page, resp->mlx5.tot_bfregs ));
848 0 : errno = EPROTONOSUPPORT;
849 0 : return -1;
850 0 : }
851 :
852 0 : caps->max_send_wqe = resp->mlx5.max_send_wqebb;
853 0 : caps->max_recv_wr = resp->mlx5.max_recv_wr;
854 0 : switch( resp->mlx5.eth_min_inline ) {
855 0 : case MLX5_USER_INLINE_MODE_NONE: caps->eth_min_inline_hdr_sz = 0U; break;
856 0 : case MLX5_USER_INLINE_MODE_L2: caps->eth_min_inline_hdr_sz = FD_MLX5_ETH_INLINE_HDR_SZ; break;
857 0 : default:
858 0 : FD_LOG_WARNING(( "unsupported mlx5 minimum inline mode %u", (uint)resp->mlx5.eth_min_inline ));
859 0 : errno = EPROTONOSUPPORT;
860 0 : return -1;
861 0 : }
862 0 : return 0;
863 0 : }
864 :
865 : static int
866 : fd_uverbs_query_device( fd_uverbs_ctx_t * ctx,
867 0 : fd_mlx5_caps_t * caps ) {
868 0 : fd_uverbs_query_device_req_t req [1];
869 0 : fd_uverbs_query_device_resp_t resp[1];
870 0 : fd_memset( req, 0, sizeof(req ) );
871 0 : fd_memset( resp, 0, sizeof(resp) );
872 :
873 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_QUERY_DEVICE,
874 0 : sizeof(req), sizeof(req), resp,
875 0 : sizeof(resp->core), sizeof(resp) ) );
876 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
877 :
878 0 : struct ib_uverbs_query_device_resp const * core = &resp->core.base;
879 0 : caps->max_mr_size = core->max_mr_size;
880 0 : caps->max_cqe = core->max_cqe;
881 0 : if( FD_UNLIKELY( resp->core.response_length<sizeof(*core) ||
882 0 : !caps->max_mr_size ||
883 0 : !core->max_qp_wr ||
884 0 : !core->max_sge ||
885 0 : !caps->max_cqe ||
886 0 : core->phys_port_cnt<ctx->port_num ) ) {
887 0 : FD_LOG_WARNING(( "unsupported mlx5 device capabilities (max MR size %lu, max QP WRs %u, max SGEs %u, "
888 0 : "max CQEs %u, physical ports %u, requested port %u)",
889 0 : caps->max_mr_size, core->max_qp_wr, core->max_sge, caps->max_cqe,
890 0 : (uint)core->phys_port_cnt, ctx->port_num ));
891 0 : errno = EPROTONOSUPPORT;
892 0 : return -1;
893 0 : }
894 :
895 0 : ulong const rss_caps_sz = offsetof( struct ib_uverbs_ex_query_device_resp, rss_caps )+
896 0 : sizeof(resp->core.rss_caps);
897 0 : if( FD_UNLIKELY( resp->core.response_length<rss_caps_sz ) ) {
898 0 : FD_LOG_WARNING(( "mlx5 device query did not return RSS capabilities (response length %u, required %lu)",
899 0 : resp->core.response_length, rss_caps_sz ));
900 0 : errno = EPROTONOSUPPORT;
901 0 : return -1;
902 0 : }
903 0 : if( FD_UNLIKELY( resp->core.rss_caps.max_rwq_indirection_table_size<FD_MLX5_INDIRECTION_COUNT ) ) {
904 0 : FD_LOG_WARNING(( "mlx5 RSS indirection table requires %lu entries, device supports only %u",
905 0 : FD_MLX5_INDIRECTION_COUNT, resp->core.rss_caps.max_rwq_indirection_table_size ));
906 0 : errno = EPROTONOSUPPORT;
907 0 : return -1;
908 0 : }
909 :
910 0 : ulong const tunnel_caps_sz = offsetof( struct mlx5_ib_query_device_resp, tunnel_offloads_caps )+
911 0 : sizeof(resp->mlx5.tunnel_offloads_caps);
912 0 : ulong const required_hash_fields = FD_MLX5_RX_HASH_IPV4_UDP | FD_MLX5_RX_HASH_INNER;
913 :
914 0 : if( FD_UNLIKELY( resp->mlx5.response_length<tunnel_caps_sz ||
915 0 : !(resp->mlx5.rss_caps.rx_hash_function & FD_MLX5_RX_HASH_FUNC_TOEPLITZ) ||
916 0 : (resp->mlx5.rss_caps.rx_hash_fields_mask & required_hash_fields)!=required_hash_fields ||
917 0 : !(resp->mlx5.tunnel_offloads_caps & FD_MLX5_TUNNEL_OFFLOADS_GRE) ) ) {
918 0 : FD_LOG_WARNING(( "unsupported mlx5 GRE RSS capabilities (response length %u, hash functions %#x, "
919 0 : "hash fields %#lx, tunnel offloads %#x)",
920 0 : resp->mlx5.response_length, (uint)resp->mlx5.rss_caps.rx_hash_function,
921 0 : (ulong)resp->mlx5.rss_caps.rx_hash_fields_mask, resp->mlx5.tunnel_offloads_caps ));
922 0 : errno = EPROTONOSUPPORT;
923 0 : return -1;
924 0 : }
925 0 : return 0;
926 0 : }
927 :
928 : static int
929 0 : fd_uverbs_query_port( fd_uverbs_ctx_t * ctx ) {
930 0 : fd_uverbs_query_port_req_t req [1];
931 0 : struct ib_uverbs_query_port_resp resp[1];
932 0 : fd_memset( req, 0, sizeof(req ) );
933 0 : fd_memset( resp, 0, sizeof(resp) );
934 :
935 0 : req->response = (ulong)resp;
936 0 : req->port_num = (uchar)ctx->port_num;
937 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_QUERY_PORT,
938 0 : sizeof(req), sizeof(resp) ) );
939 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
940 :
941 0 : if( FD_UNLIKELY( resp->link_layer!=FD_MLX5_LINK_LAYER_ETHERNET ) ) {
942 0 : FD_LOG_WARNING(( "unsupported mlx5 link layer %u, expected Ethernet (%u)",
943 0 : (uint)resp->link_layer, FD_MLX5_LINK_LAYER_ETHERNET ));
944 0 : errno = EPROTONOSUPPORT;
945 0 : return -1;
946 0 : }
947 0 : return 0;
948 0 : }
949 :
950 : static fd_uverbs_ctx_t *
951 : fd_uverbs_open_context( fd_uverbs_ctx_t * ctx,
952 : fd_mlx5_caps_t * caps,
953 : char const * rdma_name,
954 0 : uint port_num ) {
955 0 : fd_memset( ctx, 0, sizeof(*ctx) );
956 0 : fd_memset( caps, 0, sizeof(*caps) );
957 0 : ctx->cmd_fd = -1;
958 0 : ctx->async_fd = -1;
959 0 : if( FD_UNLIKELY( !port_num || port_num>(uint)UCHAR_MAX ) ) {
960 0 : errno = EINVAL;
961 0 : return NULL;
962 0 : }
963 0 : ctx->port_num = port_num;
964 :
965 0 : char uverbs_name[ FD_UVERBS_NAME_MAX ];
966 0 : if( FD_UNLIKELY( fd_uverbs_resolve( uverbs_name, rdma_name ) ) ) return NULL;
967 :
968 0 : char path[ FD_RDMA_PATH_MAX ];
969 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/dev/infiniband/%s", uverbs_name ) );
970 0 : ctx->cmd_fd = open( path, O_RDWR | O_CLOEXEC );
971 0 : if( FD_UNLIKELY( ctx->cmd_fd<0 ) ) return NULL;
972 :
973 0 : struct stat cmd_fd_stat;
974 0 : if( FD_UNLIKELY( fstat( ctx->cmd_fd, &cmd_fd_stat ) ) ) return NULL;
975 0 : if( FD_UNLIKELY( !S_ISCHR( cmd_fd_stat.st_mode ) ) ) {
976 0 : errno = ENODEV;
977 0 : return NULL;
978 0 : }
979 :
980 0 : if( FD_UNLIKELY( fd_uverbs_get_context( ctx, caps ) ||
981 0 : fd_uverbs_query_device( ctx, caps ) ||
982 0 : fd_uverbs_query_port( ctx ) ) ) return NULL;
983 0 : return ctx;
984 0 : }
985 :
986 : static fd_mlx5_pd_t *
987 : fd_uverbs_alloc_pd( fd_mlx5_pd_t * pd,
988 0 : fd_uverbs_ctx_t * ctx ) {
989 0 : if( FD_UNLIKELY( !pd ) ) {
990 0 : errno = EINVAL;
991 0 : return NULL;
992 0 : }
993 0 : fd_memset( pd, 0, sizeof(*pd) );
994 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 ) ) {
995 0 : errno = EINVAL;
996 0 : return NULL;
997 0 : }
998 :
999 0 : fd_uverbs_alloc_pd_req_t req [1];
1000 0 : fd_uverbs_alloc_pd_resp_t resp[1];
1001 0 : fd_memset( req, 0, sizeof(req ) );
1002 0 : fd_memset( resp, 0, sizeof(resp) );
1003 0 : req->response = (ulong)resp;
1004 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_ALLOC_PD,
1005 0 : sizeof(req), sizeof(resp) ) );
1006 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1007 :
1008 0 : pd->ctx = ctx;
1009 0 : pd->handle = resp->pd_handle;
1010 0 : return pd;
1011 0 : }
1012 :
1013 : static uint *
1014 : fd_uverbs_register_mr( uint * lkey,
1015 : fd_mlx5_pd_t * pd,
1016 : void * memory,
1017 : ulong memory_sz,
1018 : ulong iova,
1019 0 : ulong max_mr_size ) {
1020 0 : ulong memory_addr = (ulong)memory;
1021 0 : if( FD_UNLIKELY( !memory || !memory_sz || memory_sz>ULONG_MAX-memory_addr || memory_sz>ULONG_MAX-iova ||
1022 0 : memory_sz>max_mr_size ||
1023 0 : (memory_addr & (FD_MLX5_PAGE_SZ-1UL))!=(iova & (FD_MLX5_PAGE_SZ-1UL)) ) ) {
1024 0 : errno = EINVAL;
1025 0 : return NULL;
1026 0 : }
1027 :
1028 0 : fd_uverbs_reg_mr_req_t req [1];
1029 0 : fd_uverbs_reg_mr_resp_t resp[1];
1030 0 : fd_memset( req, 0, sizeof(req ) );
1031 0 : fd_memset( resp, 0, sizeof(resp) );
1032 0 : req->response = (ulong)resp;
1033 0 : req->start = memory_addr;
1034 0 : req->length = memory_sz;
1035 0 : req->hca_va = iova;
1036 0 : req->pd_handle = pd->handle;
1037 0 : req->access_flags = IB_UVERBS_ACCESS_LOCAL_WRITE;
1038 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_REG_MR,
1039 0 : sizeof(req), sizeof(resp) ) );
1040 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( pd->ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1041 :
1042 0 : *lkey = resp->lkey;
1043 0 : return lkey;
1044 0 : }
1045 :
1046 : static void
1047 : fd_uverbs_init_ioctl_attr( struct ib_uverbs_attr * attr,
1048 : ushort attr_id,
1049 : ushort len,
1050 0 : ulong data ) {
1051 0 : fd_memset( attr, 0, sizeof(*attr) );
1052 0 : attr->attr_id = attr_id;
1053 0 : attr->len = len;
1054 0 : attr->flags = UVERBS_ATTR_F_MANDATORY;
1055 0 : attr->data = data;
1056 0 : }
1057 :
1058 : static int
1059 : fd_uverbs_init_alloc_uar_req( fd_uverbs_alloc_uar_req_t * req,
1060 : ulong * mmap_offset,
1061 : uint * mmap_sz,
1062 0 : uint * page_id ) {
1063 0 : if( FD_UNLIKELY( !req || !mmap_offset || !mmap_sz || !page_id ) ) return -1;
1064 0 : fd_memset( req, 0, sizeof(*req) );
1065 0 : req->hdr.length = (ushort)sizeof(*req);
1066 0 : req->hdr.object_id = MLX5_IB_OBJECT_UAR;
1067 0 : req->hdr.method_id = MLX5_IB_METHOD_UAR_OBJ_ALLOC;
1068 0 : req->hdr.num_attrs = 5U;
1069 0 : req->hdr.driver_id = RDMA_DRIVER_MLX5;
1070 0 : fd_uverbs_init_ioctl_attr( req->attrs+0, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE, 0U, 0UL );
1071 0 : fd_uverbs_init_ioctl_attr( req->attrs+1, MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE, 8U, MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC );
1072 0 : fd_uverbs_init_ioctl_attr( req->attrs+2, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET, 8U, (ulong)mmap_offset );
1073 0 : fd_uverbs_init_ioctl_attr( req->attrs+3, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH, 4U, (ulong)mmap_sz );
1074 0 : fd_uverbs_init_ioctl_attr( req->attrs+4, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID, 4U, (ulong)page_id );
1075 0 : return 0;
1076 0 : }
1077 :
1078 : static int
1079 : fd_uverbs_init_destroy_uar_req( fd_uverbs_destroy_uar_req_t * req,
1080 0 : uint handle ) {
1081 0 : if( FD_UNLIKELY( !req ) ) return -1;
1082 0 : fd_memset( req, 0, sizeof(*req) );
1083 0 : req->hdr.length = (ushort)sizeof(*req);
1084 0 : req->hdr.object_id = MLX5_IB_OBJECT_UAR;
1085 0 : req->hdr.method_id = MLX5_IB_METHOD_UAR_OBJ_DESTROY;
1086 0 : req->hdr.num_attrs = 1U;
1087 0 : req->hdr.driver_id = RDMA_DRIVER_MLX5;
1088 0 : fd_uverbs_init_ioctl_attr( req->attrs, MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE, 0U, handle );
1089 0 : return 0;
1090 0 : }
1091 :
1092 : static int
1093 : fd_uverbs_destroy_uar( fd_uverbs_ctx_t * ctx,
1094 0 : uint handle ) {
1095 0 : fd_uverbs_destroy_uar_req_t req[1];
1096 0 : if( FD_UNLIKELY( fd_uverbs_init_destroy_uar_req( req, handle ) ) ) {
1097 0 : errno = EINVAL;
1098 0 : return -1;
1099 0 : }
1100 0 : return (int)syscall( SYS_ioctl, ctx->cmd_fd, RDMA_VERBS_IOCTL, &req->hdr );
1101 0 : }
1102 :
1103 : static ulong *
1104 : fd_uverbs_alloc_uar( fd_uverbs_ctx_t * ctx,
1105 : uint * page_id,
1106 0 : ulong * mmap_offset_out ) {
1107 0 : ulong mmap_offset = 0UL;
1108 0 : uint mmap_sz = 0U;
1109 0 : uint uar_page_id = 0U;
1110 0 : fd_uverbs_alloc_uar_req_t req[1];
1111 0 : if( FD_UNLIKELY( fd_uverbs_init_alloc_uar_req( req, &mmap_offset, &mmap_sz, &uar_page_id ) ) ) {
1112 0 : errno = EINVAL;
1113 0 : return NULL;
1114 0 : }
1115 0 : if( FD_UNLIKELY( syscall( SYS_ioctl, ctx->cmd_fd, RDMA_VERBS_IOCTL, &req->hdr ) ) ) return NULL;
1116 :
1117 0 : ulong handle_raw = req->attrs[0].data;
1118 0 : if( FD_UNLIKELY( handle_raw>UINT_MAX || mmap_sz!=FD_MLX5_PAGE_SZ ||
1119 0 : (mmap_offset & (FD_MLX5_PAGE_SZ-1UL)) || mmap_offset>(ulong)LONG_MAX ) ) {
1120 0 : FD_LOG_WARNING(( "invalid mlx5 UAR response (handle %lu, mapping size %u, mapping offset %#lx)",
1121 0 : handle_raw, mmap_sz, mmap_offset ));
1122 0 : int err = EPROTO;
1123 0 : if( handle_raw<=UINT_MAX ) fd_uverbs_destroy_uar( ctx, (uint)handle_raw );
1124 0 : errno = err;
1125 0 : return NULL;
1126 0 : }
1127 :
1128 0 : *page_id = uar_page_id;
1129 0 : *mmap_offset_out = mmap_offset;
1130 0 : return mmap_offset_out;
1131 0 : }
1132 :
1133 : volatile uchar *
1134 : fd_uverbs_map_uar( fd_uverbs_ctx_t * ctx,
1135 0 : ulong mmap_offset ) {
1136 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 ||
1137 0 : (mmap_offset & (FD_MLX5_PAGE_SZ-1UL)) || mmap_offset>(ulong)LONG_MAX ) ) {
1138 0 : errno = EINVAL;
1139 0 : return NULL;
1140 0 : }
1141 0 : void * uar_mapping = mmap( NULL, FD_MLX5_PAGE_SZ, PROT_WRITE, MAP_SHARED,
1142 0 : ctx->cmd_fd, (off_t)mmap_offset );
1143 0 : if( FD_UNLIKELY( uar_mapping==MAP_FAILED ) ) return NULL;
1144 0 : return (volatile uchar *)uar_mapping;
1145 0 : }
1146 :
1147 : static int
1148 0 : fd_uverbs_create_comp_channel( fd_uverbs_ctx_t * ctx ) {
1149 0 : fd_uverbs_create_comp_channel_req_t req [1];
1150 0 : struct ib_uverbs_create_comp_channel_resp resp[1];
1151 0 : fd_memset( req, 0, sizeof(req ) );
1152 0 : fd_memset( resp, 0, sizeof(resp) );
1153 :
1154 0 : req->response = (ulong)resp;
1155 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL,
1156 0 : sizeof(req), sizeof(resp) ) );
1157 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return -1;
1158 0 : return (int)resp->fd;
1159 0 : }
1160 :
1161 : static uint *
1162 : fd_uverbs_create_cq( uint * handle,
1163 : fd_uverbs_ctx_t * ctx,
1164 : fd_mlx5_cq_t * cq,
1165 : int comp_channel_fd,
1166 : uint page_id,
1167 0 : uint max_cqe ) {
1168 0 : if( FD_UNLIKELY( !handle ) ) {
1169 0 : errno = EINVAL;
1170 0 : return NULL;
1171 0 : }
1172 0 : *handle = 0U;
1173 0 : if( FD_UNLIKELY( !ctx || ctx->cmd_fd<0 || !cq || !cq->entries || !cq->control ||
1174 0 : !fd_uint_is_pow2( cq->depth ) || cq->depth-1U>max_cqe || page_id>(uint)USHRT_MAX ) ) {
1175 0 : errno = EINVAL;
1176 0 : return NULL;
1177 0 : }
1178 :
1179 0 : fd_uverbs_create_cq_req_t req [1];
1180 0 : fd_uverbs_create_cq_resp_t resp[1];
1181 0 : fd_memset( req, 0, sizeof(req ) );
1182 0 : fd_memset( resp, 0, sizeof(resp) );
1183 :
1184 0 : req->response = (ulong)resp;
1185 0 : req->user_handle = (ulong)cq;
1186 0 : req->cqe = cq->depth-1U;
1187 0 : req->comp_channel = comp_channel_fd;
1188 0 : req->mlx5.fields.buf_addr = (ulong)cq->entries;
1189 0 : req->mlx5.fields.db_addr = (ulong)cq->control;
1190 0 : req->mlx5.fields.cqe_size = sizeof(fd_mlx5_cqe_t);
1191 0 : req->mlx5.fields.flags = MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX;
1192 0 : req->mlx5.uar.uar_page_index = (ushort)page_id;
1193 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_CQ,
1194 0 : sizeof(req), sizeof(resp) ) );
1195 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( ctx->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1196 0 : if( FD_UNLIKELY( resp->cqe!=cq->depth-1U ) ) {
1197 0 : fd_uverbs_destroy_cq_req_t destroy[1];
1198 0 : struct ib_uverbs_destroy_cq_resp destroy_resp[1];
1199 0 : fd_memset( destroy, 0, sizeof(destroy) );
1200 0 : fd_memset( destroy_resp, 0, sizeof(destroy_resp) );
1201 :
1202 0 : destroy->response = (ulong)destroy_resp;
1203 0 : destroy->cq_handle = resp->cq_handle;
1204 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &destroy->hdr, IB_USER_VERBS_CMD_DESTROY_CQ,
1205 0 : sizeof(destroy), sizeof(destroy_resp) ) );
1206 0 : fd_uverbs_write_cmd( ctx->cmd_fd, destroy, sizeof(destroy) );
1207 0 : FD_LOG_WARNING(( "invalid mlx5 CQ response (CQEs %u, requested %u)", resp->cqe, cq->depth-1U ));
1208 0 : errno = EPROTO;
1209 0 : return NULL;
1210 0 : }
1211 :
1212 0 : cq->cqn = resp->mlx5.cqn;
1213 0 : *handle = resp->cq_handle;
1214 0 : return handle;
1215 0 : }
1216 :
1217 : static int
1218 : fd_uverbs_modify_qp( fd_uverbs_ctx_t * uverbs,
1219 : fd_mlx5_tx_qp_t * tx_qp,
1220 0 : uint state ) {
1221 0 : fd_uverbs_modify_qp_req_t req[1];
1222 0 : fd_memset( req, 0, sizeof(req) );
1223 0 : req->core.qp_handle = tx_qp->handle;
1224 0 : req->core.attr_mask = FD_MLX5_QP_ATTR_STATE;
1225 0 : req->core.qp_state = (uchar)state;
1226 0 : if( state==FD_MLX5_QPS_INIT ) {
1227 0 : req->core.attr_mask |= FD_MLX5_QP_ATTR_PORT;
1228 0 : req->core.port_num = (uchar)uverbs->port_num;
1229 0 : }
1230 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_MODIFY_QP,
1231 0 : sizeof(req), 0UL ) );
1232 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1233 0 : return 0;
1234 0 : }
1235 :
1236 : static fd_mlx5_tx_qp_t *
1237 : fd_uverbs_create_tx_qp( fd_uverbs_ctx_t * uverbs,
1238 : fd_mlx5_tx_qp_t * tx_qp,
1239 : fd_mlx5_pd_t * pd,
1240 : uint tx_cq_handle,
1241 0 : uint uar_page_id ) {
1242 0 : if( FD_UNLIKELY( !uverbs || !tx_qp || !tx_qp->tx_cq || !tx_qp->sq ||
1243 0 : !tx_qp->control || !pd || pd->ctx!=uverbs ) ) {
1244 0 : errno = EINVAL;
1245 0 : return NULL;
1246 0 : }
1247 :
1248 0 : fd_uverbs_create_tx_qp_req_t req [1];
1249 0 : fd_uverbs_create_tx_qp_resp_t resp[1];
1250 0 : fd_memset( req, 0, sizeof(req ) );
1251 0 : fd_memset( resp, 0, sizeof(resp) );
1252 :
1253 0 : req->response = (ulong)resp;
1254 0 : req->user_handle = (ulong)tx_qp;
1255 0 : req->pd_handle = pd->handle;
1256 0 : req->send_cq_handle = tx_cq_handle;
1257 : /* Linux requires both CQ handles even when max_recv_wr is zero. */
1258 0 : req->recv_cq_handle = tx_cq_handle;
1259 0 : req->max_send_wr = tx_qp->tx_depth;
1260 0 : req->max_send_sge = 1U;
1261 0 : req->qp_type = IB_UVERBS_QPT_RAW_PACKET;
1262 0 : req->mlx5.db_addr = (ulong)tx_qp->control;
1263 0 : req->mlx5.sq_wqe_count = tx_qp->tx_depth;
1264 0 : req->mlx5.flags = MLX5_QP_FLAG_UAR_PAGE_INDEX;
1265 0 : req->mlx5.uidx = 0U;
1266 0 : req->mlx5.bfreg_index = uar_page_id;
1267 0 : req->mlx5.sq_buf_addr = (ulong)tx_qp->sq;
1268 0 : FD_TEST( !fd_uverbs_init_cmd_hdr( &req->hdr, IB_USER_VERBS_CMD_CREATE_QP,
1269 0 : sizeof(req), sizeof(resp) ) );
1270 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1271 :
1272 0 : if( FD_UNLIKELY( resp->core.max_send_wr<tx_qp->tx_depth ||
1273 0 : resp->core.max_recv_wr ||
1274 0 : resp->core.max_send_sge<1U ||
1275 0 : resp->core.max_recv_sge ||
1276 0 : resp->core.qpn>0xffffffU ) ) {
1277 0 : FD_LOG_WARNING(( "invalid mlx5 QP response (max send WRs %u, max receive WRs %u, max send SGEs %u, "
1278 0 : "max receive SGEs %u, QPN %u)",
1279 0 : resp->core.max_send_wr, resp->core.max_recv_wr, resp->core.max_send_sge,
1280 0 : resp->core.max_recv_sge, resp->core.qpn ));
1281 0 : errno = EPROTO;
1282 0 : return NULL;
1283 0 : }
1284 :
1285 0 : tx_qp->handle = resp->core.qp_handle;
1286 0 : tx_qp->qpn = resp->core.qpn;
1287 0 : return tx_qp;
1288 0 : }
1289 :
1290 : static fd_mlx5_tx_qp_t *
1291 : fd_uverbs_start_tx_qp( fd_uverbs_ctx_t * uverbs,
1292 0 : fd_mlx5_tx_qp_t * tx_qp ) {
1293 0 : if( FD_UNLIKELY( fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_INIT ) ||
1294 0 : fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_RTR ) ||
1295 0 : fd_uverbs_modify_qp( uverbs, tx_qp, FD_MLX5_QPS_RTS ) ) ) return NULL;
1296 0 : return tx_qp;
1297 0 : }
1298 :
1299 : static fd_mlx5_rx_wq_t *
1300 : fd_uverbs_create_rx_wq( fd_uverbs_ctx_t * uverbs,
1301 : fd_mlx5_rx_wq_t * rx_wq,
1302 : fd_mlx5_pd_t * pd,
1303 0 : uint rx_cq_handle ) {
1304 0 : fd_uverbs_create_rx_wq_req_t req [1];
1305 0 : fd_uverbs_create_rx_wq_resp_t resp[1];
1306 0 : fd_memset( req, 0, sizeof(req ) );
1307 0 : fd_memset( resp, 0, sizeof(resp) );
1308 :
1309 0 : req->core.wq_type = IB_UVERBS_WQT_RQ;
1310 0 : req->core.user_handle = (ulong)rx_wq;
1311 0 : req->core.pd_handle = pd->handle;
1312 0 : req->core.cq_handle = rx_cq_handle;
1313 0 : req->core.max_wr = rx_wq->rx_depth;
1314 0 : req->core.max_sge = 1U;
1315 0 : req->mlx5.buf_addr = (ulong)rx_wq->rq;
1316 0 : req->mlx5.db_addr = (ulong)rx_wq->control;
1317 0 : req->mlx5.rq_wqe_count = rx_wq->rx_depth;
1318 0 : req->mlx5.rq_wqe_shift = (uint)fd_ulong_find_lsb( sizeof(fd_mlx5_rx_wqe_t) );
1319 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_rx_wq_req_t, mlx5 );
1320 0 : ulong const core_resp_sz = offsetof( fd_uverbs_create_rx_wq_resp_t, mlx5 );
1321 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_WQ,
1322 0 : core_req_sz, sizeof(req), resp,
1323 0 : core_resp_sz, sizeof(resp) ) );
1324 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1325 0 : if( FD_UNLIKELY( resp->core.max_wr<rx_wq->rx_depth ||
1326 0 : resp->core.max_sge<1U ||
1327 0 : resp->core.wqn>0xffffffU ) ) {
1328 0 : FD_LOG_WARNING(( "invalid mlx5 receive WQ response (max WRs %u, max SGEs %u, WQN %u)",
1329 0 : resp->core.max_wr, resp->core.max_sge, resp->core.wqn ));
1330 0 : errno = EPROTO;
1331 0 : return NULL;
1332 0 : }
1333 :
1334 0 : rx_wq->handle = resp->core.wq_handle;
1335 0 : rx_wq->wqn = resp->core.wqn;
1336 0 : return rx_wq;
1337 0 : }
1338 :
1339 : static fd_mlx5_rx_wq_t *
1340 : fd_uverbs_start_rx_wq( fd_uverbs_ctx_t * uverbs,
1341 0 : fd_mlx5_rx_wq_t * rx_wq ) {
1342 0 : fd_uverbs_modify_rx_wq_req_t req[1];
1343 0 : fd_memset( req, 0, sizeof(req) );
1344 0 : req->core.attr_mask = FD_MLX5_WQ_ATTR_STATE | FD_MLX5_WQ_ATTR_CUR_STATE;
1345 0 : req->core.wq_handle = rx_wq->handle;
1346 0 : req->core.wq_state = FD_MLX5_WQS_RDY;
1347 0 : req->core.curr_wq_state = FD_MLX5_WQS_RESET;
1348 0 : ulong const core_req_sz = offsetof( fd_uverbs_modify_rx_wq_req_t, mlx5 );
1349 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_MODIFY_WQ,
1350 0 : core_req_sz, sizeof(req), NULL, 0UL, 0UL ) );
1351 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1352 0 : return rx_wq;
1353 0 : }
1354 :
1355 : static uint *
1356 : fd_uverbs_create_rwq_ind_table( uint * handle,
1357 : fd_uverbs_ctx_t * uverbs,
1358 : fd_mlx5_uverbs_tile_t const * tiles,
1359 0 : ulong tile_cnt ) {
1360 0 : if( FD_UNLIKELY( 0==tile_cnt || tile_cnt>FD_MLX5_TILE_MAX ) ) {
1361 0 : errno = EINVAL;
1362 0 : return NULL;
1363 0 : }
1364 :
1365 0 : ulong const req_sz = fd_ulong_align_up( sizeof(fd_uverbs_create_rwq_ind_table_req_t)+FD_MLX5_INDIRECTION_COUNT*sizeof(uint), 8UL );
1366 0 : fd_uverbs_create_rwq_ind_table_req_t * req = aligned_alloc( 8UL, req_sz );
1367 0 : if( FD_UNLIKELY( !req ) ) return NULL;
1368 0 : struct ib_uverbs_ex_create_rwq_ind_table_resp resp[1];
1369 0 : fd_memset( req, 0, req_sz );
1370 0 : fd_memset( resp, 0, sizeof(resp) );
1371 0 : req->log_ind_tbl_size = (uint)fd_ulong_find_lsb( FD_MLX5_INDIRECTION_COUNT );
1372 0 : for( ulong i=0UL; i<FD_MLX5_INDIRECTION_COUNT; i++ ) req->wq_handles[ i ] = tiles[ i%tile_cnt ].rx_wq->handle;
1373 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
1374 0 : req_sz, req_sz, resp,
1375 0 : sizeof(resp), sizeof(resp) ) );
1376 0 : int err = fd_uverbs_write_cmd( uverbs->cmd_fd, req, req_sz );
1377 0 : int saved_errno = errno;
1378 0 : free( req );
1379 0 : if( FD_UNLIKELY( err ) ) {
1380 0 : errno = saved_errno;
1381 0 : return NULL;
1382 0 : }
1383 0 : *handle = resp->ind_tbl_handle;
1384 0 : return handle;
1385 0 : }
1386 :
1387 : static fd_mlx5_rss_qp_t *
1388 : fd_uverbs_create_rss_qp( fd_uverbs_ctx_t * uverbs,
1389 : fd_mlx5_rss_qp_t * rss_qp,
1390 : fd_mlx5_pd_t * pd,
1391 : uint rwq_ind_table_handle,
1392 : ulong hash_fields,
1393 0 : uint flags ) {
1394 0 : fd_uverbs_create_rss_qp_req_t req [1];
1395 0 : fd_uverbs_create_rss_qp_resp_t resp[1];
1396 0 : fd_memset( req, 0, sizeof(req ) );
1397 0 : fd_memset( resp, 0, sizeof(resp) );
1398 0 : req->core.user_handle = (ulong)rss_qp;
1399 0 : req->core.pd_handle = pd->handle;
1400 0 : req->core.qp_type = IB_UVERBS_QPT_RAW_PACKET;
1401 0 : req->core.comp_mask = IB_UVERBS_CREATE_QP_MASK_IND_TABLE;
1402 0 : req->core.rwq_ind_tbl_handle = rwq_ind_table_handle;
1403 0 : req->mlx5.rx_hash_fields_mask = hash_fields;
1404 0 : req->mlx5.rx_hash_function = FD_MLX5_RX_HASH_FUNC_TOEPLITZ;
1405 0 : req->mlx5.rx_key_len = FD_MLX5_RX_HASH_KEY_SZ;
1406 0 : req->mlx5.flags = flags;
1407 :
1408 0 : if( FD_UNLIKELY( !fd_rng_secure( req->mlx5.rx_hash_key, FD_MLX5_RX_HASH_KEY_SZ ) ) ) return NULL;
1409 :
1410 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_rss_qp_req_t, mlx5 );
1411 0 : ulong const core_resp_sz = offsetof( fd_uverbs_create_rss_qp_resp_t, mlx5 );
1412 :
1413 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_QP,
1414 0 : core_req_sz, sizeof(req), resp,
1415 0 : core_resp_sz, sizeof(resp) ) );
1416 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return NULL;
1417 0 : rss_qp->handle = resp->core.base.qp_handle;
1418 0 : return rss_qp;
1419 0 : }
1420 :
1421 : static int
1422 : fd_uverbs_init_udp_flow_req( fd_uverbs_create_udp_flow_req_t * req,
1423 : struct ib_uverbs_create_flow_resp * resp,
1424 : uint qp_handle,
1425 : uint port_num,
1426 : uint dst_ip,
1427 0 : ushort dst_port ) {
1428 0 : if( FD_UNLIKELY( !req || !resp || !port_num || port_num>UCHAR_MAX || !dst_port ) ) return -1;
1429 :
1430 0 : fd_memset( req, 0, sizeof(*req ) );
1431 0 : fd_memset( resp, 0, sizeof(*resp) );
1432 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_udp_flow_req_t, mlx5_ncounters_data );
1433 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_FLOW,
1434 0 : core_req_sz, sizeof(*req), resp,
1435 0 : sizeof(*resp), sizeof(*resp) ) );
1436 :
1437 0 : req->qp_handle = qp_handle;
1438 0 : req->size = sizeof(req->eth)+sizeof(req->ipv4)+sizeof(req->udp);
1439 0 : req->num_of_specs = 3U;
1440 0 : req->port = (uchar)port_num;
1441 0 : req->eth.type = FD_MLX5_FLOW_SPEC_ETH;
1442 0 : req->eth.size = sizeof(req->eth);
1443 0 : req->eth.val.ether_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP );
1444 0 : req->eth.mask.ether_type = USHORT_MAX;
1445 0 : req->ipv4.type = FD_MLX5_FLOW_SPEC_IPV4;
1446 0 : req->ipv4.size = sizeof(req->ipv4);
1447 0 : req->ipv4.val.dst_ip = dst_ip;
1448 0 : req->ipv4.mask.dst_ip = dst_ip ? UINT_MAX : 0U;
1449 0 : req->udp.type = FD_MLX5_FLOW_SPEC_UDP;
1450 0 : req->udp.size = sizeof(req->udp);
1451 0 : req->udp.val.dst_port = fd_ushort_bswap( dst_port );
1452 0 : req->udp.mask.dst_port = USHORT_MAX;
1453 0 : return 0;
1454 0 : }
1455 :
1456 : int
1457 : fd_uverbs_create_udp_flow( fd_uverbs_ctx_t * uverbs,
1458 : fd_mlx5_rss_qp_t const * rss_qp,
1459 : uint dst_ip,
1460 0 : ushort dst_port ) {
1461 0 : if( FD_UNLIKELY( !uverbs || uverbs->cmd_fd<0 || !rss_qp || !dst_port ) ) {
1462 0 : errno = EINVAL;
1463 0 : return -1;
1464 0 : }
1465 :
1466 0 : fd_uverbs_create_udp_flow_req_t req [1];
1467 0 : struct ib_uverbs_create_flow_resp resp[1];
1468 0 : if( FD_UNLIKELY( fd_uverbs_init_udp_flow_req( req, resp, rss_qp->handle,
1469 0 : uverbs->port_num, dst_ip,
1470 0 : dst_port ) ) ) {
1471 0 : errno = EINVAL;
1472 0 : return -1;
1473 0 : }
1474 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1475 0 : return 0;
1476 0 : }
1477 :
1478 : static int
1479 : fd_uverbs_init_gre_flow_req( fd_uverbs_create_gre_flow_req_t * req,
1480 : struct ib_uverbs_create_flow_resp * resp,
1481 : uint qp_handle,
1482 : uint port_num,
1483 : uint inner_dst_ip,
1484 0 : ushort inner_dst_port ) {
1485 0 : if( FD_UNLIKELY( !req || !resp || !port_num || port_num>UCHAR_MAX || !inner_dst_port ) ) return -1;
1486 :
1487 0 : fd_memset( req, 0, sizeof(*req ) );
1488 0 : fd_memset( resp, 0, sizeof(*resp) );
1489 0 : ulong const core_req_sz = offsetof( fd_uverbs_create_gre_flow_req_t, mlx5_ncounters_data );
1490 0 : FD_TEST( !fd_uverbs_init_ex_hdr( &req->hdr, IB_USER_VERBS_EX_CMD_CREATE_FLOW,
1491 0 : core_req_sz, sizeof(*req), resp,
1492 0 : sizeof(*resp), sizeof(*resp) ) );
1493 :
1494 0 : req->qp_handle = qp_handle;
1495 0 : req->size = sizeof(req->eth)+sizeof(req->outer_ipv4)+sizeof(req->gre)+
1496 0 : sizeof(req->inner_ipv4)+sizeof(req->inner_udp);
1497 0 : req->num_of_specs = 5U;
1498 0 : req->port = (uchar)port_num;
1499 0 : req->eth = (struct ib_uverbs_flow_spec_eth) {
1500 0 : .type = FD_MLX5_FLOW_SPEC_ETH,
1501 0 : .size = sizeof(req->eth),
1502 0 : .val = { .ether_type = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) },
1503 0 : .mask = { .ether_type = USHORT_MAX },
1504 0 : };
1505 0 : req->outer_ipv4 = (struct ib_uverbs_flow_spec_ipv4) {
1506 0 : .type = FD_MLX5_FLOW_SPEC_IPV4,
1507 0 : .size = sizeof(req->outer_ipv4),
1508 0 : .val = { .proto = FD_IP4_HDR_PROTOCOL_GRE },
1509 0 : .mask = { .proto = UCHAR_MAX },
1510 0 : };
1511 0 : req->gre = (struct ib_uverbs_flow_spec_gre) {
1512 0 : .type = FD_MLX5_FLOW_SPEC_GRE,
1513 0 : .size = sizeof(req->gre),
1514 0 : .val = { .protocol = fd_ushort_bswap( FD_ETH_HDR_TYPE_IP ) },
1515 0 : .mask = { .protocol = USHORT_MAX },
1516 0 : };
1517 0 : req->inner_ipv4 = (struct ib_uverbs_flow_spec_ipv4) {
1518 0 : .type = FD_MLX5_FLOW_SPEC_INNER | FD_MLX5_FLOW_SPEC_IPV4,
1519 0 : .size = sizeof(req->inner_ipv4),
1520 0 : .val = { .dst_ip = inner_dst_ip, .proto = FD_IP4_HDR_PROTOCOL_UDP },
1521 0 : .mask = { .dst_ip = inner_dst_ip ? UINT_MAX : 0U, .proto = UCHAR_MAX },
1522 0 : };
1523 0 : req->inner_udp = (struct ib_uverbs_flow_spec_tcp_udp) {
1524 0 : .type = FD_MLX5_FLOW_SPEC_INNER | FD_MLX5_FLOW_SPEC_UDP,
1525 0 : .size = sizeof(req->inner_udp),
1526 0 : .val = { .dst_port = fd_ushort_bswap( inner_dst_port ) },
1527 0 : .mask = { .dst_port = USHORT_MAX },
1528 0 : };
1529 0 : return 0;
1530 0 : }
1531 :
1532 : int
1533 : fd_uverbs_create_gre_udp_flow( fd_uverbs_ctx_t * uverbs,
1534 : fd_mlx5_rss_qp_t const * rss_qp,
1535 : uint inner_dst_ip,
1536 0 : ushort inner_dst_port ) {
1537 0 : if( FD_UNLIKELY( !uverbs || uverbs->cmd_fd<0 || !rss_qp || !inner_dst_port ) ) {
1538 0 : errno = EINVAL;
1539 0 : return -1;
1540 0 : }
1541 :
1542 0 : fd_uverbs_create_gre_flow_req_t req [1];
1543 0 : struct ib_uverbs_create_flow_resp resp[1];
1544 0 : if( FD_UNLIKELY( fd_uverbs_init_gre_flow_req( req, resp, rss_qp->handle,
1545 0 : uverbs->port_num, inner_dst_ip,
1546 0 : inner_dst_port ) ) ) {
1547 0 : errno = EINVAL;
1548 0 : return -1;
1549 0 : }
1550 0 : if( FD_UNLIKELY( fd_uverbs_write_cmd( uverbs->cmd_fd, req, sizeof(req) ) ) ) return -1;
1551 0 : return 0;
1552 0 : }
1553 :
1554 : fd_mlx5_rss_qp_t *
1555 : fd_uverbs_init( fd_uverbs_ctx_t * uverbs,
1556 : fd_mlx5_uverbs_tile_t * tiles,
1557 : ulong tile_cnt,
1558 : fd_mlx5_rss_qp_t * outer_rss_qp,
1559 : fd_mlx5_rss_qp_t * gre_rss_qp,
1560 : char const * rdma_name,
1561 0 : uint port_num ) {
1562 0 : if( FD_UNLIKELY( !uverbs || !tiles || 0==tile_cnt || tile_cnt>FD_MLX5_TILE_MAX || !outer_rss_qp || !gre_rss_qp ) ) {
1563 0 : errno = EINVAL;
1564 0 : return NULL;
1565 0 : }
1566 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1567 0 : fd_mlx5_uverbs_tile_t const * tile = tiles+i;
1568 0 : if( FD_UNLIKELY( !tile->rx_cq || !tile->tx_cq || !tile->rx_wq || !tile->tx_qp || !tile->lkey ||
1569 0 : !tile->packet_memory || !tile->packet_memory_sz || !tile->rx_wq->rq ||
1570 0 : !tile->rx_wq->control || tile->rx_wq->rx_cq!=tile->rx_cq || tile->tx_qp->tx_cq!=tile->tx_cq ||
1571 0 : tile->rx_wq->rx_depth!=tile->rx_cq->depth || tile->tx_qp->tx_depth!=tile->tx_cq->depth ) ) {
1572 0 : errno = EINVAL;
1573 0 : return NULL;
1574 0 : }
1575 0 : }
1576 :
1577 0 : fd_mlx5_caps_t caps[1];
1578 0 : fd_mlx5_pd_t pd[1];
1579 0 : uint rwq_ind_table_handle;
1580 0 : if( FD_UNLIKELY( !fd_uverbs_open_context( uverbs, caps, rdma_name, port_num ) ) ) return NULL;
1581 0 : if( FD_UNLIKELY( !fd_uverbs_alloc_pd( pd, uverbs ) ) ) return NULL;
1582 :
1583 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
1584 0 : fd_mlx5_uverbs_tile_t * tile = tiles+i;
1585 0 : fd_mlx5_rx_wq_t * rx_wq = tile->rx_wq;
1586 0 : fd_mlx5_tx_qp_t * tx_qp = tile->tx_qp;
1587 0 : if( FD_UNLIKELY( !fd_uint_is_pow2( rx_wq->rx_depth ) ||
1588 0 : !fd_uint_is_pow2( tx_qp->tx_depth ) ||
1589 0 : rx_wq->rx_depth>caps->max_recv_wr ||
1590 0 : tx_qp->tx_depth>caps->max_send_wqe ||
1591 0 : rx_wq->rx_depth-1U>caps->max_cqe ||
1592 0 : tx_qp->tx_depth-1U>caps->max_cqe ) ) {
1593 0 : errno = EINVAL;
1594 0 : return NULL;
1595 0 : }
1596 :
1597 0 : int rx_comp_channel_fd = -1;
1598 0 : if( tile->rx_comp_channel_fd ) {
1599 0 : rx_comp_channel_fd = fd_uverbs_create_comp_channel( uverbs );
1600 0 : if( FD_UNLIKELY( rx_comp_channel_fd<0 ) ) return NULL;
1601 0 : *tile->rx_comp_channel_fd = rx_comp_channel_fd;
1602 0 : }
1603 :
1604 0 : uint uar_page_id;
1605 0 : uint rx_cq_handle;
1606 0 : uint tx_cq_handle;
1607 0 : if( FD_UNLIKELY( !fd_uverbs_alloc_uar( uverbs, &uar_page_id, &tx_qp->uar_mmap_offset ) ||
1608 0 : !fd_uverbs_create_cq( &rx_cq_handle, uverbs, tile->rx_cq, rx_comp_channel_fd, uar_page_id, caps->max_cqe ) ||
1609 0 : !fd_uverbs_create_cq( &tx_cq_handle, uverbs, tile->tx_cq, -1, uar_page_id, caps->max_cqe ) ||
1610 0 : !fd_uverbs_register_mr( tile->lkey, pd, tile->packet_memory, tile->packet_memory_sz, tile->packet_iova, caps->max_mr_size ) ||
1611 0 : !fd_uverbs_create_tx_qp( uverbs, tx_qp, pd, tx_cq_handle, uar_page_id ) ||
1612 0 : !fd_uverbs_start_tx_qp( uverbs, tx_qp ) ||
1613 0 : !fd_uverbs_create_rx_wq( uverbs, rx_wq, pd, rx_cq_handle ) ||
1614 0 : !fd_uverbs_start_rx_wq( uverbs, rx_wq ) ) ) {
1615 0 : return NULL;
1616 0 : }
1617 0 : tx_qp->tx_inline_hdr_sz = caps->eth_min_inline_hdr_sz;
1618 0 : }
1619 :
1620 0 : if( FD_UNLIKELY( !fd_uverbs_create_rwq_ind_table( &rwq_ind_table_handle, uverbs, tiles, tile_cnt ) ||
1621 0 : !fd_uverbs_create_rss_qp( uverbs, outer_rss_qp, pd, rwq_ind_table_handle,
1622 0 : FD_MLX5_RX_HASH_IPV4_UDP, 0U ) ||
1623 0 : !fd_uverbs_create_rss_qp( uverbs, gre_rss_qp, pd, rwq_ind_table_handle,
1624 0 : FD_MLX5_RX_HASH_IPV4_UDP | FD_MLX5_RX_HASH_INNER,
1625 0 : FD_MLX5_QP_TUNNEL_OFFLOADS ) ) ) return NULL;
1626 0 : return outer_rss_qp;
1627 0 : }
|
