Line data    Source code

       1             : #ifndef HEADER_fd_src_disco_net_mlx5_fd_mlx5_private_h
       2             : #define HEADER_fd_src_disco_net_mlx5_fd_mlx5_private_h
       3             : 
       4             : #if defined(__linux__)
       5             : 
       6             : #include "fd_mlx5.h"
       7             : 
       8             : /* fd_mlx5 sets up raw mlx5 resources through Linux uverbs.  It does not
       9             :    depend on libibverbs or libmlx5.
      10             : 
      11             :    fd_uverbs_init opens one uverbs context, validates the selected device
      12             :    and port, registers packet memory, and creates a shared protection domain
      13             :    and receive indirection table over caller-owned queue memory.  Each tile
      14             :    gets a UAR, RX CQ, TX CQ, receive WQ, and send-only raw-packet QP.  Linux
      15             :    creates and tracks these resources, while the packet path uses the mapped
      16             :    queues and doorbells directly.
      17             : 
      18             :    mlx5 queue fields are big-endian.  This implementation reserves 64
      19             :    bytes for each TX WQE and 16 bytes for each RX WQE. */
      20             : 
      21             : #define FD_MLX5_TX_WQE_SZ     (64UL)
      22             : #define FD_MLX5_RQ_WQE_SZ     (16UL)
      23             : #define FD_MLX5_CQE_SZ        (64UL)
      24           0 : #define FD_MLX5_PAGE_SZ       (4096UL)
      25             : 
      26             : /* mlx5 CQ consumer indices are 24 bits. */
      27           0 : #define FD_MLX5_CQ_CONS_IDX_MASK (0xffffffU)
      28             : 
      29             : struct __attribute__((aligned(FD_MLX5_TX_WQE_SZ))) fd_mlx5_tx_wqe {
      30             :   uchar bytes[ FD_MLX5_TX_WQE_SZ ];
      31             : };
      32             : typedef struct fd_mlx5_tx_wqe fd_mlx5_tx_wqe_t;
      33             : 
      34             : struct __attribute__((aligned(FD_MLX5_RQ_WQE_SZ))) fd_mlx5_rx_wqe {
      35             :   uchar bytes[ FD_MLX5_RQ_WQE_SZ ];
      36             : };
      37             : typedef struct fd_mlx5_rx_wqe fd_mlx5_rx_wqe_t;
      38             : 
      39             : struct __attribute__((aligned(FD_MLX5_CQE_SZ))) fd_mlx5_cqe {
      40             :   uchar bytes[ FD_MLX5_CQE_SZ ];
      41             : };
      42             : typedef struct fd_mlx5_cqe fd_mlx5_cqe_t;
      43             : 
      44             : FD_STATIC_ASSERT( sizeof(fd_mlx5_tx_wqe_t)==FD_MLX5_TX_WQE_SZ, mlx5_tx_wqe_sz );
      45             : FD_STATIC_ASSERT( sizeof(fd_mlx5_rx_wqe_t)==FD_MLX5_RQ_WQE_SZ, mlx5_rx_wqe_sz );
      46             : FD_STATIC_ASSERT( sizeof(fd_mlx5_cqe_t   )==FD_MLX5_CQE_SZ,    mlx5_cqe_sz    );
      47             : 
      48             : /* fd_uverbs_ctx owns a uverbs command descriptor and the async
      49             :    event descriptor returned with it. */
      50             : struct fd_uverbs_ctx {
      51             :   int  cmd_fd;
      52             :   int  async_fd;
      53             :   uint port_num;
      54             : };
      55             : typedef struct fd_uverbs_ctx fd_uverbs_ctx_t;
      56             : 
      57             : /* fd_mlx5_cq_control is the mlx5 CQ doorbell record.  Linux names the
      58             :    words set_ci_db and arm_db.  Only the RX CQ requests notifications,
      59             :    when the tile prepares to park in efficient mode. */
      60             : struct __attribute__((aligned(8UL))) fd_mlx5_cq_control {
      61             :   uint consumer_idx;
      62             :   uint request_notification;
      63             : };
      64             : typedef struct fd_mlx5_cq_control fd_mlx5_cq_control_t;
      65             : 
      66             : /* fd_mlx5_qp_control is the mlx5 QP doorbell record in host memory,
      67             :    separate from UAR MMIO.  Linux names these words MLX5_RCV_DBR and
      68             :    MLX5_SND_DBR. */
      69             : struct __attribute__((aligned(8UL))) fd_mlx5_qp_control {
      70             :   uint reserved;
      71             :   uint sq_prod;
      72             : };
      73             : typedef struct fd_mlx5_qp_control fd_mlx5_qp_control_t;
      74             : 
      75             : /* fd_mlx5_rx_wq_control is the mlx5 receive WQ doorbell record. */
      76             : struct __attribute__((aligned(8UL))) fd_mlx5_rx_wq_control {
      77             :   uint rq_prod;
      78             :   uint reserved;
      79             : };
      80             : typedef struct fd_mlx5_rx_wq_control fd_mlx5_rx_wq_control_t;
      81             : 
      82             : FD_STATIC_ASSERT( sizeof(fd_mlx5_cq_control_t)==8UL, mlx5_cq_control_sz );
      83             : FD_STATIC_ASSERT( sizeof(fd_mlx5_qp_control_t)==8UL, mlx5_qp_control_sz );
      84             : FD_STATIC_ASSERT( sizeof(fd_mlx5_rx_wq_control_t)==8UL, mlx5_rx_wq_control_sz );
      85             : 
      86             : struct fd_mlx5_cq {
      87             :   fd_mlx5_cqe_t *        entries;
      88             :   fd_mlx5_cq_control_t * control;
      89             :   /* mlx5 CQ arm doorbell. */
      90             :   volatile uchar *       request_notification_doorbell;
      91             :   uint                   depth;
      92             :   uint                   cons_idx;
      93             :   uint                   cqn;
      94             :   uint                   comp_channel_event_seq;
      95             :   uint                   comp_channel_armed; /* cleared when its event is read */
      96             : };
      97             : typedef struct fd_mlx5_cq fd_mlx5_cq_t;
      98             : 
      99             : /* fd_mlx5_rx_wq owns one standalone Linux receive WQ. */
     100             : struct fd_mlx5_rx_wq {
     101             :   fd_mlx5_rx_wqe_t * rq;
     102             :   fd_mlx5_cq_t *     rx_cq;
     103             :   uint               rx_depth;
     104             :   uint               rq_prod;
     105             :   uint               rq_cons;
     106             :   uint *             rq_wqe_buf_chunk;
     107             : 
     108             :   fd_mlx5_rx_wq_control_t * control;
     109             :   uint                      handle;
     110             :   uint                      wqn;
     111             : };
     112             : typedef struct fd_mlx5_rx_wq fd_mlx5_rx_wq_t;
     113             : 
     114             : /* fd_mlx5_tx_qp owns one send-only raw-packet QP. */
     115             : struct fd_mlx5_tx_qp {
     116             :   fd_mlx5_tx_wqe_t *     sq;
     117             :   fd_mlx5_cq_t *         tx_cq;
     118             :   uint                   tx_depth;
     119             :   uint                   sq_prod;
     120             :   uint                   sq_posted;
     121             :   uint                   sq_cons;
     122             :   uint *                 sq_wqe_frame_sz;
     123             : 
     124             :   fd_mlx5_qp_control_t * control;
     125             :   volatile uchar *       sq_doorbell;      /* points to the SQ doorbell register in a non-cached UAR mapping. */
     126             :   ulong                  uar_mmap_offset;  /* maps this QP's non-cached mlx5 UAR page. */
     127             :   uint                   handle;           /* identifies this QP in later Linux uverbs commands. */
     128             :   uint                   qpn;              /* identifies this QP to the NIC. */
     129             :   uchar                  tx_inline_hdr_sz; /* number of Ethernet header bytes copied into each TX WQE. */
     130             : 
     131             : };
     132             : typedef struct fd_mlx5_tx_qp fd_mlx5_tx_qp_t;
     133             : 
     134             : /* fd_mlx5_rss_qp identifies the receive-only QP used as a flow target. */
     135             : struct fd_mlx5_rss_qp {
     136             :   uint handle;
     137             : };
     138             : typedef struct fd_mlx5_rss_qp fd_mlx5_rss_qp_t;
     139             : 
     140             : /* fd_mlx5_uverbs_tile identifies one tile's queues and packet-memory region
     141             :    during shared uverbs initialization. */
     142             : struct fd_mlx5_uverbs_tile {
     143             :   fd_mlx5_cq_t *    rx_cq;
     144             :   fd_mlx5_cq_t *    tx_cq;
     145             :   fd_mlx5_rx_wq_t * rx_wq;
     146             :   fd_mlx5_tx_qp_t * tx_qp;
     147             :   uint *            lkey;
     148             :   int *             rx_comp_channel_fd; /* NULL in performance mode */
     149             :   void *            packet_memory;
     150             :   ulong             packet_memory_sz;
     151             :   ulong             packet_iova;
     152             : };
     153             : typedef struct fd_mlx5_uverbs_tile fd_mlx5_uverbs_tile_t;
     154             : 
     155             : FD_PROTOTYPES_BEGIN
     156             : 
     157             : /* fd_uverbs_init creates one shared context, PD, receive indirection table,
     158             :    and two RSS QPs, one hashing outer headers and one hashing inner tunnel
     159             :    headers.  It creates an MR, UAR, RX CQ, TX CQ, receive WQ, and send-only QP
     160             :    for each entry in tiles.  For entries with a non-NULL rx_comp_channel_fd, it also
     161             :    attaches a uverbs completion channel to the RX CQ. */
     162             : fd_mlx5_rss_qp_t *
     163             : fd_uverbs_init( fd_uverbs_ctx_t *       uverbs,
     164             :                 fd_mlx5_uverbs_tile_t * tiles,
     165             :                 ulong                   tile_cnt,
     166             :                 fd_mlx5_rss_qp_t *      outer_rss_qp,
     167             :                 fd_mlx5_rss_qp_t *      gre_rss_qp,
     168             :                 char const *            rdma_name,
     169             :                 uint                    port_num );
     170             : 
     171             : /* fd_uverbs_map_uar maps a previously allocated non-cached mlx5 UAR page. */
     172             : volatile uchar *
     173             : fd_uverbs_map_uar( fd_uverbs_ctx_t * uverbs,
     174             :                    ulong             mmap_offset );
     175             : 
     176             : /* fd_uverbs_create_udp_flow and fd_uverbs_create_gre_udp_flow steer matching
     177             :    IPv4 traffic to an RSS QP.  The GRE destination IP and port select the inner
     178             :    packet.  A zero destination IP matches every IPv4 address. */
     179             : int
     180             : fd_uverbs_create_udp_flow( fd_uverbs_ctx_t *        uverbs,
     181             :                            fd_mlx5_rss_qp_t const * rss_qp,
     182             :                            uint                     dst_ip,
     183             :                            ushort                   dst_port );
     184             : 
     185             : int
     186             : fd_uverbs_create_gre_udp_flow( fd_uverbs_ctx_t *        uverbs,
     187             :                                fd_mlx5_rss_qp_t const * rss_qp,
     188             :                                uint                     inner_dst_ip,
     189             :                                ushort                   inner_dst_port );
     190             : 
     191             : FD_PROTOTYPES_END
     192             : 
     193             : #endif /* defined(__linux__) */
     194             : 
     195             : #endif /* HEADER_fd_src_disco_net_mlx5_fd_mlx5_private_h */