Line data Source code
1 : #if !defined(__linux__)
2 : #error "fd_xsk requires Linux operating system with XDP support"
3 : #endif
4 :
5 : #define _GNU_SOURCE
6 :
7 : #include <errno.h>
8 : #include <stdio.h> /* snprintf */
9 : #include <string.h> /* strcmp */
10 : #include <unistd.h>
11 : #include <fcntl.h>
12 : #include <sys/mman.h> /* mmap */
13 : #include <sys/types.h>
14 : #include <sys/socket.h> /* sendto */
15 : #include <sys/syscall.h> /* SYS_mlock */
16 :
17 : #include "../../util/log/fd_log.h"
18 : #include "fd_xsk.h"
19 :
20 : /* Support for older kernels */
21 :
22 : #ifndef SO_BUSY_POLL
23 : #define SO_BUSY_POLL 46
24 : #endif
25 :
26 : #ifndef SO_PREFER_BUSY_POLL
27 : #define SO_PREFER_BUSY_POLL 69
28 : #endif
29 :
30 : #ifndef SO_BUSY_POLL_BUDGET
31 : #define SO_BUSY_POLL_BUDGET 70
32 : #endif
33 :
34 : /* Join/leave *********************************************************/
35 :
36 : /* fd_xsk_mmap_offset_cstr: Returns a cstr describing the given offset
37 : param (6th argument of mmap(2)) assuming fd (5th param of mmap(2)) is
38 : an XSK file descriptor. Returned cstr is valid until next call. */
39 : static char const *
40 0 : fd_xsk_mmap_offset_cstr( long mmap_off ) {
41 0 : switch( mmap_off ) {
42 0 : case XDP_PGOFF_RX_RING: return "XDP_PGOFF_RX_RING";
43 0 : case XDP_PGOFF_TX_RING: return "XDP_PGOFF_TX_RING";
44 0 : case XDP_UMEM_PGOFF_FILL_RING: return "XDP_UMEM_PGOFF_FILL_RING";
45 0 : case XDP_UMEM_PGOFF_COMPLETION_RING: return "XDP_UMEM_PGOFF_COMPLETION_RING";
46 0 : default: {
47 0 : static char buf[ 19UL ];
48 0 : snprintf( buf, 19UL, "0x%lx", (ulong)mmap_off );
49 0 : return buf;
50 0 : }
51 0 : }
52 0 : }
53 :
54 : /* fd_xsk_mmap_ring maps the given XSK ring into the local address space
55 : and populates fd_xdp_ring_t. Every successful call to this function
56 : should eventually be paired with a call to fd_xsk_munmap_ring(). */
57 : static int
58 : fd_xsk_mmap_ring( fd_xdp_ring_t * ring,
59 : int xsk_fd,
60 : long map_off,
61 : ulong elem_sz,
62 : ulong depth,
63 0 : struct xdp_ring_offset const * ring_offset ) {
64 : /* TODO what is ring_offset->desc ? */
65 :
66 : /* sanity check */
67 0 : if( depth > (ulong)UINT_MAX ) {
68 0 : return -1;
69 0 : }
70 :
71 0 : ulong map_sz = ring_offset->desc + depth*elem_sz;
72 :
73 0 : void * res = mmap( NULL, map_sz, PROT_READ|PROT_WRITE, MAP_SHARED, xsk_fd, map_off );
74 0 : if( FD_UNLIKELY( res==MAP_FAILED ) ) {
75 0 : FD_LOG_WARNING(( "mmap(NULL, %lu, PROT_READ|PROT_WRITE, MAP_SHARED, xsk_fd, %s) failed (%i-%s)",
76 0 : map_sz, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
77 0 : return -1;
78 0 : }
79 :
80 : /* Lock descriptor rings to prevent swapping. Reimplements syscall
81 : logic of fd_numa_mlock() from fd_shmem_private.h to circumvent the
82 : ASan interceptor and avoid private header dependencies. */
83 :
84 0 : if( FD_UNLIKELY( (int)syscall( SYS_mlock, res, map_sz ) ) )
85 0 : FD_LOG_WARNING(( "syscall(SYS_mlock, %p, %lu KiB) on %s ring failed (%i-%s); attempting to continue",
86 0 : res, map_sz>>10, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
87 :
88 : /* We would ordinarily set up core dump config (MADV_DONTDUMP) here,
89 : but the kernel already makes XDP metadata rings non-dumpable. */
90 :
91 : /* TODO add unit test asserting that cached prod/cons seq gets
92 : cleared on join */
93 0 : fd_memset( ring, 0, sizeof(fd_xdp_ring_t) );
94 :
95 0 : ring->mem = res;
96 0 : ring->map_sz = map_sz;
97 0 : ring->depth = (uint)depth;
98 0 : ring->ptr = (void *)( (ulong)res + ring_offset->desc );
99 0 : ring->flags = (uint *)( (ulong)res + ring_offset->flags );
100 0 : ring->prod = (uint *)( (ulong)res + ring_offset->producer );
101 0 : ring->cons = (uint *)( (ulong)res + ring_offset->consumer );
102 :
103 0 : return 0;
104 0 : }
105 :
106 : /* fd_xsk_munmap_ring unmaps the given XSK ring from the local address
107 : space and zeroes fd_xdp_ring_t. */
108 : static void
109 : fd_xsk_munmap_ring( fd_xdp_ring_t * ring,
110 0 : long map_off ) {
111 0 : if( FD_UNLIKELY( !ring->mem ) ) return;
112 :
113 0 : void * mem = ring->mem;
114 0 : ulong sz = ring->map_sz;
115 :
116 0 : fd_memset( ring, 0, sizeof(fd_xdp_ring_t) );
117 :
118 0 : if( FD_UNLIKELY( 0!=munmap( mem, sz ) ) )
119 0 : FD_LOG_WARNING(( "munmap(%p, %lu) on %s ring failed (%i-%s)",
120 0 : mem, sz, fd_xsk_mmap_offset_cstr( map_off ), errno, fd_io_strerror( errno ) ));
121 0 : }
122 :
123 : /* fd_xsk_fini undoes a (partial) join by releasing all active kernel
124 : objects, such as mapped memory regions and file descriptors. Assumes
125 : that no join to `xsk` is currently being used. */
126 :
127 : static fd_xsk_t *
128 0 : fd_xsk_fini( fd_xsk_t * xsk ) {
129 : /* Undo memory mappings */
130 :
131 0 : fd_xsk_munmap_ring( &xsk->ring_rx, XDP_PGOFF_RX_RING );
132 0 : fd_xsk_munmap_ring( &xsk->ring_tx, XDP_PGOFF_TX_RING );
133 0 : fd_xsk_munmap_ring( &xsk->ring_fr, XDP_UMEM_PGOFF_FILL_RING );
134 0 : fd_xsk_munmap_ring( &xsk->ring_cr, XDP_UMEM_PGOFF_COMPLETION_RING );
135 :
136 : /* Release XSK */
137 :
138 0 : if( FD_LIKELY( xsk->xsk_fd>=0 ) ) {
139 : /* Clear XSK descriptors */
140 0 : fd_memset( &xsk->offsets, 0, sizeof(struct xdp_mmap_offsets) );
141 : /* Close XSK */
142 0 : close( xsk->xsk_fd );
143 0 : xsk->xsk_fd = -1;
144 0 : }
145 :
146 0 : return xsk;
147 0 : }
148 :
149 : /* fd_xsk_setup_umem: Initializes xdp_umem_reg and hooks up XSK with
150 : UMEM rings via setsockopt(). Retrieves xdp_mmap_offsets via
151 : getsockopt(). Returns 0 on success, -1 on failure. */
152 : static int
153 : fd_xsk_setup_umem( fd_xsk_t * xsk,
154 0 : fd_xsk_params_t const * params ) {
155 :
156 : /* Initialize xdp_umem_reg */
157 0 : struct xdp_umem_reg umem_reg = {
158 0 : .addr = (ulong)params->umem_addr,
159 0 : .len = params->umem_sz,
160 0 : .chunk_size = (uint)params->frame_sz,
161 0 : };
162 :
163 : /* Register UMEM region */
164 0 : int res;
165 0 : res = setsockopt( xsk->xsk_fd, SOL_XDP, XDP_UMEM_REG,
166 0 : &umem_reg, sizeof(struct xdp_umem_reg) );
167 0 : if( FD_UNLIKELY( res!=0 ) ) {
168 0 : FD_LOG_WARNING(( "setsockopt(SOL_XDP,XDP_UMEM_REG(addr=%p,len=%lu,chunk_size=%lu)) failed (%i-%s)",
169 0 : (void *)umem_reg.addr, (ulong)umem_reg.len, (ulong)umem_reg.chunk_size,
170 0 : errno, fd_io_strerror( errno ) ));
171 0 : return -1;
172 0 : }
173 :
174 : /* Set ring frame counts */
175 0 : # define FD_SET_XSK_RING_DEPTH(name, var) \
176 0 : do { \
177 0 : res = setsockopt( xsk->xsk_fd, SOL_XDP, name, &(var), 8UL ); \
178 0 : if( FD_UNLIKELY( res!=0 ) ) { \
179 0 : FD_LOG_WARNING(( "setsockopt(SOL_XDP," #name ",%lu) failed (%i-%s)", \
180 0 : var, errno, fd_io_strerror( errno ) )); \
181 0 : return -1; \
182 0 : } \
183 0 : } while(0)
184 0 : FD_SET_XSK_RING_DEPTH( XDP_UMEM_FILL_RING, params->fr_depth );
185 0 : FD_SET_XSK_RING_DEPTH( XDP_RX_RING, params->rx_depth );
186 0 : FD_SET_XSK_RING_DEPTH( XDP_TX_RING, params->tx_depth );
187 0 : FD_SET_XSK_RING_DEPTH( XDP_UMEM_COMPLETION_RING, params->cr_depth );
188 0 : # undef FD_SET_XSK_RING_DEPTH
189 :
190 : /* Request ring offsets */
191 0 : socklen_t offsets_sz = sizeof(struct xdp_mmap_offsets);
192 0 : res = getsockopt( xsk->xsk_fd, SOL_XDP, XDP_MMAP_OFFSETS,
193 0 : &xsk->offsets, &offsets_sz );
194 0 : if( FD_UNLIKELY( res!=0 ) ) {
195 0 : FD_LOG_WARNING(( "getsockopt(SOL_XDP, XDP_MMAP_OFFSETS) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
196 0 : return -1;
197 0 : }
198 :
199 : /* OK */
200 0 : return 0;
201 0 : }
202 :
203 : /* fd_xsk_setup_poll: Setup preferred busy polling */
204 :
205 : static void
206 : fd_xsk_setup_poll( fd_xsk_t * xsk,
207 0 : fd_xsk_params_t const * params ) {
208 0 : xsk->prefbusy_poll_enabled = 0;
209 0 : if( 0!=strcmp( params->poll_mode, "prefbusy" ) ) return;
210 :
211 : /* Configure socket options for preferred busy polling */
212 :
213 0 : int prefbusy_poll = 1;
214 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_PREFER_BUSY_POLL, &prefbusy_poll, sizeof(int) ) ) ) {
215 0 : int err = errno;
216 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: setsockopt(xsk_fd,SOL_SOCKET,SO_PREFER_BUSY_POLL,1) failed (%i-%s)", err, fd_io_strerror( err ) ));
217 0 : if( err==EINVAL ) {
218 0 : FD_LOG_NOTICE(( "Hint: Does your kernel support preferred busy polling? SO_PREFER_BUSY_POLL is available from Linux 5.11 onwards" ));
219 0 : }
220 0 : return;
221 0 : }
222 :
223 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_BUSY_POLL, ¶ms->prefbusy_time_budget_micros, sizeof(int) ) ) ) {
224 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: setsockopt(xsk_fd,SOL_SOCKET,SO_BUSY_POLL,%i) failed (%i-%s)",
225 0 : params->prefbusy_time_budget_micros, errno, fd_io_strerror( errno ) ));
226 0 : return;
227 0 : }
228 :
229 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_BUSY_POLL_BUDGET, ¶ms->prefbusy_rx_budget, sizeof(int) ) ) ) {
230 0 : FD_LOG_WARNING(( "setsockopt(xsk_fd,SOL_SOCKET,SO_BUSY_POLL_BUDGET,%i) failed (%i-%s)",
231 0 : params->prefbusy_rx_budget , errno, fd_io_strerror( errno ) ));
232 0 : return;
233 0 : }
234 :
235 : /* Set socket non blocking */
236 :
237 0 : int sk_flags = fcntl( xsk->xsk_fd, F_GETFL, 0 );
238 0 : if( FD_UNLIKELY( -1==sk_flags ) ) {
239 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: fcntl(xsk->xsk_fd, F_GETFL, 0) failed (%i-%s)",
240 0 : errno, fd_io_strerror( errno ) ));
241 0 : return;
242 0 : }
243 0 : if( FD_UNLIKELY( -1==fcntl( xsk->xsk_fd, F_SETFL, sk_flags | O_NONBLOCK ) ) ) {
244 0 : FD_LOG_WARNING(( "failed to enable XDP prefbusy polling: fcntl(xsk->xsk_fd, F_SETFL, sk_flags | O_NONBLOCK) failed (%i-%s)",
245 0 : errno, fd_io_strerror( errno ) ));
246 0 : return;
247 0 : }
248 :
249 : /* Successfully finished setting up prefbusy polling */
250 0 : xsk->prefbusy_poll_enabled = 1U;
251 0 : }
252 :
253 : /* fd_xsk_init: Creates and configures an XSK socket object, and
254 : attaches to a preinstalled XDP program. The various steps are
255 : implemented in fd_xsk_setup_{...}. */
256 :
257 : fd_xsk_t *
258 : fd_xsk_init( fd_xsk_t * xsk,
259 0 : fd_xsk_params_t const * params ) {
260 :
261 0 : if( FD_UNLIKELY( !xsk ) ) { FD_LOG_WARNING(( "NULL xsk" )); return NULL; }
262 0 : memset( xsk, 0, sizeof(fd_xsk_t) );
263 :
264 0 : if( FD_UNLIKELY( !params->if_idx ) ) { FD_LOG_WARNING(( "zero if_idx" )); return NULL; }
265 0 : if( FD_UNLIKELY( (!params->fr_depth) | (!params->rx_depth) |
266 0 : (!params->tx_depth) | (!params->cr_depth) ) ) {
267 0 : FD_LOG_WARNING(( "invalid {fr,rx,tx,cr}_depth" ));
268 0 : return NULL;
269 0 : }
270 0 : if( FD_UNLIKELY( !params->umem_addr ) ) {
271 0 : FD_LOG_WARNING(( "NULL umem_addr" ));
272 0 : return NULL;
273 0 : }
274 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)params->umem_addr, 4096UL ) ) ) {
275 0 : FD_LOG_WARNING(( "misaligned params->umem_addr" ));
276 0 : return NULL;
277 0 : }
278 0 : if( FD_UNLIKELY( !params->frame_sz || !fd_ulong_is_pow2( params->frame_sz ) ) ) {
279 0 : FD_LOG_WARNING(( "invalid frame_sz" ));
280 0 : return NULL;
281 0 : }
282 :
283 0 : xsk->if_idx = params->if_idx;
284 0 : xsk->if_queue_id = params->if_queue_id;
285 :
286 : /* Create XDP socket (XSK) */
287 :
288 0 : xsk->xsk_fd = socket( AF_XDP, SOCK_RAW, 0 );
289 0 : if( FD_UNLIKELY( xsk->xsk_fd<0 ) ) {
290 0 : FD_LOG_WARNING(( "Failed to create XSK (%i-%s)", errno, fd_io_strerror( errno ) ));
291 0 : return NULL;
292 0 : }
293 :
294 : /* In copy mode every TX frame in flight is an skb charged to the
295 : socket send buffer, so the default (wmem_default, often 64 KiB)
296 : caps the kernel at a few dozen frames. Size it to the TX ring. */
297 :
298 0 : int sndbuf = (int)fd_ulong_min( params->tx_depth*params->frame_sz, (ulong)INT_MAX );
299 0 : if( FD_UNLIKELY( 0!=setsockopt( xsk->xsk_fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(int) ) ) ) {
300 0 : FD_LOG_WARNING(( "setsockopt(SOL_SOCKET,SO_SNDBUF,%i) failed (%i-%s)", sndbuf, errno, fd_io_strerror( errno ) ));
301 0 : goto fail;
302 0 : }
303 :
304 : /* Associate UMEM region of fd_xsk_t with XSK via setsockopt() */
305 :
306 0 : if( FD_UNLIKELY( 0!=fd_xsk_setup_umem( xsk, params ) ) ) goto fail;
307 :
308 : /* Map XSK rings into local address space */
309 :
310 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_rx, xsk->xsk_fd, XDP_PGOFF_RX_RING, sizeof(struct xdp_desc), params->rx_depth, &xsk->offsets.rx ) ) ) goto fail;
311 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_tx, xsk->xsk_fd, XDP_PGOFF_TX_RING, sizeof(struct xdp_desc), params->tx_depth, &xsk->offsets.tx ) ) ) goto fail;
312 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_fr, xsk->xsk_fd, XDP_UMEM_PGOFF_FILL_RING, sizeof(ulong), params->fr_depth, &xsk->offsets.fr ) ) ) goto fail;
313 0 : if( FD_UNLIKELY( 0!=fd_xsk_mmap_ring( &xsk->ring_cr, xsk->xsk_fd, XDP_UMEM_PGOFF_COMPLETION_RING, sizeof(ulong), params->cr_depth, &xsk->offsets.cr ) ) ) goto fail;
314 :
315 : /* Bind XSK to queue on network interface */
316 :
317 0 : uint flags = XDP_USE_NEED_WAKEUP | params->bind_flags;
318 0 : struct sockaddr_xdp sa = {
319 0 : .sxdp_family = PF_XDP,
320 0 : .sxdp_ifindex = xsk->if_idx,
321 0 : .sxdp_queue_id = xsk->if_queue_id,
322 : /* See extended commentary below for details on XDP_USE_NEED_WAKEUP
323 : flag. */
324 0 : .sxdp_flags = (ushort)flags
325 0 : };
326 :
327 0 : char if_name[ IF_NAMESIZE ] = {0};
328 :
329 0 : if( FD_UNLIKELY( 0!=bind( xsk->xsk_fd, (void *)&sa, sizeof(struct sockaddr_xdp) ) ) ) {
330 0 : FD_LOG_WARNING(( "bind( PF_XDP, ifindex=%u (%s), queue_id=%u, flags=%x ) failed (%i-%s)",
331 0 : xsk->if_idx, if_indextoname( xsk->if_idx, if_name ),
332 0 : xsk->if_queue_id, flags,
333 0 : errno, fd_io_strerror( errno ) ));
334 0 : goto fail;
335 0 : }
336 :
337 : /* We've seen that some popular Intel NICs seem to have a bug that
338 : prevents them from working in SKB mode with certain kernel
339 : versions. We can identify them by sendto returning ENXIO or EINVAL
340 : in newer versions. The core of the problem is that the kernel
341 : calls the generic ndo_bpf pointer instead of the driver-specific
342 : version. This means that the driver's pointer to the BPF program
343 : never gets set, yet the driver's wakeup function gets called. */
344 0 : if( FD_UNLIKELY( -1==sendto( xsk->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0 ) ) ) {
345 0 : if( FD_LIKELY( errno==ENXIO || errno==EINVAL ) ) {
346 0 : FD_LOG_ERR(( "xsk sendto failed xsk_fd=%d (%i-%s). This likely indicates "
347 0 : "a bug with your NIC driver. Try switching XDP mode using "
348 0 : "net.xdp.xdp_mode in the configuration TOML.\n"
349 0 : "Certain Intel NICs with certain driver/kernel combinations "
350 0 : "are known to exhibit this issue in skb mode but work in drv "
351 0 : "mode.", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
352 0 : } else {
353 0 : FD_LOG_WARNING(( "xsk sendto failed xsk_fd=%d (%i-%s)", xsk->xsk_fd, errno, fd_io_strerror( errno ) ));
354 0 : }
355 0 : }
356 :
357 : /* XSK successfully configured. Traffic will arrive in XSK after
358 : configuring an XDP program to forward packets via XDP_REDIRECT.
359 : This requires providing the XSK file descriptor to the program via
360 : XSKMAP and is done in a separate step. */
361 :
362 0 : FD_LOG_INFO(( "AF_XDP socket initialized: bind( PF_XDP, ifindex=%u (%s), queue_id=%u, flags=%x ) success",
363 0 : xsk->if_idx, if_indextoname( xsk->if_idx, if_name ), xsk->if_queue_id, flags ));
364 :
365 : /* If requested, enable preferred busy polling */
366 :
367 0 : fd_xsk_setup_poll( xsk, params );
368 :
369 0 : return xsk;
370 :
371 0 : fail:
372 0 : fd_xsk_fini( xsk );
373 0 : return NULL;
374 0 : }
|
