Line data Source code
1 : #ifndef HEADER_fd_src_waltz_xdp_fd_xsk_h
2 : #define HEADER_fd_src_waltz_xdp_fd_xsk_h
3 :
4 : #if defined(__linux__)
5 :
6 : /* fd_xsk manages an XSK file descriptor and provides RX/TX buffers.
7 :
8 : ### Background
9 :
10 : AF_XDP is a Linux API providing kernel-bypass networking in the form
11 : of shared memory ring buffers accessible from userspace. The kernel
12 : redirects packets from/to these buffers with the appropriate XDP
13 : configuration (XDP_REDIRECT). AF_XDP is hardware-agnostic and allows
14 : sharing a NIC with the Linux networking stack (unlike e.g. DPDK).
15 : This allows for deployment in existing, heterogeneous networks. An
16 : AF_XDP socket is called "XSK". The shared memory region storing the
17 : packet data flowing through an XSK is called "UMEM".
18 :
19 : XDP (eXpress Data Path) is a framework for installing hooks in the
20 : form of eBPF programs at an early stage of packet processing (i.e.
21 : before tc and netfilter). eBPF is user-deployable JIT-compiled
22 : bytecode that usually runs inside the kernel. Some hardware/driver
23 : combinations optionally allow offloading eBPF processing to NICs.
24 : This is not to be confused with other BPF-derived ISAs such as sBPF
25 : (Solana BPF).
26 :
27 : +--- Figure 1: AF_XDP RX Block Diagram -----------------+
28 : | |
29 : | ┌─────┐ ┌────────┐ ┌─────┐ XDP_PASS ┌─────────┐ |
30 : | │ NIC ├──> Driver ├──> XDP ├──────────> sk_buff │ |
31 : | └─────┘ └────────┘ └─┬───┘ └─────────┘ |
32 : | │ |
33 : | │ XDP_REDIRECT |
34 : | │ |
35 : | ┌──▼───────┐ ┌─────────┐ |
36 : | │ XSK/UMEM ├──────> app │ |
37 : | └──────────┘ └─────────┘ |
38 : | |
39 : +-------------------------------------------------------+
40 :
41 : Figure 1 shows a simplified block diagram of RX packet flow within
42 : the kernel in `XDP_FLAGS_DRV_MODE` mode. Notably, the chain of eBPF
43 : programs installed in the XDP facility get invoked for every incoming
44 : packet. If all programs return the `XDP_PASS` action, the packet
45 : continues its usual path to the Linux networking stack, where it will
46 : be allocated in sk_buff, and eventually flow through ip_rcv(), tc,
47 : and netfilter before reaching downstream sockets.
48 : If the `XDP_REDIRECT` action is taken however, the packet is copied
49 : to the UMEM of an XSK, and a RX queue entry is allocated.
50 : The more generic `XDP_FLAGS_SKB_MODE` XDP mode falls back to sk_buff-
51 : based memory mgmt (still skipping the rest of the generic path), but
52 : is more widely available.
53 :
54 : +--- Figure 2: AF_XDP TX Block Diagram -------------+
55 : | |
56 : | ┌────────┐ ┌──────────┐ ┌────────┐ ┌─────┐ |
57 : | │ app ├──> XSK/UMEM ├──> Driver ├──> NIC │ |
58 : | └────────┘ └──────────┘ └────────┘ └─────┘ |
59 : | |
60 : +---------------------------------------------------+
61 :
62 : Figure 2 shows a simplified block diagram of the TX packet flow.
63 : Userspace applications deliver packets to the XSK/UMEM buffers. The
64 : kernel then forwards these packets to the NIC. This also means that
65 : the application is responsible for maintaining a routing table to
66 : resolve layer-3 dest addrs to NICs and layer-2 addrs. As in the RX
67 : flow, netfilter (iptables, nftables) is not available.
68 :
69 : ### Memory Management
70 :
71 : The UMEM area is allocated from userspace. It is recommended to use
72 : the fd_util shmem/wksp APIs to obtain large page-backed memory. UMEM
73 : is divided into equally sized frames. At any point in time, each
74 : frame is either owned by userspace or the kernel. On initialization,
75 : all frames are owned by userspace.
76 :
77 : Changes in UMEM frame ownership and packet RX/TX events are
78 : transmitted via four rings allocated by the kernel (mmap()ed in by
79 : the user). This allows for out-of-order processing of packets.
80 :
81 : Data flow:
82 : (U->K) is userspace-to-kernel communication, and
83 : (K->U) is kernel-to-userspace.
84 :
85 : FILL Free frames are provided to the kernel using the FILL
86 : (U->K) ring. The kernel may populate these frames with RX
87 : packet data.
88 :
89 : RX Once the kernel has populated a FILL frame with RX
90 : (K->U) packet data, it passes back the frame to userspace
91 : via the RX queue.
92 :
93 : TX TX frames sent by userspace are provided to the
94 : (U->K) kernel using the TX ring.
95 :
96 : COMPLETION Once the kernel has processed a TX frame, it passes
97 : (K->U) back the frame to the userspace via the COMPLETION
98 : queue.
99 :
100 : Combined, the FILL-RX and TX-COMPLETION rings form two pairs. The
101 : kernel will not move frames between the pairs. */
102 :
103 : #include <linux/if_link.h>
104 : #include <linux/if_xdp.h>
105 : #include <net/if.h>
106 :
107 : #include "../../util/fd_util_base.h"
108 :
109 : /* FD_XSK_UMEM_ALIGN: byte alignment of UMEM area within fd_xsk_t.
110 : This requirement is set by the kernel as of Linux 4.18. */
111 : #define FD_XSK_UMEM_ALIGN (4096UL)
112 :
113 : /* fd_xdp_ring_t describes an XSK descriptor ring in the thread group's
114 : local address space. All pointers fall into kernel-managed XSK
115 : descriptor buffer at [mem;mem+mem_sz) that are valid during the
116 : lifetime of an fd_xsk_t join. The ring producer and consumer are
117 : synchronized via incrementing 32-bit sequence numbers that wrap at 2^32. */
118 :
119 : struct __attribute__((aligned(64UL))) fd_xdp_ring {
120 : /* This point is 64-byte aligned */
121 :
122 : /* mmap() params, only used during join/leave for munmap() */
123 :
124 : void * mem; /* Points to start of shared descriptor ring mmap region */
125 : ulong map_sz; /* Size of shared descriptor ring mmap region */
126 : ulong _pad_0x10;
127 : ulong _pad_0x18;
128 :
129 : /* This point is 64-byte aligned */
130 :
131 : /* Pointers to fields opaque XSK ring structure.
132 : This indirection is required because the memory layout of the
133 : kernel-provided descriptor rings is unstable. The field offsets
134 : can be queried using getsockopt(SOL_XDP, XDP_MMAP_OFFSETS). */
135 :
136 : union {
137 : void * ptr; /* Opaque pointer */
138 : struct xdp_desc * packet_ring; /* For RX, TX rings */
139 : ulong * frame_ring; /* For FILL, COMPLETION rings */
140 : };
141 : uint * flags; /* Points to flags in shared descriptor ring */
142 : uint * prod; /* Points to producer seq in shared descriptor ring */
143 : uint * cons; /* Points to consumer seq in shared descriptor ring */
144 :
145 : /* This point is 64-byte aligned */
146 :
147 : /* Managed by fd_xsk_t */
148 :
149 : uint depth; /* Capacity of ring in no of entries */
150 : uint cached_prod; /* Cached value of *prod */
151 : uint cached_cons; /* Cached value of *cons */
152 : };
153 : typedef struct fd_xdp_ring fd_xdp_ring_t;
154 :
155 0 : #define FD_XDP_RING_ROLE_PROD 0
156 0 : #define FD_XDP_RING_ROLE_CONS 1
157 :
158 : /* fd_xdp_ring_empty returns 1 if the ring is empty, 0 otherwise.
159 : 'role' is FD_XDP_RING_ROLE_PROD if userspace is the producer (fill, tx),
160 : and FD_XDP_RING_ROLE_CONS if userspace is the consumer (rx, completion). */
161 :
162 : static inline int
163 0 : fd_xdp_ring_empty( fd_xdp_ring_t * ring, uint role ) {
164 0 : if( role == FD_XDP_RING_ROLE_PROD ) {
165 : /* If potentially stale cached_cons says everything consumed,
166 : it's definitely empty. Else, refresh cached seq. */
167 0 : if( FD_UNLIKELY( ring->cached_prod == ring->cached_cons ) ) return 1;
168 0 : ring->cached_cons = __atomic_load_n( ring->cons, __ATOMIC_ACQUIRE );
169 0 : } else {
170 : /* If potentially stale cached_prod says we have more to read,
171 : it's definitely non-empty. Else, refresh cached seq. */
172 0 : if( FD_LIKELY( ring->cached_cons < ring->cached_prod ) ) return 0;
173 0 : ring->cached_prod = __atomic_load_n( ring->prod, __ATOMIC_ACQUIRE );
174 0 : }
175 0 : return ring->cached_prod == ring->cached_cons;
176 0 : }
177 :
178 : /* fd_xdp_ring_full returns 1 if the ring is full, 0 otherwise.
179 : Assumes caller is the producer to this ring (fill, tx). */
180 : static inline int
181 0 : fd_xdp_ring_full( fd_xdp_ring_t * ring ) {
182 : /* If potentially stale cached_cons says we have more space,
183 : it's definitely not full. Else, refresh cached seq. */
184 0 : if( FD_LIKELY( ring->cached_prod - ring->cached_cons < ring->depth ) ) return 0;
185 0 : ring->cached_cons = __atomic_load_n( ring->cons, __ATOMIC_ACQUIRE );
186 0 : return ring->cached_prod - ring->cached_cons >= ring->depth;
187 0 : }
188 :
189 : /* fd_xsk_params_t: XSK poll configuration and memory layout
190 : parameters passed to fd_xsk_init. */
191 :
192 : struct fd_xsk_params {
193 : /* {fr,rx,tx,cr}_depth: Number of frames allocated for the Fill, RX,
194 : TX, Completion XSK rings respectively. */
195 : ulong fr_depth;
196 : ulong rx_depth;
197 : ulong tx_depth;
198 : ulong cr_depth;
199 :
200 : /* umem_addr: Pointer to UMEM in local address space */
201 : void * umem_addr;
202 :
203 : /* frame_sz: Controls the frame size used in the UMEM ring buffers. */
204 : ulong frame_sz;
205 :
206 : /* umem_sz: Total size of XSK ring shared memory area (contiguous).
207 : Aligned by FD_XSK_UMEM_ALIGN. */
208 : ulong umem_sz;
209 :
210 : /* Linux interface index */
211 : uint if_idx;
212 :
213 : /* Interface queue index */
214 : uint if_queue_id;
215 :
216 : /* sockaddr_xdp.sxdp_flags additional params, e.g. XDP_ZEROCOPY */
217 : uint bind_flags;
218 :
219 : /* Whether to use softirqs to poll napi or prefbusy poll */
220 : char poll_mode[ 16 ];
221 :
222 : /* Max time during prefbusy napi poll. Referred to as
223 : 'busy_poll_usecs' within Linux. */
224 : int prefbusy_time_budget_micros;
225 :
226 : /* Max RX processing budget during prefbusy napi poll.
227 : Referred to as 'busy_poll_budget' within Linux. */
228 : int prefbusy_rx_budget;
229 : };
230 :
231 : typedef struct fd_xsk_params fd_xsk_params_t;
232 :
233 : struct fd_xsk {
234 : /* Informational */
235 : uint if_idx; /* index of net device */
236 : uint if_queue_id; /* net device combined queue index */
237 : long log_suppress_until_ns; /* suppress log messages until this time */
238 :
239 : /* Kernel descriptor of XSK rings in local address space
240 : returned by getsockopt(SOL_XDP, XDP_MMAP_OFFSETS) */
241 : struct xdp_mmap_offsets offsets;
242 :
243 : /* AF_XDP socket file descriptor */
244 : int xsk_fd;
245 :
246 : /* Whether preferred busy polling was successfully enabled
247 : during XSK socket setup. */
248 : int prefbusy_poll_enabled;
249 :
250 : /* ring_{rx,tx,fr,cr}: XSK ring descriptors */
251 :
252 : fd_xdp_ring_t ring_rx;
253 : fd_xdp_ring_t ring_tx;
254 : fd_xdp_ring_t ring_fr;
255 : fd_xdp_ring_t ring_cr;
256 : };
257 :
258 : typedef struct fd_xsk fd_xsk_t;
259 :
260 : FD_PROTOTYPES_BEGIN
261 :
262 : /* fd_xsk_init creates an XSK, registers UMEM, maps rings, and binds the
263 : socket to the given interface queue. This is a potentially
264 : destructive operation. As of 2024-Jun, AF_XDP zero copy support is
265 : still buggy in some device drivers.
266 :
267 : Assume that all traffic sent to this interface is compromised. On
268 : some devices, the NIC is instructed to DMA all incoming packets into
269 : UMEM, even ones not belonging to Firedancer. Those are then later
270 : on software-copied out to skbs again. This further implies that
271 : enabling AF_XDP can slow down the regular kernel receive path.
272 :
273 : Requires CAP_SYS_ADMIN. May issue the following syscalls:
274 :
275 : - socket( AF_XDP, SOCK_RAW, 0 ) = fd
276 : - setsockopt( fd, SOL_XDP or SOL_SOCKET, ... )
277 : - getsockopt( fd, SOL_XDP, ... )
278 : - mmap( ..., fd, ... )
279 : - bind( fd, ... )
280 : - mlock( ... )
281 : - sendto( fd, ... )
282 : - fcntl( fd, F_GETFL or F_SETFL, ... )
283 : - munmap ; on fail
284 : - close ; on fail */
285 :
286 : fd_xsk_t *
287 : fd_xsk_init( fd_xsk_t * xsk,
288 : fd_xsk_params_t const * params );
289 :
290 : /* fd_xsk_rx_need_wakeup: returns whether a wakeup is required to
291 : complete a rx operation */
292 :
293 : static inline int
294 0 : fd_xsk_rx_need_wakeup( fd_xsk_t * xsk ) {
295 0 : return !!( *xsk->ring_fr.flags & XDP_RING_NEED_WAKEUP );
296 0 : }
297 :
298 : /* fd_xsk_tx_need_wakeup: returns whether a wakeup is required to
299 : complete a tx operation */
300 :
301 : static inline int
302 0 : fd_xsk_tx_need_wakeup( fd_xsk_t * xsk ) {
303 0 : return !!( *xsk->ring_tx.flags & XDP_RING_NEED_WAKEUP );
304 0 : }
305 :
306 :
307 : FD_PROTOTYPES_END
308 :
309 : #endif /* defined(__linux__) */
310 : #endif /* HEADER_fd_src_waltz_xdp_fd_xsk_h */
|
