Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_xdp1.h"
3 :
4 : #include "fd_xdp_license.h"
5 : #include "../ebpf/fd_linux_bpf.h"
6 : #include "../ebpf/fd_ebpf_asm.h"
7 :
8 : #include <errno.h>
9 : #include <unistd.h>
10 : #include <net/if.h>
11 : #include <sys/syscall.h>
12 : #include <linux/bpf.h>
13 : #include <linux/if_link.h>
14 :
15 : /* Define some kernel uapi constants in case the user is compiling
16 : with older kernel headers. This is especially a problem on Ubuntu
17 : 20.04 which supports these functions, but doesn't have them in
18 : the default headers. */
19 :
20 : #ifndef BPF_LINK_CREATE
21 0 : #define BPF_LINK_CREATE (28)
22 : #endif
23 :
24 : #ifndef BPF_XDP
25 0 : #define BPF_XDP (37)
26 : #endif
27 :
28 : struct __attribute__((aligned(8))) bpf_link_create {
29 : uint prog_fd;
30 : uint target_ifindex;
31 : uint attach_type;
32 : uint flags;
33 : };
34 :
35 :
36 : /*
37 :
38 : ┌─────────────────────────────────────────┐
39 : │ Packet Classification Control Flow │
40 : | |
41 : │ ┌────────┐ ┌────┐ │
42 : │ │Ethernet│──►│VLAN│ │
43 : │ └───┬────┘ └─┬──┘ │
44 : │ │ │ │
45 : │ ┌───▼────┐ │ │
46 : │ │ IPV4 │◄────┘ │
47 : │ └──┬───┬─┘ │
48 : │ │ │ │
49 : │ │ └────►──────┐ │
50 : │ │ │ GRE │ │
51 : │ │ └───┬──┘ │
52 : │ │ │ │
53 : │ │ ┌───▼────────┐ │
54 : │ │ │ Inner IPV4 │ │
55 : │ │ ├────────────┘ │
56 : │ │ │ │
57 : │ ┌───▼──◄─────┘ │
58 : │ │ UDP │ │
59 : │ └──┬───┘ │
60 : | │ |
61 : | ┌──────▼───────┐ ┌──────────────┐ |
62 : | | LBL_REDIRECT | | LBL_PASS | |
63 : | └──────────────┘ └──────────────┘ |
64 : └─────────────────────────────────────────┘
65 :
66 : fd_xdp_gen_program dynamically generates an eBPF bytecode program to
67 : classify incoming network packets in an XDP context. Each box in the above
68 : diagram represents a phase during the classification. NON-GRE packets follow
69 : the path of Ethernet-->IPV4-->UDP. GRE packets follow the path of
70 : Ethernet-->IPV4-->GRE-->Inner IPV4-->UDP. Packets carrying a single
71 : 802.1Q tag (any VLAN ID) whose inner Ethertype is IPV4 take the VLAN
72 : phase between Ethernet and IPV4, then follow the same paths.
73 : Double-tagged packets are passed to the kernel. Based on the packet's
74 : protocol and specific header values, the function decides whether to
75 : redirect the packet to the firedancer net tile (LBL_REDIRECT) or pass
76 : the packet to the kernel (LBL_PASS).
77 : */
78 : ulong
79 : fd_xdp_gen_program( ulong code_buf[ 512 ],
80 : int xsks_fd,
81 : uint listen_ip4_addr,
82 : ushort const * ports,
83 : ulong ports_cnt,
84 0 : int allowed_gre ) {
85 :
86 0 : #define LBL_PASS 1 // Pass the packet to the kernel
87 0 : #define LBL_REDIRECT 2 // Redirect the packet to firedancer software
88 :
89 0 : #define LBL_GRE_CHECK 3 // Ethernet-->IPV4-->GRE-->Inner IPV4-->UDP
90 0 : #define LBL_UDP_CHECK 4 // Ethernet-->IPV4-->UDP
91 0 : #define LBL_ETH_IP4 5 // Advance past the Ethernet hdr to the first ip4_hdr
92 :
93 0 : if( FD_UNLIKELY( ports_cnt>16UL ) ) {
94 0 : FD_LOG_ERR(( "Too many XDP UDP ports (%lu)", ports_cnt ));
95 0 : }
96 :
97 0 : ulong * code = code_buf;
98 0 : *(code++) = FD_EBPF( ldxw, r2, r1, 0 ); // r2 = xdp_md->data
99 0 : *(code++) = FD_EBPF( ldxw, r3, r1, 4 ); // r3 = xdp_md->data_end
100 :
101 0 : *(code++) = FD_EBPF( mov64_reg, r5, r2 );
102 0 : *(code++) = FD_EBPF( add64_imm, r5, 34 ); // Bound check accessing the eth_hdr (14 bytes) and the ip4_hdr (20 bytes)
103 0 : *(code++) = FD_EBPF( jgt_reg, r5, r3, LBL_PASS ); // if r2+34 > r3 goto LBL_PASS
104 :
105 0 : *(code++) = FD_EBPF( ldxh, r5, r2, 12 ); // r5 = eth_hdr->net_type
106 0 : *(code++) = FD_EBPF( jeq_imm, r5, 0x0008, LBL_ETH_IP4 ); // if eth_hdr->net_type == IP4 goto eth_ip4
107 0 : *(code++) = FD_EBPF( jne_imm, r5, 0x0081, LBL_PASS ); // if eth_hdr->net_type != VLAN goto LBL_PASS
108 :
109 : /* next hdr is a single 802.1Q vlan_tag */
110 :
111 : /* Bound check accessing the eth_hdr, vlan_tag and ip4_hdr */
112 0 : *(code++) = FD_EBPF( mov64_reg, r5, r2 );
113 0 : *(code++) = FD_EBPF( add64_imm, r5, 38 ); // sizeof(eth_hdr) + sizeof(vlan_tag) + sizeof(ip4_hdr) = 14 + 4 + 20
114 0 : *(code++) = FD_EBPF( jgt_reg, r5, r3, LBL_PASS ); // if r2+38 > r3 goto LBL_PASS
115 :
116 0 : *(code++) = FD_EBPF( ldxh, r5, r2, 16 ); // r5 = vlan_tag->net_type
117 0 : *(code++) = FD_EBPF( jne_imm, r5, 0x0008, LBL_PASS ); // if vlan_tag->net_type != IP4 goto LBL_PASS
118 :
119 : /* Advance r2 past the vlan_tag. eth_ip4 then advances it past the
120 : eth_hdr, so r2 points at the first ip4_hdr on both paths. */
121 0 : *(code++) = FD_EBPF( add64_imm, r2, 4 ); // r2 += sizeof(vlan_tag)
122 :
123 : /* Advance r2 to the start of first ip4_hdr */
124 0 : ulong * eth_ip4 = code;
125 0 : *(code++) = FD_EBPF( add64_imm, r2, 14 );
126 :
127 : /* Calculate the start of next hdr and store in r4 */
128 0 : *(code++) = FD_EBPF( ldxb, r4, r2, 0 ); // r4 = ip4_hdr->verihl
129 0 : *(code++) = FD_EBPF( and64_imm, r4, 0x0f ); // r4 = ip4_hdr->ihl (lsb of ip4_hrd->verihl)
130 0 : *(code++) = FD_EBPF( lsh64_imm, r4, 2 ); // r4 = ip4_hdr->ihl*4 (length of ipv4 header)
131 0 : *(code++) = FD_EBPF( jlt_imm, r4, 20, LBL_PASS ); // if r4<20 goto LBL_PASS
132 0 : *(code++) = FD_EBPF( add64_reg, r4, r2 ); // r4 = &ip4_hdr + length of ip4_hdr = start of next hdr
133 :
134 : /* Check if the next hdr is udp or gre */
135 0 : *(code++) = FD_EBPF( ldxb, r5, r2, 9 ); // r5 = ip4_hdr->protocol
136 :
137 0 : if( allowed_gre==1 ) {
138 0 : *(code++) = FD_EBPF( jeq_imm, r5, 47, LBL_GRE_CHECK ); // if ip4_hdr->protocol == GRE goto gre_check
139 0 : }
140 :
141 0 : *(code++) = FD_EBPF( jeq_imm, r5, 17, LBL_UDP_CHECK ); // if ip4_hdr->protocol == UDP goto udp_check
142 0 : *(code++) = FD_EBPF( ja, LBL_PASS ); // goto LBL_PASS
143 :
144 :
145 : /* next hdr is gre */
146 0 : ulong * gre_check = code;
147 :
148 : /* Advance r2 to start of gre_hdr */
149 0 : *(code++) = FD_EBPF( mov64_reg, r2, r4 );
150 :
151 : /*
152 : * At this point:
153 : * r1: xdp_md
154 : * r2: start of next header (gre_hdr)
155 : * r3: xdp_md->data_end
156 : * r4: clobber
157 : * r5: clobber
158 : */
159 :
160 : /* Bound check GRE and inner ip4_hdr access */
161 0 : *(code++) = FD_EBPF( mov64_reg, r5, r2 );
162 0 : *(code++) = FD_EBPF( add64_imm, r5, 24 ); // r5 = 1 byte past inner ip4_hdr. sizeof(gre_hdr) + sizeof(ip4_hdr) = 4 + 20
163 0 : *(code++) = FD_EBPF( jgt_reg, r5, r3, LBL_PASS ); // if (end of inner ip4_hdr + 1) > r3, goto LBL_PASS
164 :
165 :
166 : /* Verify GRE fields */
167 0 : *(code++) = FD_EBPF( ldxh, r5, r2, 0 ); // r5 = gre_hdr->flags/version
168 0 : *(code++) = FD_EBPF( jne_imm, r5, 0x0000, LBL_PASS ); // if gre_hdr->flags/version != 0, goto LBL_PASS
169 0 : *(code++) = FD_EBPF( ldxh, r5, r2, 2 ); // r5 = gre_hdr->protocol
170 0 : *(code++) = FD_EBPF( jne_imm, r5, 0x0008, LBL_PASS ); // if gre_hdr->protocol != IP, goto LBL_PASS
171 :
172 :
173 : /* Advance r2 to start of inner ip4_hdr */
174 0 : *(code++) = FD_EBPF( add64_imm, r2, 4 ); // r2 = start of inner ip4_hdr
175 :
176 : /* Check inner ip4's encapsulated protocol */
177 0 : *(code++) = FD_EBPF( ldxb, r5, r2, 9 ); // r5 = inner ip4_hdr->protocol
178 0 : *(code++) = FD_EBPF( jne_imm, r5, 17, LBL_PASS ); // if r5!=UDP, goto LBL_PASS
179 :
180 : /* Calculate the start of udp_hdr and store in r4 */
181 0 : *(code++) = FD_EBPF( ldxb, r4, r2, 0 ); // r4 = inner ip4_hdr->verihl
182 0 : *(code++) = FD_EBPF( and64_imm, r4, 0x0f ); // r4 = inner ip4_hdr->ihl
183 0 : *(code++) = FD_EBPF( lsh64_imm, r4, 2 ); // r4 = ip4_hdr->ihl*4 (length of ipv4 header)
184 0 : *(code++) = FD_EBPF( jlt_imm, r4, 20, LBL_PASS ); // if r4<20 goto LBL_PASS
185 0 : *(code++) = FD_EBPF( add64_reg, r4, r2 ); // r4 = start of udp_hdr
186 :
187 : /*
188 : * At this point:
189 : * r1: &xdp_md
190 : * r2: start of ip4_hdr (inner ip4_hdr for gre)
191 : * r3: xdp_md->data_end
192 : * r4: start of udp_hdr
193 : * r5: clobber
194 : */
195 :
196 : /* udp check */
197 0 : ulong * udp_check = code;
198 :
199 : /* check ip4's dst addr */
200 0 : if( listen_ip4_addr!=0 ) {
201 0 : *(code++) = FD_EBPF( ldxw, r5, r2, 16 );
202 0 : *(code++) = FD_EBPF( jne32_imm, r5, listen_ip4_addr, LBL_PASS ); // if ip4->daddr != listen_ip4_addr goto LBL_PASS
203 0 : }
204 :
205 : /* Advance r2 to start of udp_hdr */
206 0 : *(code++) = FD_EBPF( mov64_reg, r2, r4 );
207 :
208 : /* bound check udp hdr access */
209 0 : *(code++) = FD_EBPF( add64_imm, r4, 8 ); // r4 += sizeof(udp_hdr) = 1 byte pass the end of udp_hdr
210 0 : *(code++) = FD_EBPF( jgt_reg, r4, r3, LBL_PASS ); // if (end of udp_hdr + 1) > r3 goto LBL_PASS
211 :
212 : /* get destination port from udp_hdr */
213 0 : *(code++) = FD_EBPF( ldxh, r4, r2, 2 ); // r4 = udp_hdr->dst_port
214 :
215 : /* loop through the ports array and find a match with dst_port */
216 0 : for( ulong i=0UL; i<ports_cnt; i++ ) {
217 0 : ushort port = (ushort)fd_ushort_bswap( ports[ i ] );
218 0 : if( !port ) continue;
219 0 : *(code++) = FD_EBPF( jeq_imm, r4, port, LBL_REDIRECT ); // if dst_port == ports[i] goto LBL_REDIRECT
220 0 : }
221 :
222 0 : ulong * lbl_pass = code;
223 0 : *(code++) = FD_EBPF( mov64_imm, r0, XDP_PASS );
224 0 : *(code++) = FD_EBPF_exit; // return XDP_PASS
225 0 : ulong * lbl_redirect = code;
226 0 : *(code++) = FD_EBPF( ldxw, r2, r1, 16 ); // r2 = xdp_md->rx_queue_index
227 0 : *(code++) = FD_EBPF( lddw, r1, xsks_fd ); // r1 = xsk_map_fd ll
228 0 : *(code++) = 0;
229 0 : *(code++) = FD_EBPF( mov64_imm, r3, 0 ); // r3 = 0
230 0 : *(code++) = FD_EBPF( call, 0x33 );
231 0 : *(code++) = FD_EBPF_exit; // return bpf_redirect_map(r1,r2,r3)
232 :
233 0 : ulong * code_end = code;
234 0 : ulong code_cnt = (ulong)( code_end-code_buf );
235 :
236 : /* Fill in jump labels */
237 :
238 0 : for( ulong i=0UL; i<code_cnt; i++ ) {
239 0 : ulong ix_class = code_buf[ i ] & 0x07UL;
240 0 : if( (ix_class==5UL) | (ix_class==6UL) ) {
241 0 : ulong * jmp_target = 0;
242 0 : uint jmp_label = (code_buf[ i ]>>16) & 0xFFFF;
243 0 : switch( jmp_label ) {
244 0 : case 0: continue;
245 0 : case LBL_PASS: jmp_target = lbl_pass; break;
246 0 : case LBL_REDIRECT: jmp_target = lbl_redirect; break;
247 0 : case LBL_GRE_CHECK: jmp_target = gre_check; break;
248 0 : case LBL_UDP_CHECK: jmp_target = udp_check; break;
249 0 : case LBL_ETH_IP4: jmp_target = eth_ip4; break;
250 0 : default: FD_LOG_ERR(( "Invalid jump instruction (%016lx)", fd_ulong_bswap( code_buf[ i ] ) ));
251 0 : }
252 0 : long off = jmp_target-code_buf-(long)i-1;
253 0 : ushort off_u = (ushort)(short)off;
254 0 : code_buf[ i ] = (code_buf[ i ] & 0xFFFFFFFF0000FFFF) | ((ulong)off_u<<16UL);
255 0 : }
256 0 : }
257 :
258 0 : FD_LOG_HEXDUMP_DEBUG(( "XDP program", code_buf, code_cnt*sizeof(ulong) ));
259 :
260 :
261 0 : #undef LBL_PASS
262 0 : #undef LBL_REDIRECT
263 :
264 0 : #undef LBL_GRE_CHECK
265 0 : #undef LBL_UDP_CHECK
266 0 : #undef LBL_ETH_IP4
267 0 : return code_cnt;
268 0 : }
269 :
270 : fd_xdp_fds_t
271 : fd_xdp_install( uint if_idx,
272 : uint listen_ip4_addr,
273 : ulong ports_cnt,
274 : ushort const * ports,
275 0 : char const * xdp_mode ) {
276 : /* Check args */
277 :
278 0 : uint uxdp_mode = 0;
279 0 : if( !strcmp( xdp_mode, "skb" ) ) uxdp_mode = XDP_FLAGS_SKB_MODE;
280 0 : else if( !strcmp( xdp_mode, "drv" ) ) uxdp_mode = XDP_FLAGS_DRV_MODE;
281 0 : else if( !strcmp( xdp_mode, "hw" ) ) uxdp_mode = XDP_FLAGS_HW_MODE;
282 0 : else if( !strcmp( xdp_mode, "default" ) ) uxdp_mode = 0U;
283 0 : else FD_LOG_ERR(( "unknown XDP mode `%s`", xdp_mode ));
284 :
285 0 : uint true_port_cnt = 0U;
286 0 : for( ulong i=0UL; i<ports_cnt; i++ ) true_port_cnt += !!ports[ i ];
287 0 : if( FD_UNLIKELY( !true_port_cnt ) ) FD_LOG_ERR(( "XDP program is not listening on any UDP ports" ));
288 :
289 : /* Create XSK map */
290 :
291 0 : union bpf_attr attr2;
292 0 : memset( &attr2, 0, sizeof(union bpf_attr) );
293 0 : attr2.map_type = BPF_MAP_TYPE_XSKMAP;
294 0 : attr2.key_size = 4U;
295 0 : attr2.value_size = 4U;
296 0 : attr2.max_entries = 256U;
297 0 : fd_memcpy( attr2.map_name, "fd_xdp_xsks", sizeof("fd_xdp_xsks") );
298 0 : int xsk_map_fd = (int)bpf( BPF_MAP_CREATE, &attr2, sizeof(union bpf_attr) );
299 0 : if( FD_UNLIKELY( -1==xsk_map_fd ) ) FD_LOG_ERR(( "Failed to create XSKMAP (%i-%s)", errno, fd_io_strerror( errno ) ));
300 :
301 : /* Load eBPF program into kernel */
302 :
303 0 : ulong code_buf[ 512 ];
304 0 : ulong code_cnt = fd_xdp_gen_program( code_buf, xsk_map_fd, listen_ip4_addr, ports, ports_cnt, 1 );
305 :
306 0 : char ebpf_kern_log[ 32768UL ];
307 :
308 0 : union bpf_attr attr;
309 0 : memset( &attr, 0, sizeof(union bpf_attr) );
310 0 : attr.prog_type = BPF_PROG_TYPE_XDP;
311 0 : attr.insn_cnt = (uint)code_cnt;
312 0 : attr.insns = (ulong)code_buf;
313 0 : attr.license = (ulong)FD_LICENSE;
314 0 : attr.log_level = 6;
315 0 : attr.log_size = 32768UL;
316 0 : attr.log_buf = (ulong)ebpf_kern_log;
317 :
318 0 : int prog_fd = (int)bpf( BPF_PROG_LOAD, &attr, sizeof(union bpf_attr) );
319 0 : if( FD_UNLIKELY( -1==prog_fd ) ) {
320 0 : FD_LOG_WARNING(( "bpf(BPF_PROG_LOAD) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
321 0 : FD_LOG_ERR(( "eBPF verifier log:\n%s", ebpf_kern_log ));
322 0 : }
323 :
324 : /* Install program to device */
325 :
326 0 : struct bpf_link_create link_create = {
327 0 : .prog_fd = (uint)prog_fd,
328 0 : .target_ifindex = if_idx,
329 0 : .attach_type = BPF_XDP,
330 0 : .flags = uxdp_mode
331 0 : };
332 :
333 0 : int prog_link_fd = (int)bpf( BPF_LINK_CREATE, fd_type_pun( &link_create ), sizeof(struct bpf_link_create) );
334 0 : if( FD_UNLIKELY( -1==prog_link_fd ) ) {
335 0 : if( FD_LIKELY( errno==ENOSYS ) ) {
336 0 : FD_LOG_ERR(( "BPF_LINK_CREATE is not supported by your kernel (%i-%s). Firedancer requires a Linux "
337 0 : "kernel version of v5.7 or newer to support fast XDP networking. Please upgrade to a newer "
338 0 : "kernel version.", errno, fd_io_strerror( errno ) ));
339 0 : } else if( FD_LIKELY( errno==EINVAL ) ) {
340 0 : char if_name[ IF_NAMESIZE ] = {0};
341 0 : FD_LOG_ERR(( "BPF_LINK_CREATE failed on interface %s (%i-%s). This likely means the network device "
342 0 : "does not have support for XDP. If the device is a bonding device, you will need "
343 0 : "a kernel version of v5.15 or newer. For other devices, see the list of kernel "
344 0 : "support at https://github.com/iovisor/bcc/blob/master/docs/kernel-versions.md#xdp",
345 0 : if_indextoname( if_idx, if_name ), errno, fd_io_strerror( errno ) ));
346 0 : } else {
347 0 : FD_LOG_ERR(( "BPF_LINK_CREATE(if_idx=%u) failed (%i-%s)", if_idx, errno, fd_io_strerror( errno ) ));
348 0 : }
349 0 : }
350 :
351 0 : if( FD_UNLIKELY( -1==close( prog_fd ) ) ) FD_LOG_ERR(( "close(%d) failed (%i-%s)", prog_fd, errno, fd_io_strerror( errno ) ));
352 :
353 0 : return (fd_xdp_fds_t){
354 0 : .xsk_map_fd = xsk_map_fd,
355 0 : .prog_link_fd = prog_link_fd,
356 0 : };
357 0 : }
|
