Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 0 : # pragma GCC diagnostic push
13 0 : # pragma GCC diagnostic ignored "-Wpedantic"
14 0 : # endif
15 :
16 0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 0 : # pragma clang diagnostic push
18 0 : # pragma clang diagnostic ignored "-Wpedantic"
19 0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 0 : # endif
21 :
22 : /* Include the jump table */
23 :
24 4223 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 0 : ulong sbpf_version = vm->sbpf_version;
29 :
30 : /* Unpack the VM state */
31 :
32 0 : ulong pc = vm->pc;
33 0 : ulong ic = vm->ic;
34 0 : ulong cu = vm->cu;
35 0 : ulong frame_cnt = vm->frame_cnt;
36 :
37 0 : void const * const * const version_interp_jump_table = interp_jump_table[ sbpf_version ];
38 :
39 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
40 : pc, parses it, fetches the associated register values and then
41 : jumps to the code that executes the instruction. On normal
42 : instruction execution, the pc will be updated and
43 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
44 : instruction. After a normal halt, this will branch to interp_halt.
45 : Otherwise, it will branch to the appropriate normal termination. */
46 :
47 0 : ulong instr;
48 0 : ulong opcode;
49 0 : ulong dst;
50 0 : ulong src;
51 0 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
52 0 : uint imm;
53 0 : ulong reg_dst;
54 0 : ulong reg_src;
55 :
56 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
57 :
58 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
59 :
60 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
61 : */
62 305506 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 63 ) ))
63 :
64 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
65 :
66 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
67 : */
68 69717 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
69 :
70 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
71 :
72 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
73 : */
74 126309 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 31 ) ))
75 :
76 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
77 :
78 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
79 : */
80 688 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
81 :
82 : /* i32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
83 :
84 : https://doc.rust-lang.org/std/primitive.i32.html#method.wrapping_shr
85 : */
86 23222 : #define FD_RUST_INT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
87 :
88 : /* i64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
89 :
90 : https://doc.rust-lang.org/std/primitive.i64.html#method.wrapping_shr
91 : */
92 10703 : #define FD_RUST_LONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
93 :
94 :
95 0 : # define FD_VM_INTERP_INSTR_EXEC \
96 13423024 : if( FD_UNLIKELY( pc>=block_text_limit ) ) goto sigtext_or_sigcost; /* Note: untaken branches don't consume BTB */ \
97 13423024 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
98 13421965 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
99 13421965 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
100 13421965 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
101 13421965 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
102 13421965 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
103 13421965 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
104 13421965 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
105 13421965 : goto *version_interp_jump_table[ opcode ] /* Guaranteed in-bounds */
106 :
107 : /* FD_VM_INTERP_SYSCALL_EXEC
108 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
109 :
110 : Setup.
111 : Update the vm with the current vm execution state for the
112 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
113 : already updated ic and cu to reflect all instructions up to
114 : and including the syscall instruction itself.
115 :
116 : Execution.
117 : Do the syscall. The return value is placed into vm->reg[0].
118 : At this point, vm->cu is positive.
119 :
120 : Error handling.
121 : If we trust syscall implementations to handle the vm state
122 : correctly, the below could be implemented as unpacking the vm
123 : state and jumping to sigsys on error. But we provide some
124 : extra protection to make various strong guarantees:
125 :
126 : - We do not let the syscall modify pc currently as nothing
127 : requires this and it reduces risk of a syscall bug mucking
128 : up the interpreter. If there ever was a syscall that
129 : needed to modify the pc (e.g. a syscall that has execution
130 : resume from a different location than the instruction
131 : following the syscall), do "pc = vm->pc" below.
132 :
133 : - We do not let the syscall modify ic currently as nothing
134 : requires this and it keeps the ic precise. If a future
135 : syscall needs this, do "ic = vm->ic" below.
136 :
137 : - We do not let the syscall increase cu as nothing requires
138 : this and it guarantees the interpreter will halt in a
139 : reasonable finite amount of time. If a future syscall
140 : needs this, do "cu = vm->cu" below.
141 :
142 : - A syscall that returns SIGCOST is always treated as though
143 : it also zerod cu.
144 :
145 : At this point, vm->cu is whatever the syscall tried to set
146 : and cu is positive.
147 :
148 : Exit
149 : At this point, cu is positive and err is clear.
150 : */
151 :
152 0 : # define FD_VM_INTERP_SYSCALL_EXEC_DUMP \
153 : /* Dumping for debugging purposes */ \
154 10039 : if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) { \
155 0 : fd_dump_vm_syscall_to_protobuf( vm, syscall->name ); \
156 0 : }
157 :
158 0 : # define FD_VM_INTERP_SYSCALL_EXEC \
159 : /* Setup */ \
160 10039 : vm->pc = pc; \
161 10039 : vm->ic = ic; \
162 10039 : vm->cu = cu; \
163 10039 : vm->frame_cnt = frame_cnt; \
164 10039 : FD_VM_INTERP_SYSCALL_EXEC_DUMP \
165 : /* Execution */ \
166 10039 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5] ); \
167 : /* Error handling */ \
168 10039 : ulong cu_req = vm->cu; \
169 10039 : cu = fd_ulong_min( cu_req, cu ); \
170 10039 : if( FD_UNLIKELY( err ) ) { \
171 947 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
172 947 : FD_VM_TEST_ERR_EXISTS( vm ); \
173 947 : goto sigsyscall; \
174 947 : } \
175 : /* Exit */
176 :
177 :
178 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
179 : implementation for an opcode that does not branch. On entry, the
180 : instruction word has been unpacked into dst / src / offset / imm
181 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
182 :
183 8224169 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
184 :
185 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
186 8222737 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
187 : # else /* Use this version when tracing or optimizing code footprint */
188 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
189 : # endif
190 :
191 : /* Instead of doing a lot of compute budget calcs and tests every
192 : instruction, we note that the program counter increases
193 : monotonically after a branch (or a program start) until the next
194 : branch (or program termination). We save the program counter of
195 : the start of such a segment in pc0. Whenever we encounter a branch
196 : (or a program termination) at pc, we know we processed pc-pc0+1
197 : text words (including the text word for the branch instruction
198 : itself as all branch instructions are single word).
199 :
200 : Each instruction costs 1 cu (syscalls can cost extra on top of
201 : this that is accounted separately in CALL_IMM below). Since there
202 : could have been multiword instructions in this segment, at start of
203 : such a segment, we zero out the accumulator ic_correction and have
204 : every multiword instruction in the segment accumulate the number of
205 : extra text words it has to this variable. (Sigh ... it would be a
206 : lot simpler to bill based on text words processed but this would be
207 : very difficult to make this protocol change at this point.)
208 :
209 : When we encounter a branch at pc, the number of instructions
210 : processed (and thus the number of compute units to bill for that
211 : segment) is thus:
212 :
213 : pc - pc0 + 1 - ic_correction */
214 :
215 0 : ulong pc0 = pc;
216 0 : ulong ic_correction = 0UL;
217 :
218 : /* Compute the text limit for a basic block. This bounds the number
219 : of instructions that the basic block can execute linearly.
220 :
221 : There are two factors: the remaining CU budget as of the start of
222 : the basic block, and the text_cnt. The tightest bound is the
223 : lesser of the two.
224 :
225 : We use 2*(the remaining CU budget) because each LDDW instruction
226 : costs 1 CU, but occupies two text words, so in the worst case (a
227 : block full of LDDW instructions) the block text limit should be
228 : doubled.
229 :
230 : It is safe to not use saturating addition here. Both addends are
231 : bounded far below ULONG_MAX before any block executes:
232 :
233 : pc0 <= text_cnt <= 1310720 (see fd_sbpf_loader.h)
234 : cu <= 1400000 (FD_MAX_COMPUTE_UNIT_LIMIT)
235 :
236 : pc0 + cu + cu + 1 < 1310720 + 1400000 + 1400000 + 1 < ULONG_MAX */
237 :
238 5199236 : # define FD_VM_INTERP_BLOCK_TEXT_LIMIT fd_ulong_min( text_cnt, pc0+cu+cu+1UL )
239 :
240 4223 : ulong block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT;
241 :
242 0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
243 5197776 : interp_##opcode: \
244 : /* Bill linear text segment and this branch instruction as per the above */ \
245 5197776 : ic_correction = pc - pc0 + 1UL - ic_correction; \
246 5197776 : ic += ic_correction; \
247 5197776 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
248 5197776 : cu -= ic_correction; \
249 : /* At this point, cu>=0 */ \
250 5197751 : ic_correction = 0UL;
251 :
252 : /* FIXME: debatable if it is better to do pc++ here or have the
253 : instruction implementations do it in their code path. */
254 :
255 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
256 : # define FD_VM_INTERP_BRANCH_END \
257 5195013 : pc++; \
258 5195013 : pc0 = pc; /* Start a new linear segment */ \
259 5195013 : block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT; \
260 5196064 : FD_VM_INTERP_INSTR_EXEC
261 : # else /* Use this version when tracing or optimizing code footprint */
262 : # define FD_VM_INTERP_BRANCH_END \
263 0 : pc++; \
264 0 : pc0 = pc; /* Start a new linear segment */ \
265 0 : block_text_limit = FD_VM_INTERP_BLOCK_TEXT_LIMIT; \
266 : /* FIXME: TEST sigsplit HERE */ \
267 0 : goto interp_exec
268 : # endif
269 :
270 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
271 : advances reg[10] to a new user stack frame. If there are no more
272 : stack frames available, will do a SIGSTACK. */
273 :
274 : /* FIXME: double check faulting is desired on stack overflow. */
275 :
276 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
277 : to prevent byte code from accessing the stack outside its current
278 : stack frame. But this would break the common practice of a
279 : function passing a pointer to something on its stack into a
280 : function that it calls:
281 :
282 : void foo( ... ) {
283 : ...
284 : int ret;
285 : bar( &ret );
286 : ...
287 : }
288 :
289 : So this probably shouldn't be done. But, if it is in fact
290 : necessary, the TLB updates would be here and in pop. */
291 :
292 : /* FIXME: unvalidated code mucking with r10 */
293 :
294 0 : # define FD_VM_INTERP_STACK_PUSH \
295 47418 : shadow[ frame_cnt ].r6 = reg[6]; \
296 47418 : shadow[ frame_cnt ].r7 = reg[7]; \
297 47418 : shadow[ frame_cnt ].r8 = reg[8]; \
298 47418 : shadow[ frame_cnt ].r9 = reg[9]; \
299 47418 : shadow[ frame_cnt ].r10 = reg[10]; \
300 47418 : shadow[ frame_cnt ].pc = pc; \
301 47418 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
302 47418 : reg[10] += vm->stack_frame_sz * vm->stack_push_frame_count; \
303 : /* We subtract the heap cost in the BPF loader */
304 :
305 0 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
306 4223 : interp_exec:
307 :
308 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
309 : /* Note: when tracing or optimizing for code footprint, all
310 : instruction execution starts here such that this is only point
311 : where exe tracing diagnostics are needed. */
312 0 : if( FD_UNLIKELY( pc>=block_text_limit ) ) goto sigtext_or_sigcost;
313 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
314 0 : # endif
315 :
316 4223 : FD_VM_INTERP_INSTR_EXEC;
317 :
318 : /* 0x00 - 0x0f ******************************************************/
319 :
320 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
321 65 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
322 65 : FD_VM_INTERP_INSTR_END;
323 :
324 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
325 482937 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
326 482937 : FD_VM_INTERP_INSTR_END;
327 :
328 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
329 876333 : pc += offset;
330 876333 : FD_VM_INTERP_BRANCH_END;
331 :
332 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
333 493011 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
334 493011 : FD_VM_INTERP_INSTR_END;
335 :
336 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
337 56 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
338 56 : FD_VM_INTERP_INSTR_END;
339 :
340 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
341 19881 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
342 19881 : FD_VM_INTERP_INSTR_END;
343 :
344 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
345 240420 : reg[ dst ] = reg_dst + reg_src;
346 240420 : FD_VM_INTERP_INSTR_END;
347 :
348 : /* 0x10 - 0x1f ******************************************************/
349 :
350 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
351 95 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
352 95 : FD_VM_INTERP_INSTR_END;
353 :
354 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
355 20166 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
356 20166 : FD_VM_INTERP_INSTR_END;
357 :
358 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
359 108222 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
360 108222 : FD_VM_INTERP_BRANCH_END;
361 :
362 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
363 89 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
364 89 : FD_VM_INTERP_INSTR_END;
365 :
366 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
367 306 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
368 306 : FD_VM_INTERP_INSTR_END;
369 :
370 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
371 721815 : pc++;
372 721815 : ic_correction++;
373 : /* No need to check pc because it's already checked during validation.
374 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
375 721815 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
376 721815 : FD_VM_INTERP_INSTR_END;
377 :
378 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
379 77 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
380 77 : FD_VM_INTERP_INSTR_END;
381 :
382 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
383 215203 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
384 215203 : FD_VM_INTERP_INSTR_END;
385 :
386 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
387 15211 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
388 15211 : FD_VM_INTERP_BRANCH_END;
389 :
390 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
391 22453 : reg[ dst ] = reg_dst - reg_src;
392 22453 : FD_VM_INTERP_INSTR_END;
393 :
394 : /* 0x20 - 0x2f ******************************************************/
395 :
396 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
397 209272 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
398 209272 : FD_VM_INTERP_INSTR_END;
399 :
400 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
401 30076 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
402 30076 : FD_VM_INTERP_BRANCH_END;
403 :
404 31547 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
405 31547 : ulong vaddr = reg_dst + offset;
406 31547 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
407 31547 : if( FD_UNLIKELY( !haddr ) ) {
408 6 : vm->segv_vaddr = vaddr;
409 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
410 6 : vm->segv_access_len = 1UL;
411 6 : goto sigsegv;
412 6 : } /* Note: untaken branches don't consume BTB */
413 31541 : fd_vm_mem_st_1( haddr, (uchar)imm );
414 31541 : }
415 31541 : FD_VM_INTERP_INSTR_END;
416 :
417 96707 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
418 96707 : ulong vaddr = reg_src + offset;
419 96707 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL );
420 96707 : if( FD_UNLIKELY( !haddr ) ) {
421 9 : vm->segv_vaddr = vaddr;
422 9 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
423 9 : vm->segv_access_len = 1UL;
424 9 : goto sigsegv;
425 9 : } /* Note: untaken branches don't consume BTB */
426 96698 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
427 96698 : }
428 96698 : FD_VM_INTERP_INSTR_END;
429 :
430 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
431 126556 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
432 126556 : FD_VM_INTERP_BRANCH_END;
433 :
434 89053 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
435 89053 : ulong vaddr = reg_dst + offset;
436 89053 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
437 89053 : if( FD_UNLIKELY( !haddr ) ) {
438 16 : vm->segv_vaddr = vaddr;
439 16 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
440 16 : vm->segv_access_len = 1UL;
441 16 : goto sigsegv;
442 16 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
443 89037 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
444 89037 : }
445 89037 : FD_VM_INTERP_INSTR_END;
446 :
447 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
448 15741 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
449 15741 : FD_VM_INTERP_INSTR_END;
450 :
451 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
452 58223 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
453 58223 : FD_VM_INTERP_INSTR_END;
454 :
455 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
456 74746 : reg[ dst ] = reg_dst * reg_src;
457 74746 : FD_VM_INTERP_INSTR_END;
458 :
459 : /* 0x30 - 0x3f ******************************************************/
460 :
461 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
462 49993 : /* FIXME: convert to a multiply at validation time (usually probably
463 49993 : not worth it) */
464 49993 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
465 49993 : FD_VM_INTERP_INSTR_END;
466 :
467 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
468 612 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
469 612 : FD_VM_INTERP_BRANCH_END;
470 :
471 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
472 54 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
473 54 : FD_VM_INTERP_INSTR_END;
474 :
475 36571 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
476 36571 : ulong vaddr = reg_dst + offset;
477 36571 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
478 36571 : int sigsegv = !haddr;
479 36571 : if( FD_UNLIKELY( sigsegv ) ) {
480 4 : vm->segv_vaddr = vaddr;
481 4 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
482 4 : vm->segv_access_len = 2UL;
483 4 : goto sigsegv;
484 4 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
485 36567 : fd_vm_mem_st_2( haddr, (ushort)imm );
486 36567 : }
487 36567 : FD_VM_INTERP_INSTR_END;
488 :
489 2939 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
490 2939 : ulong vaddr = reg_src + offset;
491 2939 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL );
492 2939 : int sigsegv = !haddr;
493 2939 : if( FD_UNLIKELY( sigsegv ) ) {
494 10 : vm->segv_vaddr = vaddr;
495 10 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
496 10 : vm->segv_access_len = 2UL;
497 10 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
498 10 : }
499 2929 : reg[ dst ] = fd_vm_mem_ld_2( haddr );
500 2929 : }
501 2929 : FD_VM_INTERP_INSTR_END;
502 :
503 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
504 584701 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
505 584701 : FD_VM_INTERP_BRANCH_END;
506 :
507 1931 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
508 1931 : ulong vaddr = reg_dst + offset;
509 1931 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
510 1931 : int sigsegv = !haddr;
511 1931 : if( FD_UNLIKELY( sigsegv ) ) {
512 5 : vm->segv_vaddr = vaddr;
513 5 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
514 5 : vm->segv_access_len = 2UL;
515 5 : goto sigsegv;
516 5 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
517 1926 : fd_vm_mem_st_2( haddr, (ushort)reg_src );
518 1926 : }
519 1926 : FD_VM_INTERP_INSTR_END;
520 :
521 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
522 87 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
523 87 : FD_VM_INTERP_INSTR_END;
524 :
525 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
526 55046 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
527 55046 : FD_VM_INTERP_INSTR_END;
528 :
529 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
530 549340 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
531 549119 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
532 549119 : FD_VM_INTERP_INSTR_END;
533 :
534 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
535 156042 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
536 155831 : reg[ dst ] = reg_dst / reg_src;
537 155831 : FD_VM_INTERP_INSTR_END;
538 :
539 : /* 0x40 - 0x4f ******************************************************/
540 :
541 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
542 45035 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
543 45035 : FD_VM_INTERP_INSTR_END;
544 :
545 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
546 313340 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
547 313340 : FD_VM_INTERP_BRANCH_END;
548 :
549 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
550 64 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
551 64 : FD_VM_INTERP_INSTR_END;
552 :
553 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
554 111361 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
555 111361 : FD_VM_INTERP_INSTR_END;
556 :
557 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
558 36616 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
559 36616 : FD_VM_INTERP_INSTR_END;
560 :
561 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
562 7 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
563 7 : FD_VM_INTERP_BRANCH_END;
564 :
565 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
566 52 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
567 10 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
568 10 : FD_VM_INTERP_INSTR_END;
569 :
570 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
571 74883 : reg[ dst ] = reg_dst | reg_src;
572 74883 : FD_VM_INTERP_INSTR_END;
573 :
574 : /* 0x50 - 0x5f ******************************************************/
575 :
576 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
577 63551 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
578 63551 : FD_VM_INTERP_INSTR_END;
579 :
580 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
581 300863 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
582 300863 : FD_VM_INTERP_BRANCH_END;
583 :
584 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
585 60 : reg[ dst ] = reg_dst / (ulong)imm;
586 60 : FD_VM_INTERP_INSTR_END;
587 :
588 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
589 156671 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
590 156671 : FD_VM_INTERP_INSTR_END;
591 :
592 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
593 39150 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
594 39150 : FD_VM_INTERP_INSTR_END;
595 :
596 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
597 1133617 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
598 1133617 : FD_VM_INTERP_BRANCH_END;
599 :
600 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
601 70 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
602 25 : reg[ dst ] = reg_dst / reg_src;
603 25 : FD_VM_INTERP_INSTR_END;
604 :
605 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
606 93550 : reg[ dst ] = reg_dst & reg_src;
607 93550 : FD_VM_INTERP_INSTR_END;
608 :
609 : /* 0x60 - 0x6f ******************************************************/
610 :
611 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
612 325 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
613 325 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
614 325 : FD_VM_INTERP_INSTR_END;
615 :
616 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
617 28119 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
618 28119 : FD_VM_INTERP_BRANCH_END;
619 :
620 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
621 57 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
622 57 : FD_VM_INTERP_INSTR_END;
623 :
624 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
625 278236 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
626 278236 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
627 278236 : FD_VM_INTERP_INSTR_END;
628 :
629 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
630 125984 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
631 125984 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
632 125984 : FD_VM_INTERP_INSTR_END;
633 :
634 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
635 12991 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
636 12991 : FD_VM_INTERP_BRANCH_END;
637 :
638 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
639 65 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
640 10 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
641 10 : FD_VM_INTERP_INSTR_END;
642 :
643 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
644 27270 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
645 27270 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
646 27270 : FD_VM_INTERP_INSTR_END;
647 :
648 : /* 0x70 - 0x7f ******************************************************/
649 :
650 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
651 327 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
652 327 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
653 327 : FD_VM_INTERP_INSTR_END;
654 :
655 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
656 212 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
657 212 : FD_VM_INTERP_BRANCH_END;
658 :
659 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
660 61 : reg[ dst ] = reg_dst % (ulong)imm;
661 61 : FD_VM_INTERP_INSTR_END;
662 :
663 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
664 62345 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
665 62345 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
666 62345 : FD_VM_INTERP_INSTR_END;
667 :
668 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
669 361 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
670 361 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
671 361 : FD_VM_INTERP_INSTR_END;
672 :
673 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
674 38653 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
675 38653 : FD_VM_INTERP_BRANCH_END;
676 :
677 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
678 64 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
679 18 : reg[ dst ] = reg_dst % reg_src;
680 18 : FD_VM_INTERP_INSTR_END;
681 :
682 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
683 7372 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
684 7372 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
685 7372 : FD_VM_INTERP_INSTR_END;
686 :
687 : /* 0x80-0x8f ********************************************************/
688 :
689 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
690 393 : reg[ dst ] = (ulong)( -(uint)reg_dst );
691 393 : FD_VM_INTERP_INSTR_END;
692 :
693 54892 : FD_VM_INTERP_BRANCH_BEGIN(0x85) { /* FD_SBPF_OP_CALL_IMM */
694 :
695 54892 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
696 54892 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
697 :
698 : /* Note we do the stack push before updating the pc(*). This implies
699 : that the call stack frame gets allocated _before_ checking if the
700 : call target is valid. It would be fine to switch the order
701 : though such would change the precise faulting semantics of
702 : sigtextbr and sigstack.
703 :
704 : (*)but after checking calldests, see point below. */
705 :
706 : /* Agave's order of checks
707 : (https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L565-L572):
708 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
709 : 2. Push stack frame
710 : 3. Check PC
711 : 4. Update PC
712 :
713 : Following this precisely is impossible as our PC check also
714 : serves as a bounds check for the calldests_test call. So we
715 : have to perform step 3 before step 1. The following
716 : is a best-effort implementation that should match the VM state
717 : in all ways except error code. */
718 :
719 : /* Special case to handle entrypoint.
720 : ebpf::hash_symbol_name(b"entrypoint") = 0x71e3cf81, and
721 : fd_pchash_inverse( 0x71e3cf81U ) = 0xb00c380U */
722 46130 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
723 0 : FD_VM_INTERP_STACK_PUSH;
724 0 : pc = entry_pc - 1;
725 46130 : } else {
726 46130 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
727 46130 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
728 90 : goto sigillbr; /* different return between 0x85 and 0x8d */
729 90 : }
730 46040 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
731 0 : goto sigillbr;
732 0 : }
733 92079 : FD_VM_INTERP_STACK_PUSH;
734 92079 : pc = target_pc - 1;
735 92079 : }
736 :
737 46130 : } else {
738 :
739 8762 : FD_VM_INTERP_SYSCALL_EXEC;
740 :
741 7937 : }
742 54892 : } FD_VM_INTERP_BRANCH_END;
743 :
744 : /* SIMD-0178: Static syscalls (SBPF V3+)
745 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L542-L577 */
746 53976 : FD_VM_INTERP_BRANCH_BEGIN(0x85_static) { /* FD_SBPF_OP_CALL_IMM (static syscalls) */
747 :
748 1300 : if( src == 0 ) {
749 : /* External syscall
750 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L545-L553 */
751 1290 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
752 1290 : if( FD_UNLIKELY( !syscall ) ) goto sigillbr;
753 2432 : FD_VM_INTERP_SYSCALL_EXEC;
754 2432 : } else if( src == 1 ) {
755 : /* Internal call
756 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L555-L563
757 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L97-L103 */
758 10 : long target_pc_l = fd_long_sat_add( (long)pc, fd_long_sat_add( (long)(int)imm, 1L ) );
759 10 : if( FD_UNLIKELY( target_pc_l<0L || (ulong)target_pc_l>=text_cnt ) ) goto sigillbr;
760 20 : FD_VM_INTERP_STACK_PUSH;
761 20 : pc = (ulong)target_pc_l - 1;
762 20 : } else {
763 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L574-L576 */
764 0 : goto sigillbr;
765 0 : }
766 :
767 1300 : } FD_VM_INTERP_BRANCH_END;
768 :
769 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
770 59 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
771 59 : FD_VM_INTERP_INSTR_END;
772 :
773 1559 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
774 1559 : ulong vaddr = reg_dst + offset;
775 1559 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
776 1559 : int sigsegv = !haddr;
777 1559 : if( FD_UNLIKELY( sigsegv ) ) {
778 5 : vm->segv_vaddr = vaddr;
779 5 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
780 5 : vm->segv_access_len = 4UL;
781 5 : goto sigsegv;
782 5 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
783 1554 : fd_vm_mem_st_4( haddr, imm );
784 1554 : } FD_VM_INTERP_INSTR_END;
785 :
786 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
787 6556 : reg[ dst ] = -reg_dst;
788 6556 : FD_VM_INTERP_INSTR_END;
789 :
790 8500 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
791 8500 : ulong vaddr = reg_src + offset;
792 8500 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL );
793 8500 : int sigsegv = !haddr;
794 8500 : if( FD_UNLIKELY( sigsegv ) ) {
795 7 : vm->segv_vaddr = vaddr;
796 7 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
797 7 : vm->segv_access_len = 4UL;
798 7 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
799 7 : }
800 8493 : reg[ dst ] = fd_vm_mem_ld_4( haddr );
801 8493 : }
802 8493 : FD_VM_INTERP_INSTR_END;
803 :
804 8493 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
805 :
806 1368 : FD_VM_INTERP_STACK_PUSH;
807 :
808 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L528-L540 */
809 1368 : ulong vaddr = fd_sbpf_callx_uses_src_reg_enabled( sbpf_version ) ? reg_src
810 1368 : : fd_sbpf_callx_uses_dst_reg_enabled( sbpf_version ) ? reg[ dst ]
811 1355 : : reg[ imm & 15U ];
812 :
813 : /* Notes: Agave checks region and target_pc before updating the pc.
814 : To match their state, we do the same, even though we could simply
815 : update the pc and let BRANCH_END fail.
816 : Also, Agave doesn't check alignment. */
817 :
818 1368 : ulong region = vaddr >> 32;
819 : /* ulong align = vaddr & 7UL; */
820 1368 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
821 1368 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
822 1312 : pc = target_pc - 1;
823 :
824 1312 : } FD_VM_INTERP_BRANCH_END;
825 :
826 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
827 61 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
828 61 : FD_VM_INTERP_INSTR_END;
829 :
830 17115 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
831 17115 : ulong vaddr = reg_dst + offset;
832 17115 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
833 17115 : int sigsegv = !haddr;
834 17115 : if( FD_UNLIKELY( sigsegv ) ) {
835 16 : vm->segv_vaddr = vaddr;
836 16 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
837 16 : vm->segv_access_len = 4UL;
838 16 : goto sigsegv;
839 16 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
840 17099 : fd_vm_mem_st_4( haddr, (uint)reg_src );
841 17099 : }
842 17099 : FD_VM_INTERP_INSTR_END;
843 :
844 : /* 0x90 - 0x9f ******************************************************/
845 :
846 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
847 2857 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
848 2857 : FD_VM_INTERP_INSTR_END;
849 :
850 : FD_VM_INTERP_BRANCH_BEGIN(0x95) /* FD_SBPF_OP_EXIT */
851 48147 : /* Agave JIT VM exit implementation analysis below.
852 48147 :
853 48147 : Agave references:
854 48147 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
855 48147 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
856 48147 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
857 46516 : frame_cnt--;
858 46516 : reg[6] = shadow[ frame_cnt ].r6;
859 46516 : reg[7] = shadow[ frame_cnt ].r7;
860 46516 : reg[8] = shadow[ frame_cnt ].r8;
861 46516 : reg[9] = shadow[ frame_cnt ].r9;
862 46516 : reg[10] = shadow[ frame_cnt ].r10;
863 46516 : pc = shadow[ frame_cnt ].pc;
864 46516 : FD_VM_INTERP_BRANCH_END;
865 :
866 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
867 77 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
868 77 : FD_VM_INTERP_INSTR_END;
869 :
870 6738 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
871 6738 : ulong vaddr = reg_dst + offset;
872 6738 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
873 6738 : int sigsegv = !haddr;
874 6738 : if( FD_UNLIKELY( sigsegv ) ) {
875 7 : vm->segv_vaddr = vaddr;
876 7 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
877 7 : vm->segv_access_len = 8UL;
878 7 : goto sigsegv;
879 7 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
880 6731 : fd_vm_mem_st_8( haddr, (ulong)(long)(int)imm );
881 6731 : }
882 6731 : FD_VM_INTERP_INSTR_END;
883 :
884 289623 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
885 289623 : ulong vaddr = reg_src + offset;
886 289623 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL );
887 289623 : int sigsegv = !haddr;
888 289623 : if( FD_UNLIKELY( sigsegv ) ) {
889 84 : vm->segv_vaddr = vaddr;
890 84 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
891 84 : vm->segv_access_len = 8UL;
892 84 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
893 84 : }
894 289539 : reg[ dst ] = fd_vm_mem_ld_8( haddr );
895 289539 : }
896 289539 : FD_VM_INTERP_INSTR_END;
897 :
898 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
899 92 : reg[ dst ] = reg_dst * reg_src;
900 92 : FD_VM_INTERP_INSTR_END;
901 :
902 537174 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
903 537174 : ulong vaddr = reg_dst + offset;
904 537174 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
905 537174 : int sigsegv = !haddr;
906 537174 : if( FD_UNLIKELY( sigsegv ) ) {
907 18 : vm->segv_vaddr = vaddr;
908 18 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
909 18 : vm->segv_access_len = 8UL;
910 18 : goto sigsegv;
911 18 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
912 537156 : fd_vm_mem_st_8( haddr, reg_src );
913 537156 : }
914 537156 : FD_VM_INTERP_INSTR_END;
915 :
916 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
917 56507 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
918 56507 : FD_VM_INTERP_INSTR_END;
919 :
920 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
921 36034 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
922 35786 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
923 35786 : FD_VM_INTERP_INSTR_END;
924 :
925 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
926 36635 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
927 36446 : reg[ dst ] = reg_dst % reg_src;
928 36446 : FD_VM_INTERP_INSTR_END;
929 :
930 : /* 0xa0 - 0xaf ******************************************************/
931 :
932 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
933 120507 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
934 120507 : FD_VM_INTERP_INSTR_END;
935 :
936 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
937 133798 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
938 133798 : FD_VM_INTERP_BRANCH_END;
939 :
940 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
941 6428 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
942 6428 : FD_VM_INTERP_INSTR_END;
943 :
944 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
945 25225 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
946 25225 : FD_VM_INTERP_INSTR_END;
947 :
948 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
949 22935 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
950 22935 : FD_VM_INTERP_BRANCH_END;
951 :
952 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
953 284383 : reg[ dst ] = reg_dst ^ reg_src;
954 284383 : FD_VM_INTERP_INSTR_END;
955 :
956 : /* 0xb0 - 0xbf ******************************************************/
957 :
958 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
959 353044 : reg[ dst ] = (ulong)imm;
960 353044 : FD_VM_INTERP_INSTR_END;
961 :
962 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
963 379810 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
964 379810 : FD_VM_INTERP_BRANCH_END;
965 :
966 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
967 42 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
968 42 : FD_VM_INTERP_INSTR_END;
969 :
970 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
971 588424 : reg[ dst ] = (ulong)(long)(int)imm;
972 588424 : FD_VM_INTERP_INSTR_END;
973 :
974 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
975 377 : reg[ dst ] = (ulong)(long)(int)reg_src;
976 377 : FD_VM_INTERP_INSTR_END;
977 :
978 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
979 228259 : reg[ dst ] = (ulong)(uint)reg_src;
980 228259 : FD_VM_INTERP_INSTR_END;
981 :
982 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
983 784 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
984 784 : FD_VM_INTERP_BRANCH_END;
985 :
986 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
987 46 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
988 46 : FD_VM_INTERP_INSTR_END;
989 :
990 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
991 409960 : reg[ dst ] = reg_src;
992 409960 : FD_VM_INTERP_INSTR_END;
993 :
994 : /* 0xc0 - 0xcf ******************************************************/
995 :
996 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
997 373 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, imm ) );
998 373 : FD_VM_INTERP_INSTR_END;
999 :
1000 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
1001 3394 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
1002 3394 : FD_VM_INTERP_BRANCH_END;
1003 :
1004 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
1005 56 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1006 56 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
1007 56 : FD_VM_INTERP_INSTR_END;
1008 :
1009 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
1010 10341 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, imm ) );
1011 10341 : FD_VM_INTERP_INSTR_END;
1012 :
1013 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
1014 22849 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, (uint)reg_src ) );
1015 22849 : FD_VM_INTERP_INSTR_END;
1016 :
1017 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
1018 607153 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
1019 607153 : FD_VM_INTERP_BRANCH_END;
1020 :
1021 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
1022 52 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1023 12 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1024 12 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
1025 12 : FD_VM_INTERP_INSTR_END;
1026 :
1027 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
1028 362 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, reg_src ) );
1029 362 : FD_VM_INTERP_INSTR_END;
1030 :
1031 : /* 0xd0 - 0xdf ******************************************************/
1032 :
1033 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1034 112422 : switch( imm ) {
1035 105 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1036 36371 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1037 75946 : case 64U: break;
1038 0 : default: goto siginv;
1039 112422 : }
1040 112422 : FD_VM_INTERP_INSTR_END;
1041 :
1042 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1043 5050 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1044 5050 : FD_VM_INTERP_BRANCH_END;
1045 :
1046 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1047 64 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1048 64 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1049 64 : FD_VM_INTERP_INSTR_END;
1050 :
1051 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1052 144080 : switch( imm ) {
1053 33255 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1054 66296 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1055 44540 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1056 0 : default: goto siginv;
1057 144080 : }
1058 144080 : FD_VM_INTERP_INSTR_END;
1059 :
1060 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1061 369594 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1062 369594 : FD_VM_INTERP_BRANCH_END;
1063 :
1064 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1065 73 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1066 16 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1067 16 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1068 16 : FD_VM_INTERP_INSTR_END;
1069 :
1070 : /* 0xe0 - 0xef ******************************************************/
1071 :
1072 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1073 64 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1074 64 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1075 64 : FD_VM_INTERP_INSTR_END;
1076 :
1077 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1078 53 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1079 8 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1080 8 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1081 8 : FD_VM_INTERP_INSTR_END;
1082 :
1083 : /* 0xf0 - 0xff ******************************************************/
1084 :
1085 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1086 74 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1087 74 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1088 74 : FD_VM_INTERP_INSTR_END;
1089 :
1090 : FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1091 149179 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1092 149179 : FD_VM_INTERP_INSTR_END;
1093 :
1094 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1095 56 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1096 10 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1097 10 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1098 10 : FD_VM_INTERP_INSTR_END;
1099 :
1100 : /* SIMD-0377: JMP32
1101 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/interpreter.rs#L480-L501
1102 :
1103 : 0x16 - 0xde ******************************************************/
1104 :
1105 : FD_VM_INTERP_BRANCH_BEGIN(0x16_jmp32) /* FD_SBPF_OP_JEQ32_IMM */
1106 1 : pc += fd_ulong_if( (uint)reg_dst==(uint)imm, offset, 0UL );
1107 1 : FD_VM_INTERP_BRANCH_END;
1108 :
1109 : FD_VM_INTERP_BRANCH_BEGIN(0x1e_jmp32) /* FD_SBPF_OP_JEQ32_REG */
1110 0 : pc += fd_ulong_if( (uint)reg_dst==(uint)reg_src, offset, 0UL );
1111 0 : FD_VM_INTERP_BRANCH_END;
1112 :
1113 : FD_VM_INTERP_BRANCH_BEGIN(0x26_jmp32) /* FD_SBPF_OP_JGT32_IMM */
1114 0 : pc += fd_ulong_if( (uint)reg_dst>(uint)imm, offset, 0UL );
1115 0 : FD_VM_INTERP_BRANCH_END;
1116 :
1117 : FD_VM_INTERP_BRANCH_BEGIN(0x2e_jmp32) /* FD_SBPF_OP_JGT32_REG */
1118 1 : pc += fd_ulong_if( (uint)reg_dst>(uint)reg_src, offset, 0UL );
1119 1 : FD_VM_INTERP_BRANCH_END;
1120 :
1121 : FD_VM_INTERP_BRANCH_BEGIN(0x36_jmp32) /* FD_SBPF_OP_JGE32_IMM */
1122 0 : pc += fd_ulong_if( (uint)reg_dst>=(uint)imm, offset, 0UL );
1123 0 : FD_VM_INTERP_BRANCH_END;
1124 :
1125 : FD_VM_INTERP_BRANCH_BEGIN(0x3e_jmp32) /* FD_SBPF_OP_JGE32_REG */
1126 0 : pc += fd_ulong_if( (uint)reg_dst>=(uint)reg_src, offset, 0UL );
1127 0 : FD_VM_INTERP_BRANCH_END;
1128 :
1129 : FD_VM_INTERP_BRANCH_BEGIN(0x46_jmp32) /* FD_SBPF_OP_JSET32_IMM */
1130 2 : pc += fd_ulong_if( !!((uint)reg_dst & (uint)imm), offset, 0UL );
1131 2 : FD_VM_INTERP_BRANCH_END;
1132 :
1133 : FD_VM_INTERP_BRANCH_BEGIN(0x4e_jmp32) /* FD_SBPF_OP_JSET32_REG */
1134 0 : pc += fd_ulong_if( !!((uint)reg_dst & (uint)reg_src), offset, 0UL );
1135 0 : FD_VM_INTERP_BRANCH_END;
1136 :
1137 : FD_VM_INTERP_BRANCH_BEGIN(0x56_jmp32) /* FD_SBPF_OP_JNE32_IMM */
1138 0 : pc += fd_ulong_if( (uint)reg_dst!=(uint)imm, offset, 0UL );
1139 0 : FD_VM_INTERP_BRANCH_END;
1140 :
1141 : FD_VM_INTERP_BRANCH_BEGIN(0x5e_jmp32) /* FD_SBPF_OP_JNE32_REG */
1142 0 : pc += fd_ulong_if( (uint)reg_dst!=(uint)reg_src, offset, 0UL );
1143 0 : FD_VM_INTERP_BRANCH_END;
1144 :
1145 : FD_VM_INTERP_BRANCH_BEGIN(0x66_jmp32) /* FD_SBPF_OP_JSGT32_IMM */
1146 0 : pc += fd_ulong_if( (int)reg_dst>(int)imm, offset, 0UL );
1147 0 : FD_VM_INTERP_BRANCH_END;
1148 :
1149 : FD_VM_INTERP_BRANCH_BEGIN(0x6e_jmp32) /* FD_SBPF_OP_JSGT32_REG */
1150 0 : pc += fd_ulong_if( (int)reg_dst>(int)reg_src, offset, 0UL );
1151 0 : FD_VM_INTERP_BRANCH_END;
1152 :
1153 : FD_VM_INTERP_BRANCH_BEGIN(0x76_jmp32) /* FD_SBPF_OP_JSGE32_IMM */
1154 4 : pc += fd_ulong_if( (int)reg_dst>=(int)imm, offset, 0UL );
1155 4 : FD_VM_INTERP_BRANCH_END;
1156 :
1157 : FD_VM_INTERP_BRANCH_BEGIN(0x7e_jmp32) /* FD_SBPF_OP_JSGE32_REG */
1158 0 : pc += fd_ulong_if( (int)reg_dst>=(int)reg_src, offset, 0UL );
1159 0 : FD_VM_INTERP_BRANCH_END;
1160 :
1161 : FD_VM_INTERP_BRANCH_BEGIN(0xa6_jmp32) /* FD_SBPF_OP_JLT32_IMM */
1162 0 : pc += fd_ulong_if( (uint)reg_dst<(uint)imm, offset, 0UL );
1163 0 : FD_VM_INTERP_BRANCH_END;
1164 :
1165 : FD_VM_INTERP_BRANCH_BEGIN(0xae_jmp32) /* FD_SBPF_OP_JLT32_REG */
1166 1 : pc += fd_ulong_if( (uint)reg_dst<(uint)reg_src, offset, 0UL );
1167 1 : FD_VM_INTERP_BRANCH_END;
1168 :
1169 : FD_VM_INTERP_BRANCH_BEGIN(0xb6_jmp32) /* FD_SBPF_OP_JLE32_IMM */
1170 0 : pc += fd_ulong_if( (uint)reg_dst<=(uint)imm, offset, 0UL );
1171 0 : FD_VM_INTERP_BRANCH_END;
1172 :
1173 : FD_VM_INTERP_BRANCH_BEGIN(0xbe_jmp32) /* FD_SBPF_OP_JLE32_REG */
1174 0 : pc += fd_ulong_if( (uint)reg_dst<=(uint)reg_src, offset, 0UL );
1175 0 : FD_VM_INTERP_BRANCH_END;
1176 :
1177 : FD_VM_INTERP_BRANCH_BEGIN(0xc6_jmp32) /* FD_SBPF_OP_JSLT32_IMM */
1178 2 : pc += fd_ulong_if( (int)reg_dst<(int)imm, offset, 0UL );
1179 2 : FD_VM_INTERP_BRANCH_END;
1180 :
1181 : FD_VM_INTERP_BRANCH_BEGIN(0xce_jmp32) /* FD_SBPF_OP_JSLT32_REG */
1182 1 : pc += fd_ulong_if( (int)reg_dst<(int)reg_src, offset, 0UL );
1183 1 : FD_VM_INTERP_BRANCH_END;
1184 :
1185 : FD_VM_INTERP_BRANCH_BEGIN(0xd6_jmp32) /* FD_SBPF_OP_JSLE32_IMM */
1186 0 : pc += fd_ulong_if( (int)reg_dst<=(int)imm, offset, 0UL );
1187 0 : FD_VM_INTERP_BRANCH_END;
1188 :
1189 : FD_VM_INTERP_BRANCH_BEGIN(0xde_jmp32) /* FD_SBPF_OP_JSLE32_REG */
1190 1 : pc += fd_ulong_if( (int)reg_dst<=(int)reg_src, offset, 0UL );
1191 1 : FD_VM_INTERP_BRANCH_END;
1192 :
1193 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1194 : currently but could be enabled if desired. */
1195 :
1196 : /* Note: sigtextbr is for sigtext errors that occur on branching
1197 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1198 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1199 :
1200 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1201 : instructions preceding a fault generated by a non-branching
1202 : instruction. When a non-branching instruction faults, pc is at the
1203 : instruction and the number of non-branching instructions that have
1204 : not yet been reflected in ic and cu is:
1205 :
1206 : pc - pc0 + 1 - ic_correction
1207 :
1208 : as per the accounting described above. +1 to include the faulting
1209 : instruction itself.
1210 :
1211 : Note that, for a sigtext caused by a branch instruction, pc0==pc
1212 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
1213 : such that the below does not change the already current values in
1214 : ic and cu. Thus it also "does the right thing" in both the
1215 : non-branching and branching cases for sigtext. The same applies to
1216 : sigsplit. */
1217 :
1218 0 : #define FD_VM_INTERP_FAULT \
1219 1454 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1220 1454 : ic += ic_correction; \
1221 1454 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
1222 1454 : cu -= fd_ulong_min( ic_correction, cu )
1223 :
1224 8 : sigtext_or_sigcost:
1225 : /* If the block text limit is exceeded, sigtext_or_sigcost will be
1226 : thrown. This could either be because the pc has exceeded the text
1227 : section, or we have exhausted our CU budget. The block text limit
1228 : is used to combine both of these checks into one, primarily for
1229 : performance reasons. We then disambiguate the scenarios in this
1230 : signal handler, as hitting either of these cases is rare. */
1231 8 : if( pc<text_cnt ) goto sigcost; else goto sigtext;
1232 1 : sigtext: err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN; FD_VM_INTERP_FAULT; goto interp_halt;
1233 56 : sigtextbr: err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT; /* ic current */ /* cu current */ goto interp_halt;
1234 1 : sigstack: err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED; /* ic current */ /* cu current */ goto interp_halt;
1235 21 : sigill: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; FD_VM_INTERP_FAULT; goto interp_halt;
1236 103 : sigillbr: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1237 0 : siginv: err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1238 187 : sigsegv: err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version ); FD_VM_INTERP_FAULT; goto interp_halt;
1239 32 : sigcost: err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; /* ic current */ cu = 0UL; goto interp_halt;
1240 947 : sigsyscall: err = FD_VM_ERR_EBPF_SYSCALL_ERROR; /* ic current */ /* cu current */ goto interp_halt;
1241 1245 : sigfpe: err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO; FD_VM_INTERP_FAULT; goto interp_halt;
1242 0 : sigfpeof: err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW; FD_VM_INTERP_FAULT; goto interp_halt;
1243 1631 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1244 :
1245 0 : #undef FD_VM_INTERP_FAULT
1246 :
1247 4223 : interp_halt:
1248 :
1249 : /* Pack the unpacked execution state into vm to give a precise view of
1250 : the execution when the vm halted. */
1251 :
1252 4223 : vm->pc = pc;
1253 4223 : vm->ic = ic;
1254 4223 : vm->cu = cu;
1255 4223 : vm->frame_cnt = frame_cnt;
1256 :
1257 4223 : # undef FD_VM_INTERP_STACK_PUSH
1258 :
1259 4223 : # undef FD_VM_INTERP_BRANCH_END
1260 4223 : # undef FD_VM_INTERP_BRANCH_BEGIN
1261 :
1262 4223 : # undef FD_VM_INTERP_BLOCK_TEXT_LIMIT
1263 :
1264 4223 : # undef FD_VM_INTERP_INSTR_END
1265 4223 : # undef FD_VM_INTERP_INSTR_BEGIN
1266 4223 : # undef FD_VM_INTERP_INSTR_EXEC
1267 :
1268 4223 : # if defined(__clang__)
1269 4223 : # pragma clang diagnostic pop
1270 4223 : # endif
1271 :
1272 4223 : # if defined(__GNUC__)
1273 4223 : # pragma GCC diagnostic pop
1274 4223 : # endif
1275 :
1276 : /* Agave/JIT CU model analysis (and why we are conformant!):
1277 :
1278 : The Agave JIT employs a similar strategy of accumulating instructions
1279 : in a linear run and processing them at the start of a new linear
1280 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1281 : that jumps pc + 2).
1282 :
1283 : In what is assumed to be an act of register conservation, the JIT
1284 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1285 : that represents two different interpretations of the question
1286 : "how many instructions can I execute?".
1287 :
1288 : The IM, depending on where we are in the execution, either represents:
1289 : 1. IM => The number of instructions remaining before exhausting CU
1290 : budget. This is analogous to vm->cu in our interpreter.
1291 : 2. IM' => The last pc you can execute in the current linear run before
1292 : exhausting CU budget. Mathematically, IM' = IM + pc0
1293 : where pc0, just like our definition, is the start of the linear run.
1294 :
1295 : Note: IM' can go past the actual basic block/segment. In-fact,
1296 : it typically does, and implies we can execute the full block without
1297 : exhausting CU budget (reminder that LDQ is treated as a branch).
1298 :
1299 : By default, the IM' form is used during execution. The IM form is used:
1300 : - (transiently) during the processing of a branch instruction
1301 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1302 :
1303 : When a branch instruction is encountered, the JIT checks
1304 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
1305 : because as described above, IM' is the largest PC you can reach.
1306 :
1307 : If we haven't exhausted our CU limit, it updates IM':
1308 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1309 : # where pc0' is the start of the current linear run.
1310 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1311 :
1312 : Code (that does the above in one ALU instruction):
1313 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1314 :
1315 :
1316 : ### How does this relate to our interpreter?
1317 :
1318 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1319 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1320 : If we break down step 1 from above with what we know about IM and IM',
1321 : we get the following:
1322 : 1. IM = IM' - (pc + 1)
1323 : IM = (IM + pc0') - (pc + 1)
1324 : IM = IM + (pc0' - (pc + 1))
1325 : IM = IM - ((pc + 1) - pc0')
1326 : IM = IM - ic_correction
1327 : Here, ((pc + 1) - pc0') is the number of instructions executed in the current
1328 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1329 :
1330 : If we replace IM with cu, this effectively becomes the
1331 : cu -= ic_correction
1332 : line in FD_VM_INTERP_BRANCH_BEGIN.
1333 :
1334 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1335 : accumulator or the number of instructions executed in the current linear run.
1336 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1337 : FD_VM_INTERP_FAULT, and the former form otherwise.
1338 : */
1339 :
1340 : /* (WIP) Precise faulting and the Agave JIT:
1341 :
1342 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1343 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1344 : section covers the various faults that the respective VMs implement and how they match.
1345 :
1346 : # Normal VM exit (sigexit):
1347 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1348 :
1349 : Pseudocode (with FD semantics):
1350 : ```
1351 : # pc is at the exit instruction
1352 : # pc0 is the start of the current linear run
1353 : if (frame_cnt == 0) {
1354 : goto sigexit;
1355 : }
1356 : ...
1357 :
1358 : sigexit:
1359 : if IM' <= pc {
1360 : goto sigcost;
1361 : } else {
1362 : goto interp_halt;
1363 : }
1364 : ```
1365 :
1366 : Breaking down the IM' < pc check:
1367 : - IM' = IM + pc0
1368 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1369 :
1370 : IM' <= pc
1371 : IM + pc0 <= ic + pc0
1372 : IM <= ic
1373 : IM <= pc - pc0
1374 : IM < pc - pc0 + 1 # all unsigned integers
1375 : IM < ic_correction
1376 :
1377 : This is analogous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1378 :
1379 : # (TODO) Text Overrun (sigtext/sigsplit):
1380 :
1381 : */
|
