Line data Source code
1 : /* This is the VM SBPF interpreter core. The caller unpacks the VM
2 : state and then just lets execution continue into this (or jumps to
3 : interp_exec) to start running. The VM will run until it halts or
4 : faults. On normal termination, it will branch to interp_halt to
5 : exit. Each fault has its own exit label to allow the caller to
6 : handle individually. */
7 :
8 : /* FIXME: SIGILLS FOR VARIOUS THINGS THAT HAVE UNNECESSARY BITS IN IMM
9 : SET? (LIKE WIDE SHIFTS?) */
10 :
11 0 : # if defined(__GNUC__) /* -Wpedantic rejects labels as values and rejects goto *expr */
12 0 : # pragma GCC diagnostic push
13 0 : # pragma GCC diagnostic ignored "-Wpedantic"
14 0 : # endif
15 :
16 0 : # if defined(__clang__) /* Clang is differently picky about labels as values and goto *expr */
17 0 : # pragma clang diagnostic push
18 0 : # pragma clang diagnostic ignored "-Wpedantic"
19 0 : # pragma clang diagnostic ignored "-Wgnu-label-as-value"
20 0 : # endif
21 :
22 : /* Include the jump table */
23 :
24 6354 : # include "fd_vm_interp_jump_table.c"
25 :
26 : /* Update the jump table based on SBPF version */
27 :
28 0 : ulong sbpf_version = vm->sbpf_version;
29 :
30 : /* Unpack the VM state */
31 :
32 0 : ulong pc = vm->pc;
33 0 : ulong ic = vm->ic;
34 0 : ulong cu = vm->cu;
35 0 : ulong frame_cnt = vm->frame_cnt;
36 :
37 0 : void const * const * const version_interp_jump_table = interp_jump_table[ sbpf_version ];
38 :
39 : /* FD_VM_INTERP_INSTR_EXEC loads the first word of the instruction at
40 : pc, parses it, fetches the associated register values and then
41 : jumps to the code that executes the instruction. On normal
42 : instruction execution, the pc will be updated and
43 : FD_VM_INTERP_INSTR_EXEC will be invoked again to do the next
44 : instruction. After a normal halt, this will branch to interp_halt.
45 : Otherwise, it will branch to the appropriate normal termination. */
46 :
47 0 : ulong instr;
48 0 : ulong opcode;
49 0 : ulong dst;
50 0 : ulong src;
51 0 : ulong offset; /* offset is 16-bit but always sign extended, so we handle cast once */
52 0 : uint imm;
53 0 : ulong reg_dst;
54 0 : ulong reg_src;
55 :
56 : /* These mimic the exact Rust semantics for wrapping_shl and wrapping_shr. */
57 :
58 : /* u64::wrapping_shl: a.unchecked_shl(b & (64 - 1))
59 :
60 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shl
61 : */
62 284675 : #define FD_RUST_ULONG_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 63 ) ))
63 :
64 : /* u64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
65 :
66 : https://doc.rust-lang.org/std/primitive.u64.html#method.wrapping_shr
67 : */
68 34139 : #define FD_RUST_ULONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
69 :
70 : /* u32::wrapping_shl: a.unchecked_shl(b & (32 - 1))
71 :
72 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shl
73 : */
74 117912 : #define FD_RUST_UINT_WRAPPING_SHL( a, b ) ((a) << ( (b) & ( 31 ) ))
75 :
76 : /* u32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
77 :
78 : https://doc.rust-lang.org/std/primitive.u32.html#method.wrapping_shr
79 : */
80 14 : #define FD_RUST_UINT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
81 :
82 : /* i32::wrapping_shr: a.unchecked_shr(b & (32 - 1))
83 :
84 : https://doc.rust-lang.org/std/primitive.i32.html#method.wrapping_shr
85 : */
86 22475 : #define FD_RUST_INT_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 31 ) ))
87 :
88 : /* i64::wrapping_shr: a.unchecked_shr(b & (64 - 1))
89 :
90 : https://doc.rust-lang.org/std/primitive.i64.html#method.wrapping_shr
91 : */
92 7279 : #define FD_RUST_LONG_WRAPPING_SHR( a, b ) ((a) >> ( (b) & ( 63 ) ))
93 :
94 :
95 0 : # define FD_VM_INTERP_INSTR_EXEC \
96 15598333 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext; /* Note: untaken branches don't consume BTB */ \
97 15598333 : instr = text[ pc ]; /* Guaranteed in-bounds */ \
98 15594763 : opcode = fd_vm_instr_opcode( instr ); /* in [0,256) even if malformed */ \
99 15594763 : dst = fd_vm_instr_dst ( instr ); /* in [0, 16) even if malformed */ \
100 15594763 : src = fd_vm_instr_src ( instr ); /* in [0, 16) even if malformed */ \
101 15594763 : offset = fd_vm_instr_offset( instr ); /* in [-2^15,2^15) even if malformed */ \
102 15594763 : imm = fd_vm_instr_imm ( instr ); /* in [0,2^32) even if malformed */ \
103 15594763 : reg_dst = reg[ dst ]; /* Guaranteed in-bounds */ \
104 15594763 : reg_src = reg[ src ]; /* Guaranteed in-bounds */ \
105 15594763 : goto *version_interp_jump_table[ opcode ] /* Guaranteed in-bounds */
106 :
107 : /* FD_VM_INTERP_SYSCALL_EXEC
108 : (macro to handle the logic of 0x85 pre- and post- SIMD-0178: static syscalls)
109 :
110 : Setup.
111 : Update the vm with the current vm execution state for the
112 : syscall. Note that BRANCH_BEGIN has pc at the syscall and
113 : already updated ic and cu to reflect all instructions up to
114 : and including the syscall instruction itself.
115 :
116 : Execution.
117 : Do the syscall. We use ret reduce the risk of the syscall
118 : accidentally modifying other registers (note however since a
119 : syscall has the vm handle it still do arbitrary modifications
120 : to the vm state) and the risk of a pointer escape on reg from
121 : inhibiting compiler optimizations (this risk is likely low in
122 : as this is the only point in the whole interpreter core that
123 : calls outside this translation unit).
124 : At this point, vm->cu is positive.
125 :
126 : Error handling.
127 : If we trust syscall implementations to handle the vm state
128 : correctly, the below could be implemented as unpacking the vm
129 : state and jumping to sigsys on error. But we provide some
130 : extra protection to make various strong guarantees:
131 :
132 : - We do not let the syscall modify pc currently as nothing
133 : requires this and it reduces risk of a syscall bug mucking
134 : up the interpreter. If there ever was a syscall that
135 : needed to modify the pc (e.g. a syscall that has execution
136 : resume from a different location than the instruction
137 : following the syscall), do "pc = vm->pc" below.
138 :
139 : - We do not let the syscall modify ic currently as nothing
140 : requires this and it keeps the ic precise. If a future
141 : syscall needs this, do "ic = vm->ic" below.
142 :
143 : - We do not let the syscall increase cu as nothing requires
144 : this and it guarantees the interpreter will halt in a
145 : reasonable finite amount of time. If a future syscall
146 : needs this, do "cu = vm->cu" below.
147 :
148 : - A syscall that returns SIGCOST is always treated as though
149 : it also zerod cu.
150 :
151 : At this point, vm->cu is whatever the syscall tried to set
152 : and cu is positive.
153 :
154 : Exit
155 : At this point, cu is positive and err is clear.
156 : */
157 :
158 0 : # if FD_HAS_FLATCC
159 0 : # define FD_VM_INTERP_SYSCALL_EXEC_DUMP \
160 : /* Dumping for debugging purposes */ \
161 7558 : if( FD_UNLIKELY( vm->dump_syscall_to_pb ) ) { \
162 0 : fd_dump_vm_syscall_to_protobuf( vm, syscall->name ); \
163 0 : }
164 : # else
165 : # define FD_VM_INTERP_SYSCALL_EXEC_DUMP
166 : # endif
167 0 : # define FD_VM_INTERP_SYSCALL_EXEC \
168 : /* Setup */ \
169 7558 : vm->pc = pc; \
170 7558 : vm->ic = ic; \
171 7558 : vm->cu = cu; \
172 7558 : vm->frame_cnt = frame_cnt; \
173 7558 : FD_VM_INTERP_SYSCALL_EXEC_DUMP \
174 : /* Execution */ \
175 7558 : ulong ret[1]; \
176 7558 : err = syscall->func( vm, reg[1], reg[2], reg[3], reg[4], reg[5], ret ); \
177 7558 : reg[0] = ret[0]; \
178 : /* Error handling */ \
179 7558 : ulong cu_req = vm->cu; \
180 7558 : cu = fd_ulong_min( cu_req, cu ); \
181 7558 : if( FD_UNLIKELY( err ) ) { \
182 3562 : if( err==FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED ) cu = 0UL; /* cmov */ \
183 3562 : FD_VM_TEST_ERR_EXISTS( vm ); \
184 3562 : goto sigsyscall; \
185 3562 : } \
186 : /* Exit */
187 :
188 :
189 : /* FD_VM_INTERP_INSTR_BEGIN / FD_VM_INTERP_INSTR_END bracket opcode's
190 : implementation for an opcode that does not branch. On entry, the
191 : instruction word has been unpacked into dst / src / offset / imm
192 : and reg[dst] / reg[src] has been prefetched into reg_dst / reg_src. */
193 :
194 10296292 : # define FD_VM_INTERP_INSTR_BEGIN(opcode) interp_##opcode:
195 :
196 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~0.3% faster in some benchmarks, slower in others but more code footprint */
197 10296108 : # define FD_VM_INTERP_INSTR_END pc++; FD_VM_INTERP_INSTR_EXEC
198 : # else /* Use this version when tracing or optimizing code footprint */
199 0 : # define FD_VM_INTERP_INSTR_END pc++; goto interp_exec
200 : # endif
201 :
202 : /* Instead of doing a lot of compute budget calcs and tests every
203 : instruction, we note that the program counter increases
204 : monotonically after a branch (or a program start) until the next
205 : branch (or program termination). We save the program counter of
206 : the start of such a segment in pc0. Whenever we encounter a branch
207 : (or a program termination) at pc, we know we processed pc-pc0+1
208 : text words (including the text word for the branch instruction
209 : itself as all branch instructions are single word).
210 :
211 : Each instruction costs 1 cu (syscalls can cost extra on top of
212 : this that is accounted separately in CALL_IMM below). Since there
213 : could have been multiword instructions in this segment, at start of
214 : such a segment, we zero out the accumulator ic_correction and have
215 : every multiword instruction in the segment accumulate the number of
216 : extra text words it has to this variable. (Sigh ... it would be a
217 : lot simpler to bill based on text words processed but this would be
218 : very difficult to make this protocol change at this point.)
219 :
220 : When we encounter a branch at pc, the number of instructions
221 : processed (and thus the number of compute units to bill for that
222 : segment) is thus:
223 :
224 : pc - pc0 + 1 - ic_correction
225 :
226 : IMPORTANT SAFETY TIP! This implies the worst case interval before
227 : checking the cu budget is the worst case text_cnt. But since all
228 : such instructions are cheap 1 cu instructions and processed fast
229 : and text max is limited in size, this should be acceptable in
230 : practice. FIXME: DOUBLE CHECK THE MATH ABOVE AGAINST PROTOCOL
231 : LIMITS. */
232 :
233 0 : ulong pc0 = pc;
234 0 : ulong ic_correction = 0UL;
235 :
236 0 : # define FD_VM_INTERP_BRANCH_BEGIN(opcode) \
237 5298450 : interp_##opcode: \
238 : /* Bill linear text segment and this branch instruction as per the above */ \
239 5298450 : ic_correction = pc - pc0 + 1UL - ic_correction; \
240 5298450 : ic += ic_correction; \
241 5298450 : if( FD_UNLIKELY( ic_correction>cu ) ) goto sigcost; /* Note: untaken branches don't consume BTB */ \
242 5298450 : cu -= ic_correction; \
243 : /* At this point, cu>=0 */ \
244 5297118 : ic_correction = 0UL;
245 :
246 : /* FIXME: debatable if it is better to do pc++ here or have the
247 : instruction implementations do it in their code path. */
248 :
249 : # ifndef FD_VM_INTERP_EXE_TRACING_ENABLED /* Non-tracing path only, ~4% faster in some benchmarks, slower in others but more code footprint */
250 : # define FD_VM_INTERP_BRANCH_END \
251 5292303 : pc++; \
252 5292303 : pc0 = pc; /* Start a new linear segment */ \
253 5295872 : FD_VM_INTERP_INSTR_EXEC
254 : # else /* Use this version when tracing or optimizing code footprint */
255 : # define FD_VM_INTERP_BRANCH_END \
256 0 : pc++; \
257 0 : pc0 = pc; /* Start a new linear segment */ \
258 : /* FIXME: TEST sigsplit HERE */ \
259 0 : goto interp_exec
260 : # endif
261 :
262 : /* FD_VM_INTERP_STACK_PUSH pushes reg[6:9] onto the shadow stack and
263 : advances reg[10] to a new user stack frame. If there are no more
264 : stack frames available, will do a SIGSTACK. */
265 :
266 : /* FIXME: double check faulting is desired on stack overflow. */
267 :
268 : /* FIXME: a pre-belt-sanding FIXME implied the TLB should be updated
269 : to prevent byte code from accessing the stack outside its current
270 : stack frame. But this would break the common practice of a
271 : function passing a pointer to something on its stack into a
272 : function that it calls:
273 :
274 : void foo( ... ) {
275 : ...
276 : int ret;
277 : bar( &ret );
278 : ...
279 : }
280 :
281 : So this probably shouldn't be done. But, if it is in fact
282 : necessary, the TLB updates would be here and in pop. */
283 :
284 : /* FIXME: unvalidated code mucking with r10 */
285 :
286 0 : # define FD_VM_INTERP_STACK_PUSH \
287 91992 : shadow[ frame_cnt ].r6 = reg[6]; \
288 91992 : shadow[ frame_cnt ].r7 = reg[7]; \
289 91992 : shadow[ frame_cnt ].r8 = reg[8]; \
290 91992 : shadow[ frame_cnt ].r9 = reg[9]; \
291 91992 : shadow[ frame_cnt ].r10 = reg[10]; \
292 91992 : shadow[ frame_cnt ].pc = pc; \
293 91992 : if( FD_UNLIKELY( ++frame_cnt>=frame_max ) ) goto sigstack; /* Note: untaken branches don't consume BTB */ \
294 91992 : if( !fd_sbpf_dynamic_stack_frames_enabled( sbpf_version ) ) reg[10] += FD_VM_STACK_FRAME_SZ * 2UL; \
295 0 :
296 : /* We subtract the heap cost in the BPF loader */
297 :
298 0 : goto interp_exec; /* Silly but to avoid unused label warning in some configurations */
299 6353 : interp_exec:
300 :
301 : # ifdef FD_VM_INTERP_EXE_TRACING_ENABLED
302 : /* Note: when tracing or optimizing for code footprint, all
303 : instruction execution starts here such that this is only point
304 : where exe tracing diagnostics are needed. */
305 0 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigtext;
306 0 : fd_vm_trace_event_exe( vm->trace, pc, ic + ( pc - pc0 - ic_correction ), cu, reg, vm->text + pc, vm->text_cnt - pc, ic_correction, frame_cnt );
307 0 : # endif
308 :
309 6353 : FD_VM_INTERP_INSTR_EXEC;
310 :
311 : /* 0x00 - 0x0f ******************************************************/
312 :
313 : FD_VM_INTERP_INSTR_BEGIN(0x04) /* FD_SBPF_OP_ADD_IMM */
314 3 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)imm );
315 3 : FD_VM_INTERP_INSTR_END;
316 :
317 : FD_VM_INTERP_INSTR_BEGIN(0x04depr) /* FD_SBPF_OP_ADD_IMM deprecated SIMD-0174 */
318 482602 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)imm );
319 482602 : FD_VM_INTERP_INSTR_END;
320 :
321 : FD_VM_INTERP_BRANCH_BEGIN(0x05) /* FD_SBPF_OP_JA */
322 836357 : pc += offset;
323 836357 : FD_VM_INTERP_BRANCH_END;
324 :
325 : FD_VM_INTERP_INSTR_BEGIN(0x07) /* FD_SBPF_OP_ADD64_IMM */
326 642495 : reg[ dst ] = reg_dst + (ulong)(long)(int)imm;
327 642495 : FD_VM_INTERP_INSTR_END;
328 :
329 : FD_VM_INTERP_INSTR_BEGIN(0x0c) /* FD_SBPF_OP_ADD_REG */
330 0 : reg[ dst ] = (ulong)(uint)( (int)reg_dst + (int)reg_src );
331 0 : FD_VM_INTERP_INSTR_END;
332 :
333 : FD_VM_INTERP_INSTR_BEGIN(0x0cdepr) /* FD_SBPF_OP_ADD_REG deprecated SIMD-0174 */
334 19579 : reg[ dst ] = (ulong)(long)( (int)reg_dst + (int)reg_src );
335 19579 : FD_VM_INTERP_INSTR_END;
336 :
337 : FD_VM_INTERP_INSTR_BEGIN(0x0f) /* FD_SBPF_OP_ADD64_REG */
338 286759 : reg[ dst ] = reg_dst + reg_src;
339 286759 : FD_VM_INTERP_INSTR_END;
340 :
341 : /* 0x10 - 0x1f ******************************************************/
342 :
343 : FD_VM_INTERP_INSTR_BEGIN(0x14) /* FD_SBPF_OP_SUB_IMM */
344 0 : reg[ dst ] = (ulong)(uint)( (int)imm - (int)reg_dst );
345 0 : FD_VM_INTERP_INSTR_END;
346 :
347 : FD_VM_INTERP_INSTR_BEGIN(0x14depr) /* FD_SBPF_OP_SUB_IMM deprecated SIMD-0174 */
348 19882 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)imm );
349 19882 : FD_VM_INTERP_INSTR_END;
350 :
351 : FD_VM_INTERP_BRANCH_BEGIN(0x15) /* FD_SBPF_OP_JEQ_IMM */
352 163483 : pc += fd_ulong_if( reg_dst==(ulong)(long)(int)imm, offset, 0UL );
353 163483 : FD_VM_INTERP_BRANCH_END;
354 :
355 : FD_VM_INTERP_INSTR_BEGIN(0x17) /* FD_SBPF_OP_SUB64_IMM */
356 0 : reg[ dst ] = (ulong)(long)(int)imm - reg_dst;
357 0 : FD_VM_INTERP_INSTR_END;
358 :
359 : FD_VM_INTERP_INSTR_BEGIN(0x17depr) /* FD_SBPF_OP_SUB64_IMM deprecated SIMD-0174 */
360 7 : reg[ dst ] = reg_dst - (ulong)(long)(int)imm;
361 7 : FD_VM_INTERP_INSTR_END;
362 :
363 : FD_VM_INTERP_INSTR_BEGIN(0x18) /* FD_SBPF_OP_LDQ */
364 812797 : pc++;
365 812797 : ic_correction++;
366 : /* No need to check pc because it's already checked during validation.
367 : if( FD_UNLIKELY( pc>=text_cnt ) ) goto sigsplit; // Note: untaken branches don't consume BTB */
368 812797 : reg[ dst ] = (ulong)((ulong)imm | ((ulong)fd_vm_instr_imm( text[ pc ] ) << 32));
369 812797 : FD_VM_INTERP_INSTR_END;
370 :
371 : FD_VM_INTERP_INSTR_BEGIN(0x1c) /* FD_SBPF_OP_SUB_REG */
372 2 : reg[ dst ] = (ulong)(uint)( (int)reg_dst - (int)reg_src );
373 2 : FD_VM_INTERP_INSTR_END;
374 :
375 : FD_VM_INTERP_INSTR_BEGIN(0x1cdepr) /* FD_SBPF_OP_SUB_REG deprecated SIMD-0174 */
376 214882 : reg[ dst ] = (ulong)(long)( (int)reg_dst - (int)reg_src );
377 214882 : FD_VM_INTERP_INSTR_END;
378 :
379 : FD_VM_INTERP_BRANCH_BEGIN(0x1d) /* FD_SBPF_OP_JEQ_REG */
380 9894 : pc += fd_ulong_if( reg_dst==reg_src, offset, 0UL );
381 9894 : FD_VM_INTERP_BRANCH_END;
382 :
383 : FD_VM_INTERP_INSTR_BEGIN(0x1f) /* FD_SBPF_OP_SUB64_REG */
384 64245 : reg[ dst ] = reg_dst - reg_src;
385 64245 : FD_VM_INTERP_INSTR_END;
386 :
387 : /* 0x20 - 0x2f ******************************************************/
388 :
389 : FD_VM_INTERP_INSTR_BEGIN(0x24) /* FD_SBPF_OP_MUL_IMM */
390 215754 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)imm );
391 215754 : FD_VM_INTERP_INSTR_END;
392 :
393 : FD_VM_INTERP_BRANCH_BEGIN(0x25) /* FD_SBPF_OP_JGT_IMM */
394 28703 : pc += fd_ulong_if( reg_dst>(ulong)(long)(int)imm, offset, 0UL );
395 28703 : FD_VM_INTERP_BRANCH_END;
396 :
397 81 : FD_VM_INTERP_INSTR_BEGIN(0x27) { /* FD_SBPF_OP_STB */
398 81 : ulong vaddr = reg_dst + offset;
399 81 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
400 81 : if( FD_UNLIKELY( !haddr ) ) {
401 6 : vm->segv_vaddr = vaddr;
402 6 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
403 6 : vm->segv_access_len = 1UL;
404 6 : goto sigsegv;
405 6 : } /* Note: untaken branches don't consume BTB */
406 75 : fd_vm_mem_st_1( haddr, (uchar)imm );
407 75 : }
408 75 : FD_VM_INTERP_INSTR_END;
409 :
410 181063 : FD_VM_INTERP_INSTR_BEGIN(0x2c) { /* FD_SBPF_OP_LDXB */
411 181063 : ulong vaddr = reg_src + offset;
412 181063 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_ld_sz, 0, 0UL );
413 181063 : if( FD_UNLIKELY( !haddr ) ) {
414 9 : vm->segv_vaddr = vaddr;
415 9 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
416 9 : vm->segv_access_len = 1UL;
417 9 : goto sigsegv;
418 9 : } /* Note: untaken branches don't consume BTB */
419 181054 : reg[ dst ] = fd_vm_mem_ld_1( haddr );
420 181054 : }
421 181054 : FD_VM_INTERP_INSTR_END;
422 :
423 : FD_VM_INTERP_BRANCH_BEGIN(0x2d) /* FD_SBPF_OP_JGT_REG */
424 257184 : pc += fd_ulong_if( reg_dst>reg_src, offset, 0UL );
425 257184 : FD_VM_INTERP_BRANCH_END;
426 :
427 156490 : FD_VM_INTERP_INSTR_BEGIN(0x2f) { /* FD_SBPF_OP_STXB */
428 156490 : ulong vaddr = reg_dst + offset;
429 156490 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uchar), region_haddr, region_st_sz, 1, 0UL );
430 156490 : if( FD_UNLIKELY( !haddr ) ) {
431 16 : vm->segv_vaddr = vaddr;
432 16 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
433 16 : vm->segv_access_len = 1UL;
434 16 : goto sigsegv;
435 16 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigrdonly */
436 156474 : fd_vm_mem_st_1( haddr, (uchar)reg_src );
437 156474 : }
438 156474 : FD_VM_INTERP_INSTR_END;
439 :
440 : FD_VM_INTERP_INSTR_BEGIN(0x27depr) /* FD_SBPF_OP_MUL64_IMM */
441 40212 : reg[ dst ] = (ulong)( (long)reg_dst * (long)(int)imm );
442 40212 : FD_VM_INTERP_INSTR_END;
443 :
444 : FD_VM_INTERP_INSTR_BEGIN(0x2cdepr) /* FD_SBPF_OP_MUL_REG */
445 57928 : reg[ dst ] = (ulong)(long)( (int)reg_dst * (int)reg_src );
446 57928 : FD_VM_INTERP_INSTR_END;
447 :
448 : FD_VM_INTERP_INSTR_BEGIN(0x2fdepr) /* FD_SBPF_OP_MUL64_REG */
449 58162 : reg[ dst ] = reg_dst * reg_src;
450 58162 : FD_VM_INTERP_INSTR_END;
451 :
452 : /* 0x30 - 0x3f ******************************************************/
453 :
454 : FD_VM_INTERP_INSTR_BEGIN(0x34) /* FD_SBPF_OP_DIV_IMM */
455 49683 : /* FIXME: convert to a multiply at validation time (usually probably
456 49683 : not worth it) */
457 49683 : reg[ dst ] = (ulong)((uint)reg_dst / imm);
458 49683 : FD_VM_INTERP_INSTR_END;
459 :
460 : FD_VM_INTERP_BRANCH_BEGIN(0x35) /* FD_SBPF_OP_JGE_IMM */
461 381 : pc += fd_ulong_if( reg_dst>=(ulong)(long)(int)imm, offset, 0UL );
462 381 : FD_VM_INTERP_BRANCH_END;
463 :
464 : FD_VM_INTERP_INSTR_BEGIN(0x36) /* FD_SBPF_OP_UHMUL64_IMM */
465 0 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)(ulong)imm ) >> 64 );
466 0 : FD_VM_INTERP_INSTR_END;
467 :
468 36247 : FD_VM_INTERP_INSTR_BEGIN(0x37) { /* FD_SBPF_OP_STH */
469 36247 : ulong vaddr = reg_dst + offset;
470 36247 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
471 36247 : int sigsegv = !haddr;
472 36247 : if( FD_UNLIKELY( sigsegv ) ) {
473 4 : vm->segv_vaddr = vaddr;
474 4 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
475 4 : vm->segv_access_len = 2UL;
476 4 : goto sigsegv;
477 4 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
478 36243 : fd_vm_mem_st_2( haddr, (ushort)imm );
479 36243 : }
480 36243 : FD_VM_INTERP_INSTR_END;
481 :
482 1167 : FD_VM_INTERP_INSTR_BEGIN(0x3c) { /* FD_SBPF_OP_LDXH */
483 1167 : ulong vaddr = reg_src + offset;
484 1167 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_ld_sz, 0, 0UL );
485 1167 : int sigsegv = !haddr;
486 1167 : if( FD_UNLIKELY( sigsegv ) ) {
487 10 : vm->segv_vaddr = vaddr;
488 10 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
489 10 : vm->segv_access_len = 2UL;
490 10 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
491 10 : }
492 1157 : reg[ dst ] = fd_vm_mem_ld_2( haddr );
493 1157 : }
494 1157 : FD_VM_INTERP_INSTR_END;
495 :
496 : FD_VM_INTERP_BRANCH_BEGIN(0x3d) /* FD_SBPF_OP_JGE_REG */
497 519074 : pc += fd_ulong_if( reg_dst>=reg_src, offset, 0UL );
498 519074 : FD_VM_INTERP_BRANCH_END;
499 :
500 1170 : FD_VM_INTERP_INSTR_BEGIN(0x3f) { /* FD_SBPF_OP_STXH */
501 1170 : ulong vaddr = reg_dst + offset;
502 1170 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ushort), region_haddr, region_st_sz, 1, 0UL );
503 1170 : int sigsegv = !haddr;
504 1170 : if( FD_UNLIKELY( sigsegv ) ) {
505 5 : vm->segv_vaddr = vaddr;
506 5 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
507 5 : vm->segv_access_len = 2UL;
508 5 : goto sigsegv;
509 5 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
510 1165 : fd_vm_mem_st_2( haddr, (ushort)reg_src );
511 1165 : }
512 1165 : FD_VM_INTERP_INSTR_END;
513 :
514 : FD_VM_INTERP_INSTR_BEGIN(0x3e) /* FD_SBPF_OP_UHMUL64_REG */
515 3 : reg[ dst ] = (ulong)(( (uint128)reg_dst * (uint128)reg_src ) >> 64 );
516 3 : FD_VM_INTERP_INSTR_END;
517 :
518 : FD_VM_INTERP_INSTR_BEGIN(0x37depr) /* FD_SBPF_OP_DIV64_IMM */
519 50041 : reg[ dst ] = reg_dst / (ulong)(long)(int)imm;
520 50041 : FD_VM_INTERP_INSTR_END;
521 :
522 : FD_VM_INTERP_INSTR_BEGIN(0x3cdepr) /* FD_SBPF_OP_DIV_REG */
523 549028 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
524 549028 : reg[ dst ] = (ulong)((uint)reg_dst / (uint)reg_src);
525 549028 : FD_VM_INTERP_INSTR_END;
526 :
527 : FD_VM_INTERP_INSTR_BEGIN(0x3fdepr) /* FD_SBPF_OP_DIV64_REG */
528 155173 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
529 155173 : reg[ dst ] = reg_dst / reg_src;
530 155173 : FD_VM_INTERP_INSTR_END;
531 :
532 : /* 0x40 - 0x4f ******************************************************/
533 :
534 : FD_VM_INTERP_INSTR_BEGIN(0x44) /* FD_SBPF_OP_OR_IMM */
535 44673 : reg[ dst ] = (ulong)( (uint)reg_dst | imm );
536 44673 : FD_VM_INTERP_INSTR_END;
537 :
538 : FD_VM_INTERP_BRANCH_BEGIN(0x45) /* FD_SBPF_OP_JSET_IMM */
539 112071 : pc += fd_ulong_if( !!(reg_dst & (ulong)(long)(int)imm), offset, 0UL );
540 112071 : FD_VM_INTERP_BRANCH_END;
541 :
542 : FD_VM_INTERP_INSTR_BEGIN(0x46) /* FD_SBPF_OP_UDIV32_IMM */
543 0 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)imm );
544 0 : FD_VM_INTERP_INSTR_END;
545 :
546 : FD_VM_INTERP_INSTR_BEGIN(0x47) /* FD_SBPF_OP_OR64_IMM */
547 111217 : reg[ dst ] = reg_dst | (ulong)(long)(int)imm;
548 111217 : FD_VM_INTERP_INSTR_END;
549 :
550 : FD_VM_INTERP_INSTR_BEGIN(0x4c) /* FD_SBPF_OP_OR_REG */
551 36238 : reg[ dst ] = (ulong)(uint)( reg_dst | reg_src );
552 36238 : FD_VM_INTERP_INSTR_END;
553 :
554 : FD_VM_INTERP_BRANCH_BEGIN(0x4d) /* FD_SBPF_OP_JSET_REG */
555 2 : pc += fd_ulong_if( !!(reg_dst & reg_src), offset, 0UL );
556 2 : FD_VM_INTERP_BRANCH_END;
557 :
558 : FD_VM_INTERP_INSTR_BEGIN(0x4e) /* FD_SBPF_OP_UDIV32_REG */
559 0 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
560 0 : reg[ dst ] = (ulong)( (uint)reg_dst / (uint)reg_src );
561 0 : FD_VM_INTERP_INSTR_END;
562 :
563 : FD_VM_INTERP_INSTR_BEGIN(0x4f) /* FD_SBPF_OP_OR64_REG */
564 61779 : reg[ dst ] = reg_dst | reg_src;
565 61779 : FD_VM_INTERP_INSTR_END;
566 :
567 : /* 0x50 - 0x5f ******************************************************/
568 :
569 : FD_VM_INTERP_INSTR_BEGIN(0x54) /* FD_SBPF_OP_AND_IMM */
570 62899 : reg[ dst ] = (ulong)( (uint)reg_dst & imm );
571 62899 : FD_VM_INTERP_INSTR_END;
572 :
573 : FD_VM_INTERP_BRANCH_BEGIN(0x55) /* FD_SBPF_OP_JNE_IMM */
574 463145 : pc += fd_ulong_if( reg_dst!=(ulong)(long)(int)imm, offset, 0UL );
575 463145 : FD_VM_INTERP_BRANCH_END;
576 :
577 : FD_VM_INTERP_INSTR_BEGIN(0x56) /* FD_SBPF_OP_UDIV64_IMM */
578 0 : reg[ dst ] = reg_dst / (ulong)imm;
579 0 : FD_VM_INTERP_INSTR_END;
580 :
581 : FD_VM_INTERP_INSTR_BEGIN(0x57) /* FD_SBPF_OP_AND64_IMM */
582 170823 : reg[ dst ] = reg_dst & (ulong)(long)(int)imm;
583 170823 : FD_VM_INTERP_INSTR_END;
584 :
585 : FD_VM_INTERP_INSTR_BEGIN(0x5c) /* FD_SBPF_OP_AND_REG */
586 35934 : reg[ dst ] = (ulong)(uint)( reg_dst & reg_src );
587 35934 : FD_VM_INTERP_INSTR_END;
588 :
589 : FD_VM_INTERP_BRANCH_BEGIN(0x5d) /* FD_SBPF_OP_JNE_REG */
590 1144483 : pc += fd_ulong_if( reg_dst!=reg_src, offset, 0UL );
591 1144483 : FD_VM_INTERP_BRANCH_END;
592 :
593 : FD_VM_INTERP_INSTR_BEGIN(0x5e) /* FD_SBPF_OP_UDIV64_REG */
594 0 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
595 0 : reg[ dst ] = reg_dst / reg_src;
596 0 : FD_VM_INTERP_INSTR_END;
597 :
598 : FD_VM_INTERP_INSTR_BEGIN(0x5f) /* FD_SBPF_OP_AND64_REG */
599 139454 : reg[ dst ] = reg_dst & reg_src;
600 139454 : FD_VM_INTERP_INSTR_END;
601 :
602 : /* 0x60 - 0x6f ******************************************************/
603 :
604 : /* FIXME: CHECK THE CU COST MODEL FOR THESE (IS IT LIKE
605 : FD_VM_CONSUME_MEM AND NOT JUST FIXED) */
606 : /* FIXME: MEM TRACING DIAGNOSTICS GO IN HERE */
607 :
608 : FD_VM_INTERP_INSTR_BEGIN(0x64) /* FD_SBPF_OP_LSH_IMM */
609 4 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L291 */
610 4 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, (uint)imm ) );
611 4 : FD_VM_INTERP_INSTR_END;
612 :
613 : FD_VM_INTERP_BRANCH_BEGIN(0x65) /* FD_SBPF_OP_JSGT_IMM */
614 28236 : pc += fd_ulong_if( (long)reg_dst>(long)(int)imm, offset, 0UL );
615 28236 : FD_VM_INTERP_BRANCH_END;
616 :
617 : FD_VM_INTERP_INSTR_BEGIN(0x66) /* FD_SBPF_OP_UREM32_IMM */
618 4 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)imm );
619 4 : FD_VM_INTERP_INSTR_END;
620 :
621 : FD_VM_INTERP_INSTR_BEGIN(0x67) /* FD_SBPF_OP_LSH64_IMM */
622 259581 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L376 */
623 259581 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, imm );
624 259581 : FD_VM_INTERP_INSTR_END;
625 :
626 : FD_VM_INTERP_INSTR_BEGIN(0x6c) /* FD_SBPF_OP_LSH_REG */
627 117908 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L292 */
628 117908 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHL( (uint)reg_dst, reg_src ) );
629 117908 : FD_VM_INTERP_INSTR_END;
630 :
631 : FD_VM_INTERP_BRANCH_BEGIN(0x6d) /* FD_SBPF_OP_JSGT_REG */
632 11462 : pc += fd_ulong_if( (long)reg_dst>(long)reg_src, offset, 0UL );
633 11462 : FD_VM_INTERP_BRANCH_END;
634 :
635 : FD_VM_INTERP_INSTR_BEGIN(0x6e) /* FD_SBPF_OP_UREM32_REG */
636 0 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
637 0 : reg[ dst ] = (ulong)( (uint)reg_dst % (uint)reg_src );
638 0 : FD_VM_INTERP_INSTR_END;
639 :
640 : FD_VM_INTERP_INSTR_BEGIN(0x6f) /* FD_SBPF_OP_LSH64_REG */
641 25094 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L377 */
642 25094 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHL( reg_dst, reg_src );
643 25094 : FD_VM_INTERP_INSTR_END;
644 :
645 : /* 0x70 - 0x7f ******************************************************/
646 :
647 : FD_VM_INTERP_INSTR_BEGIN(0x74) /* FD_SBPF_OP_RSH_IMM */
648 6 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L293 */
649 6 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, imm ) );
650 6 : FD_VM_INTERP_INSTR_END;
651 :
652 : FD_VM_INTERP_BRANCH_BEGIN(0x75) /* FD_SBPF_OP_JSGE_IMM */
653 18 : pc += fd_ulong_if( (long)reg_dst>=(long)(int)imm, offset, 0UL );
654 18 : FD_VM_INTERP_BRANCH_END;
655 :
656 : FD_VM_INTERP_INSTR_BEGIN(0x76) /* FD_SBPF_OP_UREM64_IMM */
657 1 : reg[ dst ] = reg_dst % (ulong)imm;
658 1 : FD_VM_INTERP_INSTR_END;
659 :
660 : FD_VM_INTERP_INSTR_BEGIN(0x77) /* FD_SBPF_OP_RSH64_IMM */
661 28948 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L378 */
662 28948 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, imm );
663 28948 : FD_VM_INTERP_INSTR_END;
664 :
665 : FD_VM_INTERP_INSTR_BEGIN(0x7c) /* FD_SBPF_OP_RSH_REG */
666 8 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L294 */
667 8 : reg[ dst ] = (ulong)( FD_RUST_UINT_WRAPPING_SHR( (uint)reg_dst, (uint)reg_src ) );
668 8 : FD_VM_INTERP_INSTR_END;
669 :
670 : FD_VM_INTERP_BRANCH_BEGIN(0x7d) /* FD_SBPF_OP_JSGE_REG */
671 39781 : pc += fd_ulong_if( (long)reg_dst>=(long)reg_src, offset, 0UL );
672 39781 : FD_VM_INTERP_BRANCH_END;
673 :
674 : FD_VM_INTERP_INSTR_BEGIN(0x7e) /* FD_SBPF_OP_UREM64_REG */
675 0 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
676 0 : reg[ dst ] = reg_dst % reg_src;
677 0 : FD_VM_INTERP_INSTR_END;
678 :
679 : FD_VM_INTERP_INSTR_BEGIN(0x7f) /* FD_SBPF_OP_RSH64_REG */
680 5191 : /* https://github.com/solana-labs/rbpf/blob/8d36530b7071060e2837ebb26f25590db6816048/src/interpreter.rs#L379 */
681 5191 : reg[ dst ] = FD_RUST_ULONG_WRAPPING_SHR( reg_dst, reg_src );
682 5191 : FD_VM_INTERP_INSTR_END;
683 :
684 : /* 0x80-0x8f ********************************************************/
685 :
686 : FD_VM_INTERP_INSTR_BEGIN(0x84) /* FD_SBPF_OP_NEG */
687 8 : reg[ dst ] = (ulong)( -(uint)reg_dst );
688 8 : FD_VM_INTERP_INSTR_END;
689 :
690 98367 : FD_VM_INTERP_BRANCH_BEGIN(0x85) { /* FD_SBPF_OP_CALL_IMM */
691 :
692 98371 : fd_sbpf_syscalls_t const * syscall = imm!=fd_sbpf_syscalls_key_null() ? fd_sbpf_syscalls_query_const( syscalls, (ulong)imm, NULL ) : NULL;
693 98367 : if( FD_UNLIKELY( !syscall ) ) { /* Optimize for the syscall case */
694 :
695 : /* Note we do the stack push before updating the pc(*). This implies
696 : that the call stack frame gets allocated _before_ checking if the
697 : call target is valid. It would be fine to switch the order
698 : though such would change the precise faulting semantics of
699 : sigtextbr and sigstack.
700 :
701 : (*)but after checking calldests, see point below. */
702 :
703 : /* Agave's order of checks
704 : (https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L486):
705 : 1. Lookup imm hash in FunctionRegistry (calldests_test is our equivalent)
706 : 2. Push stack frame
707 : 3. Check PC
708 : 4. Update PC
709 :
710 : Following this precisely is impossible as our PC check also
711 : serves as a bounds check for the calldests_test call. So we
712 : have to perform step 3 before step 1. The following
713 : is a best-effort implementation that should match the VM state
714 : in all ways except error code. */
715 :
716 : /* Special case to handle entrypoint.
717 : ebpf::hash_symbol_name(b"entrypoint") = 0xb00c380, and
718 : fd_pchash_inverse( 0xb00c380U ) = 0x71e3cf81U */
719 90809 : if( FD_UNLIKELY( imm==0x71e3cf81U ) ) {
720 0 : FD_VM_INTERP_STACK_PUSH;
721 0 : pc = entry_pc - 1;
722 90809 : } else {
723 90809 : ulong target_pc = (ulong)fd_pchash_inverse( imm );
724 90809 : if( FD_UNLIKELY( target_pc>=text_cnt ) ) {
725 6 : goto sigillbr; /* different return between 0x85 and 0x8d */
726 6 : }
727 90803 : if( FD_UNLIKELY( !fd_sbpf_calldests_test( calldests, target_pc ) ) ) {
728 0 : goto sigillbr;
729 0 : }
730 181605 : FD_VM_INTERP_STACK_PUSH;
731 181605 : pc = target_pc - 1;
732 181605 : }
733 :
734 90809 : } else {
735 :
736 7558 : FD_VM_INTERP_SYSCALL_EXEC;
737 :
738 3996 : }
739 98367 : } FD_VM_INTERP_BRANCH_END;
740 :
741 : FD_VM_INTERP_INSTR_BEGIN(0x86) /* FD_SBPF_OP_LMUL32_IMM */
742 2 : reg[ dst ] = (ulong)( (uint)reg_dst * imm );
743 2 : FD_VM_INTERP_INSTR_END;
744 :
745 148 : FD_VM_INTERP_INSTR_BEGIN(0x87) { /* FD_SBPF_OP_STW */
746 148 : ulong vaddr = reg_dst + offset;
747 148 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
748 148 : int sigsegv = !haddr;
749 148 : if( FD_UNLIKELY( sigsegv ) ) {
750 5 : vm->segv_vaddr = vaddr;
751 5 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
752 5 : vm->segv_access_len = 4UL;
753 5 : goto sigsegv;
754 5 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
755 143 : fd_vm_mem_st_4( haddr, imm );
756 143 : } FD_VM_INTERP_INSTR_END;
757 :
758 : FD_VM_INTERP_INSTR_BEGIN(0x87depr) /* FD_SBPF_OP_NEG64 deprecated */
759 58738 : reg[ dst ] = -reg_dst;
760 58738 : FD_VM_INTERP_INSTR_END;
761 :
762 4674 : FD_VM_INTERP_INSTR_BEGIN(0x8c) { /* FD_SBPF_OP_LDXW */
763 4674 : ulong vaddr = reg_src + offset;
764 4674 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_ld_sz, 0, 0UL );
765 4674 : int sigsegv = !haddr;
766 4674 : if( FD_UNLIKELY( sigsegv ) ) {
767 7 : vm->segv_vaddr = vaddr;
768 7 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
769 7 : vm->segv_access_len = 4UL;
770 7 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
771 7 : }
772 4667 : reg[ dst ] = fd_vm_mem_ld_4( haddr );
773 4667 : }
774 4667 : FD_VM_INTERP_INSTR_END;
775 :
776 4667 : FD_VM_INTERP_BRANCH_BEGIN(0x8d) { /* FD_SBPF_OP_CALL_REG */
777 :
778 1189 : FD_VM_INTERP_STACK_PUSH;
779 :
780 1189 : ulong vaddr = fd_sbpf_callx_uses_src_reg_enabled( sbpf_version ) ? reg_src : reg[ imm & 15U ];
781 :
782 : /* Notes: Agave checks region and target_pc before updating the pc.
783 : To match their state, we do the same, even though we could simply
784 : update the pc and let BRANCH_END fail.
785 : Also, Agave doesn't check alignment. */
786 :
787 1189 : ulong region = vaddr >> 32;
788 : /* ulong align = vaddr & 7UL; */
789 1189 : ulong target_pc = ((vaddr & FD_VM_OFFSET_MASK) - vm->text_off) / 8UL;
790 1189 : if( FD_UNLIKELY( (region!=1UL) | (target_pc>=text_cnt) ) ) goto sigtextbr; /* Note: untaken branches don't consume BTB */
791 1187 : pc = target_pc - 1;
792 :
793 1187 : } FD_VM_INTERP_BRANCH_END;
794 :
795 : FD_VM_INTERP_INSTR_BEGIN(0x8e) /* FD_SBPF_OP_LMUL32_REG */
796 1 : reg[ dst ] = (ulong)( (uint)reg_dst * (uint)reg_src );
797 1 : FD_VM_INTERP_INSTR_END;
798 :
799 39837 : FD_VM_INTERP_INSTR_BEGIN(0x8f) { /* FD_SBPF_OP_STXW */
800 39837 : ulong vaddr = reg_dst + offset;
801 39837 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(uint), region_haddr, region_st_sz, 1, 0UL );
802 39837 : int sigsegv = !haddr;
803 39837 : if( FD_UNLIKELY( sigsegv ) ) {
804 16 : vm->segv_vaddr = vaddr;
805 16 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
806 16 : vm->segv_access_len = 4UL;
807 16 : goto sigsegv;
808 16 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
809 39821 : fd_vm_mem_st_4( haddr, (uint)reg_src );
810 39821 : }
811 39821 : FD_VM_INTERP_INSTR_END;
812 :
813 : /* 0x90 - 0x9f ******************************************************/
814 :
815 : FD_VM_INTERP_INSTR_BEGIN(0x94) /* FD_SBPF_OP_MOD_IMM */
816 2593 : reg[ dst ] = (ulong)( (uint)reg_dst % imm );
817 2593 : FD_VM_INTERP_INSTR_END;
818 :
819 : FD_VM_INTERP_BRANCH_BEGIN(0x95) /* FD_SBPF_OP_EXIT */
820 86021 : /* Agave JIT VM exit implementation analysis below.
821 86021 :
822 86021 : Agave references:
823 86021 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/interpreter.rs#L503-L509
824 86021 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L697-L702 */
825 86021 : if( FD_UNLIKELY( !frame_cnt ) ) goto sigexit; /* Exit program */
826 84777 : frame_cnt--;
827 84777 : reg[6] = shadow[ frame_cnt ].r6;
828 84777 : reg[7] = shadow[ frame_cnt ].r7;
829 84777 : reg[8] = shadow[ frame_cnt ].r8;
830 84777 : reg[9] = shadow[ frame_cnt ].r9;
831 84777 : reg[10] = shadow[ frame_cnt ].r10;
832 84777 : pc = shadow[ frame_cnt ].pc;
833 84777 : FD_VM_INTERP_BRANCH_END;
834 :
835 : FD_VM_INTERP_INSTR_BEGIN(0x96) /* FD_SBPF_OP_LMUL64_IMM */
836 0 : reg[ dst ] = reg_dst * (ulong)(long)(int)imm;
837 0 : FD_VM_INTERP_INSTR_END;
838 :
839 348 : FD_VM_INTERP_INSTR_BEGIN(0x97) { /* FD_SBPF_OP_STQ */
840 348 : ulong vaddr = reg_dst + offset;
841 348 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
842 348 : int sigsegv = !haddr;
843 348 : if( FD_UNLIKELY( sigsegv ) ) {
844 7 : vm->segv_vaddr = vaddr;
845 7 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
846 7 : vm->segv_access_len = 8UL;
847 7 : goto sigsegv;
848 7 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
849 341 : fd_vm_mem_st_8( haddr, (ulong)(long)(int)imm );
850 341 : }
851 341 : FD_VM_INTERP_INSTR_END;
852 :
853 671392 : FD_VM_INTERP_INSTR_BEGIN(0x9c) { /* FD_SBPF_OP_LDXQ */
854 671392 : ulong vaddr = reg_src + offset;
855 671392 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_ld_sz, 0, 0UL );
856 671392 : int sigsegv = !haddr;
857 671392 : if( FD_UNLIKELY( sigsegv ) ) {
858 84 : vm->segv_vaddr = vaddr;
859 84 : vm->segv_access_type = FD_VM_ACCESS_TYPE_LD;
860 84 : vm->segv_access_len = 8UL;
861 84 : goto sigsegv; /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
862 84 : }
863 671308 : reg[ dst ] = fd_vm_mem_ld_8( haddr );
864 671308 : }
865 671308 : FD_VM_INTERP_INSTR_END;
866 :
867 : FD_VM_INTERP_INSTR_BEGIN(0x9e) /* FD_SBPF_OP_LMUL64_REG */
868 4 : reg[ dst ] = reg_dst * reg_src;
869 4 : FD_VM_INTERP_INSTR_END;
870 :
871 1037084 : FD_VM_INTERP_INSTR_BEGIN(0x9f) { /* FD_SBPF_OP_STXQ */
872 1037084 : ulong vaddr = reg_dst + offset;
873 1037084 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sizeof(ulong), region_haddr, region_st_sz, 1, 0UL );
874 1037084 : int sigsegv = !haddr;
875 1037084 : if( FD_UNLIKELY( sigsegv ) ) {
876 14 : vm->segv_vaddr = vaddr;
877 14 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
878 14 : vm->segv_access_len = 8UL;
879 14 : goto sigsegv;
880 14 : } /* Note: untaken branches don't consume BTB */ /* FIXME: sigbus */
881 1037070 : fd_vm_mem_st_8( haddr, reg_src );
882 1037070 : }
883 1037070 : FD_VM_INTERP_INSTR_END;
884 :
885 : FD_VM_INTERP_INSTR_BEGIN(0x97depr) /* FD_SBPF_OP_MOD64_IMM */
886 56173 : reg[ dst ] = reg_dst % (ulong)(long)(int)imm;
887 56173 : FD_VM_INTERP_INSTR_END;
888 :
889 : FD_VM_INTERP_INSTR_BEGIN(0x9cdepr) /* FD_SBPF_OP_MOD_REG */
890 35714 : if( FD_UNLIKELY( !(uint)reg_src ) ) goto sigfpe;
891 35714 : reg[ dst ] = (ulong)( ((uint)reg_dst % (uint)reg_src) );
892 35714 : FD_VM_INTERP_INSTR_END;
893 :
894 : FD_VM_INTERP_INSTR_BEGIN(0x9fdepr) /* FD_SBPF_OP_MOD64_REG */
895 36260 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
896 36259 : reg[ dst ] = reg_dst % reg_src;
897 36259 : FD_VM_INTERP_INSTR_END;
898 :
899 : /* 0xa0 - 0xaf ******************************************************/
900 :
901 : FD_VM_INTERP_INSTR_BEGIN(0xa4) /* FD_SBPF_OP_XOR_IMM */
902 120121 : reg[ dst ] = (ulong)( (uint)reg_dst ^ imm );
903 120121 : FD_VM_INTERP_INSTR_END;
904 :
905 : FD_VM_INTERP_BRANCH_BEGIN(0xa5) /* FD_SBPF_OP_JLT_IMM */
906 132491 : pc += fd_ulong_if( reg_dst<(ulong)(long)(int)imm, offset, 0UL );
907 132491 : FD_VM_INTERP_BRANCH_END;
908 :
909 : FD_VM_INTERP_INSTR_BEGIN(0xa7) /* FD_SBPF_OP_XOR64_IMM */
910 4269 : reg[ dst ] = reg_dst ^ (ulong)(long)(int)imm;
911 4269 : FD_VM_INTERP_INSTR_END;
912 :
913 : FD_VM_INTERP_INSTR_BEGIN(0xac) /* FD_SBPF_OP_XOR_REG */
914 24851 : reg[ dst ] = (ulong)(uint)( reg_dst ^ reg_src );
915 24851 : FD_VM_INTERP_INSTR_END;
916 :
917 : FD_VM_INTERP_BRANCH_BEGIN(0xad) /* FD_SBPF_OP_JLT_REG */
918 892 : pc += fd_ulong_if( reg_dst<reg_src, offset, 0UL );
919 892 : FD_VM_INTERP_BRANCH_END;
920 :
921 : FD_VM_INTERP_INSTR_BEGIN(0xaf) /* FD_SBPF_OP_XOR64_REG */
922 291452 : reg[ dst ] = reg_dst ^ reg_src;
923 291452 : FD_VM_INTERP_INSTR_END;
924 :
925 : /* 0xb0 - 0xbf ******************************************************/
926 :
927 : FD_VM_INTERP_INSTR_BEGIN(0xb4) /* FD_SBPF_OP_MOV_IMM */
928 374545 : reg[ dst ] = (ulong)imm;
929 374545 : FD_VM_INTERP_INSTR_END;
930 :
931 : FD_VM_INTERP_BRANCH_BEGIN(0xb5) /* FD_SBPF_OP_JLE_IMM */
932 379531 : pc += fd_ulong_if( reg_dst<=(ulong)(long)(int)imm, offset, 0UL );
933 379531 : FD_VM_INTERP_BRANCH_END;
934 :
935 : FD_VM_INTERP_INSTR_BEGIN(0xb6) /* FD_SBPF_OP_SHMUL64_IMM */
936 2 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)(int)imm ) >> 64 );
937 2 : FD_VM_INTERP_INSTR_END;
938 :
939 : FD_VM_INTERP_INSTR_BEGIN(0xb7) /* FD_SBPF_OP_MOV64_IMM */
940 1017453 : reg[ dst ] = (ulong)(long)(int)imm;
941 1017453 : FD_VM_INTERP_INSTR_END;
942 :
943 : FD_VM_INTERP_INSTR_BEGIN(0xbc) /* FD_SBPF_OP_MOV_REG */
944 1 : reg[ dst ] = (ulong)(long)(int)reg_src;
945 1 : FD_VM_INTERP_INSTR_END;
946 :
947 : FD_VM_INTERP_INSTR_BEGIN(0xbcdepr) /* FD_SBPF_OP_MOV_REG deprecated SIMD-1074 */
948 212971 : reg[ dst ] = (ulong)(uint)reg_src;
949 212971 : FD_VM_INTERP_INSTR_END;
950 :
951 : FD_VM_INTERP_BRANCH_BEGIN(0xbd) /* FD_SBPF_OP_JLE_REG */
952 54 : pc += fd_ulong_if( reg_dst<=reg_src, offset, 0UL );
953 54 : FD_VM_INTERP_BRANCH_END;
954 :
955 : FD_VM_INTERP_INSTR_BEGIN(0xbe) /* FD_SBPF_OP_SHMUL64_REG */
956 0 : reg[ dst ] = (ulong)(( (int128)(long)reg_dst * (int128)(long)reg_src ) >> 64 );
957 0 : FD_VM_INTERP_INSTR_END;
958 :
959 : FD_VM_INTERP_INSTR_BEGIN(0xbf) /* FD_SBPF_OP_MOV64_REG */
960 687547 : reg[ dst ] = reg_src;
961 687547 : FD_VM_INTERP_INSTR_END;
962 :
963 : /* 0xc0 - 0xcf ******************************************************/
964 :
965 : FD_VM_INTERP_INSTR_BEGIN(0xc4) /* FD_SBPF_OP_ARSH_IMM */
966 16 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, imm ) );
967 16 : FD_VM_INTERP_INSTR_END;
968 :
969 : FD_VM_INTERP_BRANCH_BEGIN(0xc5) /* FD_SBPF_OP_JSLT_IMM */ /* FIXME: CHECK IMM SIGN EXTENSION */
970 2775 : pc += fd_ulong_if( (long)reg_dst<(long)(int)imm, offset, 0UL );
971 2775 : FD_VM_INTERP_BRANCH_END;
972 :
973 : FD_VM_INTERP_INSTR_BEGIN(0xc6) /* FD_SBPF_OP_SDIV32_IMM */
974 0 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
975 0 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)imm );
976 0 : FD_VM_INTERP_INSTR_END;
977 :
978 : FD_VM_INTERP_INSTR_BEGIN(0xc7) /* FD_SBPF_OP_ARSH64_IMM */
979 7266 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, imm ) );
980 7266 : FD_VM_INTERP_INSTR_END;
981 :
982 : FD_VM_INTERP_INSTR_BEGIN(0xcc) /* FD_SBPF_OP_ARSH_REG */
983 22459 : reg[ dst ] = (ulong)(uint)( FD_RUST_INT_WRAPPING_SHR( (int)reg_dst, (uint)reg_src ) );
984 22459 : FD_VM_INTERP_INSTR_END;
985 :
986 : FD_VM_INTERP_BRANCH_BEGIN(0xcd) /* FD_SBPF_OP_JSLT_REG */
987 607015 : pc += fd_ulong_if( (long)reg_dst<(long)reg_src, offset, 0UL );
988 607015 : FD_VM_INTERP_BRANCH_END;
989 :
990 : FD_VM_INTERP_INSTR_BEGIN(0xce) /* FD_SBPF_OP_SDIV32_REG */
991 0 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
992 0 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
993 0 : reg[ dst ] = (ulong)(uint)( (int)reg_dst / (int)reg_src );
994 0 : FD_VM_INTERP_INSTR_END;
995 :
996 : FD_VM_INTERP_INSTR_BEGIN(0xcf) /* FD_SBPF_OP_ARSH64_REG */
997 13 : reg[ dst ] = (ulong)( FD_RUST_LONG_WRAPPING_SHR( (long)reg_dst, reg_src ) );
998 13 : FD_VM_INTERP_INSTR_END;
999 :
1000 : /* 0xd0 - 0xdf ******************************************************/
1001 :
1002 : FD_VM_INTERP_INSTR_BEGIN(0xd4) /* FD_SBPF_OP_END_LE */
1003 112065 : switch( imm ) {
1004 3 : case 16U: reg[ dst ] = (ushort)reg_dst; break;
1005 36231 : case 32U: reg[ dst ] = (uint) reg_dst; break;
1006 75831 : case 64U: break;
1007 0 : default: goto siginv;
1008 112065 : }
1009 112065 : FD_VM_INTERP_INSTR_END;
1010 :
1011 : FD_VM_INTERP_BRANCH_BEGIN(0xd5) /* FD_SBPF_OP_JSLE_IMM */
1012 4983 : pc += fd_ulong_if( (long)reg_dst<=(long)(int)imm, offset, 0UL );
1013 4983 : FD_VM_INTERP_BRANCH_END;
1014 :
1015 : FD_VM_INTERP_INSTR_BEGIN(0xd6) /* FD_SBPF_OP_SDIV64_IMM */
1016 0 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1017 0 : reg[ dst ] = (ulong)( (long)reg_dst / (long)(int)imm );
1018 0 : FD_VM_INTERP_INSTR_END;
1019 :
1020 : FD_VM_INTERP_INSTR_BEGIN(0xdc) /* FD_SBPF_OP_END_BE */
1021 143615 : switch( imm ) {
1022 33121 : case 16U: reg[ dst ] = (ulong)fd_ushort_bswap( (ushort)reg_dst ); break;
1023 66150 : case 32U: reg[ dst ] = (ulong)fd_uint_bswap ( (uint) reg_dst ); break;
1024 44344 : case 64U: reg[ dst ] = fd_ulong_bswap ( (ulong) reg_dst ); break;
1025 0 : default: goto siginv;
1026 143615 : }
1027 143615 : FD_VM_INTERP_INSTR_END;
1028 :
1029 : FD_VM_INTERP_BRANCH_BEGIN(0xdd) /* FD_SBPF_OP_JSLE_REG */
1030 369526 : pc += fd_ulong_if( (long)reg_dst<=(long)reg_src, offset, 0UL );
1031 369526 : FD_VM_INTERP_BRANCH_END;
1032 :
1033 : FD_VM_INTERP_INSTR_BEGIN(0xde) /* FD_SBPF_OP_SDIV64_REG */
1034 0 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1035 0 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1036 0 : reg[ dst ] = (ulong)( (long)reg_dst / (long)reg_src );
1037 0 : FD_VM_INTERP_INSTR_END;
1038 :
1039 : /* 0xe0 - 0xef ******************************************************/
1040 :
1041 : FD_VM_INTERP_INSTR_BEGIN(0xe6) /* FD_SBPF_OP_SREM32_IMM */
1042 0 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)imm==-1) ) ) goto sigfpeof;
1043 0 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)imm );
1044 0 : FD_VM_INTERP_INSTR_END;
1045 :
1046 : FD_VM_INTERP_INSTR_BEGIN(0xee) /* FD_SBPF_OP_SREM32_REG */
1047 0 : if( FD_UNLIKELY( !(int)reg_src ) ) goto sigfpe;
1048 0 : if( FD_UNLIKELY( ((int)reg_dst==INT_MIN) & ((int)reg_src==-1) ) ) goto sigfpeof;
1049 0 : reg[ dst ] = (ulong)(uint)( (int)reg_dst % (int)reg_src );
1050 0 : FD_VM_INTERP_INSTR_END;
1051 :
1052 : /* 0xf0 - 0xff ******************************************************/
1053 :
1054 : FD_VM_INTERP_INSTR_BEGIN(0xf6) /* FD_SBPF_OP_SREM64_IMM */
1055 1 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)(int)imm==-1L) ) ) goto sigfpeof;
1056 1 : reg[ dst ] = (ulong)( (long)reg_dst % (long)(int)imm );
1057 1 : FD_VM_INTERP_INSTR_END;
1058 :
1059 : FD_VM_INTERP_INSTR_BEGIN(0xf7) /* FD_SBPF_OP_HOR64 */
1060 139452 : reg[ dst ] = reg_dst | (((ulong)imm) << 32);
1061 139452 : FD_VM_INTERP_INSTR_END;
1062 :
1063 : FD_VM_INTERP_INSTR_BEGIN(0xfe) /* FD_SBPF_OP_SREM64_REG */
1064 0 : if( FD_UNLIKELY( !reg_src ) ) goto sigfpe;
1065 0 : if( FD_UNLIKELY( ((long)reg_dst==LONG_MIN) & ((long)reg_src==-1L) ) ) goto sigfpeof;
1066 0 : reg[ dst ] = (ulong)( (long)reg_dst % (long)reg_src );
1067 0 : FD_VM_INTERP_INSTR_END;
1068 :
1069 : /* FIXME: sigbus/sigrdonly are mapped to sigsegv for simplicity
1070 : currently but could be enabled if desired. */
1071 :
1072 : /* Note: sigtextbr is for sigtext errors that occur on branching
1073 : instructions (i.e., prefixed with FD_VM_INTERP_BRANCH_BEGIN).
1074 : We skip a repeat ic accumulation in FD_VM_INTERP_FAULT */
1075 :
1076 : /* FD_VM_INTERP_FAULT accumulates to ic and cu all non-faulting
1077 : instructions preceeding a fault generated by a non-branching
1078 : instruction. When a non-branching instruction faults, pc is at the
1079 : instruction and the number of non-branching instructions that have
1080 : not yet been reflected in ic and cu is:
1081 :
1082 : pc - pc0 + 1 - ic_correction
1083 :
1084 : as per the accounting described above. +1 to include the faulting
1085 : instruction itself.
1086 :
1087 : Note that, for a sigtext caused by a branch instruction, pc0==pc
1088 : (from the BRANCH_END) and ic_correction==0 (from the BRANCH_BEGIN)
1089 : such that the below does not change the already current values in
1090 : ic and cu. Thus it also "does the right thing" in both the
1091 : non-branching and branching cases for sigtext. The same applies to
1092 : sigsplit. */
1093 :
1094 0 : #define FD_VM_INTERP_FAULT \
1095 207 : ic_correction = pc - pc0 + 1UL - ic_correction; \
1096 207 : ic += ic_correction; \
1097 207 : if ( FD_UNLIKELY( ic_correction > cu ) ) err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; \
1098 207 : cu -= fd_ulong_min( ic_correction, cu )
1099 :
1100 1 : sigtext: err = FD_VM_ERR_EBPF_EXECUTION_OVERRUN; FD_VM_INTERP_FAULT; goto interp_halt;
1101 2 : sigtextbr: err = FD_VM_ERR_EBPF_CALL_OUTSIDE_TEXT_SEGMENT; /* ic current */ /* cu current */ goto interp_halt;
1102 1 : sigstack: err = FD_VM_ERR_EBPF_CALL_DEPTH_EXCEEDED; /* ic current */ /* cu current */ goto interp_halt;
1103 22 : sigill: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; FD_VM_INTERP_FAULT; goto interp_halt;
1104 6 : sigillbr: err = FD_VM_ERR_EBPF_UNSUPPORTED_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1105 0 : siginv: err = FD_VM_ERR_EBPF_INVALID_INSTRUCTION; /* ic current */ /* cu current */ goto interp_halt;
1106 183 : sigsegv: err = fd_vm_generate_access_violation( vm->segv_vaddr, vm->sbpf_version ); FD_VM_INTERP_FAULT; goto interp_halt;
1107 1332 : sigcost: err = FD_VM_ERR_EBPF_EXCEEDED_MAX_INSTRUCTIONS; /* ic current */ cu = 0UL; goto interp_halt;
1108 3562 : sigsyscall: err = FD_VM_ERR_EBPF_SYSCALL_ERROR; /* ic current */ /* cu current */ goto interp_halt;
1109 1 : sigfpe: err = FD_VM_ERR_EBPF_DIVIDE_BY_ZERO; FD_VM_INTERP_FAULT; goto interp_halt;
1110 0 : sigfpeof: err = FD_VM_ERR_EBPF_DIVIDE_OVERFLOW; FD_VM_INTERP_FAULT; goto interp_halt;
1111 1244 : sigexit: /* err current */ /* ic current */ /* cu current */ goto interp_halt;
1112 :
1113 0 : #undef FD_VM_INTERP_FAULT
1114 :
1115 6354 : interp_halt:
1116 :
1117 : /* Pack the unpacked execution state into vm to give a precise view of
1118 : the execution when the vm halted. */
1119 :
1120 6354 : vm->pc = pc;
1121 6354 : vm->ic = ic;
1122 6354 : vm->cu = cu;
1123 6354 : vm->frame_cnt = frame_cnt;
1124 :
1125 6354 : # undef FD_VM_INTERP_STACK_PUSH
1126 :
1127 6354 : # undef FD_VM_INTERP_BRANCH_END
1128 6354 : # undef FD_VM_INTERP_BRANCH_BEGIN
1129 :
1130 6354 : # undef FD_VM_INTERP_INSTR_END
1131 6354 : # undef FD_VM_INTERP_INSTR_BEGIN
1132 6354 : # undef FD_VM_INTERP_INSTR_EXEC
1133 :
1134 6354 : # if defined(__clang__)
1135 6354 : # pragma clang diagnostic pop
1136 6354 : # endif
1137 :
1138 6354 : # if defined(__GNUC__)
1139 6354 : # pragma GCC diagnostic pop
1140 6354 : # endif
1141 :
1142 : /* Agave/JIT CU model analysis (and why we are conformant!):
1143 :
1144 : The Agave JIT employs a similar strategy of accumulating instructions
1145 : in a linear run and processing them at the start of a new linear
1146 : run/branch (side note: the JIT treats the LDQ instruction as a "branch"
1147 : that jumps pc + 2).
1148 :
1149 : In what is assumed to be an act of register conservation, the JIT
1150 : uses a catch-all "instruction meter" (IM) register (REGISTER_INSTRUCTION_METER)
1151 : that represents two different interpretations of the question
1152 : "how many instructions can I execute?".
1153 :
1154 : The IM, depending on where we are in the execution, either represents:
1155 : 1. IM => The number of instructions remaining before exhausting CU
1156 : budget. This is analagous to vm->cu in our interpreter.
1157 : 2. IM' => The last pc you can execute in the current linear run before
1158 : exhausting CU budget. Mathematically, IM' = IM + pc0
1159 : where pc0, just like our definition, is the start of the linear run.
1160 :
1161 : Note: IM' can go past the actual basic block/segment. In-fact,
1162 : it typically does, and implies we can execute the full block without
1163 : exhausting CU budget (reminder that LDQ is treated as a branch).
1164 :
1165 : By default, the IM' form is used during execution. The IM form is used:
1166 : - (transiently) during the processing of a branch instruction
1167 : - in post-VM cleanup (updates EbpfVm::previous_instruction_meter).
1168 :
1169 : When a branch instruction is encountered, the JIT checks
1170 : for CU exhaustion with pc > IM', and throws an exception if so. This is valid,
1171 : because as described above, IM' is the largest PC you can reach.
1172 :
1173 : If we haven't exhausted our CU limit, it updates IM':
1174 : 1. IM = IM' - (pc + 1) # Note that IM' at this point is IM + pc0',
1175 : # where pc0' is the start of the current linear run.
1176 : 2. IM' = IM + pc0 # pc0 is the start of the new linear run (typically the target pc)
1177 :
1178 : Code (that does the above in one ALU instruction):
1179 : https://github.com/solana-labs/rbpf/blob/v0.8.5/src/jit.rs#L891
1180 :
1181 :
1182 : ### How does this relate to our interpreter?
1183 :
1184 : This process is similar to FD_VM_INTERP_BRANCH_BEGIN.
1185 : We just deal with the IM form throughout (with vm->cu and ic_correction).
1186 : If we break down step 1 from above with what we know about IM and IM',
1187 : we get the following:
1188 : 1. IM = IM' - (pc + 1)
1189 : IM = (IM + pc0') - (pc + 1)
1190 : IM = IM + (pc0' - (pc + 1))
1191 : IM = IM - ((pc + 1) - pc0')
1192 : IM = IM - ic_correction
1193 : Here, ((pc + 1) - pc0') is the number of instrutions executed in the current
1194 : linear run. This is the same as our ic_correction(*) in FD_VM_INTERP_BRANCH_BEGIN.
1195 :
1196 : If we replace IM with cu, this effectively becomes the
1197 : cu -= ic_correction
1198 : line in FD_VM_INTERP_BRANCH_BEGIN.
1199 :
1200 : (*) Note: ic_correction (also) takes two forms. It is either the instruction
1201 : accumulator or the number of instructions executed in the current linear run.
1202 : It (transiently) takes the latter form during FD_VM_INTERP_BRANCH_BEGIN and
1203 : FD_VM_INTERP_FAULT, and the former form otherwise.
1204 : */
1205 :
1206 : /* (WIP) Precise faulting and the Agave JIT:
1207 :
1208 : Since the cost model is a part of consensus, we need to conform with the Agave/JIT
1209 : cost model 1:1. To achieve that, our faulting model also needs to match precisely. This
1210 : section covers the various faults that the respective VMs implement and how they match.
1211 :
1212 : # Normal VM exit (sigexit):
1213 : VM exit instruction entrypoint: https://github.com/solana-labs/rbpf/blob/12237895305ab38514be865ebed6268553e4f589/src/jit.rs#L698-L708
1214 :
1215 : Pseudocode (with FD semantics):
1216 : ```
1217 : # pc is at the exit instruction
1218 : # pc0 is the start of the current linear run
1219 : if (frame_cnt == 0) {
1220 : goto sigexit;
1221 : }
1222 : ...
1223 :
1224 : sigexit:
1225 : if IM' <= pc {
1226 : goto sigcost;
1227 : } else {
1228 : goto interp_halt;
1229 : }
1230 : ```
1231 :
1232 : Breaking down the IM' < pc check:
1233 : - IM' = IM + pc0
1234 : - pc = ic + pc0, where (ic + 1) is the number of instructions executed in the current linear run
1235 :
1236 : IM' <= pc
1237 : IM + pc0 <= ic + pc0
1238 : IM <= ic
1239 : IM <= pc - pc0
1240 : IM < pc - pc0 + 1 # all unsigned integers
1241 : IM < ic_correction
1242 :
1243 : This is analagous to the ic_correction>cu check in VM_INTERP_BRANCH_BEGIN.
1244 :
1245 : # (TODO) Text Overrun (sigtext/sigsplit):
1246 :
1247 : */
|
