Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_fd_vm_private_h
2 : #define HEADER_fd_src_flamenco_vm_fd_vm_private_h
3 :
4 : #include "fd_vm.h"
5 :
6 : #include "../runtime/fd_runtime_const.h"
7 : #include "../runtime/fd_runtime.h"
8 : #include "fd_vm_base.h"
9 :
10 : /* FD_VM_ALIGN_RUST_{} define the alignments for relevant rust types.
11 : Alignments are derived with std::mem::align_of::<T>() and are enforced
12 : by the VM (with the exception of v1 loader).
13 :
14 : In our implementation, when calling FD_VM_MEM_HADDR_ST / FD_VM_MEM_HADDR_LD,
15 : we need to make sure we're passing the correct alignment based on the Rust
16 : type in the corresponding mapping in Agave.
17 :
18 : FD_VM_ALIGN_RUST_{} has been generated with this Rust code:
19 : ```rust
20 : pub type Epoch = u64;
21 : pub struct Pubkey(pub [u8; 32]);
22 : pub struct AccountMeta {
23 : pub lamports: u64,
24 : pub rent_epoch: Epoch,
25 : pub owner: Pubkey,
26 : pub executable: bool,
27 : }
28 :
29 : pub struct PodScalar(pub [u8; 32]);
30 :
31 : fn main() {
32 : println!("u8: {}", std::mem::align_of::<u8>());
33 : println!("u32: {}", std::mem::align_of::<u32>());
34 : println!("u64: {}", std::mem::align_of::<u64>());
35 : println!("u128: {}", std::mem::align_of::<u128>());
36 : println!("&[u8]: {}", std::mem::align_of::<&[u8]>());
37 : println!("AccountMeta: {}", std::mem::align_of::<AccountMeta>());
38 : println!("PodScalar: {}", std::mem::align_of::<PodScalar>());
39 : println!("Pubkey: {}", std::mem::align_of::<Pubkey>());
40 : }
41 : ``` */
42 :
43 8048 : #define FD_VM_ALIGN_RUST_U8 (1UL)
44 : #define FD_VM_ALIGN_RUST_U32 (4UL)
45 1647 : #define FD_VM_ALIGN_RUST_I32 (4UL)
46 : #define FD_VM_ALIGN_RUST_U64 (8UL)
47 : #define FD_VM_ALIGN_RUST_SLICE_U8_REF (8UL)
48 1051 : #define FD_VM_ALIGN_RUST_POD_U8_ARRAY (1UL)
49 36 : #define FD_VM_ALIGN_RUST_PUBKEY (1UL)
50 143 : #define FD_VM_ALIGN_RUST_SYSVAR_CLOCK (8UL)
51 73 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_SCHEDULE (8UL)
52 147 : #define FD_VM_ALIGN_RUST_SYSVAR_RENT (8UL)
53 90 : #define FD_VM_ALIGN_RUST_SYSVAR_LAST_RESTART_SLOT (8UL)
54 : #define FD_VM_ALIGN_RUST_SYSVAR_EPOCH_REWARDS (16UL)
55 :
56 : /* fd_vm_vec_t is the in-memory representation of a vector descriptor.
57 : Equal in layout to the Rust slice header &[_] and various vector
58 : types in the C version of the syscall API. */
59 : /* FIXME: WHEN IS VADDR NULL AND/OR SZ 0 OKAY? */
60 : /* FIXME: MOVE FD_VM_RUST_VEC_T FROM SYSCALL/FD_VM_CPI.H HERE TOO? */
61 :
62 : #define FD_VM_VEC_ALIGN FD_VM_ALIGN_RUST_SLICE_U8_REF
63 0 : #define FD_VM_VEC_SIZE (16UL)
64 :
65 : struct __attribute__((packed)) fd_vm_vec {
66 : ulong addr; /* FIXME: NAME -> VADDR */
67 : ulong len; /* FIXME: NAME -> SZ */
68 : };
69 :
70 : typedef struct fd_vm_vec fd_vm_vec_t;
71 :
72 : FD_STATIC_ASSERT( sizeof(fd_vm_vec_t)==FD_VM_VEC_SIZE, fd_vm_vec size mismatch );
73 :
74 : /* SBPF version and features
75 : https://github.com/anza-xyz/sbpf/blob/v0.12.2/src/program.rs#L28
76 : Note: SIMDs enable or disable features, e.g. BPF instructions.
77 : If we have macros with names ENABLE vs DISABLE, we have the advantage that
78 : the condition is always pretty clear: sbpf_version <= activation_version,
79 : but the disadvantage of inconsistent names.
80 : Vice versa, calling everything ENABLE has the risk to invert a <= with a >=
81 : and create a huge mess.
82 : We define both, so hopefully it's foolproof. */
83 :
84 : /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L28-L93 */
85 : /* SIMD-0166 */
86 12588703 : #define FD_VM_SBPF_MANUAL_STACK_FRAME_BUMP(v) ( v == FD_SBPF_V1 || v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L32-L34 */
87 : #define FD_VM_SBPF_STACK_FRAME_GAPS(v) ( v == FD_SBPF_V0 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L36-L38 */
88 : /* SIMD-0174 */
89 842807 : #define FD_VM_SBPF_ENABLE_PQR(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L41-L43 */
90 67568 : #define FD_VM_SBPF_EXPLICIT_SIGN_EXTENSION_OF_RESULTS(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L45-L47 */
91 33784 : #define FD_VM_SBPF_SWAP_SUB_REG_IMM_OPERANDS(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L49-L51 */
92 21205 : #define FD_VM_SBPF_DISABLE_NEG(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L53-L55 */
93 : /* SIMD-0173 */
94 4313 : #define FD_VM_SBPF_CALLX_USES_SRC_REG(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L58-L60 */
95 42410 : #define FD_VM_SBPF_DISABLE_LDDW(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L62-L64 */
96 21205 : #define FD_VM_SBPF_DISABLE_LE(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L66-L68 */
97 508920 : #define FD_VM_SBPF_MOVE_MEMORY_IX_CLASSES(v) ( v == FD_SBPF_V2 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L70-L72 */
98 : /* SIMD-0178 */
99 16892 : #define FD_VM_SBPF_STATIC_SYSCALLS(v) ( v >= FD_SBPF_V3 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L75-L77 */
100 : /* SIMD-0189 */
101 : #define FD_VM_SBPF_ENABLE_STRICTER_ELF_HEADERS(v) ( v >= FD_SBPF_V3 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L79-L81 */
102 32164 : #define FD_VM_SBPF_ENABLE_LOWER_RODATA_VADDR(v) ( v >= FD_SBPF_V3 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L83-L85 */
103 : /* SIMD-0377 */
104 375937 : #define FD_VM_SBPF_ENABLE_JMP32(v) ( v >= FD_SBPF_V3 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L87-L89 */
105 3117 : #define FD_VM_SBPF_CALLX_USES_DST_REG(v) ( v >= FD_SBPF_V3 ) /* https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/program.rs#L91-L93 */
106 :
107 1179019 : #define FD_VM_OFFSET_MASK (0xffffffffUL)
108 :
109 : /* https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L32 */
110 0 : #define FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION ((long)(FD_RUNTIME_ACC_SZ_MAX * 2UL))
111 :
112 : /* Kept equal to the canonical constant in fd_runtime_const.h, which the
113 : bpf loader serialization buffer bound relies on. */
114 : FD_STATIC_ASSERT( FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION==(long)FD_RUNTIME_ACC_DATA_GROWTH_MAX_PER_TXN, account_data_growth_cap );
115 :
116 : FD_PROTOTYPES_BEGIN
117 :
118 : /* Error logging handholding assertions */
119 :
120 : #ifdef FD_RUNTIME_ERR_HANDHOLDING
121 : /* Asserts that the error and error kind are populated (non-zero) */
122 : #define FD_VM_TEST_ERR_EXISTS( vm ) \
123 : FD_TEST( vm->instr_ctx->txn_out->err.exec_err ); \
124 : FD_TEST( vm->instr_ctx->txn_out->err.exec_err_kind )
125 :
126 : /* Used prior to a FD_VM_ERR_FOR_LOG_INSTR call to deliberately
127 : bypass overwrite handholding checks.
128 : Only use this if you know what you're doing. */
129 : #define FD_VM_PREPARE_ERR_OVERWRITE( vm ) \
130 : vm->instr_ctx->txn_out->err.exec_err = 0; \
131 : vm->instr_ctx->txn_out->err.exec_err_kind = 0
132 :
133 : /* Asserts that the error and error kind are not populated (zero) */
134 : #define FD_VM_TEST_ERR_OVERWRITE( vm ) \
135 : FD_TEST( !vm->instr_ctx->txn_out->err.exec_err ); \
136 : FD_TEST( !vm->instr_ctx->txn_out->err.exec_err_kind )
137 : #else
138 947 : #define FD_VM_TEST_ERR_EXISTS( vm ) ( ( void )0 )
139 2585 : #define FD_VM_PREPARE_ERR_OVERWRITE( vm ) ( ( void )0 )
140 4018 : #define FD_VM_TEST_ERR_OVERWRITE( vm ) ( ( void )0 )
141 : #endif
142 :
143 : /* Log error within the instr_ctx to match Agave/Rust error. */
144 :
145 2228 : #define FD_VM_ERR_FOR_LOG_EBPF( vm, err_ ) (__extension__({ \
146 2228 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
147 2228 : vm->instr_ctx->txn_out->err.exec_err = err_; \
148 2228 : vm->instr_ctx->txn_out->err.exec_err_kind = FD_EXECUTOR_ERR_KIND_EBPF; \
149 2228 : }))
150 :
151 1048 : #define FD_VM_ERR_FOR_LOG_SYSCALL( vm, err_ ) (__extension__({ \
152 1048 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
153 1048 : vm->instr_ctx->txn_out->err.exec_err = err_; \
154 1048 : vm->instr_ctx->txn_out->err.exec_err_kind = FD_EXECUTOR_ERR_KIND_SYSCALL; \
155 1048 : }))
156 :
157 742 : #define FD_VM_ERR_FOR_LOG_INSTR( vm, err_ ) (__extension__({ \
158 742 : FD_VM_TEST_ERR_OVERWRITE( vm ); \
159 742 : vm->instr_ctx->txn_out->err.exec_err = err_; \
160 742 : vm->instr_ctx->txn_out->err.exec_err_kind = FD_EXECUTOR_ERR_KIND_INSTR; \
161 742 : }))
162 :
163 1177651 : #define FD_VADDR_TO_REGION( _vaddr ) fd_ulong_min( (_vaddr) >> FD_VM_MEM_MAP_REGION_VIRT_ADDR_BITS, FD_VM_HIGH_REGION )
164 :
165 : /* fd_vm_instr APIs ***************************************************/
166 :
167 : /* FIXME: MIGRATE FD_SBPF_INSTR_T STUFF TO THIS API */
168 :
169 : /* fd_vm_instr returns the SBPF instruction word corresponding to the
170 : given fields. */
171 :
172 : FD_FN_CONST static inline ulong
173 : fd_vm_instr( ulong opcode, /* Assumed valid */
174 : ulong dst, /* Assumed in [0,FD_VM_REG_CNT) */
175 : ulong src, /* Assumed in [0,FD_VM_REG_CNT) */
176 : short offset,
177 0 : uint imm ) {
178 0 : return opcode | (dst<<8) | (src<<12) | (((ulong)(ushort)offset)<<16) | (((ulong)imm)<<32);
179 0 : }
180 :
181 : /* fd_vm_instr_* return the SBPF instruction field for the given word.
182 : fd_vm_instr_{normal,mem}_* only apply to {normal,mem} opclass
183 : instructions. */
184 :
185 10799603 : FD_FN_CONST static inline ulong fd_vm_instr_opcode( ulong instr ) { return instr & 255UL; } /* In [0,256) */
186 10728886 : FD_FN_CONST static inline ulong fd_vm_instr_dst ( ulong instr ) { return ((instr>> 8) & 15UL); } /* In [0,16) */
187 10663975 : FD_FN_CONST static inline ulong fd_vm_instr_src ( ulong instr ) { return ((instr>>12) & 15UL); } /* In [0,16) */
188 10666197 : FD_FN_CONST static inline ulong fd_vm_instr_offset( ulong instr ) { return (ulong)(long)(short)(ushort)(instr>>16); }
189 11284743 : FD_FN_CONST static inline uint fd_vm_instr_imm ( ulong instr ) { return (uint)(instr>>32); }
190 :
191 0 : FD_FN_CONST static inline ulong fd_vm_instr_opclass ( ulong instr ) { return instr & 7UL; } /* In [0,8) */
192 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opsrc ( ulong instr ) { return (instr>>3) & 1UL; } /* In [0,2) */
193 0 : FD_FN_CONST static inline ulong fd_vm_instr_normal_opmode ( ulong instr ) { return (instr>>4) & 15UL; } /* In [0,16) */
194 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opsize ( ulong instr ) { return (instr>>3) & 3UL; } /* In [0,4) */
195 0 : FD_FN_CONST static inline ulong fd_vm_instr_mem_opaddrmode( ulong instr ) { return (instr>>5) & 7UL; } /* In [0,16) */
196 :
197 : /* fd_vm_mem API ******************************************************/
198 :
199 : /* fd_vm_mem APIs support the fast mapping of virtual address ranges to
200 : host address ranges. The SBPF virtual address space consists of
201 : 5 consecutive 4 GiB regions (see fd_vm_base.h for layout). The
202 : mappable size of each region is less than 4 GiB (as implied by
203 : FD_VM_MEM_MAP_REGION_SZ==2^32-1 and that Solana protocol limits are
204 : much smaller still), so a valid virtual address range cannot span
205 : multiple regions. */
206 :
207 : /* fd_vm_mem_cfg configures the vm's tlb arrays. Assumes vm is valid
208 : and vm already has configured the rodata, stack, heap and input
209 : regions. Returns vm. */
210 :
211 : static inline fd_vm_t *
212 16081 : fd_vm_mem_cfg( fd_vm_t * vm ) {
213 16081 : if( FD_VM_SBPF_ENABLE_LOWER_RODATA_VADDR( vm->sbpf_version ) ) {
214 : /* In SBPF V3, rodata is at vaddr 0:
215 : [rodata@0, empty@0x100000000, stack@0x200000000, heap@0x300000000, input@0x400000000]
216 :
217 : This is so that we don't need to do any relocations - all rodata
218 : accesses are direct offsets from 0.
219 :
220 : https://github.com/anza-xyz/sbpf/blob/v0.14.4/src/elf.rs#L358-L362
221 : https://github.com/anza-xyz/agave/blob/v4.0.0-beta.4/syscalls/src/lib.rs#L346 */
222 1414 : vm->region_haddr[0] = (ulong)vm->rodata; vm->region_ld_sz[0] = (uint)vm->rodata_sz; vm->region_st_sz[0] = (uint)0UL;
223 1414 : vm->region_haddr[FD_VM_PROG_REGION] = 0UL; vm->region_ld_sz[FD_VM_PROG_REGION] = (uint)0UL; vm->region_st_sz[FD_VM_PROG_REGION] = (uint)0UL;
224 14667 : } else {
225 : /* V0-V2: region 0 unused, rodata at region 1 (vaddr 0x100000000) */
226 14667 : vm->region_haddr[0] = 0UL; vm->region_ld_sz[0] = (uint)0UL; vm->region_st_sz[0] = (uint)0UL;
227 14667 : vm->region_haddr[FD_VM_PROG_REGION] = (ulong)vm->rodata; vm->region_ld_sz[FD_VM_PROG_REGION] = (uint)vm->rodata_sz; vm->region_st_sz[FD_VM_PROG_REGION] = (uint)0UL;
228 14667 : }
229 16081 : vm->region_haddr[FD_VM_STACK_REGION] = (ulong)vm->stack; vm->region_ld_sz[FD_VM_STACK_REGION] = (uint)vm->stack_clean; vm->region_st_sz[FD_VM_STACK_REGION] = (uint)vm->stack_clean;
230 16081 : vm->region_haddr[FD_VM_HEAP_REGION] = (ulong)vm->heap; vm->region_ld_sz[FD_VM_HEAP_REGION] = (uint)vm->heap_clean; vm->region_st_sz[FD_VM_HEAP_REGION] = (uint)vm->heap_clean;
231 16081 : vm->region_haddr[5] = 0UL; vm->region_ld_sz[5] = (uint)0UL; vm->region_st_sz[5] = (uint)0UL;
232 16081 : if( vm->direct_mapping || !vm->input_mem_regions_cnt ) {
233 : /* When direct mapping is enabled, we don't use these fields because
234 : the load and stores are fragmented. */
235 4970 : vm->region_haddr[FD_VM_INPUT_REGION] = 0UL;
236 4970 : vm->region_ld_sz[FD_VM_INPUT_REGION] = 0U;
237 4970 : vm->region_st_sz[FD_VM_INPUT_REGION] = 0U;
238 11111 : } else {
239 11111 : vm->region_haddr[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].haddr;
240 11111 : vm->region_ld_sz[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].region_sz;
241 11111 : vm->region_st_sz[FD_VM_INPUT_REGION] = vm->input_mem_regions[0].region_sz;
242 11111 : }
243 16081 : return vm;
244 16081 : }
245 :
246 : /* Simplified version of Agave's `generate_access_violation()` function
247 : that simply returns either FD_VM_ERR_EBPF_ACCESS_VIOLATION or
248 : FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION. This has no consensus
249 : effects and is purely for logging purposes for fuzzing. Returns
250 : FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION if the provided vaddr is in the
251 : stack (0x200000000) and FD_VM_ERR_EBPF_ACCESS_VIOLATION otherwise.
252 :
253 : https://github.com/anza-xyz/sbpf/blob/v0.11.1/src/memory_region.rs#L834-L869 */
254 : static FD_FN_PURE inline int
255 769 : fd_vm_generate_access_violation( ulong vaddr, ulong sbpf_version ) {
256 : /* rel_offset can be negative because there is an edge case where the
257 : first "frame" right before the stack region should also throw a
258 : stack access violation. */
259 769 : long rel_offset = fd_long_sat_sub( (long)vaddr, (long)FD_VM_MEM_MAP_STACK_REGION_START );
260 769 : long stack_frame = rel_offset / (long)FD_VM_STACK_FRAME_SZ;
261 769 : if( !fd_sbpf_manual_stack_frame_bump_enabled( sbpf_version ) &&
262 769 : stack_frame>=-1L && stack_frame<=(long)FD_VM_MAX_CALL_DEPTH ) {
263 39 : return FD_VM_ERR_EBPF_STACK_ACCESS_VIOLATION;
264 39 : }
265 730 : return FD_VM_ERR_EBPF_ACCESS_VIOLATION;
266 769 : }
267 :
268 : /* fd_vm_mem_haddr translates the vaddr range [vaddr,vaddr+sz) (in
269 : infinite precision math) into the non-wrapping haddr range
270 : [haddr,haddr+sz). On success, returns haddr and every byte in the
271 : haddr range is a valid address. On failure, returns sentinel and
272 : there was at least one byte in the virtual address range that did not
273 : have a corresponding byte in the host address range.
274 :
275 : IMPORTANT SAFETY TIP! When sz==0, the return value currently is
276 : arbitrary. This is often fine as there should be no
277 : actual accesses to a sz==0 region. However, this also means that
278 : testing return for sentinel is insufficient to tell if mapping
279 : failed. That is, assuming sentinel is a location that could never
280 : happen on success:
281 :
282 : sz!=0 and ret!=sentinel -> success
283 : sz!=0 and ret==sentinel -> failure
284 : sz==0 -> ignore ret, application specific handling
285 :
286 : With ~O(2) extra fast branchless instructions, the below could be
287 : tweaked in the sz==0 case to return NULL or return a non-NULL
288 : sentinel value. What is most optimal practically depends on how
289 : empty ranges and NULL vaddr handling is defined in the application. */
290 :
291 :
292 : /* fd_vm_get_input_mem_region_idx returns the index into the input memory
293 : region array with the largest region offset that is <= the offset that
294 : is passed in. This function makes NO guarantees about the input being
295 : a valid input region offset; the caller is responsible for safely handling
296 : it. */
297 : static inline ulong
298 71125 : fd_vm_get_input_mem_region_idx( fd_vm_t const * vm, ulong offset ) {
299 71125 : uint left = 0U;
300 71125 : uint right = vm->input_mem_regions_cnt - 1U;
301 71125 : uint mid = 0U;
302 :
303 85436 : while( left<right ) {
304 14311 : mid = (left+right) / 2U;
305 14311 : if( offset>=vm->input_mem_regions[ mid ].vaddr_offset+vm->input_mem_regions[ mid ].address_space_reserved ) {
306 6032 : left = mid + 1U;
307 8279 : } else {
308 8279 : right = mid;
309 8279 : }
310 14311 : }
311 71125 : return left;
312 71125 : }
313 :
314 : /* If the region is an account, handle the resizing logic. This logic
315 : corresponds to
316 : solana_transaction_context::TransactionContext::access_violation_handler
317 :
318 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L510-L581 */
319 : static inline void
320 : fd_vm_handle_input_mem_region_oob( fd_vm_t const * vm,
321 : ulong offset,
322 : ulong sz,
323 : ulong region_idx,
324 167 : uchar write ) {
325 : /* If virtual_address_space_adjustments is not enabled, we don't need to
326 : do anything */
327 167 : if( FD_UNLIKELY( !vm->virtual_address_space_adjustments ) ) {
328 152 : return;
329 152 : }
330 :
331 : /* If the access is not a write, we don't need to do anything
332 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L523-L525 */
333 15 : if( FD_UNLIKELY( !write ) ) {
334 15 : return;
335 15 : }
336 :
337 0 : fd_vm_input_region_t * region = &vm->input_mem_regions[ region_idx ];
338 : /* If the region is not writable, we don't need to do anything
339 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L526-L529 */
340 0 : if( FD_UNLIKELY( !region->is_writable ) ) {
341 0 : return;
342 0 : }
343 :
344 : /* Calculate the requested length
345 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L532-L535 */
346 0 : ulong requested_len = fd_ulong_sat_sub( fd_ulong_sat_add( offset, sz ), region->vaddr_offset );
347 0 : if( FD_UNLIKELY( requested_len > region->address_space_reserved ) ) {
348 0 : return;
349 0 : }
350 :
351 : /* Calculate the remaining allowed growth
352 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L549-L551 */
353 0 : long remaining_growth_signed = fd_long_sat_sub(
354 0 : FD_MAX_ACCOUNT_DATA_GROWTH_PER_TRANSACTION,
355 0 : vm->instr_ctx->txn_out->details.accounts_resize_delta );
356 0 : ulong remaining_allowed_growth = (remaining_growth_signed > 0L)
357 0 : ? (ulong)remaining_growth_signed
358 0 : : 0UL;
359 :
360 : /* If the requested length is greater than the size of the region,
361 : resize the region
362 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L553-L571 */
363 0 : if( FD_UNLIKELY( requested_len > region->region_sz ) ) {
364 : /* Calculate the new region size
365 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L558-L560 */
366 0 : ulong new_region_sz = fd_ulong_min(
367 0 : fd_ulong_min( region->address_space_reserved, FD_RUNTIME_ACC_SZ_MAX ),
368 0 : fd_ulong_sat_add( region->region_sz, remaining_allowed_growth ) );
369 :
370 : /* Resize the account and the region
371 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L569-L570 */
372 0 : if( FD_UNLIKELY( new_region_sz > region->region_sz ) ) {
373 : /* Safe because new_region_sz > region->region_sz */
374 0 : long growth = (long)(new_region_sz - region->region_sz);
375 0 : vm->instr_ctx->txn_out->details.accounts_resize_delta = fd_long_sat_add(
376 0 : vm->instr_ctx->txn_out->details.accounts_resize_delta, growth );
377 :
378 0 : ulong memset_sz = fd_ulong_sat_sub( new_region_sz, vm->acc_region_metas[ region->acc_region_meta_idx ].acc->data_len );
379 0 : fd_memset( vm->acc_region_metas[ region->acc_region_meta_idx ].acc->data+vm->acc_region_metas[ region->acc_region_meta_idx ].acc->data_len, 0, memset_sz );
380 0 : vm->acc_region_metas[ region->acc_region_meta_idx ].acc->data_len = new_region_sz;
381 : /* https://github.com/anza-xyz/agave/blob/v4.2.0-beta.1/transaction-context/src/transaction.rs#L546 */
382 0 : *vm->acc_region_metas[ region->acc_region_meta_idx ].touched = 1;
383 0 : region->region_sz = (uint)new_region_sz;
384 0 : }
385 0 : }
386 0 : }
387 :
388 : /* fd_vm_find_input_mem_region returns the translated haddr for a given
389 : offset into the input region. If an offset/sz is invalid or if an
390 : illegal write is performed, the sentinel value is returned. If the offset
391 : provided is too large, it will choose the upper-most region as the
392 : region_idx. However, it will get caught for being too large of an access
393 : in the multi-region checks. */
394 : static inline ulong
395 : fd_vm_find_input_mem_region( fd_vm_t const * vm,
396 : ulong offset,
397 : ulong sz,
398 : uchar write,
399 71123 : ulong sentinel ) {
400 71123 : if( FD_UNLIKELY( vm->input_mem_regions_cnt==0 ) ) {
401 0 : return sentinel; /* Access is too large */
402 0 : }
403 :
404 : /* Binary search to find the correct memory region. If direct mapping is not
405 : enabled, then there is only 1 memory region which spans the input region. */
406 71123 : ulong region_idx = fd_vm_get_input_mem_region_idx( vm, offset );
407 71123 : if( FD_UNLIKELY( region_idx>=vm->input_mem_regions_cnt ) ) {
408 0 : return sentinel; /* Region not found */
409 0 : }
410 :
411 71123 : ulong bytes_in_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
412 71123 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
413 :
414 : /* If the access is out of bounds, invoke the callback to handle the out of bounds access.
415 : This potentially resizes the region if necessary. */
416 71123 : if( FD_UNLIKELY( sz>bytes_in_region ) ) {
417 167 : fd_vm_handle_input_mem_region_oob( vm, offset, sz, region_idx, write );
418 167 : }
419 :
420 : /* After potentially resizing, re-check the bounds */
421 71123 : bytes_in_region = fd_ulong_sat_sub( vm->input_mem_regions[ region_idx ].region_sz,
422 71123 : fd_ulong_sat_sub( offset, vm->input_mem_regions[ region_idx ].vaddr_offset ) );
423 : /* If the access is still out of bounds, return the sentinel */
424 71123 : if( FD_UNLIKELY( sz>bytes_in_region ) ) {
425 167 : return sentinel;
426 167 : }
427 :
428 70956 : if( FD_UNLIKELY( write && vm->input_mem_regions[ region_idx ].is_writable==0U ) ) {
429 142 : return sentinel; /* Illegal write */
430 142 : }
431 :
432 : /* https://github.com/anza-xyz/agave/blob/v4.2.0-beta.1/transaction-context/src/transaction.rs#L546 */
433 70814 : if( FD_UNLIKELY( write && vm->acc_region_metas &&
434 70814 : vm->input_mem_regions[ region_idx ].acc_region_meta_idx!=ULONG_MAX ) ) {
435 0 : uchar * touched = vm->acc_region_metas[ vm->input_mem_regions[ region_idx ].acc_region_meta_idx ].touched;
436 0 : if( FD_LIKELY( touched ) ) *touched = 1;
437 0 : }
438 :
439 70814 : ulong start_region_idx = region_idx;
440 :
441 70814 : ulong adjusted_haddr = vm->input_mem_regions[ start_region_idx ].haddr + offset - vm->input_mem_regions[ start_region_idx ].vaddr_offset;
442 70814 : return adjusted_haddr;
443 70956 : }
444 :
445 : /* FD_VM_GROW_MIN amortizes {stack,heap} growth by rounding up request
446 : sizes. */
447 :
448 20872 : #define FD_VM_GROW_MIN (16384UL)
449 :
450 : /* fd_vm_{stack,heap}_grow lazily grow the stack and heap */
451 :
452 : static inline void
453 : fd_vm_region_grow( fd_vm_t * vm,
454 : ulong region,
455 : uchar * base,
456 : ulong * pclean,
457 : ulong cap,
458 20887 : ulong sz ) {
459 20887 : ulong clean = *pclean;
460 20887 : if( FD_LIKELY( sz<=clean ) ) return; /* nothing to do */
461 :
462 20872 : sz = fd_ulong_min( fd_ulong_align_up( sz, FD_VM_GROW_MIN ), cap );
463 :
464 20872 : fd_memset( base + clean, 0, sz-clean );
465 :
466 20872 : *pclean = sz;
467 20872 : vm->region_ld_sz[ region ] = (uint)sz;
468 20872 : vm->region_st_sz[ region ] = (uint)sz;
469 20872 : }
470 :
471 : static inline void
472 : fd_vm_stack_grow( fd_vm_t * vm,
473 10476 : ulong sz ) {
474 10476 : fd_vm_region_grow( vm, FD_VM_STACK_REGION, vm->stack, &vm->stack_clean, FD_VM_STACK_MAX, sz );
475 10476 : }
476 :
477 : static inline void
478 : fd_vm_heap_grow( fd_vm_t * vm,
479 10414 : ulong sz ) {
480 10414 : fd_vm_region_grow( vm, FD_VM_HEAP_REGION, vm->heap, &vm->heap_clean, vm->heap_max, sz );
481 10414 : }
482 :
483 : /* fd_vm_mem_init_full eagerly initializes the whole stack and heap. */
484 :
485 : static inline void
486 7544 : fd_vm_mem_init_full( fd_vm_t * vm ) {
487 7544 : fd_vm_stack_grow( vm, FD_VM_STACK_MAX );
488 7544 : fd_vm_heap_grow ( vm, vm->heap_max );
489 7544 : }
490 :
491 : static inline ulong
492 : fd_vm_mem_haddr( fd_vm_t * vm,
493 : ulong vaddr,
494 : ulong sz,
495 : ulong const * vm_region_haddr, /* indexed [0,6) */
496 : uint const * vm_region_sz, /* indexed [0,6) */
497 : uchar write, /* 1 if the access is a write, 0 if it is a read */
498 1177651 : ulong sentinel ) {
499 1177651 : ulong region = FD_VADDR_TO_REGION( vaddr );
500 1177651 : ulong offset = vaddr & FD_VM_OFFSET_MASK;
501 :
502 : /* Some configurations of the vm have unmapped gaps between each
503 : stack frame. If this is the case, we need to check that the access
504 : is not in a gap region. */
505 1177651 : int stack_frame_gaps_enabled = vm->stack_push_frame_count > 1;
506 1177651 : if( FD_UNLIKELY( region==FD_VM_STACK_REGION && stack_frame_gaps_enabled ) ) {
507 : /* If an access starts in a gap region, that is an access violation */
508 568679 : if( FD_UNLIKELY( !!(vaddr & 0x1000) ) ) {
509 49 : return sentinel;
510 49 : }
511 :
512 : /* To account for the fact that we have gaps in the virtual address space but not in the
513 : physical address space, we need to subtract from the offset the size of all the virtual
514 : gap frames underneath it.
515 :
516 : https://github.com/solana-labs/rbpf/blob/b503a1867a9cfa13f93b4d99679a17fe219831de/src/memory_region.rs#L147-L149 */
517 568630 : ulong gap_mask = 0xFFFFFFFFFFFFF000;
518 568630 : offset = ( ( offset & gap_mask ) >> 1 ) | ( offset & ~gap_mask );
519 568630 : }
520 :
521 1177602 : ulong region_sz = (ulong)vm_region_sz[ region ];
522 1177602 : ulong sz_max = region_sz - fd_ulong_min( offset, region_sz );
523 :
524 : /* If the region is an account, handle the resizing logic. This logic corresponds to
525 : solana_transaction_context::TransactionContext::access_violation_handler
526 :
527 : https://github.com/anza-xyz/agave/blob/v3.0.1/transaction-context/src/lib.rs#L510-L581 */
528 1177602 : if( region==FD_VM_INPUT_REGION ) {
529 71110 : return fd_vm_find_input_mem_region( vm, offset, sz, write, sentinel );
530 71110 : }
531 :
532 1106492 : if( FD_UNLIKELY( sz>sz_max && (region==FD_VM_STACK_REGION || region==FD_VM_HEAP_REGION) ) ) {
533 : /* {stack,heap} are lazily initialized. faults here mean that we
534 : need to do some zero initialization work. */
535 5876 : ulong need = fd_ulong_sat_add( offset, sz );
536 5876 : ulong cap = fd_ulong_if( region==FD_VM_STACK_REGION, FD_VM_STACK_MAX, vm->heap_max );
537 5876 : if( FD_LIKELY( need<=cap ) ) {
538 5768 : if( region==FD_VM_STACK_REGION ) fd_vm_stack_grow( vm, need );
539 2847 : else fd_vm_heap_grow ( vm, need );
540 5768 : region_sz = (ulong)vm_region_sz[ region ]; /* republished by grow */
541 5768 : sz_max = region_sz - fd_ulong_min( offset, region_sz );
542 5768 : }
543 5876 : }
544 :
545 1106492 : return fd_ulong_if( sz<=sz_max, vm_region_haddr[ region ] + offset, sentinel );
546 1177602 : }
547 :
548 :
549 96699 : FD_FN_PURE static inline ulong fd_vm_mem_ld_1( ulong haddr ) {
550 96699 : return (ulong)*(uchar const *)haddr;
551 96699 : }
552 :
553 2926 : FD_FN_PURE static inline ulong fd_vm_mem_ld_2( ulong haddr ) {
554 2926 : ushort t;
555 2926 : memcpy( &t, (void const *)haddr, sizeof(ushort) );
556 2926 : return (ulong)t;
557 2926 : }
558 :
559 8496 : FD_FN_PURE static inline ulong fd_vm_mem_ld_4( ulong haddr ) {
560 8496 : uint t;
561 8496 : memcpy( &t, (void const *)haddr, sizeof(uint) );
562 8496 : return (ulong)t;
563 8496 : }
564 :
565 289581 : FD_FN_PURE static inline ulong fd_vm_mem_ld_8( ulong haddr ) {
566 289581 : ulong t;
567 289581 : memcpy( &t, (void const *)haddr, sizeof(ulong) );
568 289581 : return t;
569 289581 : }
570 :
571 120552 : static inline void fd_vm_mem_st_1( ulong haddr, uchar val ) {
572 120552 : *(uchar *)haddr = val;
573 120552 : }
574 :
575 : static inline void fd_vm_mem_st_2( ulong haddr,
576 38494 : ushort val ) {
577 38494 : memcpy( (void *)haddr, &val, sizeof(ushort) );
578 38494 : }
579 :
580 : static inline void fd_vm_mem_st_4( ulong haddr,
581 18651 : uint val ) {
582 18651 : memcpy( (void *)haddr, &val, sizeof(uint) );
583 18651 : }
584 :
585 : static inline void fd_vm_mem_st_8( ulong haddr,
586 543810 : ulong val ) {
587 543810 : memcpy( (void *)haddr, &val, sizeof(ulong) );
588 543810 : }
589 :
590 : FD_PROTOTYPES_END
591 :
592 : #endif /* HEADER_fd_src_flamenco_vm_fd_vm_private_h */
|
