Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
2 : #define HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
3 : #include "../fd_vm_private.h"
4 :
5 : /* fd_vm_cu API *******************************************************/
6 :
7 : /* FD_VM_CU_UPDATE charges the vm cost compute units.
8 :
9 : If the vm has fewer than cost cu available, this will cause
10 : the caller to zero out the vm->cu and return with FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED.
11 : This macro is robust.
12 : This is meant to be used by syscall implementations and strictly
13 : conforms with the vm-syscall ABI interface.
14 :
15 : Note: in Agave a syscall can return success leaving 0 available CUs.
16 : The instruction will fail at the next instruction (e.g., exit).
17 : To reproduce the same behavior, we do not return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED
18 : when cu == 0.
19 :
20 : FD_VM_CU_MEM_OP_UPDATE charges the vm for a memory operation on sz
21 : bytes: sz / FD_VM_CPI_BYTES_PER_UNIT compute units, but at least
22 : FD_VM_MEM_OP_BASE_COST. Behavior is otherwise identical to
23 : FD_VM_CU_UPDATE. */
24 :
25 35607 : #define FD_VM_CU_UPDATE( vm, cost ) (__extension__({ \
26 35607 : fd_vm_t * _vm = (vm); \
27 35607 : ulong _cost = (cost); \
28 35607 : ulong _cu = _vm->cu; \
29 35607 : if( FD_UNLIKELY( _cost>_cu ) ) { \
30 99 : _vm->cu = 0UL; \
31 99 : FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_COMPUTE_BUDGET_EXCEEDED ); \
32 99 : return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED; \
33 99 : } \
34 35607 : _vm->cu = _cu - _cost; \
35 35508 : }))
36 :
37 : /* https://github.com/anza-xyz/agave/blob/5263c9d61f3af060ac995956120bef11c1bbf182/programs/bpf_loader/src/syscalls/mem_ops.rs#L7 */
38 : #define FD_VM_CU_MEM_OP_UPDATE( vm, sz ) \
39 3748 : FD_VM_CU_UPDATE( vm, fd_ulong_max( FD_VM_MEM_OP_BASE_COST, sz / FD_VM_CPI_BYTES_PER_UNIT ) )
40 :
41 :
42 : /* fd_vm_mem API *****************************************************/
43 :
44 : /* fd_vm_haddr_query is a struct that contains information about a vaddr, align, sz, and whether it is a slice.
45 : The translated haddr is written into the `haddr` field of the struct on success. This struct is primarily used
46 : by the FD_VM_TRANSLATE_MUT macro. See more details in the macro's documentation. */
47 : struct fd_vm_haddr_query {
48 : ulong vaddr;
49 : ulong align;
50 : ulong sz;
51 : uchar is_slice;
52 : void * haddr; /* out field */
53 : };
54 : typedef struct fd_vm_haddr_query fd_vm_haddr_query_t;
55 :
56 : /* FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY returns the haddr corresponding
57 : to _vaddr, when _vaddr is an u8 array of size _sz, and handles the
58 : difference between slice (_is_slice=1) or array (_is_slice=0).
59 : This is a common pattern in cryptographic syscalls, see for example
60 : fd_vm_syscall_crypto.c, fd_vm_syscall_curve.c */
61 6066 : #define FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY(_vm,_vaddr,_sz,_is_slice) (__extension__({ \
62 6066 : fd_vm_haddr_query_t _result = { \
63 6066 : .vaddr = (_vaddr), \
64 6066 : .align = (_is_slice) ? FD_VM_ALIGN_RUST_U8 : FD_VM_ALIGN_RUST_POD_U8_ARRAY, \
65 6066 : .sz = (_sz), \
66 6066 : .is_slice = (_is_slice), \
67 6066 : }; \
68 6066 : fd_vm_haddr_query_t * queries[] = { &_result }; \
69 6066 : FD_VM_TRANSLATE_MUT( (_vm), queries ); \
70 6066 : _result.haddr; }));
71 :
72 5015 : #define FD_VM_HADDR_QUERY_U8_SLICE(_vm,_vaddr,_sz) FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY(_vm,_vaddr,_sz,1)
73 1051 : #define FD_VM_HADDR_QUERY_U8_ARRAY(_vm,_vaddr,_sz) FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY(_vm,_vaddr,_sz,0)
74 :
75 : /* FD_VM_MEM_HADDR_LD returns a read only pointer to the first byte
76 : in the host address space corresponding to vm's virtual address range
77 : [vaddr,vaddr+sz). If the vm has check_align enabled, the vaddr
78 : should be aligned to align and the returned pointer will be similarly
79 : aligned. Align is assumed to be a power of two <= 8 (FIXME: CHECK
80 : THIS LIMIT).
81 :
82 : If the virtual address range cannot be mapped to the host address
83 : space completely and/or (when applicable) vaddr is not appropriately
84 : aligned, this will cause the caller to return FD_VM_SYSCALL_ERR_SEGFAULT.
85 : This macro is robust. This is meant to be used by syscall
86 : implementations and strictly conforms with the vm-syscall ABI
87 : interface.
88 :
89 : FD_VM_MEM_HADDR_ST returns a read-write pointer but is otherwise
90 : identical to FD_VM_MEM_HADDR_LD.
91 :
92 : These macros intentionally don't support multi region loads/stores.
93 : The load/store macros are used by vm syscalls and mirror the use
94 : of translate_slice{_mut}. However, this check does not allow for
95 : multi region accesses. So if there is an attempt at a multi region
96 : translation, an error will be returned.
97 :
98 : FD_VM_MEM_HADDR_ST_UNCHECKED checks region writability during
99 : address translation, but omits the size-limit, translation-failure and
100 : alignment error handling in FD_VM_MEM_HADDR_ST.
101 :
102 : FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED has all of the checks of a
103 : store, but intentionally omits the is_writable checks for the
104 : input region that are done during memory translation.
105 :
106 : FD_VM_MEM_HADDR_ST_NO_SZ_CHECK does all of the checks of a load,
107 : except for a check on the validity of the size of a load. It only
108 : checks that the specific vaddr that is being translated is valid. */
109 :
110 49688 : #define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
111 49688 : fd_vm_t * _vm = (vm); \
112 49688 : ulong _vaddr = (vaddr); \
113 49688 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL ); \
114 49688 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
115 49688 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
116 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
117 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
118 0 : } \
119 49688 : if( FD_UNLIKELY( (!_haddr) ) ) { \
120 318 : _vm->segv_vaddr = _vaddr; \
121 318 : _vm->segv_access_len = (sz); \
122 318 : _vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; \
123 318 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \
124 318 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
125 318 : } \
126 49688 : if ( FD_UNLIKELY( _sigbus ) ) { \
127 5 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
128 5 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
129 5 : } \
130 49370 : (void const *)_haddr; \
131 49365 : }))
132 :
133 : #define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
134 : fd_vm_t const * _vm = (vm); \
135 : ulong _vaddr = (vaddr); \
136 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL ); \
137 : (void const *)_haddr; \
138 : }))
139 :
140 :
141 : #define FD_VM_MEM_HADDR_LD_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
142 : FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
143 : }))
144 :
145 : static inline void *
146 15102 : FD_VM_MEM_HADDR_ST_( fd_vm_t *vm, ulong vaddr, ulong align, ulong sz, int *err ) {
147 15102 : fd_vm_t * _vm = (vm);
148 15102 : ulong _vaddr = (vaddr);
149 15102 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL );
150 15102 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) ));
151 15102 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) {
152 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH );
153 0 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
154 0 : return NULL;
155 0 : }
156 15102 : if( FD_UNLIKELY( (!_haddr) ) ) {
157 3 : vm->segv_vaddr = vaddr;
158 3 : vm->segv_access_len = (sz);
159 3 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
160 3 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) );
161 3 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
162 3 : return NULL;
163 3 : }
164 15099 : if ( FD_UNLIKELY( _sigbus ) ) {
165 10 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER );
166 10 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
167 10 : return NULL;
168 10 : }
169 15089 : return (void *)_haddr;
170 15099 : }
171 :
172 15096 : #define FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
173 15096 : int _err = 0; \
174 15096 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, sz, &_err ); \
175 15096 : if ( FD_UNLIKELY( 0 != _err )) \
176 15096 : return _err; \
177 15096 : ret; \
178 15083 : }))
179 :
180 : #define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
181 : fd_vm_t * _vm = (vm); \
182 : ulong _vaddr = (vaddr); \
183 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL ); \
184 : (void const *)_haddr; \
185 : }))
186 :
187 : #define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
188 : fd_vm_t * _vm = (vm); \
189 : ulong _vaddr = (vaddr); \
190 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL ); \
191 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
192 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
193 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
194 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
195 : } \
196 : if( FD_UNLIKELY( !_haddr ) ) { \
197 : _vm->segv_vaddr = _vaddr; \
198 : _vm->segv_access_len = (sz); \
199 : _vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; \
200 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \
201 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
202 : } \
203 : if ( FD_UNLIKELY( _sigbus ) ) { \
204 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
205 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
206 : } \
207 : (void *)_haddr; \
208 : }))
209 :
210 :
211 : #define FD_VM_MEM_HADDR_ST_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
212 : int _err = 0; \
213 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, 1UL, &_err ); \
214 : if ( FD_UNLIKELY( 0 != _err )) \
215 : return _err; \
216 : ret; \
217 : }))
218 :
219 :
220 : /* FD_VM_MEM_SLICE_HADDR_[LD, ST] macros return an arbitrary value if sz == 0. This is because
221 : Agave's translate_slice function returns an empty array if the sz == 0.
222 :
223 : Users of this macro should be aware that they should never access the returned value if sz==0.
224 :
225 : https://github.com/solana-labs/solana/blob/767d24e5c10123c079e656cdcf9aeb8a5dae17db/programs/bpf_loader/src/syscalls/mod.rs#L560
226 :
227 : LONG_MAX check: https://github.com/anza-xyz/agave/blob/dc4b9dcbbf859ff48f40d00db824bde063fdafcc/programs/bpf_loader/src/syscalls/mod.rs#L580
228 : Technically, the check in Agave is against
229 : "pointer-sized signed integer type ... The size of this primitive is
230 : how many bytes it takes to reference any location in memory. For
231 : example, on a 32 bit target, this is 4 bytes and on a 64 bit target,
232 : this is 8 bytes."
233 : Realistically, given the amount of memory that a validator consumes,
234 : no one is going to be running on a 32 bit target. So, we don't bother
235 : with conditionally compiling in an INT_MAX check. We just assume
236 : LONG_MAX. */
237 30612 : #define FD_VM_MEM_SLICE_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
238 30612 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
239 14 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
240 14 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
241 14 : } \
242 30612 : void const * haddr = 0UL; \
243 30598 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
244 87346 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ); \
245 87346 : } \
246 30598 : haddr; \
247 30452 : }))
248 :
249 8556 : #define FD_VM_MEM_SLICE_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
250 8556 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
251 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
252 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
253 0 : } \
254 8556 : void * haddr = 0UL; \
255 8556 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
256 8220 : haddr = FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ); \
257 8220 : } \
258 8556 : haddr; \
259 8556 : }))
260 :
261 : /* FIXME: use overlap logic from runtime? */
262 1924 : #define FD_VM_MEM_CHECK_NON_OVERLAPPING( vm, addr0, sz0, addr1, sz1 ) do { \
263 1924 : if( FD_UNLIKELY(( ((addr0> addr1) && (fd_ulong_sat_sub(addr0, addr1) < sz1)) ) || \
264 1924 : ( ((addr1>=addr0) && (fd_ulong_sat_sub(addr1, addr0) < sz0)) ) )) { \
265 9 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_COPY_OVERLAPPING ); \
266 9 : return FD_VM_SYSCALL_ERR_COPY_OVERLAPPING; \
267 9 : } \
268 1924 : } while(0)
269 :
270 : /* https://github.com/anza-xyz/agave/blob/v4.4.0-beta.0/syscalls/src/lib.rs#L630-L656 */
271 : static inline int
272 : fd_vm_mem_touch_st( fd_vm_t * vm,
273 : ulong vaddr,
274 11048 : ulong sz ) {
275 11048 : ulong haddr = fd_vm_mem_haddr( vm, vaddr, sz, vm->region_haddr, vm->region_st_sz, 1, 0UL );
276 11048 : if( FD_UNLIKELY( !haddr ) ) {
277 261 : vm->segv_vaddr = vaddr;
278 261 : vm->segv_access_len = sz;
279 261 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
280 261 : FD_VM_ERR_FOR_LOG_EBPF( vm, fd_vm_generate_access_violation( vaddr, vm->sbpf_version ) );
281 261 : return FD_VM_SYSCALL_ERR_SEGFAULT;
282 261 : }
283 10787 : return FD_VM_SUCCESS;
284 11048 : }
285 :
286 : /* Mimics Agave's `translate_mut!` macro by taking in a variable number
287 : of (vaddr, align, sz) entries and translates each of them, failing
288 : if any one of the translations fail, or if any of the vaddrs have
289 : overlapping haddr regions. The caller is responsible for creating
290 : each `fd_vm_haddr_query_t` object containing information about the
291 : vaddr, align, sz, and whether it is a slice. Takes in any number
292 : of queries, provided into the input as an array of pointers to each
293 : query. The translated haddr is written into the `haddr` field
294 : of each of the `fd_vm_haddr_query_t` objects on success.
295 :
296 : https://github.com/anza-xyz/agave/blob/v4.4.0-beta.0/syscalls/src/lib.rs#L662-L713 */
297 10760 : #define FD_VM_TRANSLATE_MUT( _vm, _queries ) do { \
298 10760 : ulong _n = sizeof(_queries)/sizeof(fd_vm_haddr_query_t *); \
299 10760 : /* https://github.com/anza-xyz/agave/blob/v4.4.0-beta.0/syscalls/src/lib.rs#L699 */ \
300 21581 : for( ulong i=0UL; i<(_n); i++ ) { \
301 11082 : fd_vm_haddr_query_t * query = _queries[i]; \
302 11082 : if( query->is_slice && !query->sz ) continue; \
303 11082 : int touch_err = fd_vm_mem_touch_st( _vm, query->vaddr, query->sz ); \
304 11051 : if( FD_UNLIKELY( touch_err ) ) return touch_err; \
305 11051 : } \
306 10760 : /* https://github.com/anza-xyz/agave/blob/v4.4.0-beta.0/syscalls/src/lib.rs#L700 */ \
307 21318 : for( ulong i=0UL; i<(_n); i++ ) { \
308 10819 : fd_vm_haddr_query_t * query = _queries[i]; \
309 10819 : if( query->is_slice ) { \
310 14876 : query->haddr = FD_VM_MEM_SLICE_HADDR_ST( _vm, query->vaddr, query->align, query->sz ); \
311 14876 : } else { \
312 3381 : query->haddr = FD_VM_MEM_HADDR_ST( _vm, query->vaddr, query->align, query->sz ); \
313 3381 : } \
314 10819 : } \
315 10499 : /* https://github.com/anza-xyz/agave/blob/v4.4.0-beta.0/syscalls/src/lib.rs#L701-L710 */ \
316 21325 : for( ulong i=0UL; i<(_n); i++ ) { \
317 10829 : fd_vm_haddr_query_t * query = _queries[i]; \
318 11148 : for( ulong j=0UL; j<i; j++ ) { \
319 322 : fd_vm_haddr_query_t * other_query = _queries[j]; \
320 322 : FD_VM_MEM_CHECK_NON_OVERLAPPING( _vm, (ulong)query->haddr, query->sz, (ulong)other_query->haddr, other_query->sz ); \
321 322 : } \
322 10829 : } \
323 10499 : } while(0)
324 :
325 : #endif /* HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h */
|
