Line data Source code
1 : #ifndef HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
2 : #define HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h
3 : #include "../fd_vm_private.h"
4 :
5 : /* fd_vm_cu API *******************************************************/
6 :
7 : /* FD_VM_CU_UPDATE charges the vm cost compute units.
8 :
9 : If the vm does not have more than cost cu available, this will cause
10 : the caller to zero out the vm->cu and return with FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED.
11 : This macro is robust.
12 : This is meant to be used by syscall implementations and strictly
13 : conforms with the vm-syscall ABI interface.
14 :
15 : Note: in Agave a syscall can return success leaving 0 available CUs.
16 : The instruction will fail at the next instruction (e.g., exit).
17 : To reproduce the same behavior, we do not return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED
18 : when cu == 0.
19 :
20 : FD_VM_CU_MEM_UPDATE charges the vm the equivalent of sz bytes of
21 : compute units. Behavior is otherwise identical to FD_VM_CU_UPDATE.
22 : FIXME: THIS API PROBABLY BELONGS IN SYSCALL CPI LAND. */
23 :
24 21353 : #define FD_VM_CU_UPDATE( vm, cost ) (__extension__({ \
25 21353 : fd_vm_t * _vm = (vm); \
26 21353 : ulong _cost = (cost); \
27 21353 : ulong _cu = _vm->cu; \
28 21353 : if( FD_UNLIKELY( _cost>_cu ) ) { \
29 90 : _vm->cu = 0UL; \
30 90 : FD_VM_ERR_FOR_LOG_INSTR( vm, FD_EXECUTOR_INSTR_ERR_COMPUTE_BUDGET_EXCEEDED ); \
31 90 : return FD_VM_SYSCALL_ERR_COMPUTE_BUDGET_EXCEEDED; \
32 90 : } \
33 21353 : _vm->cu = _cu - _cost; \
34 21263 : }))
35 :
36 : /* https://github.com/anza-xyz/agave/blob/5263c9d61f3af060ac995956120bef11c1bbf182/programs/bpf_loader/src/syscalls/mem_ops.rs#L7 */
37 : #define FD_VM_CU_MEM_OP_UPDATE( vm, sz ) \
38 1094 : FD_VM_CU_UPDATE( vm, fd_ulong_max( FD_VM_MEM_OP_BASE_COST, sz / FD_VM_CPI_BYTES_PER_UNIT ) )
39 :
40 :
41 : /* fd_vm_mem API *****************************************************/
42 :
43 : /* fd_vm_haddr_query is a struct that contains information about a vaddr, align, sz, and whether it is a slice.
44 : The translated haddr is written into the `haddr` field of the struct on success. This struct is primarily used
45 : by the FD_VM_TRANSLATE_MUT macro. See more details in the macro's documentation. */
46 : struct fd_vm_haddr_query {
47 : ulong vaddr;
48 : ulong align;
49 : ulong sz;
50 : uchar is_slice;
51 : void * haddr; /* out field */
52 : };
53 : typedef struct fd_vm_haddr_query fd_vm_haddr_query_t;
54 :
55 : /* FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY returns the haddr corresponding
56 : to _vaddr, when _vaddr is an u8 array of size _sz, and handles the
57 : difference between slice (_is_slice=1) or array (_is_slice=0).
58 : This is a common pattern in cryptographic syscalls, see for example
59 : fd_vm_syscall_crypto.c, fd_vm_syscall_curve.c */
60 3882 : #define FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY(_vm,_vaddr,_sz,_is_slice) (__extension__({ \
61 3882 : fd_vm_haddr_query_t _result = { \
62 3882 : .vaddr = (_vaddr), \
63 3882 : .align = (_is_slice) ? FD_VM_ALIGN_RUST_U8 : FD_VM_ALIGN_RUST_POD_U8_ARRAY, \
64 3882 : .sz = (_sz), \
65 3882 : .is_slice = (_is_slice), \
66 3882 : }; \
67 3882 : fd_vm_haddr_query_t * queries[] = { &_result }; \
68 3882 : FD_VM_TRANSLATE_MUT( (_vm), queries ); \
69 3882 : _result.haddr; }));
70 :
71 3668 : #define FD_VM_HADDR_QUERY_U8_SLICE(_vm,_vaddr,_sz) FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY(_vm,_vaddr,_sz,1)
72 214 : #define FD_VM_HADDR_QUERY_U8_ARRAY(_vm,_vaddr,_sz) FD_VM_HADDR_QUERY_U8_SLICE_OR_ARRAY(_vm,_vaddr,_sz,0)
73 :
74 : /* FD_VM_MEM_HADDR_LD returns a read only pointer to the first byte
75 : in the host address space corresponding to vm's virtual address range
76 : [vaddr,vaddr+sz). If the vm has check_align enabled, the vaddr
77 : should be aligned to align and the returned pointer will be similarly
78 : aligned. Align is assumed to be a power of two <= 8 (FIXME: CHECK
79 : THIS LIMIT).
80 :
81 : If the virtual address range cannot be mapped to the host address
82 : space completely and/or (when applicable) vaddr is not appropriately
83 : aligned, this will cause the caller to return FD_VM_SYSCALL_ERR_SEGFAULT.
84 : This macro is robust. This is meant to be used by syscall
85 : implementations and strictly conforms with the vm-syscall ABI
86 : interface.
87 :
88 : FD_VM_MEM_HADDR_ST returns a read-write pointer but is otherwise
89 : identical to FD_VM_MEM_HADDR_LD.
90 :
91 : FD_VM_MEM_HADDR_LD_FAST and FD_VM_HADDR_ST_FAST are for use when the
92 : corresponding vaddr region it known to correctly resolve (e.g. a
93 : syscall has already done preflight checks on them).
94 :
95 : These macros intentionally don't support multi region loads/stores.
96 : The load/store macros are used by vm syscalls and mirror the use
97 : of translate_slice{_mut}. However, this check does not allow for
98 : multi region accesses. So if there is an attempt at a multi region
99 : translation, an error will be returned.
100 :
101 : FD_VM_MEM_HADDR_ST_UNCHECKED has all of the checks of a load or a
102 : store, but intentionally omits the is_writable checks for the
103 : input region that are done during memory translation.
104 :
105 : FD_VM_MEM_HADDR_ST_NO_SZ_CHECK does all of the checks of a load,
106 : except for a check on the validity of the size of a load. It only
107 : checks that the specific vaddr that is being translated is valid. */
108 :
109 28622 : #define FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
110 28622 : fd_vm_t * _vm = (vm); \
111 28622 : ulong _vaddr = (vaddr); \
112 28622 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL ); \
113 28622 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
114 28622 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
115 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
116 0 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
117 0 : } \
118 28622 : if( FD_UNLIKELY( (!_haddr) ) ) { \
119 256 : _vm->segv_vaddr = _vaddr; \
120 256 : _vm->segv_access_len = (sz); \
121 256 : _vm->segv_access_type = FD_VM_ACCESS_TYPE_LD; \
122 256 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \
123 256 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
124 256 : } \
125 28622 : if ( FD_UNLIKELY( _sigbus ) ) { \
126 86 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
127 86 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
128 86 : } \
129 28366 : (void const *)_haddr; \
130 28280 : }))
131 :
132 : #define FD_VM_MEM_HADDR_LD_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
133 : fd_vm_t const * _vm = (vm); \
134 : ulong _vaddr = (vaddr); \
135 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_ld_sz, 0, 0UL ); \
136 : (void const *)_haddr; \
137 : }))
138 :
139 :
140 : #define FD_VM_MEM_HADDR_LD_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
141 : FD_VM_MEM_HADDR_LD( vm, vaddr, align, 1UL ); \
142 : }))
143 :
144 : static inline void *
145 10424 : FD_VM_MEM_HADDR_ST_( fd_vm_t *vm, ulong vaddr, ulong align, ulong sz, int *err ) {
146 10424 : fd_vm_t * _vm = (vm);
147 10424 : ulong _vaddr = (vaddr);
148 10424 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL );
149 10424 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) ));
150 10424 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) {
151 0 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH );
152 0 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
153 0 : return 0;
154 0 : }
155 10424 : if( FD_UNLIKELY( (!_haddr) ) ) {
156 171 : vm->segv_vaddr = vaddr;
157 171 : vm->segv_access_len = (sz);
158 171 : vm->segv_access_type = FD_VM_ACCESS_TYPE_ST;
159 171 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) );
160 171 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
161 171 : return 0;
162 171 : }
163 10253 : if ( FD_UNLIKELY( _sigbus ) ) {
164 5 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER );
165 5 : *err = FD_VM_SYSCALL_ERR_SEGFAULT;
166 5 : return 0;
167 5 : }
168 10248 : return (void *)_haddr;
169 10253 : }
170 :
171 10426 : #define FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
172 10426 : int _err = 0; \
173 10426 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, sz, &_err ); \
174 10426 : if ( FD_UNLIKELY( 0 != _err )) \
175 10426 : return _err; \
176 10426 : ret; \
177 10251 : }))
178 :
179 : #define FD_VM_MEM_HADDR_ST_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
180 : fd_vm_t * _vm = (vm); \
181 : ulong _vaddr = (vaddr); \
182 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 1, 0UL ); \
183 : (void const *)_haddr; \
184 : }))
185 :
186 : #define FD_VM_MEM_HADDR_ST_WRITE_UNCHECKED( vm, vaddr, align, sz ) (__extension__({ \
187 : fd_vm_t * _vm = (vm); \
188 : ulong _vaddr = (vaddr); \
189 : ulong _haddr = fd_vm_mem_haddr( vm, _vaddr, (sz), _vm->region_haddr, _vm->region_st_sz, 0, 0UL ); \
190 : int _sigbus = fd_vm_is_check_align_enabled( vm ) & (!fd_ulong_is_aligned( _haddr, (align) )); \
191 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
192 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
193 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
194 : } \
195 : if( FD_UNLIKELY( !_haddr ) ) { \
196 : _vm->segv_vaddr = _vaddr; \
197 : _vm->segv_access_len = (sz); \
198 : _vm->segv_access_type = FD_VM_ACCESS_TYPE_ST; \
199 : FD_VM_ERR_FOR_LOG_EBPF( _vm, fd_vm_generate_access_violation( _vaddr, _vm->sbpf_version ) ); \
200 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
201 : } \
202 : if ( FD_UNLIKELY( _sigbus ) ) { \
203 : FD_VM_ERR_FOR_LOG_SYSCALL( _vm, FD_VM_SYSCALL_ERR_UNALIGNED_POINTER ); \
204 : return FD_VM_SYSCALL_ERR_SEGFAULT; \
205 : } \
206 : (void *)_haddr; \
207 : }))
208 :
209 :
210 : #define FD_VM_MEM_HADDR_ST_NO_SZ_CHECK( vm, vaddr, align ) (__extension__({ \
211 : int _err = 0; \
212 : void * ret = FD_VM_MEM_HADDR_ST_( vm, vaddr, align, 1UL, &_err ); \
213 : if ( FD_UNLIKELY( 0 != _err )) \
214 : return _err; \
215 : ret; \
216 : }))
217 :
218 :
219 : #define FD_VM_MEM_HADDR_LD_FAST( vm, vaddr ) ((void const *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
220 : #define FD_VM_MEM_HADDR_ST_FAST( vm, vaddr ) ((void *)fd_vm_mem_haddr_fast( (vm), (vaddr), (vm)->region_haddr ))
221 :
222 :
223 : /* FD_VM_MEM_SLICE_HADDR_[LD, ST] macros return an arbitrary value if sz == 0. This is because
224 : Agave's translate_slice function returns an empty array if the sz == 0.
225 :
226 : Users of this macro should be aware that they should never access the returned value if sz==0.
227 :
228 : https://github.com/solana-labs/solana/blob/767d24e5c10123c079e656cdcf9aeb8a5dae17db/programs/bpf_loader/src/syscalls/mod.rs#L560
229 :
230 : LONG_MAX check: https://github.com/anza-xyz/agave/blob/dc4b9dcbbf859ff48f40d00db824bde063fdafcc/programs/bpf_loader/src/syscalls/mod.rs#L580
231 : Technically, the check in Agave is against
232 : "pointer-sized signed integer type ... The size of this primitive is
233 : how many bytes it takes to reference any location in memory. For
234 : example, on a 32 bit target, this is 4 bytes and on a 64 bit target,
235 : this is 8 bytes."
236 : Realistically, given the amount of memory that a validator consumes,
237 : no one is going to be running on a 32 bit target. So, we don't bother
238 : with conditionally compiling in an INT_MAX check. We just assume
239 : LONG_MAX. */
240 17369 : #define FD_VM_MEM_SLICE_HADDR_LD( vm, vaddr, align, sz ) (__extension__({ \
241 17369 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
242 18 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
243 18 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
244 18 : } \
245 17369 : void const * haddr = 0UL; \
246 17351 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
247 47998 : haddr = FD_VM_MEM_HADDR_LD( vm, vaddr, align, sz ); \
248 47998 : } \
249 17351 : haddr; \
250 17207 : }))
251 :
252 5126 : #define FD_VM_MEM_SLICE_HADDR_ST( vm, vaddr, align, sz ) (__extension__({ \
253 5126 : if ( FD_UNLIKELY( sz > LONG_MAX ) ) { \
254 0 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_INVALID_LENGTH ); \
255 0 : return FD_VM_SYSCALL_ERR_INVALID_LENGTH; \
256 0 : } \
257 5126 : void * haddr = 0UL; \
258 5126 : if ( FD_LIKELY( (ulong)sz > 0UL ) ) { \
259 5006 : haddr = FD_VM_MEM_HADDR_ST( vm, vaddr, align, sz ); \
260 4844 : } \
261 5126 : haddr; \
262 4964 : }))
263 :
264 : /* FIXME: use overlap logic from runtime? */
265 594 : #define FD_VM_MEM_CHECK_NON_OVERLAPPING( vm, addr0, sz0, addr1, sz1 ) do { \
266 594 : if( FD_UNLIKELY(( ((addr0> addr1) && (fd_ulong_sat_sub(addr0, addr1) < sz1)) ) || \
267 594 : ( ((addr1>=addr0) && (fd_ulong_sat_sub(addr1, addr0) < sz0)) ) )) { \
268 9 : FD_VM_ERR_FOR_LOG_SYSCALL( vm, FD_VM_SYSCALL_ERR_COPY_OVERLAPPING ); \
269 9 : return FD_VM_SYSCALL_ERR_COPY_OVERLAPPING; \
270 9 : } \
271 594 : } while(0)
272 :
273 : /* Mimics Agave's `translate_mut!` macro by taking in a variable number
274 : of (vaddr, align, sz) entries and translates each of them, failing
275 : if any one of the translations fail, or if any of the vaddrs have
276 : overlapping haddr regions. The caller is responsible for creating
277 : each `fd_vm_haddr_query_t` object containing information about the
278 : vaddr, align, sz, and whether it is a slice. Takes in any number
279 : of queries, provided into the input as an array of pointers to each
280 : query. The translated haddr is written into the `haddr` field
281 : of each of the `fd_vm_haddr_query_t` objects on success.
282 :
283 : https://github.com/anza-xyz/agave/blob/v2.3.1/programs/bpf_loader/src/syscalls/mod.rs#L701-L738 */
284 5645 : #define FD_VM_TRANSLATE_MUT( _vm, _queries ) do { \
285 5645 : ulong _n = sizeof(_queries)/sizeof(fd_vm_haddr_query_t *); \
286 11255 : for( ulong i=0UL; i<(_n); i++ ) { \
287 5756 : fd_vm_haddr_query_t * query = _queries[i]; \
288 5756 : if( query->is_slice ) { \
289 9522 : query->haddr = FD_VM_MEM_SLICE_HADDR_ST( _vm, query->vaddr, query->align, query->sz ); \
290 9522 : } else { \
291 927 : query->haddr = FD_VM_MEM_HADDR_ST( _vm, query->vaddr, query->align, query->sz ); \
292 920 : } \
293 5756 : for( ulong j=0UL; j<i; j++ ) { \
294 114 : fd_vm_haddr_query_t * other_query = _queries[j]; \
295 114 : FD_VM_MEM_CHECK_NON_OVERLAPPING( _vm, (ulong)query->haddr, query->sz, (ulong)other_query->haddr, other_query->sz ); \
296 114 : } \
297 5613 : } \
298 5645 : } while(0)
299 :
300 : #endif /* HEADER_fd_src_flamenco_vm_syscall_fd_vm_syscall_macros_h */
|
