Line data Source code
1 : #include "fd_bpf_loader_serialization.h"
2 : #include "../fd_borrowed_account.h"
3 : #include "../fd_runtime.h"
4 : #include "../../vm/fd_vm_base.h"
5 :
6 : /* This file is responsible for serializing and deserializing
7 : the input region of the BPF virtual machine. The input region contains
8 : instruction information, account metadata, and account data. The high level
9 : format is as follows:
10 :
11 : [ account 1 metadata, account 1 data, account 2 metadata, account 2 data, ...,
12 : account N metadata, account N data, instruction info. ]
13 :
14 : This format by no means comprehensive, but it should give an idea of how
15 : the input region is laid out. When direct mapping is not enabled, the input
16 : region is stored as a single contiguous buffer. This buffer in the host
17 : address space is then mapped to the VM virtual address space (the range
18 : starting with 0x400...). This means to serialize into the input region, we
19 : need to copy in the account metadata and account data into the buffer for
20 : each account. Everything must get copied out after execution is complete.
21 : A consequence of this is that a memcpy for the account data is required
22 : for each serialize and deserialize operation: this can potentially become
23 : expensive if there are many accounts and many nested CPI calls. Also, the
24 : entire memory region is treated as writable even though many accounts are
25 : read-only. This means that for all read-only accounts, a memcmp must be done
26 : while deserializing to make sure that the account (meta)data has not changed.
27 :
28 : Direct mapping offers a solution to this by introducing a more sophisticated
29 : memory translation protocol. Now the account data is not copied into a single
30 : contiguous buffer, but instead a borrowed account's data is directly mapped
31 : into the VM's virtual address space. The host memory for the input region is
32 : now represented by a list of fragmented memory regions. These sub regions
33 : also have different write permissions. This should solve the problem of
34 : having to memcpy/memcmp account data regions (which can be up to 10MiB each).
35 : There is some nuance to this, as the account data can be resized. This means
36 : that memcpys for account data regions can't totally be avoided.
37 :
38 : SERIALIZATION BEHAVIOR
39 : ==========================================
40 :
41 : This implementation supports three distinct serialization modes based on two
42 : feature flags: stricter_abi_and_runtime_constraints and
43 : account_data_direct_mapping.
44 :
45 : MODE 1
46 : --------------------------------------
47 : stricter_abi_and_runtime_constraints = false
48 : account_data_direct_mapping = false
49 :
50 : Memory Layout:
51 : - Single contiguous buffer in host memory
52 : - Buffer contains: [metadata1, data1, realloc_buffer1, metadata2, data2,
53 : realloc_buffer2, ..., metadataN, dataN, realloc_bufferN, instruction_info]
54 : - Each account gets: original data + MAX_PERMITTED_DATA_INCREASE (10KiB)
55 : - Padding added to maintain 16-byte alignment between accounts
56 : - Entire buffer is writable
57 :
58 : Memory Regions:
59 : - The entire input region buffer is mapped as one contiguous VM address
60 : space region
61 :
62 : Serialization Process:
63 : - Account data is memcpy'd into the buffer
64 : - 10KiB realloc buffer is zeroed out and appended after each account's data
65 : - Alignment padding is zeroed and added after realloc buffer
66 :
67 : Deserialization Process:
68 : - Account data must be memcpy'd back from buffer to borrowed account
69 : - For writable accounts: always copy data back
70 : - For read-only accounts: memcmp to verify data unchanged, error if modified
71 : - Account resizing allowed if account permissions permit it
72 :
73 : MODE 2
74 : -------------------------------------------
75 : stricter_abi_and_runtime_constraints = true
76 : account_data_direct_mapping = false
77 :
78 : Memory Layout:
79 : - Still uses a single contiguous buffer, but organized into fragmented
80 : regions.
81 : - Each account now has separate regions for metadata and data+realloc.
82 : - Buffer contains: [metadata1, data1+realloc1, metadata2, data2+realloc2, ...,
83 : metadataN, dataN+reallocN, instruction_info].
84 : - Each metadata region and data region tracked separately in
85 : input_mem_regions.
86 :
87 : Memory Regions:
88 : - For each account:
89 : * Region 0: Account metadata (writable)
90 : * Region 1: Account data + realloc space (writable if account is writable)
91 : - If the account is owned by the deprecated loader, no realloc region is
92 : created as the deprecated loader does not support resizing accounts.
93 :
94 : Serialization:
95 : - Account metadata serialized first, added as a memory region.
96 : - Account data memcpy'd into buffer - not directly mapped.
97 : - 10KiB realloc buffer zeroed and appended (not direct mapped).
98 : - Data region created pointing to copied data in buffer.
99 :
100 : MODE 3: Direct Mapping (requires stricter_abi_and_runtime_constraints)
101 : -----------------------------------------------
102 : stricter_abi_and_runtime_constraints = true
103 : account_data_direct_mapping = true
104 :
105 : This is very similar to stricter_abi_and_runtime_constraints, but account
106 : data is NOT copied into the input region buffer.
107 :
108 : Instead, the data region points directly to the staging area for the
109 : account in the transaction account's data. This staging area has enough
110 : space to hold the account data and the realloc buffer. Changes to this
111 : staging area will be written back to the account database in transaction
112 : finalization.
113 : */
114 :
115 : /* Add a new memory region to represent the input region. All of the memory
116 : regions here have sorted virtual addresses. These regions may or may not
117 : correspond to an account's data region. If it corresponds to metadata,
118 : the pubkey for the region will be NULL. */
119 : static void
120 : new_input_mem_region( fd_vm_input_region_t * input_mem_regions,
121 : uint * input_mem_regions_cnt,
122 : const uchar * buffer,
123 : ulong region_sz,
124 : ulong address_space_reserved,
125 : uchar is_writable,
126 11230 : ulong acc_region_meta_idx ) {
127 :
128 : /* The start vaddr of the new region should be equal to start of the previous
129 : region added to the address space reserved for the region. */
130 11230 : ulong vaddr_offset = *input_mem_regions_cnt==0UL ? 0UL : input_mem_regions[ *input_mem_regions_cnt-1U ].vaddr_offset +
131 124 : input_mem_regions[ *input_mem_regions_cnt-1U ].address_space_reserved;
132 11230 : input_mem_regions[ *input_mem_regions_cnt ].is_writable = is_writable;
133 11230 : input_mem_regions[ *input_mem_regions_cnt ].haddr = (ulong)buffer;
134 11230 : input_mem_regions[ *input_mem_regions_cnt ].region_sz = (uint)region_sz;
135 11230 : input_mem_regions[ *input_mem_regions_cnt ].address_space_reserved = address_space_reserved;
136 11230 : input_mem_regions[ *input_mem_regions_cnt ].vaddr_offset = vaddr_offset;
137 11230 : input_mem_regions[ *input_mem_regions_cnt ].acc_region_meta_idx = acc_region_meta_idx;
138 11230 : (*input_mem_regions_cnt)++;
139 11230 : }
140 :
141 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L127-L189 */
142 : /* This function handles casing for direct mapping being enabled as well as if
143 : the alignment is being stored. In the case where direct mapping is not
144 : enabled, we copy in the account data and a 10KiB buffer into the input region.
145 : These both go into the same memory buffer. However, when direct mapping is
146 : enabled, the account data and resizing buffers are represented by two
147 : different memory regions. In both cases, padding is used to maintain 8 byte
148 : alignment. If alignment is not required, then a resizing buffer is not used
149 : as the deprecated loader doesn't allow for resizing accounts. */
150 : static ulong
151 : write_account( fd_borrowed_account_t * account,
152 : uchar instr_acc_idx,
153 : uchar * * serialized_params,
154 : uchar * * serialized_params_start,
155 : fd_vm_input_region_t * input_mem_regions,
156 : uint * input_mem_regions_cnt,
157 : fd_vm_acc_region_meta_t * acc_region_metas,
158 : int is_loader_v1,
159 : int stricter_abi_and_runtime_constraints,
160 52807 : int direct_mapping ) {
161 :
162 52807 : uchar const * data = account ? fd_borrowed_account_get_data( account ) : NULL;
163 52807 : ulong dlen = account ? fd_borrowed_account_get_data_len( account ) : 0UL;
164 :
165 52807 : acc_region_metas[instr_acc_idx].original_data_len = dlen;
166 52807 : acc_region_metas[instr_acc_idx].meta = account->meta;
167 :
168 : /* Legacy behavior: no stricter_abi_and_runtime_constraints (also implies no direct mapping)
169 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L131-L140 */
170 52807 : if( !stricter_abi_and_runtime_constraints ) {
171 : /* Copy the account data into input region buffer */
172 52744 : fd_memcpy( *serialized_params, data, dlen );
173 52744 : *serialized_params += dlen;
174 :
175 52744 : if( FD_LIKELY( !is_loader_v1 ) ) {
176 : /* Zero out padding bytes and max permitted data increase */
177 50099 : ulong align_offset = fd_ulong_align_up( dlen, FD_BPF_ALIGN_OF_U128 ) - dlen;
178 50099 : fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE + align_offset );
179 50099 : *serialized_params += MAX_PERMITTED_DATA_INCREASE + align_offset;
180 50099 : }
181 52744 : acc_region_metas[instr_acc_idx].region_idx = UINT_MAX;
182 52744 : } else { /* stricter_abi_and_runtime_constraints == true */
183 :
184 : /* Set up account region metadata */
185 63 : acc_region_metas[instr_acc_idx].region_idx = *input_mem_regions_cnt;
186 :
187 : /* First, push on the region for the metadata that has just been serialized.
188 : This function will push the metadata in the serialized_params from
189 : serialized_params_start to serialized_params as a region to the input
190 : memory regions array.
191 :
192 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L142 */
193 63 : ulong region_sz = (ulong)(*serialized_params) - (ulong)(*serialized_params_start);
194 63 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, *serialized_params_start, region_sz, region_sz, 1U, ULONG_MAX );
195 :
196 : /* If direct mapping isn't enabled, then copy the account data in directly
197 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L144-L150 */
198 63 : if( !direct_mapping ) {
199 0 : fd_memcpy( *serialized_params, data, dlen );
200 0 : *serialized_params += dlen;
201 0 : if( FD_LIKELY( !is_loader_v1 ) ) {
202 0 : fd_memset( *serialized_params, 0, MAX_PERMITTED_DATA_INCREASE );
203 0 : *serialized_params += MAX_PERMITTED_DATA_INCREASE;
204 0 : }
205 0 : }
206 :
207 : /* Calculate address space reserved for account (data + realloc space)
208 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L151-L158 */
209 63 : ulong address_space_reserved = !is_loader_v1 ?
210 62 : fd_ulong_sat_add( dlen, MAX_PERMITTED_DATA_INCREASE ) : dlen;
211 :
212 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L159-L169 */
213 63 : if( address_space_reserved > 0 ) {
214 62 : int err = 0;
215 62 : uchar is_writable = !!(fd_borrowed_account_can_data_be_changed( account, &err ) && !err);
216 :
217 62 : if( !direct_mapping ) {
218 : /* Create region pointing to the copied data in buffer
219 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L160-L164 */
220 0 : uchar * data_start = *serialized_params - address_space_reserved;
221 0 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data_start, dlen, address_space_reserved, is_writable, instr_acc_idx );
222 62 : } else {
223 : /* Direct mapping: create region pointing directly to account data */
224 62 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, data, dlen, address_space_reserved, is_writable, instr_acc_idx );
225 62 : }
226 62 : }
227 :
228 63 : *serialized_params_start = *serialized_params;
229 :
230 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L170-L186 */
231 63 : if( FD_LIKELY( !is_loader_v1 ) ) {
232 62 : ulong align_offset = fd_ulong_align_up( dlen, FD_BPF_ALIGN_OF_U128 ) - dlen;
233 62 : if( !direct_mapping ) {
234 : /* If direct mapping is not enabled, we do not align the start of each
235 : region metadata to FD_BPF_ALIGN_OF_U128, but we do align the start
236 : of the actual contents of the metadata region.
237 :
238 : This follows Agave's logic
239 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L173-L176 */
240 0 : fd_memset( *serialized_params, 0, align_offset );
241 0 : *serialized_params += align_offset;
242 62 : } else {
243 : /* If direct mapping is enabled, we align the start of each region
244 : metadata to FD_BPF_ALIGN_OF_U128. */
245 62 : fd_memset( *serialized_params, 0, FD_BPF_ALIGN_OF_U128 );
246 62 : *serialized_params += FD_BPF_ALIGN_OF_U128;
247 62 : *serialized_params_start += fd_ulong_sat_sub( FD_BPF_ALIGN_OF_U128, align_offset );
248 62 : }
249 62 : }
250 :
251 63 : return region_sz + address_space_reserved;
252 63 : }
253 :
254 52744 : return 0UL;
255 52807 : }
256 :
257 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L466 */
258 : static int
259 : fd_bpf_loader_input_serialize_aligned( fd_exec_instr_ctx_t * ctx,
260 : ulong * pre_lens,
261 : fd_vm_input_region_t * input_mem_regions,
262 : uint * input_mem_regions_cnt,
263 : fd_vm_acc_region_meta_t * acc_region_metas,
264 : int stricter_abi_and_runtime_constraints,
265 : int direct_mapping,
266 : ulong * instr_data_offset,
267 10496 : ulong * serialized_bytes_written ) {
268 10496 : fd_pubkey_t * txn_accs = ctx->txn_out->accounts.keys;
269 :
270 : /* Transaction sanitisation limits the number of instruction accounts to
271 : FD_TXN_ACCT_ADDR_MAX. */
272 10496 : uchar acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
273 10496 : ushort dup_acc_idx[ FD_TXN_ACCT_ADDR_MAX ] = {0};
274 :
275 : /* 16-byte aligned buffer from runtime:
276 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L60 */
277 10496 : uchar * serialized_params = ctx->runtime->bpf_loader_serialization.serialization_mem[ ctx->runtime->instr.stack_sz-1UL ];
278 10496 : uchar * serialized_params_start = serialized_params;
279 10496 : uchar * curr_serialized_params_start = serialized_params;
280 10496 : ulong curr_region_vaddr = 0UL;
281 :
282 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L522 */
283 10496 : FD_STORE( ulong, serialized_params, ctx->instr->acct_cnt );
284 10496 : serialized_params += sizeof(ulong);
285 :
286 : /* Iterate over accounts in the instruction to populate input region.
287 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L523-L557 */
288 61206 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
289 50710 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
290 50710 : fd_pubkey_t * acc = &txn_accs[acc_idx];
291 :
292 50710 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] && dup_acc_idx[acc_idx] != i ) ) {
293 : /* Duplicate. Store 8 byte buffer to maintain alignment but store the
294 : account index in the first byte.
295 :
296 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L551-L555 */
297 605 : FD_STORE( ulong, serialized_params, 0UL );
298 605 : FD_STORE( uchar, serialized_params, (uchar)dup_acc_idx[acc_idx] );
299 605 : serialized_params += sizeof(ulong);
300 :
301 : /* Clone the account metadata from the original account
302 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L552 */
303 605 : acc_region_metas[i] = acc_region_metas[dup_acc_idx[acc_idx]];
304 50105 : } else {
305 50105 : acc_idx_seen[acc_idx] = 1;
306 50105 : dup_acc_idx[acc_idx] = i;
307 :
308 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L526 */
309 50105 : FD_STORE( uchar, serialized_params, FD_NON_DUP_MARKER );
310 50105 : serialized_params += sizeof(uchar);
311 :
312 : /* Borrow the account
313 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L244-L257 */
314 50105 : fd_guarded_borrowed_account_t view_acc = {0};
315 50105 : int err = fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc );
316 50105 : if( FD_UNLIKELY( err ) ) {
317 0 : return err;
318 0 : }
319 :
320 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L525 */
321 50105 : fd_account_meta_t const * metadata = fd_borrowed_account_get_acc_meta( &view_acc );
322 :
323 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L527 */
324 50105 : uchar is_signer = (uchar)fd_instr_acc_is_signer_idx( ctx->instr, (uchar)i, NULL );
325 50105 : FD_STORE( uchar, serialized_params, is_signer );
326 50105 : serialized_params += sizeof(uchar);
327 :
328 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L528 */
329 50105 : uchar is_writable = (uchar)fd_instr_acc_is_writable_idx( ctx->instr, (uchar)i );
330 50105 : FD_STORE( uchar, serialized_params, is_writable );
331 50105 : serialized_params += sizeof(uchar);
332 :
333 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L529-L530 */
334 50105 : uchar is_executable = (uchar)metadata->executable;
335 50105 : FD_STORE( uchar, serialized_params, is_executable );
336 50105 : serialized_params += sizeof(uchar);
337 :
338 : /* The original data len field is intentionally NOT populated. */
339 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L531 */
340 50105 : uint padding_0 = 0U;
341 50105 : FD_STORE( uint, serialized_params, padding_0 );
342 50105 : serialized_params += sizeof(uint);
343 :
344 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L532 */
345 50105 : fd_pubkey_t key = *acc;
346 50105 : acc_region_metas[i].vm_key_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
347 50105 : (ulong)(serialized_params - curr_serialized_params_start);
348 50105 : FD_STORE( fd_pubkey_t, serialized_params, key );
349 50105 : serialized_params += sizeof(fd_pubkey_t);
350 :
351 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L533 */
352 50105 : fd_pubkey_t owner = *(fd_pubkey_t *)&metadata->owner;
353 50105 : acc_region_metas[i].vm_owner_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
354 50105 : (ulong)(serialized_params - curr_serialized_params_start);
355 50105 : FD_STORE( fd_pubkey_t, serialized_params, owner );
356 50105 : serialized_params += sizeof(fd_pubkey_t);
357 :
358 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L534 */
359 50105 : ulong lamports = metadata->lamports;
360 50105 : acc_region_metas[i].vm_lamports_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
361 50105 : (ulong)(serialized_params - curr_serialized_params_start);
362 50105 : FD_STORE( ulong, serialized_params, lamports );
363 50105 : serialized_params += sizeof(ulong);
364 :
365 50105 : ulong acc_data_len = metadata->dlen;
366 50105 : pre_lens[i] = acc_data_len;
367 :
368 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L535 */
369 50105 : ulong data_len = acc_data_len;
370 50105 : FD_STORE( ulong, serialized_params, data_len );
371 50105 : serialized_params += sizeof(ulong);
372 :
373 : /* vm_data_addr: data is written immediately after the data_len field */
374 50105 : acc_region_metas[i].vm_data_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
375 50105 : (ulong)(serialized_params - curr_serialized_params_start);
376 :
377 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L536 */
378 50105 : write_account(
379 50105 : &view_acc,
380 50105 : (uchar)i,
381 50105 : &serialized_params,
382 50105 : &curr_serialized_params_start,
383 50105 : input_mem_regions,
384 50105 : input_mem_regions_cnt,
385 50105 : acc_region_metas,
386 50105 : 0,
387 50105 : stricter_abi_and_runtime_constraints,
388 50105 : direct_mapping );
389 :
390 : /* write_account may have pushed a new region(s) */
391 50105 : curr_region_vaddr = *input_mem_regions_cnt == 0U ? 0UL :
392 50105 : input_mem_regions[*input_mem_regions_cnt-1U].vaddr_offset +
393 103 : input_mem_regions[*input_mem_regions_cnt-1U].address_space_reserved;
394 :
395 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L537-L541 */
396 50105 : FD_STORE( ulong, serialized_params, ULONG_MAX );
397 50105 : serialized_params += sizeof(ulong);
398 50105 : }
399 :
400 50710 : }
401 :
402 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L558 */
403 10496 : ulong instr_data_len = ctx->instr->data_sz;
404 10496 : FD_STORE( ulong, serialized_params, instr_data_len );
405 10496 : serialized_params += sizeof(ulong);
406 :
407 : /* https://github.com/anza-xyz/agave/blob/v3.1.1/program-runtime/src/serialization.rs#L568 */
408 10496 : *instr_data_offset = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
409 10496 : (ulong)(serialized_params - curr_serialized_params_start);
410 :
411 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L559 */
412 10496 : fd_memcpy( serialized_params, ctx->instr->data, instr_data_len );
413 10496 : serialized_params += instr_data_len;
414 :
415 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L560 */
416 10496 : FD_STORE( fd_pubkey_t, serialized_params, txn_accs[ctx->instr->program_id] );
417 10496 : serialized_params += sizeof(fd_pubkey_t);
418 :
419 : /* Write out the final region. */
420 10496 : ulong region_sz = (ulong)(serialized_params - curr_serialized_params_start);
421 10496 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, curr_serialized_params_start,
422 10496 : region_sz, region_sz, 1U, ULONG_MAX );
423 :
424 10496 : *serialized_bytes_written = (ulong)(serialized_params - serialized_params_start);
425 10496 : return FD_EXECUTOR_INSTR_SUCCESS;
426 10496 : }
427 :
428 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L566-L653 */
429 : static int
430 : fd_bpf_loader_input_deserialize_aligned( fd_exec_instr_ctx_t * ctx,
431 : ulong const * pre_lens,
432 : uchar * buffer,
433 : ulong FD_FN_UNUSED buffer_sz,
434 : int stricter_abi_and_runtime_constraints,
435 802 : int direct_mapping ) {
436 : /* TODO: An optimization would be to skip ahead through non-writable accounts */
437 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L573 */
438 802 : ulong start = 0UL;
439 :
440 802 : uchar acc_idx_seen[256] = {0};
441 :
442 802 : start += sizeof(ulong); // number of accounts
443 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L574-L650 */
444 2700 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
445 1901 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
446 :
447 1901 : start++; // position
448 :
449 : /* get the borrowed account
450 : https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L584-L585 */
451 1901 : fd_guarded_borrowed_account_t view_acc = {0};
452 1901 : FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( ctx, i, &view_acc );
453 :
454 1901 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) {
455 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L582 */
456 64 : start += 7UL;
457 1837 : } else {
458 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L586-L590 */
459 1837 : acc_idx_seen[acc_idx] = 1;
460 1837 : start += sizeof(uchar) // is_signer
461 1837 : + sizeof(uchar) // is_writable
462 1837 : + sizeof(uchar) // executable
463 1837 : + sizeof(uint) // original_data_len
464 1837 : + sizeof(fd_pubkey_t); // key
465 :
466 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L591-L593 */
467 :
468 1837 : fd_pubkey_t * owner = (fd_pubkey_t *)(buffer+start);
469 1837 : start += sizeof(fd_pubkey_t); // owner
470 :
471 1837 : ulong lamports = FD_LOAD( ulong, buffer+start );
472 1837 : if( lamports!=fd_borrowed_account_get_lamports( &view_acc ) ) {
473 13 : int err = fd_borrowed_account_set_lamports( &view_acc, lamports );
474 13 : if( FD_UNLIKELY( err ) ) {
475 1 : return err;
476 1 : }
477 13 : }
478 1836 : start += sizeof(ulong); // lamports
479 :
480 1836 : ulong post_len = FD_LOAD( ulong, buffer+start );
481 1836 : start += sizeof(ulong); // data length
482 :
483 1836 : ulong pre_len = pre_lens[i];
484 1836 : ulong alignment_offset = fd_ulong_align_up( pre_len, FD_BPF_ALIGN_OF_U128 ) - pre_len;
485 :
486 1836 : uchar * post_data = buffer+start;
487 :
488 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L612-L616 */
489 1836 : if( FD_UNLIKELY( fd_ulong_sat_sub( post_len, pre_len )>MAX_PERMITTED_DATA_INCREASE ||
490 1836 : post_len>MAX_PERMITTED_DATA_LENGTH ) ) {
491 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_REALLOC;
492 0 : }
493 :
494 1836 : int can_data_be_changed_err = 0;
495 1836 : if( !stricter_abi_and_runtime_constraints ) {
496 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L617-L627 */
497 :
498 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L618-L620 */
499 1836 : if( FD_UNLIKELY( start + post_len > buffer_sz ) ) {
500 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
501 0 : }
502 :
503 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L621-L626 */
504 1836 : int can_data_be_resized_err = 0;
505 1836 : if( fd_borrowed_account_can_data_be_resized( &view_acc, post_len, &can_data_be_resized_err ) &&
506 1836 : fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
507 71 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len );
508 71 : if( FD_UNLIKELY( set_data_err ) ) {
509 0 : return set_data_err;
510 0 : }
511 1765 : } else {
512 1765 : if( FD_UNLIKELY( fd_borrowed_account_get_data_len( &view_acc )!=post_len ||
513 1765 : memcmp( fd_borrowed_account_get_data( &view_acc ), post_data, post_len ) ) ) {
514 2 : return can_data_be_resized_err ? can_data_be_resized_err : can_data_be_changed_err;
515 2 : }
516 1765 : }
517 :
518 1836 : } else if( !direct_mapping && fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
519 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L629-L631 */
520 0 : if( FD_UNLIKELY( start + post_len > buffer_sz ) ) {
521 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
522 0 : }
523 :
524 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L627-L633 */
525 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, post_len );
526 0 : if( FD_UNLIKELY( set_data_err ) ) {
527 0 : return set_data_err;
528 0 : }
529 0 : } else if( fd_borrowed_account_get_data_len( &view_acc ) != post_len ) {
530 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L633-L635 */
531 0 : int set_data_length_err = fd_borrowed_account_set_data_length( &view_acc, post_len );
532 0 : if( FD_UNLIKELY( set_data_length_err ) ) {
533 0 : return set_data_length_err;
534 0 : }
535 0 : }
536 :
537 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L636-L644 */
538 1834 : if( !( stricter_abi_and_runtime_constraints && direct_mapping ) ) {
539 1834 : start += fd_ulong_sat_add( MAX_PERMITTED_DATA_INCREASE, fd_ulong_sat_add( pre_len, alignment_offset ) );
540 1834 : } else {
541 0 : start += FD_BPF_ALIGN_OF_U128;
542 0 : }
543 :
544 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L645 */
545 1834 : start += sizeof(ulong); // rent epoch
546 1834 : if( memcmp( fd_borrowed_account_get_owner( &view_acc ), owner, sizeof(fd_pubkey_t) ) ) {
547 1 : int err = fd_borrowed_account_set_owner( &view_acc, owner );
548 1 : if( FD_UNLIKELY( err ) ) {
549 0 : return err;
550 0 : }
551 1 : }
552 1834 : }
553 1901 : }
554 :
555 799 : return FD_EXECUTOR_INSTR_SUCCESS;
556 802 : }
557 :
558 : static int
559 : fd_bpf_loader_input_serialize_unaligned( fd_exec_instr_ctx_t * ctx,
560 : ulong * pre_lens,
561 : fd_vm_input_region_t * input_mem_regions,
562 : uint * input_mem_regions_cnt,
563 : fd_vm_acc_region_meta_t * acc_region_metas,
564 : int stricter_abi_and_runtime_constraints,
565 : int direct_mapping,
566 : ulong * instr_data_offset,
567 614 : ulong * serialized_bytes_written ) {
568 614 : fd_pubkey_t const * txn_accs = ctx->txn_out->accounts.keys;
569 :
570 : /* Transaction sanitisation limits the number of instruction accounts to
571 : FD_TXN_ACCT_ADDR_MAX. */
572 614 : uchar acc_idx_seen[ FD_TXN_ACCT_ADDR_MAX ] = {0};
573 614 : ushort dup_acc_idx[ FD_TXN_ACCT_ADDR_MAX ] = {0};
574 :
575 : /* 16-byte aligned buffer:
576 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L60 */
577 614 : uchar * serialized_params = ctx->runtime->bpf_loader_serialization.serialization_mem[ ctx->runtime->instr.stack_sz-1UL ];
578 614 : uchar * serialized_params_start = serialized_params;
579 614 : uchar * curr_serialized_params_start = serialized_params;
580 614 : ulong curr_region_vaddr = 0UL;
581 :
582 614 : FD_STORE( ulong, serialized_params, ctx->instr->acct_cnt );
583 614 : serialized_params += sizeof(ulong);
584 :
585 3267 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
586 2653 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
587 2653 : fd_pubkey_t const * acc = &txn_accs[acc_idx];
588 :
589 2653 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] && dup_acc_idx[acc_idx] != i ) ) {
590 : // Duplicate
591 2 : FD_STORE( uchar, serialized_params, (uchar)dup_acc_idx[acc_idx] );
592 2 : serialized_params += sizeof(uchar);
593 :
594 : /* Clone the account metadata from the original account
595 : https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L376 */
596 2 : acc_region_metas[i] = acc_region_metas[dup_acc_idx[acc_idx]];
597 2651 : } else {
598 2651 : acc_idx_seen[acc_idx] = 1;
599 2651 : dup_acc_idx[acc_idx] = i;
600 :
601 2651 : FD_STORE( uchar, serialized_params, FD_NON_DUP_MARKER );
602 2651 : serialized_params += sizeof(uchar);
603 :
604 : /* Borrow the account
605 : https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L225 */
606 2651 : fd_guarded_borrowed_account_t view_acc = {0};
607 2651 : int err = fd_exec_instr_ctx_try_borrow_instr_account( ctx, i, &view_acc );
608 2651 : if( FD_UNLIKELY( err ) ) {
609 0 : return err;
610 0 : }
611 :
612 2651 : fd_account_meta_t const * metadata = fd_borrowed_account_get_acc_meta( &view_acc );
613 :
614 2651 : pre_lens[i] = metadata->dlen;
615 :
616 2651 : uchar is_signer = (uchar)fd_instr_acc_is_signer_idx( ctx->instr, (uchar)i, NULL );
617 2651 : FD_STORE( uchar, serialized_params, is_signer );
618 2651 : serialized_params += sizeof(uchar);
619 :
620 2651 : uchar is_writable = (uchar)fd_instr_acc_is_writable_idx( ctx->instr, (uchar)i );
621 2651 : FD_STORE( uchar, serialized_params, is_writable );
622 2651 : serialized_params += sizeof(uchar);
623 :
624 2651 : fd_pubkey_t key = *acc;
625 2651 : acc_region_metas[i].vm_key_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
626 2651 : (ulong)(serialized_params - curr_serialized_params_start);
627 2651 : FD_STORE( fd_pubkey_t, serialized_params, key );
628 2651 : serialized_params += sizeof(fd_pubkey_t);
629 :
630 2651 : ulong lamports = metadata->lamports;
631 2651 : acc_region_metas[i].vm_lamports_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
632 2651 : (ulong)(serialized_params - curr_serialized_params_start);
633 2651 : FD_STORE( ulong, serialized_params, lamports );
634 2651 : serialized_params += sizeof(ulong);
635 :
636 2651 : ulong acc_data_len = metadata->dlen;
637 2651 : FD_STORE( ulong, serialized_params, acc_data_len );
638 2651 : serialized_params += sizeof(ulong);
639 :
640 : /* vm_data_addr: data is written immediately after the data_len field */
641 2651 : acc_region_metas[i].vm_data_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
642 2651 : (ulong)(serialized_params - curr_serialized_params_start);
643 :
644 2651 : write_account( &view_acc, (uchar)i,
645 2651 : &serialized_params, &curr_serialized_params_start,
646 2651 : input_mem_regions, input_mem_regions_cnt, acc_region_metas, 1,
647 2651 : stricter_abi_and_runtime_constraints, direct_mapping );
648 :
649 : /* write_account may have pushed a new region(s) */
650 2651 : curr_region_vaddr = *input_mem_regions_cnt == 0U ? 0UL :
651 2651 : input_mem_regions[*input_mem_regions_cnt-1U].vaddr_offset +
652 0 : input_mem_regions[*input_mem_regions_cnt-1U].address_space_reserved;
653 :
654 2651 : fd_pubkey_t owner = *(fd_pubkey_t *)&metadata->owner;
655 2651 : acc_region_metas[i].vm_owner_addr = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
656 2651 : (ulong)(serialized_params - curr_serialized_params_start);
657 2651 : FD_STORE( fd_pubkey_t, serialized_params, owner );
658 2651 : serialized_params += sizeof(fd_pubkey_t);
659 :
660 2651 : uchar is_executable = (uchar)metadata->executable;
661 2651 : FD_STORE( uchar, serialized_params, is_executable );
662 2651 : serialized_params += sizeof(uchar);
663 :
664 2651 : FD_STORE( ulong, serialized_params, ULONG_MAX );
665 2651 : serialized_params += sizeof(ulong);
666 2651 : }
667 2653 : }
668 :
669 614 : ulong instr_data_len = ctx->instr->data_sz;
670 614 : FD_STORE( ulong, serialized_params, instr_data_len );
671 614 : serialized_params += sizeof(ulong);
672 :
673 : /* https://github.com/anza-xyz/agave/blob/v3.1.1/program-runtime/src/serialization.rs#L400 */
674 614 : *instr_data_offset = FD_VM_MEM_MAP_INPUT_REGION_START + curr_region_vaddr +
675 614 : (ulong)(serialized_params - curr_serialized_params_start);
676 :
677 614 : fd_memcpy( serialized_params, ctx->instr->data, instr_data_len );
678 614 : serialized_params += instr_data_len;
679 :
680 614 : FD_STORE( fd_pubkey_t, serialized_params, txn_accs[ctx->instr->program_id] );
681 614 : serialized_params += sizeof(fd_pubkey_t);
682 :
683 614 : *serialized_bytes_written = (ulong)(serialized_params - serialized_params_start);
684 :
685 614 : ulong region_sz = (ulong)(serialized_params - curr_serialized_params_start);
686 614 : new_input_mem_region( input_mem_regions, input_mem_regions_cnt, curr_serialized_params_start,
687 614 : region_sz, region_sz, 1U, ULONG_MAX );
688 :
689 614 : return FD_EXECUTOR_INSTR_SUCCESS;
690 614 : }
691 :
692 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L404 */
693 : static int
694 : fd_bpf_loader_input_deserialize_unaligned( fd_exec_instr_ctx_t * ctx,
695 : ulong const * pre_lens,
696 : uchar * input,
697 : ulong input_sz,
698 : int stricter_abi_and_runtime_constraints,
699 280 : int direct_mapping ) {
700 280 : uchar * input_cursor = input;
701 280 : uchar acc_idx_seen[256] = {0};
702 :
703 280 : input_cursor += sizeof(ulong);
704 :
705 834 : for( ushort i=0; i<ctx->instr->acct_cnt; i++ ) {
706 557 : uchar acc_idx = (uchar)ctx->instr->accounts[i].index_in_transaction;
707 :
708 557 : input_cursor++; /* is_dup */
709 557 : if( FD_UNLIKELY( acc_idx_seen[acc_idx] ) ) {
710 : /* no-op */
711 555 : } else {
712 555 : acc_idx_seen[acc_idx] = 1;
713 555 : input_cursor += sizeof(uchar) + /* is_signer */
714 555 : sizeof(uchar) + /* is_writable */
715 555 : sizeof(fd_pubkey_t); /* key */
716 :
717 : /* https://github.com/anza-xyz/agave/blob/v2.1.4/programs/bpf_loader/src/serialization.rs#L378 */
718 555 : fd_guarded_borrowed_account_t view_acc = {0};
719 555 : FD_TRY_BORROW_INSTR_ACCOUNT_DEFAULT_ERR_CHECK( ctx, i, &view_acc );
720 :
721 555 : ulong lamports = FD_LOAD( ulong, input_cursor );
722 555 : if( fd_borrowed_account_get_acc_meta( &view_acc ) && fd_borrowed_account_get_lamports( &view_acc )!=lamports ) {
723 0 : int err = fd_borrowed_account_set_lamports( &view_acc, lamports );
724 0 : if( FD_UNLIKELY( err ) ) {
725 0 : return err;
726 0 : }
727 0 : }
728 :
729 555 : input_cursor += sizeof(ulong); /* lamports */
730 555 : input_cursor += sizeof(ulong); /* data length */
731 :
732 555 : ulong pre_len = pre_lens[i];
733 555 : uchar * post_data = input_cursor;
734 :
735 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L436-L446 */
736 555 : int can_data_be_changed_err = 0;
737 555 : if( !stricter_abi_and_runtime_constraints ) {
738 555 : int can_data_be_resized_err = 0;
739 555 : if( fd_borrowed_account_can_data_be_resized( &view_acc, pre_len, &can_data_be_resized_err ) &&
740 555 : fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
741 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, pre_len );
742 0 : if( FD_UNLIKELY( set_data_err ) ) {
743 0 : return set_data_err;
744 0 : }
745 555 : } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len ||
746 555 : memcmp( post_data, fd_borrowed_account_get_data( &view_acc ), pre_len ) ) {
747 3 : return can_data_be_resized_err ? can_data_be_resized_err : can_data_be_changed_err;
748 3 : }
749 555 : } else if( !direct_mapping && fd_borrowed_account_can_data_be_changed( &view_acc, &can_data_be_changed_err ) ) {
750 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L446-L452 */
751 0 : int set_data_err = fd_borrowed_account_set_data_from_slice( &view_acc, post_data, pre_len );
752 0 : if( FD_UNLIKELY( set_data_err ) ) {
753 0 : return set_data_err;
754 0 : }
755 0 : } else if( fd_borrowed_account_get_data_len( &view_acc ) != pre_len ) {
756 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L452-L454 */
757 0 : int set_data_length_err = fd_borrowed_account_set_data_length( &view_acc, pre_len );
758 0 : if( FD_UNLIKELY( set_data_length_err ) ) {
759 0 : return set_data_length_err;
760 0 : }
761 0 : }
762 :
763 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L455-L457 */
764 552 : if( !( stricter_abi_and_runtime_constraints && direct_mapping ) ) {
765 552 : input_cursor += pre_len;
766 552 : }
767 552 : input_cursor += sizeof(fd_pubkey_t) + /* owner */
768 552 : sizeof(uchar) + /* executable */
769 552 : sizeof(ulong); /* rent_epoch*/
770 552 : }
771 557 : }
772 :
773 277 : if( FD_UNLIKELY( input_cursor>input+input_sz ) ) {
774 0 : return FD_EXECUTOR_INSTR_ERR_INVALID_ARG;
775 0 : }
776 :
777 277 : return 0;
778 277 : }
779 :
780 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L221 */
781 : int
782 : fd_bpf_loader_input_serialize_parameters( fd_exec_instr_ctx_t * instr_ctx,
783 : ulong * pre_lens,
784 : fd_vm_input_region_t * input_mem_regions,
785 : uint * input_mem_regions_cnt,
786 : fd_vm_acc_region_meta_t * acc_region_metas,
787 : int stricter_abi_and_runtime_constraints,
788 : int direct_mapping,
789 : uchar is_deprecated,
790 : ulong * instr_data_offset,
791 11111 : ulong * serialized_bytes_written ) {
792 :
793 : /* https://github.com/anza-xyz/agave/blob/v3.0.0/program-runtime/src/serialization.rs#L234-L237 */
794 11111 : ulong num_ix_accounts = instr_ctx->instr->acct_cnt;
795 11111 : if( FD_UNLIKELY( num_ix_accounts>FD_BPF_INSTR_ACCT_MAX ) ) {
796 0 : return FD_EXECUTOR_INSTR_ERR_MAX_ACCS_EXCEEDED;
797 0 : }
798 :
799 : /* https://github.com/anza-xyz/agave/blob/v2.1.11/programs/bpf_loader/src/serialization.rs#L237-L251 */
800 11111 : if( FD_UNLIKELY( is_deprecated ) ) {
801 614 : return fd_bpf_loader_input_serialize_unaligned( instr_ctx, pre_lens,
802 614 : input_mem_regions, input_mem_regions_cnt,
803 614 : acc_region_metas, stricter_abi_and_runtime_constraints,
804 614 : direct_mapping, instr_data_offset, serialized_bytes_written );
805 10497 : } else {
806 10497 : return fd_bpf_loader_input_serialize_aligned( instr_ctx, pre_lens,
807 10497 : input_mem_regions, input_mem_regions_cnt,
808 10497 : acc_region_metas, stricter_abi_and_runtime_constraints,
809 10497 : direct_mapping, instr_data_offset, serialized_bytes_written );
810 10497 : }
811 11111 : }
812 :
813 : /* https://github.com/anza-xyz/agave/blob/v3.0.4/program-runtime/src/serialization.rs#L284-L311 */
814 : int
815 : fd_bpf_loader_input_deserialize_parameters( fd_exec_instr_ctx_t * ctx,
816 : ulong const * pre_lens,
817 : uchar * input,
818 : ulong input_sz,
819 : int stricter_abi_and_runtime_constraints,
820 : int direct_mapping,
821 1082 : uchar is_deprecated ) {
822 1082 : if( FD_UNLIKELY( is_deprecated ) ) {
823 280 : return fd_bpf_loader_input_deserialize_unaligned(
824 280 : ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping );
825 802 : } else {
826 802 : return fd_bpf_loader_input_deserialize_aligned(
827 802 : ctx, pre_lens, input, input_sz, stricter_abi_and_runtime_constraints, direct_mapping );
828 802 : }
829 1082 : }
|
