Line data Source code
1 : #include "fd_accdb_impl_v1.h"
2 : #include "fd_accdb_lineage.h"
3 : #include "fd_accdb_funk.h"
4 : #include <stdatomic.h>
5 :
6 : FD_STATIC_ASSERT( alignof(fd_accdb_user_v1_t)<=alignof(fd_accdb_user_t), layout );
7 : FD_STATIC_ASSERT( sizeof (fd_accdb_user_v1_t)<=sizeof(fd_accdb_user_t), layout );
8 :
9 : static int
10 : fd_accdb_search_chain( fd_funk_t const * funk,
11 : fd_accdb_lineage_t const * lineage,
12 : ulong chain_idx,
13 : fd_funk_rec_key_t const * key,
14 350469 : fd_funk_rec_t ** out_rec ) {
15 350469 : *out_rec = NULL;
16 :
17 350469 : fd_funk_rec_map_shmem_t const * shmap = funk->rec_map->map;
18 350469 : fd_funk_rec_map_shmem_private_chain_t const * chain_tbl = fd_funk_rec_map_shmem_private_chain_const( shmap, 0UL );
19 350469 : fd_funk_rec_map_shmem_private_chain_t const * chain = chain_tbl + chain_idx;
20 350469 : fd_funk_rec_t * rec_tbl = funk->rec_pool->ele;
21 350469 : ulong rec_max = fd_funk_rec_pool_ele_max( funk->rec_pool );
22 :
23 350469 : uint ele_idx = chain->head_cidx;
24 350469 : ulong _Atomic * ver_cnt_p = (ulong _Atomic *)&chain->ver_cnt;
25 350469 : ulong ver_cnt = atomic_load_explicit( ver_cnt_p, memory_order_acquire );
26 :
27 : /* Start a speculative transaction for the chain containing revisions
28 : of the account key we are looking for. */
29 350469 : ulong cnt = fd_funk_rec_map_private_vcnt_cnt( ver_cnt );
30 350469 : if( FD_UNLIKELY( fd_funk_rec_map_private_vcnt_ver( ver_cnt )&1 ) ) {
31 0 : return FD_MAP_ERR_AGAIN; /* chain is locked */
32 0 : }
33 :
34 : /* Walk the map chain, bail at the first entry
35 : (Each chain is sorted newest-to-oldest) */
36 350469 : fd_funk_rec_t * best = NULL;
37 364029 : for( ulong i=0UL; i<cnt; i++ ) {
38 100161 : uint ele_next = rec_tbl[ ele_idx ].map_next;
39 100161 : if( FD_UNLIKELY( atomic_load_explicit( ver_cnt_p, memory_order_acquire )!=ver_cnt ) ) {
40 0 : return FD_MAP_ERR_AGAIN;
41 0 : }
42 :
43 100161 : if( FD_UNLIKELY( ele_idx>=rec_max ) ) {
44 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected memory corruption: invalid ele_idx at node %lu:%u (rec_max %lu)",
45 0 : chain_idx, ele_idx, rec_max ));
46 0 : }
47 :
48 100161 : fd_funk_rec_t * rec = &rec_tbl[ ele_idx ];
49 :
50 : /* Skip over unrelated records (hash collision) */
51 100161 : if( FD_UNLIKELY( !fd_funk_rec_key_eq( rec->pair.key, key ) ) ) goto next;
52 :
53 : /* Confirm that record is part of the current fork */
54 86603 : if( FD_UNLIKELY( !fd_accdb_lineage_has_xid( lineage, rec->pair.xid ) ) ) goto next;
55 :
56 86603 : if( FD_UNLIKELY( ele_next==ele_idx ) ) {
57 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected cycle" ));
58 0 : }
59 86603 : best = rec;
60 86603 : break;
61 :
62 13560 : next:
63 13560 : ele_idx = ele_next;
64 13560 : }
65 350471 : if( FD_UNLIKELY( !best && ele_idx!=FD_FUNK_REC_IDX_NULL ) ) {
66 0 : FD_LOG_CRIT(( "fd_accdb_search_chain detected malformed chain (%lu): found more nodes than chain header indicated (%lu)", chain_idx, cnt ));
67 0 : }
68 :
69 350471 : if( FD_UNLIKELY( atomic_load_explicit( ver_cnt_p, memory_order_acquire )!=ver_cnt ) ) {
70 0 : return FD_MAP_ERR_AGAIN;
71 0 : }
72 350471 : *out_rec = best;
73 350471 : return FD_MAP_SUCCESS;
74 350471 : }
75 :
76 : fd_accdb_ro_t *
77 : fd_accdb_peek_funk( fd_accdb_user_v1_t * accdb,
78 : fd_accdb_ro_t * ro,
79 : fd_funk_txn_xid_t const * xid,
80 351546 : void const * address ) {
81 351546 : fd_funk_t const * funk = accdb->funk;
82 351546 : fd_funk_rec_key_t key[1]; memcpy( key->uc, address, 32UL );
83 :
84 : /* Hash key to chain */
85 351546 : fd_funk_xid_key_pair_t pair[1];
86 351546 : fd_funk_txn_xid_copy( pair->xid, xid );
87 351546 : fd_funk_rec_key_copy( pair->key, key );
88 351546 : fd_funk_rec_map_t const * rec_map = funk->rec_map;
89 351546 : ulong hash = fd_funk_rec_map_key_hash( pair, rec_map->map->seed );
90 351546 : ulong chain_idx = (hash & (rec_map->map->chain_cnt-1UL) );
91 :
92 : /* Traverse chain for candidate */
93 351546 : fd_funk_rec_t * rec = NULL;
94 351546 : for(;;) {
95 350434 : int err = fd_accdb_search_chain( accdb->funk, accdb->lineage, chain_idx, key, &rec );
96 350434 : if( FD_LIKELY( err==FD_MAP_SUCCESS ) ) break;
97 258 : FD_SPIN_PAUSE();
98 : /* FIXME backoff */
99 258 : }
100 351546 : if( !rec ) return NULL;
101 :
102 86675 : memcpy( ro->ref->address, address, 32UL );
103 86675 : ro->ref->accdb_type = FD_ACCDB_TYPE_V1;
104 86675 : ro->ref->ref_type = FD_ACCDB_REF_RO;
105 86675 : ro->ref->user_data = (ulong)rec;
106 86675 : ro->ref->user_data2 = 0UL;
107 86675 : ro->meta = fd_funk_val( rec, funk->wksp );
108 86675 : return ro;
109 351546 : }
110 :
111 : static ulong
112 0 : fd_accdb_user_v1_batch_max( fd_accdb_user_t * accdb ) {
113 0 : (void)accdb;
114 0 : return ULONG_MAX;
115 0 : }
116 :
117 : void
118 12 : fd_accdb_user_v1_fini( fd_accdb_user_t * accdb ) {
119 12 : fd_accdb_user_v1_t * user = (fd_accdb_user_v1_t *)accdb;
120 :
121 12 : if( FD_UNLIKELY( !fd_funk_leave( user->funk, NULL, NULL ) ) ) FD_LOG_CRIT(( "fd_funk_leave failed" ));
122 12 : }
123 :
124 : void
125 : fd_accdb_user_v1_open_ro_multi( fd_accdb_user_t * accdb,
126 : fd_accdb_ro_t * ro,
127 : fd_funk_txn_xid_t const * xid,
128 : void const * address,
129 227082 : ulong cnt ) {
130 227082 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
131 227082 : fd_accdb_lineage_set_fork( v1->lineage, v1->funk, xid );
132 227082 : ulong addr_laddr = (ulong)address;
133 454086 : for( ulong i=0UL; i<cnt; i++ ) {
134 227004 : void const * addr_i = (void const *)( (ulong)addr_laddr + i*32UL );
135 227004 : if( !fd_accdb_peek_funk( v1, &ro[i], xid, addr_i ) ) {
136 141931 : fd_accdb_ro_init_empty( &ro[i], addr_i );
137 141931 : } else {
138 85073 : v1->base.ro_active++;
139 85073 : }
140 227004 : }
141 227082 : }
142 :
143 : static void
144 : fd_accdb_user_v1_close_ro( fd_accdb_user_t * accdb,
145 84500 : fd_accdb_ro_t * ro ) {
146 84500 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
147 :
148 84500 : v1->base.ro_active--;
149 84500 : (void)ro;
150 84500 : }
151 :
152 : fd_accdb_rw_t *
153 : fd_accdb_user_v1_open_rw( fd_accdb_user_t * accdb,
154 : fd_accdb_rw_t * rw,
155 : fd_funk_txn_xid_t const * xid,
156 : void const * address,
157 : ulong data_max,
158 128891 : int flags ) {
159 128891 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
160 :
161 128891 : int const flag_create = !!( flags & FD_ACCDB_FLAG_CREATE );
162 128891 : int const flag_truncate = !!( flags & FD_ACCDB_FLAG_TRUNCATE );
163 128891 : if( FD_UNLIKELY( flags & ~(FD_ACCDB_FLAG_CREATE|FD_ACCDB_FLAG_TRUNCATE) ) ) {
164 0 : FD_LOG_CRIT(( "invalid flags for open_rw: %#02x", (uint)flags ));
165 0 : }
166 :
167 : /* Pivot to different fork */
168 128891 : fd_accdb_lineage_set_fork( v1->lineage, v1->funk, xid );
169 128891 : fd_funk_txn_t * txn = fd_accdb_lineage_write_check( v1->lineage, v1->funk );
170 :
171 : /* Query old record value */
172 :
173 128891 : fd_accdb_ro_t ro[1];
174 128891 : if( FD_UNLIKELY( !fd_accdb_peek_funk( v1, ro, xid, address ) ) ) {
175 : /* Record not found */
176 125803 : if( flag_create ) return fd_accdb_funk_create( v1->funk, rw, txn, address, data_max );
177 286 : return NULL;
178 125803 : }
179 :
180 3088 : if( !ro->meta->lamports ) {
181 : /* Record previously deleted */
182 0 : if( !flag_create ) return NULL;
183 0 : }
184 :
185 3088 : fd_funk_rec_t * rec = (fd_funk_rec_t *)ro->ref->user_data;
186 3088 : if( fd_funk_txn_xid_eq( rec->pair.xid, xid ) ) {
187 :
188 : /* Mutable record found, modify in-place */
189 303 : ulong acc_orig_sz = fd_accdb_ref_data_sz( ro );
190 303 : ulong val_sz_min = sizeof(fd_account_meta_t)+fd_ulong_max( data_max, acc_orig_sz );
191 303 : void * val = fd_funk_val_truncate( rec, v1->funk->alloc, v1->funk->wksp, 16UL, val_sz_min, NULL );
192 303 : if( FD_UNLIKELY( !val ) ) {
193 0 : FD_LOG_CRIT(( "Failed to modify account: out of memory allocating %lu bytes", acc_orig_sz ));
194 0 : }
195 303 : fd_accdb_funk_prep_inplace( rw, v1->funk, rec );
196 303 : if( flag_truncate ) {
197 2 : rec->val_sz = sizeof(fd_account_meta_t);
198 2 : rw->meta->dlen = 0;
199 2 : }
200 303 : return rw;
201 :
202 2785 : } else {
203 :
204 : /* Frozen record found, copy out to new object */
205 2785 : ulong acc_orig_sz = fd_accdb_ref_data_sz( ro );
206 2785 : ulong val_sz_min = sizeof(fd_account_meta_t)+fd_ulong_max( data_max, acc_orig_sz );
207 2785 : ulong val_sz = flag_truncate ? sizeof(fd_account_meta_t) : rec->val_sz;
208 2785 : ulong val_max = 0UL;
209 2785 : void * val = fd_alloc_malloc_at_least( v1->funk->alloc, 16UL, val_sz_min, &val_max );
210 2785 : if( FD_UNLIKELY( !val ) ) {
211 0 : FD_LOG_CRIT(( "Failed to modify account: out of memory allocating %lu bytes", acc_orig_sz ));
212 0 : }
213 :
214 2785 : fd_account_meta_t * meta = val;
215 2785 : uchar * data = (uchar *)( meta+1 );
216 2785 : ulong data_max_actual = val_max - sizeof(fd_account_meta_t);
217 2785 : if( flag_truncate ) fd_accdb_funk_copy_truncated( meta, ro->meta );
218 2189 : else fd_accdb_funk_copy_account ( meta, data, ro->meta, fd_account_data( ro->meta ) );
219 2785 : if( acc_orig_sz<data_max_actual ) {
220 : /* Zero out trailing data */
221 1800 : uchar * tail = data +acc_orig_sz;
222 1800 : ulong tail_sz = data_max_actual-acc_orig_sz;
223 1800 : fd_memset( tail, 0, tail_sz );
224 1800 : }
225 :
226 2785 : accdb->base.created_cnt++;
227 2785 : return fd_accdb_funk_prep_create( rw, v1->funk, txn, address, val, val_sz, val_max );
228 :
229 2785 : }
230 3088 : }
231 :
232 : void
233 : fd_accdb_user_v1_open_rw_multi( fd_accdb_user_t * accdb,
234 : fd_accdb_rw_t * rw,
235 : fd_funk_txn_xid_t const * xid,
236 : void const * address,
237 : ulong const * data_max,
238 : int flags,
239 128978 : ulong cnt ) {
240 128978 : ulong addr_laddr = (ulong)address;
241 257988 : for( ulong i=0UL; i<cnt; i++ ) {
242 129010 : void const * addr_i = (void const *)( (ulong)addr_laddr + i*32UL );
243 129010 : ulong dmax_i = data_max[i];
244 129010 : fd_accdb_rw_t * rw_i = fd_accdb_user_v1_open_rw( accdb, &rw[i], xid, addr_i, dmax_i, flags );
245 129010 : if( !rw_i ) memset( &rw[i], 0, sizeof(fd_accdb_rw_t) );
246 128711 : else accdb->base.rw_active++;
247 129010 : }
248 128978 : }
249 :
250 : void
251 : fd_accdb_user_v1_close_rw( fd_accdb_user_t * accdb,
252 128625 : fd_accdb_rw_t * write ) {
253 128625 : if( FD_UNLIKELY( !accdb ) ) FD_LOG_CRIT(( "NULL accdb" ));
254 128625 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
255 128625 : fd_funk_rec_t * rec = (fd_funk_rec_t *)write->ref->user_data;
256 :
257 128625 : if( FD_UNLIKELY( write->ref->accdb_type!=FD_ACCDB_TYPE_V1 ) ) {
258 0 : FD_LOG_CRIT(( "invalid accdb_type %u in fd_accdb_user_v1_close_rw", (uint)write->ref->accdb_type ));
259 0 : }
260 :
261 128625 : if( FD_UNLIKELY( !v1->base.rw_active ) ) {
262 0 : FD_LOG_CRIT(( "Failed to modify account: ref count underflow" ));
263 0 : }
264 :
265 128625 : if( write->ref->user_data2 ) {
266 128283 : fd_funk_txn_t * txn = (fd_funk_txn_t *)write->ref->user_data2;
267 128283 : fd_funk_rec_prepare_t prepare = {
268 128283 : .rec = rec,
269 128283 : .rec_head_idx = &txn->rec_head_idx,
270 128283 : .rec_tail_idx = &txn->rec_tail_idx
271 128283 : };
272 128283 : fd_funk_rec_publish( v1->funk, &prepare );
273 128283 : }
274 :
275 128625 : memset( write, 0, sizeof(fd_accdb_rw_t) );
276 128625 : v1->base.rw_active--;
277 128625 : }
278 :
279 : void
280 : fd_accdb_user_v1_close_ref_multi( fd_accdb_user_t * accdb,
281 : fd_accdb_ref_t * ref0,
282 212616 : ulong cnt ) {
283 426867 : for( ulong i=0UL; i<cnt; i++ ) {
284 212399 : if( ref0[ i ].accdb_type==FD_ACCDB_TYPE_NONE ) continue;
285 212399 : switch( ref0[ i ].ref_type ) {
286 84499 : case FD_ACCDB_REF_RO:
287 84499 : fd_accdb_user_v1_close_ro( accdb, (fd_accdb_ro_t *)ref0+i );
288 84499 : break;
289 128618 : case FD_ACCDB_REF_RW:
290 128618 : fd_accdb_user_v1_close_rw( accdb, (fd_accdb_rw_t *)ref0+i );
291 128618 : break;
292 0 : default:
293 0 : FD_LOG_CRIT(( "invalid ref_type %u in fd_accdb_user_v1_close_ref", (uint)ref0[ i ].ref_type ));
294 212399 : }
295 212399 : }
296 212616 : }
297 :
298 : ulong
299 : fd_accdb_user_v1_rw_data_max( fd_accdb_user_t * accdb,
300 101448 : fd_accdb_rw_t const * rw ) {
301 101448 : (void)accdb;
302 101448 : if( rw->ref->accdb_type==FD_ACCDB_TYPE_NONE ) {
303 0 : return rw->ref->user_data; /* data_max */
304 0 : }
305 101448 : fd_funk_rec_t * rec = (fd_funk_rec_t *)rw->ref->user_data;
306 101448 : return (ulong)( rec->val_max - sizeof(fd_account_meta_t) );
307 101448 : }
308 :
309 : void
310 : fd_accdb_user_v1_rw_data_sz_set( fd_accdb_user_t * accdb,
311 : fd_accdb_rw_t * rw,
312 : ulong data_sz,
313 102936 : int flags ) {
314 102936 : int flag_dontzero = !!( flags & FD_ACCDB_FLAG_DONTZERO );
315 102936 : if( FD_UNLIKELY( flags & ~(FD_ACCDB_FLAG_DONTZERO) ) ) {
316 0 : FD_LOG_CRIT(( "invalid flags for rw_data_sz_set: %#02x", (uint)flags ));
317 0 : }
318 :
319 102936 : ulong prev_sz = rw->meta->dlen;
320 102936 : if( data_sz>prev_sz ) {
321 101445 : ulong data_max = fd_accdb_user_v1_rw_data_max( accdb, rw );
322 101445 : if( FD_UNLIKELY( data_sz>data_max ) ) {
323 0 : FD_LOG_CRIT(( "attempted to write %lu bytes into a rec with only %lu bytes of data space",
324 0 : data_sz, data_max ));
325 0 : }
326 101445 : if( !flag_dontzero ) {
327 0 : void * tail = (uchar *)fd_accdb_ref_data( rw ) + prev_sz;
328 0 : fd_memset( tail, 0, data_sz-prev_sz );
329 0 : }
330 101445 : }
331 102936 : rw->meta->dlen = (uint)data_sz;
332 :
333 102936 : if( rw->ref->accdb_type==FD_ACCDB_TYPE_V1 ) {
334 102931 : fd_funk_rec_t * rec = (fd_funk_rec_t *)rw->ref->user_data;
335 102931 : rec->val_sz = (uint)( sizeof(fd_account_meta_t)+data_sz ) & FD_FUNK_REC_VAL_MAX;
336 102931 : }
337 102936 : }
338 :
339 : fd_accdb_user_vt_t const fd_accdb_user_v1_vt = {
340 : .fini = fd_accdb_user_v1_fini,
341 : .batch_max = fd_accdb_user_v1_batch_max,
342 : .open_ro_multi = fd_accdb_user_v1_open_ro_multi,
343 : .open_rw_multi = fd_accdb_user_v1_open_rw_multi,
344 : .close_ref_multi = fd_accdb_user_v1_close_ref_multi,
345 : .rw_data_max = fd_accdb_user_v1_rw_data_max,
346 : .rw_data_sz_set = fd_accdb_user_v1_rw_data_sz_set
347 : };
348 :
349 : fd_accdb_user_t *
350 : fd_accdb_user_v1_init( fd_accdb_user_t * accdb,
351 : void * shfunk,
352 : void * shlocks,
353 12 : ulong max_depth ) {
354 12 : fd_accdb_user_v1_t * ljoin = (fd_accdb_user_v1_t *)accdb;
355 :
356 12 : if( FD_UNLIKELY( !ljoin ) ) {
357 0 : FD_LOG_WARNING(( "NULL ljoin" ));
358 0 : return NULL;
359 0 : }
360 12 : if( FD_UNLIKELY( !shfunk ) ) {
361 0 : FD_LOG_WARNING(( "NULL shfunk" ));
362 0 : return NULL;
363 0 : }
364 :
365 12 : memset( ljoin, 0, sizeof(fd_accdb_user_v1_t) );
366 12 : if( FD_UNLIKELY( !fd_funk_join( ljoin->funk, shfunk, shlocks ) ) ) {
367 0 : FD_LOG_CRIT(( "fd_funk_join failed" ));
368 0 : }
369 :
370 12 : ljoin->lineage->max_depth = max_depth;
371 12 : accdb->base.accdb_type = FD_ACCDB_TYPE_V1;
372 12 : accdb->base.vt = &fd_accdb_user_v1_vt;
373 12 : return accdb;
374 12 : }
375 :
376 : fd_funk_t *
377 5566 : fd_accdb_user_v1_funk( fd_accdb_user_t * accdb ) {
378 5566 : fd_accdb_user_v1_t * v1 = (fd_accdb_user_v1_t *)accdb;
379 5566 : uint accdb_type = accdb->base.accdb_type;
380 5566 : if( FD_UNLIKELY( accdb_type!=FD_ACCDB_TYPE_V1 && accdb_type!=FD_ACCDB_TYPE_V2 ) ) {
381 0 : FD_LOG_CRIT(( "fd_accdb_user_v1_funk called on non-v1 accdb_user (type %u)", accdb->base.accdb_type ));
382 0 : }
383 5566 : return v1->funk;
384 5566 : }
|
