Line data Source code
1 : #include "fd_vote_stakes.h"
2 : #include "../fd_rwlock.h"
3 : #include "../accdb/fd_accdb.h"
4 : #include "../fd_flamenco_base.h"
5 : #include "../runtime/fd_runtime_const.h"
6 : #include "../runtime/program/vote/fd_vote_state_versioned.h"
7 : #include "../../util/bits/fd_bits.h"
8 : #include "../../util/log/fd_log.h"
9 : #include "../../ballet/bls/fd_bls12_381.h"
10 :
11 : #include <errno.h>
12 : #include <unistd.h>
13 :
14 12 : #define FD_VOTE_STAKES_MAGIC (0xF17EDA2CE7601E70UL) /* FIREDANCER VOTE STAKES V0 */
15 82114 : #define FD_VOTE_STAKES_EPOCH_CACHE_CNT (5UL) /* t-2..t-5 sets plus one so crossing into t+1 evicts t-6 */
16 :
17 : struct vacc {
18 : fd_pubkey_t pubkey;
19 : fd_pubkey_t node_account;
20 : ulong stake;
21 : ushort commission; /* inflation rewards, basis points */
22 : ushort block_revenue_commission_bps; /* SIMD-0123 */
23 : ulong pending_delegator_rewards; /* SIMD-0123, as of the set's epoch boundary */
24 : fd_pubkey_t inflation_collector; /* SIMD-0232, defaults to pubkey */
25 : fd_pubkey_t block_collector; /* SIMD-0232, defaults to node_account */
26 : ushort rank;
27 : uchar bls_key[ FD_BLS_PUBKEY_COMPRESSED_SZ ]; /* zero if unregistered */
28 : uchar bls_key_uncompressed[ FD_BLS_PUBKEY_UNCOMPRESSED_SZ ]; /* decompressed by finalize, valid iff rank!=NULL */
29 : fd_validator_location_t location;
30 : uint left;
31 : uint right;
32 : uint next;
33 : };
34 : typedef struct vacc vacc_t;
35 :
36 : struct vacc_rank {
37 : vacc_t * vacc;
38 : ulong drop;
39 : };
40 : typedef struct vacc_rank vacc_rank_t;
41 :
42 : #define SORT_NAME vacc_rank_sort
43 0 : #define SORT_KEY_T vacc_rank_t
44 0 : #define SORT_BEFORE(a,b) (((a).vacc->stake>(b).vacc->stake) | ((a).vacc->stake==(b).vacc->stake && \
45 0 : memcmp( (a).vacc->bls_key, (b).vacc->bls_key, FD_BLS_PUBKEY_COMPRESSED_SZ )<0 ))
46 : #include "../../util/tmpl/fd_sort.c"
47 :
48 : #define SORT_NAME vacc_rank_bls_sort
49 0 : #define SORT_KEY_T vacc_rank_t
50 0 : #define SORT_BEFORE(a,b) (memcmp( (a).vacc->bls_key, (b).vacc->bls_key, FD_BLS_PUBKEY_COMPRESSED_SZ )<0)
51 : #include "../../util/tmpl/fd_sort.c"
52 :
53 : #define SORT_NAME vacc_rank_identity_sort
54 0 : #define SORT_KEY_T vacc_rank_t
55 0 : #define SORT_BEFORE(a,b) (memcmp( &(a).vacc->node_account, &(b).vacc->node_account, sizeof(fd_pubkey_t) )<0)
56 : #include "../../util/tmpl/fd_sort.c"
57 :
58 : #define HEAP_NAME vacc_heap
59 101 : #define HEAP_IDX_T uint
60 : #define HEAP_T vacc_t
61 134 : #define HEAP_LT(e0,e1) ( ((e0)->stake < (e1)->stake) | \
62 134 : (((e0)->stake==(e1)->stake) & \
63 134 : (memcmp( &(e0)->pubkey, &(e1)->pubkey, sizeof(fd_pubkey_t) )<0 ) ) )
64 : #include "../../util/tmpl/fd_heap.c"
65 :
66 : #define POOL_NAME vacc_pool
67 168 : #define POOL_T vacc_t
68 : #define POOL_IDX_T uint
69 : #define POOL_LAZY 1
70 : #include "../../util/tmpl/fd_pool.c"
71 :
72 : #define MAP_NAME vacc_map
73 : #define MAP_KEY_T fd_pubkey_t
74 : #define MAP_ELE_T vacc_t
75 2276 : #define MAP_KEY pubkey
76 2675 : #define MAP_KEY_EQ(k0,k1) (!memcmp( k0, k1, sizeof(fd_pubkey_t) ))
77 5014 : #define MAP_KEY_HASH(key,seed) (fd_hash32( key->uc, seed ))
78 79389 : #define MAP_IDX_T uint
79 : #include "../../util/tmpl/fd_map_chain.c"
80 :
81 : struct vacc_fork {
82 : uint ref_cnt;
83 : uint next;
84 : uint cache_idx; /* UINT_MAX if the t-1 set is not cached */
85 : };
86 : typedef struct vacc_fork vacc_fork_t;
87 :
88 : #define POOL_NAME vacc_fork_pool
89 24 : #define POOL_T vacc_fork_t
90 : #define POOL_IDX_T uint
91 : #define POOL_LAZY 1
92 : #include "../../util/tmpl/fd_pool.c"
93 :
94 : #define SET_NAME vacc_valid_set
95 : #define SET_MAX FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS
96 : #include "../../util/tmpl/fd_set.c"
97 :
98 : /* fork_t is a fork handed out to callers, who address it by its pool
99 : idx (the fork id). */
100 :
101 : struct fork {
102 : uint next;
103 : ushort t_1_idx; /* vacc_fork_pool idx of the fork's t-1 set */
104 : ushort epoch;
105 : };
106 : typedef struct fork fork_t;
107 :
108 : #define POOL_NAME fork_pool
109 24 : #define POOL_T fork_t
110 : #define POOL_IDX_T uint
111 : #define POOL_LAZY 1
112 : #include "../../util/tmpl/fd_pool.c"
113 :
114 : struct t_1_cache_ent {
115 : ulong set_idx; /* ULONG_MAX if the entry is empty */
116 : ulong lru;
117 : };
118 : typedef struct t_1_cache_ent t_1_cache_ent_t;
119 :
120 : struct fd_vote_stakes {
121 : ulong magic;
122 : ulong cache_cnt;
123 : ulong max_live_slots;
124 : ulong min_stake_wmark;
125 : int disk_fd;
126 : ulong lru;
127 : fd_rwlock_t lock;
128 :
129 : /* (pubkey, stake) pairs for the t-2 epoch. These are shared across
130 : forks/banks. */
131 : ulong t_2_epoch[ FD_VOTE_STAKES_EPOCH_CACHE_CNT ];
132 : ulong t_2_vacc_pool_off[ FD_VOTE_STAKES_EPOCH_CACHE_CNT ];
133 : ulong t_2_vacc_map_off [ FD_VOTE_STAKES_EPOCH_CACHE_CNT ];
134 :
135 : /* (pubkey, stake) pairs for the t-1 epoch. These can be different
136 : for every fork across the epoch boundary. Up to max_live_slots
137 : sets exist, keyed by set idx; cache_cnt of them are held in memory
138 : and the rest are spilled to disk_fd. */
139 : ulong vacc_fork_pool_off;
140 : ulong t_1_cache_off;
141 : ulong t_1_vacc_pools_mem_off;
142 : ulong t_1_vacc_pools_off;
143 : ulong t_1_vacc_maps_off;
144 : ulong vacc_heap_off;
145 :
146 : /* Forks handed out to callers, max_live_slots of them. */
147 : ulong fork_pool_off;
148 :
149 : /* Per-fork validity of each t-2 vote account: max_live_slots
150 : vacc_valid_set_t sets, keyed by fork id and indexed by t-2 pool
151 : idx. */
152 : ulong valid_off;
153 : };
154 : typedef struct fd_vote_stakes fd_vote_stakes_t;
155 :
156 : static inline vacc_t *
157 : t_2_vacc_pool( fd_vote_stakes_t const * vote_stakes,
158 15115 : ulong idx ) {
159 15115 : return fd_type_pun( (uchar *)vote_stakes + vote_stakes->t_2_vacc_pool_off[ idx ] );
160 15115 : }
161 :
162 : static inline vacc_map_t *
163 : t_2_vacc_map( fd_vote_stakes_t const * vote_stakes,
164 15114 : ulong idx ) {
165 15114 : return fd_type_pun( (uchar *)vote_stakes + vote_stakes->t_2_vacc_map_off[ idx ] );
166 15114 : }
167 :
168 : static inline vacc_fork_t *
169 264911 : vacc_fork_pool( fd_vote_stakes_t const * vote_stakes ) {
170 264911 : return fd_type_pun( (uchar *)vote_stakes + vote_stakes->vacc_fork_pool_off );
171 264911 : }
172 :
173 : static inline t_1_cache_ent_t *
174 131968 : t_1_cache( fd_vote_stakes_t const * vote_stakes ) {
175 131968 : return fd_type_pun( (uchar *)vote_stakes + vote_stakes->t_1_cache_off );
176 131968 : }
177 :
178 : static inline vacc_t *
179 : t_1_cache_pool( fd_vote_stakes_t const * vote_stakes,
180 70329 : ulong cache_idx ) {
181 70329 : ulong off = vote_stakes->t_1_vacc_pools_off + cache_idx*vacc_pool_footprint( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
182 70329 : return fd_type_pun( (uchar *)vote_stakes + off );
183 70329 : }
184 :
185 : static inline vacc_map_t *
186 : t_1_cache_map( fd_vote_stakes_t const * vote_stakes,
187 70449 : ulong cache_idx ) {
188 70449 : ulong chain_cnt = vacc_map_chain_cnt_est( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
189 70449 : ulong off = vote_stakes->t_1_vacc_maps_off + cache_idx*vacc_map_footprint( chain_cnt );
190 70449 : return fd_type_pun( (uchar *)vote_stakes + off );
191 70449 : }
192 :
193 : static void
194 : t_1_disk_io( fd_vote_stakes_t * vote_stakes,
195 : ulong cache_idx,
196 : ulong set_idx,
197 0 : int write ) {
198 0 : ulong pool_sz = vacc_pool_footprint( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
199 0 : ulong map_sz = vacc_map_footprint( vacc_map_chain_cnt_est( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
200 0 : uchar * bufs[2] = { (uchar *)vote_stakes + vote_stakes->t_1_vacc_pools_mem_off + cache_idx*pool_sz,
201 0 : (uchar *)t_1_cache_map( vote_stakes, cache_idx ) };
202 0 : ulong szs [2] = { pool_sz, map_sz };
203 0 : ulong off = set_idx*(pool_sz+map_sz);
204 0 : for( ulong b=0UL; b<2UL; b++ ) {
205 0 : ulong done = 0UL;
206 0 : while( done<szs[b] ) {
207 0 : long n = write ? pwrite( vote_stakes->disk_fd, bufs[b]+done, szs[b]-done, (long)(off+done) )
208 0 : : pread ( vote_stakes->disk_fd, bufs[b]+done, szs[b]-done, (long)(off+done) );
209 0 : if( FD_UNLIKELY( n<0L ) ) {
210 0 : if( FD_LIKELY( errno==EINTR ) ) continue;
211 0 : FD_LOG_CRIT(( "%s(vote stakes spill file, fd=%d) failed (%i-%s)", write ? "pwrite" : "pread", vote_stakes->disk_fd, errno, fd_io_strerror( errno ) ));
212 0 : }
213 0 : if( FD_UNLIKELY( !n ) ) FD_LOG_CRIT(( "%s(vote stakes spill file, fd=%d) returned 0 at offset %lu (set=%lu)",
214 0 : write ? "pwrite" : "pread", vote_stakes->disk_fd, off+done, set_idx ));
215 0 : done += (ulong)n;
216 0 : }
217 0 : off += szs[b];
218 0 : }
219 0 : }
220 :
221 : static ulong
222 : t_1_evict( fd_vote_stakes_t * vote_stakes,
223 65676 : ulong set_idx ) {
224 65676 : t_1_cache_ent_t * ent = t_1_cache( vote_stakes );
225 65676 : t_1_cache_ent_t * victim = ent;
226 131347 : for( ulong i=1UL; i<vote_stakes->cache_cnt; i++ ) if( ent[i].lru<victim->lru ) victim = ent+i;
227 :
228 65676 : ulong cache_idx = (ulong)(victim-ent);
229 65676 : vacc_fork_t * fork_pool = vacc_fork_pool( vote_stakes );
230 65676 : if( victim->set_idx!=ULONG_MAX ) {
231 0 : t_1_disk_io( vote_stakes, cache_idx, victim->set_idx, 1 );
232 0 : vacc_fork_pool_ele( fork_pool, victim->set_idx )->cache_idx = UINT_MAX;
233 0 : }
234 65676 : victim->set_idx = set_idx;
235 65676 : victim->lru = ++vote_stakes->lru;
236 65676 : vacc_fork_pool_ele( fork_pool, set_idx )->cache_idx = (uint)cache_idx;
237 65676 : return cache_idx;
238 65676 : }
239 :
240 : static ulong
241 : t_1_lock( fd_vote_stakes_t * vote_stakes,
242 : ulong set_idx,
243 3494 : int write ) {
244 3494 : if( FD_LIKELY( !write ) ) {
245 2342 : fd_rwlock_read( &vote_stakes->lock );
246 2342 : ulong cache_idx = vacc_fork_pool_ele_const( vacc_fork_pool( vote_stakes ), set_idx )->cache_idx;
247 2342 : if( FD_LIKELY( cache_idx!=UINT_MAX ) ) return cache_idx;
248 0 : fd_rwlock_unread( &vote_stakes->lock );
249 0 : }
250 1152 : fd_rwlock_write( &vote_stakes->lock );
251 1152 : ulong cache_idx = vacc_fork_pool_ele_const( vacc_fork_pool( vote_stakes ), set_idx )->cache_idx;
252 1152 : if( FD_UNLIKELY( cache_idx==UINT_MAX ) ) {
253 0 : cache_idx = t_1_evict( vote_stakes, set_idx );
254 0 : t_1_disk_io( vote_stakes, cache_idx, set_idx, 0 );
255 1152 : } else {
256 1152 : t_1_cache( vote_stakes )[ cache_idx ].lru = ++vote_stakes->lru;
257 1152 : }
258 1152 : if( FD_LIKELY( !write ) ) fd_rwlock_demote( &vote_stakes->lock );
259 1152 : return cache_idx;
260 3494 : }
261 :
262 : static void
263 : t_1_unlock( fd_vote_stakes_t * vote_stakes,
264 3492 : int write ) {
265 3492 : if( FD_UNLIKELY( write ) ) fd_rwlock_unwrite( &vote_stakes->lock );
266 2342 : else fd_rwlock_unread ( &vote_stakes->lock );
267 3492 : }
268 :
269 : static ushort
270 65666 : t_1_acquire( fd_vote_stakes_t * vote_stakes ) {
271 65666 : vacc_fork_t * fork_pool = vacc_fork_pool( vote_stakes );
272 65666 : FD_CHECK_CRIT( vacc_fork_pool_free( fork_pool ), "vote stakes t-1 set pool exhausted" );
273 65666 : ulong set_idx = vacc_fork_pool_idx_acquire( fork_pool );
274 65666 : vacc_fork_pool_ele( fork_pool, set_idx )->ref_cnt = 1U;
275 65666 : fd_rwlock_write( &vote_stakes->lock );
276 65666 : ulong cache_idx = t_1_evict( vote_stakes, set_idx );
277 65666 : vacc_map_reset ( t_1_cache_map ( vote_stakes, cache_idx ) );
278 65666 : vacc_pool_reset( t_1_cache_pool( vote_stakes, cache_idx ) );
279 65666 : fd_rwlock_unwrite( &vote_stakes->lock );
280 65666 : return (ushort)set_idx;
281 65666 : }
282 :
283 : static void
284 : t_1_release( fd_vote_stakes_t * vote_stakes,
285 64459 : ulong set_idx ) {
286 64459 : vacc_fork_t * fork_pool = vacc_fork_pool( vote_stakes );
287 64459 : vacc_fork_t * fork = vacc_fork_pool_ele( fork_pool, set_idx );
288 64459 : fd_rwlock_write( &vote_stakes->lock );
289 64507 : if( fork->cache_idx!=UINT_MAX ) {
290 64507 : t_1_cache_ent_t * ent = t_1_cache( vote_stakes ) + fork->cache_idx;
291 64507 : ent->set_idx = ULONG_MAX;
292 64507 : ent->lru = 0UL;
293 64507 : fork->cache_idx = UINT_MAX;
294 64507 : }
295 64459 : fd_rwlock_unwrite( &vote_stakes->lock );
296 64459 : vacc_fork_pool_idx_release( fork_pool, set_idx );
297 64459 : }
298 :
299 : static inline vacc_heap_t *
300 751 : vacc_heap( fd_vote_stakes_t const * vote_stakes ) {
301 751 : return fd_type_pun( (uchar *)vote_stakes + vote_stakes->vacc_heap_off );
302 751 : }
303 :
304 : static inline fork_t *
305 345565 : fork_pool( fd_vote_stakes_t const * vote_stakes ) {
306 345565 : return fd_type_pun( (uchar *)vote_stakes + vote_stakes->fork_pool_off );
307 345565 : }
308 :
309 : static inline fork_t *
310 : fork_ele( fd_vote_stakes_t const * vote_stakes,
311 148072 : ushort fork_id ) {
312 148072 : return fork_pool_ele( fork_pool( vote_stakes ), fork_id );
313 148072 : }
314 :
315 : static inline vacc_valid_set_t *
316 : valid_set( fd_vote_stakes_t const * vote_stakes,
317 70597 : ushort fork_id ) {
318 70597 : return (vacc_valid_set_t *)fd_type_pun( (uchar *)vote_stakes + vote_stakes->valid_off ) + (ulong)fork_id*vacc_valid_set_word_cnt;
319 70597 : }
320 :
321 : ulong
322 360 : fd_vote_stakes_align( void ) {
323 360 : return FD_VOTE_STAKES_ALIGN;
324 360 : }
325 :
326 : ulong
327 : fd_vote_stakes_footprint( ulong max_live_slots,
328 60 : ulong cache_cnt ) {
329 60 : if( FD_UNLIKELY( !max_live_slots || max_live_slots>USHORT_MAX ) ) return 0UL;
330 60 : if( FD_UNLIKELY( !cache_cnt ) ) return 0UL;
331 60 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
332 :
333 60 : ulong map_chain_cnt = vacc_map_chain_cnt_est( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
334 60 : ulong pool_footprint = vacc_pool_footprint( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
335 60 : ulong map_footprint = vacc_map_footprint( map_chain_cnt );
336 :
337 60 : ulong l = FD_LAYOUT_INIT;
338 60 : l = FD_LAYOUT_APPEND( l, fd_vote_stakes_align(), sizeof(fd_vote_stakes_t) );
339 360 : for( ulong i=0UL; i<FD_VOTE_STAKES_EPOCH_CACHE_CNT; i++ ) {
340 300 : l = FD_LAYOUT_APPEND( l, vacc_pool_align(), pool_footprint );
341 300 : l = FD_LAYOUT_APPEND( l, vacc_map_align(), map_footprint );
342 300 : }
343 60 : l = FD_LAYOUT_APPEND( l, vacc_fork_pool_align(), vacc_fork_pool_footprint( max_live_slots ) );
344 60 : l = FD_LAYOUT_APPEND( l, alignof(t_1_cache_ent_t), cache_cnt*sizeof(t_1_cache_ent_t) );
345 60 : l = FD_LAYOUT_APPEND( l, vacc_pool_align(), cache_cnt*pool_footprint );
346 60 : l = FD_LAYOUT_APPEND( l, vacc_map_align(), cache_cnt*map_footprint );
347 60 : l = FD_LAYOUT_APPEND( l, vacc_heap_align(), vacc_heap_footprint( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
348 60 : l = FD_LAYOUT_APPEND( l, fork_pool_align(), fork_pool_footprint( max_live_slots ) );
349 60 : l = FD_LAYOUT_APPEND( l, vacc_valid_set_align(), max_live_slots*vacc_valid_set_footprint() );
350 60 : return FD_LAYOUT_FINI( l, fd_vote_stakes_align() );
351 60 : }
352 :
353 : void *
354 : fd_vote_stakes_new( void * mem,
355 : int disk_fd,
356 : ulong max_live_slots,
357 : ulong cache_cnt,
358 12 : ulong seed ) {
359 12 : if( FD_UNLIKELY( !mem ) ) {
360 0 : FD_LOG_WARNING(( "NULL mem" ));
361 0 : return NULL;
362 0 : }
363 :
364 12 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)mem, fd_vote_stakes_align() ) ) ) {
365 0 : FD_LOG_WARNING(( "misaligned mem" ));
366 0 : return NULL;
367 0 : }
368 :
369 12 : if( FD_UNLIKELY( !max_live_slots || max_live_slots>USHORT_MAX ) ) {
370 0 : FD_LOG_WARNING(( "invalid max_live_slots" ));
371 0 : return NULL;
372 0 : }
373 :
374 12 : if( FD_UNLIKELY( !cache_cnt ) ) {
375 0 : FD_LOG_WARNING(( "invalid cache_cnt" ));
376 0 : return NULL;
377 0 : }
378 :
379 12 : if( FD_UNLIKELY( disk_fd<0 ) ) {
380 0 : FD_LOG_WARNING(( "invalid disk_fd %d", disk_fd ));
381 0 : return NULL;
382 0 : }
383 :
384 12 : ulong footprint = fd_vote_stakes_footprint( max_live_slots, cache_cnt );
385 12 : cache_cnt = fd_ulong_min( cache_cnt, max_live_slots );
386 :
387 12 : ulong map_chain_cnt = vacc_map_chain_cnt_est( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
388 12 : ulong pool_footprint = vacc_pool_footprint( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
389 12 : ulong map_footprint = vacc_map_footprint( map_chain_cnt );
390 :
391 12 : FD_SCRATCH_ALLOC_INIT( l, mem );
392 12 : fd_vote_stakes_t * vote_stakes = FD_SCRATCH_ALLOC_APPEND( l, fd_vote_stakes_align(), sizeof(fd_vote_stakes_t) );
393 0 : void * t_2_pool_mem[ FD_VOTE_STAKES_EPOCH_CACHE_CNT ];
394 12 : void * t_2_map_mem [ FD_VOTE_STAKES_EPOCH_CACHE_CNT ];
395 72 : for( ulong i=0UL; i<FD_VOTE_STAKES_EPOCH_CACHE_CNT; i++ ) {
396 60 : t_2_pool_mem[ i ] = FD_SCRATCH_ALLOC_APPEND( l, vacc_pool_align(), pool_footprint );
397 60 : t_2_map_mem [ i ] = FD_SCRATCH_ALLOC_APPEND( l, vacc_map_align(), map_footprint );
398 60 : }
399 12 : void * vacc_fork_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, vacc_fork_pool_align(), vacc_fork_pool_footprint( max_live_slots ) );
400 12 : void * t_1_cache_mem = FD_SCRATCH_ALLOC_APPEND( l, alignof(t_1_cache_ent_t), cache_cnt*sizeof(t_1_cache_ent_t) );
401 12 : void * t_1_vacc_pools_mem = FD_SCRATCH_ALLOC_APPEND( l, vacc_pool_align(), cache_cnt*pool_footprint );
402 12 : void * t_1_vacc_maps_mem = FD_SCRATCH_ALLOC_APPEND( l, vacc_map_align(), cache_cnt*map_footprint );
403 12 : void * vacc_heap_mem = FD_SCRATCH_ALLOC_APPEND( l, vacc_heap_align(), vacc_heap_footprint( FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
404 12 : void * fork_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fork_pool_align(), fork_pool_footprint( max_live_slots ) );
405 12 : void * valid_mem = FD_SCRATCH_ALLOC_APPEND( l, vacc_valid_set_align(), max_live_slots*vacc_valid_set_footprint() );
406 :
407 12 : if( FD_UNLIKELY( FD_SCRATCH_ALLOC_FINI( l, fd_vote_stakes_align() )!=(ulong)mem+footprint ) ) {
408 0 : FD_LOG_WARNING(( "fd_vote_stakes_new: bad layout" ));
409 0 : return NULL;
410 0 : }
411 :
412 72 : for( ulong i=0UL; i<FD_VOTE_STAKES_EPOCH_CACHE_CNT; i++ ) {
413 60 : vacc_t * t_2_pool = vacc_pool_join( vacc_pool_new( t_2_pool_mem[ i ], FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
414 60 : if( FD_UNLIKELY( !t_2_pool ) ) {
415 0 : FD_LOG_WARNING(( "Failed to create t-2 vote account pool" ));
416 0 : return NULL;
417 0 : }
418 :
419 60 : vacc_map_t * t_2_map = vacc_map_join( vacc_map_new( t_2_map_mem[ i ], map_chain_cnt, seed ) );
420 60 : if( FD_UNLIKELY( !t_2_map ) ) {
421 0 : FD_LOG_WARNING(( "Failed to create t-2 vote account map" ));
422 0 : return NULL;
423 0 : }
424 :
425 60 : vote_stakes->t_2_vacc_pool_off[ i ] = (ulong)t_2_pool - (ulong)mem;
426 60 : vote_stakes->t_2_vacc_map_off [ i ] = (ulong)t_2_map - (ulong)mem;
427 60 : }
428 :
429 12 : vacc_fork_t * fork_pool = vacc_fork_pool_join( vacc_fork_pool_new( vacc_fork_pool_mem, max_live_slots ) );
430 12 : if( FD_UNLIKELY( !fork_pool ) ) {
431 0 : FD_LOG_WARNING(( "Failed to create vote account fork pool" ));
432 0 : return NULL;
433 0 : }
434 12 : vote_stakes->vacc_fork_pool_off = (ulong)fork_pool - (ulong)mem;
435 :
436 12 : t_1_cache_ent_t * t_1_cache_ents = t_1_cache_mem;
437 36 : for( ulong i=0UL; i<cache_cnt; i++ ) t_1_cache_ents[ i ] = (t_1_cache_ent_t){ .set_idx = ULONG_MAX };
438 12 : vote_stakes->t_1_cache_off = (ulong)t_1_cache_mem - (ulong)mem;
439 12 : vote_stakes->t_1_vacc_pools_mem_off = (ulong)t_1_vacc_pools_mem - (ulong)mem;
440 :
441 36 : for( ulong i=0UL; i<cache_cnt; i++ ) {
442 24 : void * pool_mem = (uchar *)t_1_vacc_pools_mem + i*pool_footprint;
443 24 : vacc_t * pool = vacc_pool_join( vacc_pool_new( pool_mem, FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
444 24 : if( FD_UNLIKELY( !pool ) ) {
445 0 : FD_LOG_WARNING(( "Failed to create t-1 vote account pool" ));
446 0 : return NULL;
447 0 : }
448 24 : if( FD_UNLIKELY( !i ) ) vote_stakes->t_1_vacc_pools_off = (ulong)pool - (ulong)mem;
449 :
450 24 : void * map_mem = (uchar *)t_1_vacc_maps_mem + i*map_footprint;
451 24 : vacc_map_t * map = vacc_map_join( vacc_map_new( map_mem, map_chain_cnt, seed ) );
452 24 : if( FD_UNLIKELY( !map ) ) {
453 0 : FD_LOG_WARNING(( "Failed to create t-1 vote account map" ));
454 0 : return NULL;
455 0 : }
456 24 : }
457 12 : vote_stakes->t_1_vacc_maps_off = (ulong)t_1_vacc_maps_mem - (ulong)mem;
458 :
459 12 : vacc_heap_t * heap = vacc_heap_join( vacc_heap_new( vacc_heap_mem, FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
460 12 : if( FD_UNLIKELY( !heap ) ) {
461 0 : FD_LOG_WARNING(( "Failed to create vote account heap" ));
462 0 : return NULL;
463 0 : }
464 :
465 12 : fork_t * forks = fork_pool_join( fork_pool_new( fork_pool_mem, max_live_slots ) );
466 12 : if( FD_UNLIKELY( !forks ) ) {
467 0 : FD_LOG_WARNING(( "Failed to create vote stakes fork pool" ));
468 0 : return NULL;
469 0 : }
470 :
471 12 : vote_stakes->max_live_slots = max_live_slots;
472 12 : vote_stakes->cache_cnt = cache_cnt;
473 12 : vote_stakes->disk_fd = disk_fd;
474 12 : vote_stakes->lru = 0UL;
475 12 : vote_stakes->min_stake_wmark = 0UL;
476 12 : vote_stakes->vacc_heap_off = (ulong)heap - (ulong)mem;
477 12 : vote_stakes->fork_pool_off = (ulong)forks - (ulong)mem;
478 12 : vote_stakes->valid_off = (ulong)valid_mem - (ulong)mem;
479 12 : fd_rwlock_new( &vote_stakes->lock );
480 72 : for( ulong i=0UL; i<FD_VOTE_STAKES_EPOCH_CACHE_CNT; i++ ) vote_stakes->t_2_epoch[ i ] = ULONG_MAX;
481 :
482 12 : FD_COMPILER_MFENCE();
483 12 : FD_VOLATILE( vote_stakes->magic ) = FD_VOTE_STAKES_MAGIC;
484 12 : FD_COMPILER_MFENCE();
485 :
486 12 : return vote_stakes;
487 12 : }
488 :
489 : fd_vote_stakes_t *
490 : fd_vote_stakes_join( void * mem,
491 24 : int disk_fd ) {
492 24 : fd_vote_stakes_t * vote_stakes = (fd_vote_stakes_t *)mem;
493 :
494 24 : if( FD_UNLIKELY( !vote_stakes ) ) {
495 0 : FD_LOG_WARNING(( "NULL vote stakes" ));
496 0 : return NULL;
497 0 : }
498 :
499 24 : if( FD_UNLIKELY( vote_stakes->magic!=FD_VOTE_STAKES_MAGIC ) ) {
500 0 : FD_LOG_WARNING(( "Invalid vote stakes magic" ));
501 0 : return NULL;
502 0 : }
503 :
504 24 : if( FD_UNLIKELY( vote_stakes->disk_fd!=disk_fd ) ) {
505 0 : FD_LOG_WARNING(( "disk_fd mismatch (vote_stakes=%d join=%d)", vote_stakes->disk_fd, disk_fd ));
506 0 : return NULL;
507 0 : }
508 :
509 24 : return vote_stakes;
510 24 : }
511 :
512 : ulong
513 : fd_vote_stakes_fork_epoch( fd_vote_stakes_t const * vote_stakes,
514 611 : ushort fork_id ) {
515 611 : return (ulong)fork_ele( vote_stakes, fork_id )->epoch;
516 611 : }
517 :
518 : ushort
519 : fd_vote_stakes_init( fd_vote_stakes_t * vote_stakes,
520 65658 : ulong epoch ) {
521 65658 : FD_TEST( epoch<=USHORT_MAX );
522 65658 : FD_CHECK_CRIT( fork_pool_free( fork_pool( vote_stakes ) ), "vote stakes fork pool exhausted" );
523 65658 : ushort fork_id = (ushort)fork_pool_idx_acquire( fork_pool( vote_stakes ) );
524 65658 : fork_t * fork = fork_ele( vote_stakes, fork_id );
525 65658 : fork->t_1_idx = t_1_acquire( vote_stakes );
526 65658 : fork->epoch = (ushort)epoch;
527 65658 : vacc_valid_set_null( valid_set( vote_stakes, fork_id ) );
528 65658 : vote_stakes->t_2_epoch[ epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT ] = epoch;
529 65658 : return fork_id;
530 65658 : }
531 :
532 : void
533 622 : fd_vote_stakes_reset( fd_vote_stakes_t * vote_stakes ) {
534 3735 : for( ulong i=0UL; i<FD_VOTE_STAKES_EPOCH_CACHE_CNT; i++ ) {
535 3113 : vacc_map_reset( t_2_vacc_map( vote_stakes, i ) );
536 3113 : vacc_pool_reset( t_2_vacc_pool( vote_stakes, i ) );
537 3113 : vote_stakes->t_2_epoch[ i ] = ULONG_MAX;
538 3113 : }
539 :
540 622 : t_1_cache_ent_t * t_1_cache_ents = t_1_cache( vote_stakes );
541 1868 : for( ulong i=0UL; i<vote_stakes->cache_cnt; i++ ) {
542 1246 : t_1_cache_ents[ i ] = (t_1_cache_ent_t){ .set_idx = ULONG_MAX };
543 1246 : vacc_map_reset ( t_1_cache_map ( vote_stakes, i ) );
544 1246 : vacc_pool_reset( t_1_cache_pool( vote_stakes, i ) );
545 1246 : }
546 622 : vote_stakes->lru = 0UL;
547 :
548 622 : vacc_fork_pool_reset( vacc_fork_pool( vote_stakes ) );
549 622 : fork_pool_reset( fork_pool( vote_stakes ) );
550 622 : FD_TEST( vacc_heap_new( vacc_heap( vote_stakes ), FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
551 622 : vote_stakes->min_stake_wmark = 0UL;
552 622 : }
553 :
554 : static void
555 : vacc_init( vacc_t * vacc,
556 : fd_pubkey_t const * pubkey,
557 : fd_pubkey_t const * node_account,
558 : ulong stake,
559 : ushort commission,
560 : uchar const bls_key[ static FD_BLS_PUBKEY_COMPRESSED_SZ ],
561 : ushort block_revenue_commission_bps,
562 : ulong pending_delegator_rewards,
563 : fd_pubkey_t const * inflation_collector_opt,
564 : fd_pubkey_t const * block_collector_opt,
565 2116 : fd_validator_location_t const * location ) {
566 2116 : vacc->pubkey = *pubkey;
567 2116 : vacc->node_account = *node_account;
568 2116 : vacc->stake = stake;
569 2116 : vacc->commission = commission;
570 2116 : vacc->block_revenue_commission_bps = block_revenue_commission_bps;
571 2116 : vacc->pending_delegator_rewards = pending_delegator_rewards;
572 2116 : vacc->inflation_collector = inflation_collector_opt ? *inflation_collector_opt : *pubkey;
573 2116 : vacc->block_collector = block_collector_opt ? *block_collector_opt : *node_account;
574 2116 : vacc->rank = FD_VOTE_STAKES_ALPENGLOW_RANK_NULL;
575 2116 : vacc->location = *location;
576 2116 : memcpy( vacc->bls_key, bls_key, FD_BLS_PUBKEY_COMPRESSED_SZ );
577 2116 : }
578 :
579 : void
580 : fd_vote_stakes_snap_insert_t_1( fd_vote_stakes_t * vote_stakes,
581 : ushort fork_id,
582 : fd_pubkey_t const * pubkey,
583 : fd_pubkey_t const * node_account,
584 : ulong stake,
585 : ushort commission,
586 : uchar const bls_key[ static FD_BLS_PUBKEY_COMPRESSED_SZ ],
587 : ushort block_revenue_commission_bps,
588 : ulong pending_delegator_rewards,
589 : fd_pubkey_t const * inflation_collector_opt,
590 : fd_pubkey_t const * block_collector_opt,
591 1022 : fd_validator_location_t const * location ) {
592 1022 : if( FD_UNLIKELY( !stake ) ) return;
593 :
594 1022 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 1 );
595 1022 : vacc_t * pool = t_1_cache_pool( vote_stakes, cache_idx );
596 1022 : vacc_map_t * map = t_1_cache_map ( vote_stakes, cache_idx );
597 :
598 1022 : vacc_t * vacc = vacc_pool_ele_acquire( pool );
599 1022 : vacc_init( vacc, pubkey, node_account, stake, commission, bls_key,
600 1022 : block_revenue_commission_bps, pending_delegator_rewards, inflation_collector_opt, block_collector_opt, location );
601 1022 : FD_TEST( vacc_map_ele_insert( map, vacc, pool ) );
602 1022 : t_1_unlock( vote_stakes, 1 );
603 1022 : }
604 :
605 : void
606 : fd_vote_stakes_snap_insert_t_2( fd_vote_stakes_t * vote_stakes,
607 : ushort fork_id,
608 : fd_pubkey_t const * pubkey,
609 : fd_pubkey_t const * node_account,
610 : ulong stake,
611 : ushort commission,
612 : uchar const bls_key[ static FD_BLS_PUBKEY_COMPRESSED_SZ ],
613 : ushort block_revenue_commission_bps,
614 : ulong pending_delegator_rewards,
615 : fd_pubkey_t const * inflation_collector_opt,
616 : fd_pubkey_t const * block_collector_opt,
617 992 : fd_validator_location_t const * location ) {
618 992 : if( FD_UNLIKELY( !stake ) ) return;
619 :
620 992 : ulong epoch_idx = (ulong)fork_ele( vote_stakes, fork_id )->epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
621 992 : vacc_t * pool = t_2_vacc_pool( vote_stakes, epoch_idx );
622 992 : vacc_map_t * map = t_2_vacc_map ( vote_stakes, epoch_idx );
623 :
624 992 : vacc_t * vacc = vacc_pool_ele_acquire( pool );
625 992 : vacc_init( vacc, pubkey, node_account, stake, commission, bls_key,
626 992 : block_revenue_commission_bps, pending_delegator_rewards, inflation_collector_opt, block_collector_opt, location );
627 992 : FD_TEST( vacc_map_ele_insert( map, vacc, pool ) );
628 992 : }
629 :
630 : void
631 : fd_vote_stakes_snap_insert_t_n( fd_vote_stakes_t * vote_stakes,
632 : ushort fork_id,
633 : ulong n,
634 : fd_pubkey_t const * pubkey,
635 : fd_pubkey_t const * node_account,
636 : ulong stake,
637 : ushort commission,
638 : uchar const bls_key[ static FD_BLS_PUBKEY_COMPRESSED_SZ ],
639 : ushort block_revenue_commission_bps,
640 : ulong pending_delegator_rewards,
641 : fd_pubkey_t const * inflation_collector_opt,
642 : fd_pubkey_t const * block_collector_opt,
643 0 : fd_validator_location_t const * location ) {
644 0 : ulong epoch = (ulong)fork_ele( vote_stakes, fork_id )->epoch;
645 0 : ulong back = n-2UL;
646 0 : if( FD_UNLIKELY( epoch<back || !stake ) ) return;
647 :
648 0 : ulong t_n_epoch = epoch - back;
649 0 : ulong t_n_idx = t_n_epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
650 0 : vacc_t * pool = t_2_vacc_pool( vote_stakes, t_n_idx );
651 0 : vacc_map_t * map = t_2_vacc_map ( vote_stakes, t_n_idx );
652 :
653 0 : vacc_t * vacc = vacc_pool_ele_acquire( pool );
654 0 : vacc_init( vacc, pubkey, node_account, stake, commission, bls_key,
655 0 : block_revenue_commission_bps, pending_delegator_rewards, inflation_collector_opt, block_collector_opt, location );
656 0 : FD_TEST( vacc_map_ele_insert( map, vacc, pool ) );
657 0 : vote_stakes->t_2_epoch[ t_n_idx ] = t_n_epoch;
658 0 : }
659 :
660 : static void
661 : t_1_insert( fd_vote_stakes_t * vote_stakes,
662 : vacc_t * pool,
663 : vacc_map_t * map,
664 : fd_pubkey_t const * pubkey,
665 : fd_pubkey_t const * node_account,
666 : ulong stake,
667 : ushort commission,
668 : uchar const bls_key[ static FD_BLS_PUBKEY_COMPRESSED_SZ ],
669 : ushort block_revenue_commission_bps,
670 : ulong pending_delegator_rewards,
671 : fd_pubkey_t const * inflation_collector_opt,
672 : fd_pubkey_t const * block_collector_opt,
673 101 : fd_validator_location_t const * location ) {
674 101 : vacc_heap_t * heap = vacc_heap( vote_stakes );
675 :
676 101 : if( FD_UNLIKELY( stake==0UL || stake<=vote_stakes->min_stake_wmark ) ) return;
677 :
678 101 : if( FD_UNLIKELY( vacc_heap_ele_cnt( heap )==vacc_heap_ele_max( heap ) ) ) {
679 0 : vacc_t * ele = vacc_heap_ele_peek_min( heap, pool );
680 0 : ulong min_stake = ele->stake;
681 0 : if( stake<min_stake ) return;
682 :
683 0 : vote_stakes->min_stake_wmark = min_stake;
684 0 : while( (ele=vacc_heap_ele_peek_min( heap, pool )) && min_stake==ele->stake ) {
685 0 : vacc_heap_ele_remove_min( heap, pool );
686 0 : vacc_map_ele_remove( map, &ele->pubkey, NULL, pool );
687 0 : vacc_pool_ele_release( pool, ele );
688 0 : }
689 0 : if( FD_UNLIKELY( stake==min_stake ) ) return;
690 0 : }
691 :
692 101 : vacc_t * vacc = vacc_pool_ele_acquire( pool );
693 101 : vacc_init( vacc, pubkey, node_account, stake, commission, bls_key,
694 101 : block_revenue_commission_bps, pending_delegator_rewards, inflation_collector_opt, block_collector_opt, location );
695 101 : vacc_heap_ele_insert( heap, vacc, pool );
696 101 : FD_TEST( vacc_map_ele_insert( map, vacc, pool ) );
697 101 : }
698 :
699 : void
700 : fd_vote_stakes_insert( fd_vote_stakes_t * vote_stakes,
701 : ushort fork_id,
702 : fd_pubkey_t const * pubkey,
703 : fd_pubkey_t const * node_account,
704 : ulong stake,
705 : ushort commission,
706 : uchar const bls_key[ static FD_BLS_PUBKEY_COMPRESSED_SZ ],
707 : ushort block_revenue_commission_bps,
708 : ulong pending_delegator_rewards,
709 : fd_pubkey_t const * inflation_collector_opt,
710 : fd_pubkey_t const * block_collector_opt,
711 101 : fd_validator_location_t const * location ) {
712 101 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 1 );
713 101 : t_1_insert( vote_stakes, t_1_cache_pool( vote_stakes, cache_idx ), t_1_cache_map( vote_stakes, cache_idx ),
714 101 : pubkey, node_account, stake, commission, bls_key,
715 101 : block_revenue_commission_bps, pending_delegator_rewards, inflation_collector_opt, block_collector_opt, location );
716 101 : t_1_unlock( vote_stakes, 1 );
717 101 : }
718 :
719 : void
720 : fd_vote_stakes_finalize( fd_vote_stakes_t * vote_stakes,
721 : ushort fork_id,
722 28 : int iter_kind ) {
723 28 : vacc_t * pool;
724 28 : vacc_map_t * map;
725 28 : ulong cache_idx = ULONG_MAX;
726 28 : if( FD_LIKELY( iter_kind==FD_VOTE_STAKES_ITER_T_1 ) ) {
727 28 : cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 1 );
728 28 : pool = t_1_cache_pool( vote_stakes, cache_idx );
729 28 : map = t_1_cache_map ( vote_stakes, cache_idx );
730 28 : } else {
731 0 : FD_TEST( iter_kind>=FD_VOTE_STAKES_ITER_T_2 && iter_kind<=FD_VOTE_STAKES_ITER_T_5 );
732 0 : ulong fork_epoch = (ulong)fork_ele( vote_stakes, fork_id )->epoch;
733 0 : ulong back = (ulong)(iter_kind-FD_VOTE_STAKES_ITER_T_2);
734 0 : if( FD_UNLIKELY( fork_epoch<back ) ) return;
735 :
736 0 : ulong epoch = fork_epoch-back;
737 0 : ulong epoch_idx = epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
738 0 : if( FD_UNLIKELY( vote_stakes->t_2_epoch[ epoch_idx ]!=epoch ) ) return;
739 0 : pool = t_2_vacc_pool( vote_stakes, epoch_idx );
740 0 : map = t_2_vacc_map ( vote_stakes, epoch_idx );
741 0 : }
742 :
743 28 : vacc_rank_t rank[ FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ];
744 28 : ulong rank_cnt = 0UL;
745 28 : for( vacc_map_iter_t iter = vacc_map_iter_init( map, pool );
746 129 : !vacc_map_iter_done( iter, map, pool );
747 101 : iter = vacc_map_iter_next( iter, map, pool ) ) {
748 101 : vacc_t * vacc = vacc_map_iter_ele( iter, map, pool );
749 101 : vacc->rank = FD_VOTE_STAKES_ALPENGLOW_RANK_NULL;
750 101 : if( FD_UNLIKELY( !vacc->stake ) ) continue;
751 101 : if( FD_UNLIKELY( fd_bls12_381_g1_decompress_syscall( vacc->bls_key_uncompressed, vacc->bls_key, 1 ) ) ) continue;
752 0 : if( FD_UNLIKELY( vacc->bls_key[0]&0x40 ) ) continue; /* infinity: agave's rank map rejects the identity point */
753 0 : FD_TEST( rank_cnt<FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS );
754 0 : rank[ rank_cnt++ ] = (vacc_rank_t){ .vacc=vacc, .drop=0UL };
755 0 : }
756 :
757 28 : vacc_rank_bls_sort_inplace( rank, rank_cnt );
758 28 : for( ulong i=1UL; i<rank_cnt; i++ ) {
759 0 : if( FD_UNLIKELY( !memcmp( rank[i].vacc->bls_key, rank[i-1UL].vacc->bls_key, FD_BLS_PUBKEY_COMPRESSED_SZ ) ) ) {
760 0 : rank[i].drop = rank[i-1UL].drop = 1UL;
761 0 : }
762 0 : }
763 :
764 28 : vacc_rank_identity_sort_inplace( rank, rank_cnt );
765 28 : for( ulong i=1UL; i<rank_cnt; i++ ) {
766 0 : if( FD_UNLIKELY( fd_pubkey_eq( &rank[i].vacc->node_account, &rank[i-1UL].vacc->node_account ) ) ) {
767 0 : rank[i].drop = rank[i-1UL].drop = 1UL;
768 0 : }
769 0 : }
770 :
771 28 : ulong survivor_cnt = 0UL;
772 28 : for( ulong i=0UL; i<rank_cnt; i++ ) {
773 0 : if( FD_LIKELY( !rank[i].drop ) ) rank[ survivor_cnt++ ] = rank[i];
774 0 : }
775 28 : vacc_rank_sort_inplace( rank, survivor_cnt );
776 :
777 28 : for( ulong i=0UL; i<survivor_cnt; i++ ) rank[i].vacc->rank = (ushort)i;
778 28 : if( cache_idx!=ULONG_MAX ) t_1_unlock( vote_stakes, 1 );
779 28 : }
780 :
781 : void
782 : fd_vote_stakes_purge_fork( fd_vote_stakes_t * vote_stakes,
783 65135 : ushort fork_id ) {
784 65135 : ushort t_1_idx = fork_ele( vote_stakes, fork_id )->t_1_idx;
785 :
786 65135 : fork_pool_idx_release( fork_pool( vote_stakes ), fork_id );
787 :
788 65135 : vacc_fork_t * fork = vacc_fork_pool_ele( vacc_fork_pool( vote_stakes ), t_1_idx );
789 65135 : FD_TEST( fork->ref_cnt );
790 65135 : if( !--fork->ref_cnt ) t_1_release( vote_stakes, t_1_idx );
791 65135 : }
792 :
793 : ushort
794 : fd_vote_stakes_new_fork( fd_vote_stakes_t * vote_stakes,
795 : ushort parent_fork_id,
796 651 : ulong epoch ) {
797 651 : FD_TEST( epoch<=USHORT_MAX );
798 :
799 651 : FD_CHECK_CRIT( fork_pool_free( fork_pool( vote_stakes ) ), "vote stakes fork pool exhausted" );
800 651 : ushort fork_id = (ushort)fork_pool_idx_acquire( fork_pool( vote_stakes ) );
801 651 : fork_t * fork = fork_ele( vote_stakes, fork_id );
802 651 : fork_t const * parent = fork_ele( vote_stakes, parent_fork_id );
803 651 : vacc_valid_set_t * child_valid = valid_set( vote_stakes, fork_id );
804 :
805 651 : if( FD_LIKELY( epoch==(ulong)parent->epoch ) ) {
806 : /* Standard case: just inherit the parent's t-1 set (and increment
807 : the ref count) and copy over the t-2 validity. */
808 623 : fork->t_1_idx = parent->t_1_idx;
809 623 : vacc_fork_pool_ele( vacc_fork_pool( vote_stakes ), fork->t_1_idx )->ref_cnt++;
810 :
811 623 : vacc_valid_set_copy( child_valid, valid_set( vote_stakes, parent_fork_id ) );
812 623 : } else {
813 : /* Epoch boundary: copy the parent's t-1 set into the t-2 set and
814 : acquire new t-1 and t-2 validity. */
815 :
816 : /* Copy the t-1 set, ranks included, iff it's the first time
817 : crossing into a new epoch boundary. */
818 28 : ulong t_2_idx = epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
819 28 : if( vote_stakes->t_2_epoch[ t_2_idx ]!=epoch ) {
820 28 : ulong t_1_cache_idx = t_1_lock( vote_stakes, parent->t_1_idx, 0 );
821 28 : vacc_t * t_1_pool = t_1_cache_pool( vote_stakes, t_1_cache_idx );
822 28 : vacc_map_t * t_1_map = t_1_cache_map ( vote_stakes, t_1_cache_idx );
823 28 : vacc_t * t_2_pool = t_2_vacc_pool( vote_stakes, t_2_idx );
824 28 : vacc_map_t * t_2_map = t_2_vacc_map ( vote_stakes, t_2_idx );
825 :
826 28 : vacc_map_reset( t_2_map );
827 28 : vacc_pool_reset( t_2_pool );
828 28 : for( vacc_map_iter_t iter = vacc_map_iter_init( t_1_map, t_1_pool );
829 191 : !vacc_map_iter_done( iter, t_1_map, t_1_pool );
830 163 : iter = vacc_map_iter_next( iter, t_1_map, t_1_pool ) ) {
831 163 : vacc_t const * src = vacc_map_iter_ele_const( iter, t_1_map, t_1_pool );
832 163 : vacc_t * dst = vacc_pool_ele_acquire( t_2_pool );
833 163 : dst->pubkey = src->pubkey;
834 163 : dst->node_account = src->node_account;
835 163 : dst->stake = src->stake;
836 163 : dst->commission = src->commission;
837 163 : dst->block_revenue_commission_bps = src->block_revenue_commission_bps;
838 163 : dst->pending_delegator_rewards = src->pending_delegator_rewards;
839 163 : dst->inflation_collector = src->inflation_collector;
840 163 : dst->block_collector = src->block_collector;
841 163 : dst->location = src->location;
842 163 : dst->rank = src->rank;
843 163 : memcpy( dst->bls_key, src->bls_key, FD_BLS_PUBKEY_COMPRESSED_SZ );
844 163 : memcpy( dst->bls_key_uncompressed, src->bls_key_uncompressed, FD_BLS_PUBKEY_UNCOMPRESSED_SZ );
845 163 : FD_TEST( vacc_map_ele_insert( t_2_map, dst, t_2_pool ) );
846 163 : }
847 28 : t_1_unlock( vote_stakes, 0 );
848 28 : vote_stakes->t_2_epoch[ t_2_idx ] = epoch;
849 28 : }
850 :
851 : /* Reset the t-2 validity for the new fork and prepare the heap for
852 : insertion into the new t-1 set. */
853 28 : fork->t_1_idx = t_1_acquire( vote_stakes );
854 28 : vacc_valid_set_null( child_valid );
855 :
856 28 : FD_TEST( vacc_heap_new( vacc_heap( vote_stakes ), FD_RUNTIME_MAX_VAT_VOTE_ACCOUNTS ) );
857 28 : vote_stakes->min_stake_wmark = 0UL;
858 28 : }
859 :
860 651 : fork->epoch = (ushort)epoch;
861 651 : return fork_id;
862 651 : }
863 :
864 : void
865 : fd_vote_stakes_update_state( fd_vote_stakes_t * vote_stakes,
866 : ushort fork_id,
867 : fd_pubkey_t const * pubkey,
868 1156 : int is_valid ) {
869 1156 : ulong epoch_idx = (ulong)fork_ele( vote_stakes, fork_id )->epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
870 1156 : vacc_t * pool = t_2_vacc_pool( vote_stakes, epoch_idx );
871 1156 : vacc_map_t * map = t_2_vacc_map ( vote_stakes, epoch_idx );
872 :
873 1156 : vacc_t const * vacc = vacc_map_ele_query_const( map, pubkey, NULL, pool );
874 1156 : if( FD_UNLIKELY( !vacc ) ) return;
875 :
876 : /* Exec tiles commit transactions touching different vote accounts
877 : concurrently, and those accounts can share a bitset word. */
878 1156 : vacc_valid_set_t * valid = valid_set( vote_stakes, fork_id );
879 1156 : ulong vacc_idx = vacc_pool_idx( pool, vacc );
880 1156 : ulong * word = valid + (vacc_idx>>6);
881 1156 : ulong bit = 1UL<<(vacc_idx&63UL);
882 1156 : if( is_valid ) FD_ATOMIC_FETCH_AND_OR ( word, bit );
883 39 : else FD_ATOMIC_FETCH_AND_AND( word, ~bit );
884 1156 : }
885 :
886 : void
887 : fd_vote_stakes_refresh( fd_vote_stakes_t * vote_stakes,
888 : ushort fork_id,
889 : fd_accdb_t * accdb,
890 638 : fd_accdb_fork_id_t accdb_fork_id ) {
891 638 : uchar __attribute__((aligned(FD_VOTE_STAKES_ITER_ALIGN))) iter_mem[ FD_VOTE_STAKES_ITER_FOOTPRINT ];
892 :
893 638 : for( fd_vote_stakes_iter_t * iter = fd_vote_stakes_iter_init( vote_stakes, fork_id, FD_VOTE_STAKES_ITER_T_2, iter_mem );
894 1794 : !fd_vote_stakes_iter_done( vote_stakes, fork_id, FD_VOTE_STAKES_ITER_T_2, iter );
895 1156 : fd_vote_stakes_iter_next( vote_stakes, fork_id, FD_VOTE_STAKES_ITER_T_2, iter ) ) {
896 1156 : fd_vote_stakes_ele_t ele[1];
897 1156 : fd_vote_stakes_iter_ele( vote_stakes, fork_id, FD_VOTE_STAKES_ITER_T_2, iter, ele );
898 1156 : fd_pubkey_t const * pubkey = &ele->pubkey;
899 :
900 1156 : fd_acc_t acc = fd_accdb_read_one( accdb, accdb_fork_id, pubkey->uc );
901 1156 : int is_valid = acc.lamports && fd_vsv_is_correct_size_owner_and_init( acc.owner, acc.data, acc.data_len );
902 1156 : fd_accdb_unread_one( accdb, &acc );
903 1156 : fd_vote_stakes_update_state( vote_stakes, fork_id, pubkey, is_valid );
904 1156 : }
905 638 : }
906 :
907 : /* t_1_vacc_query_const looks up pubkey in the fork's t-1 set under
908 : the shared t-1 lock, which the caller must release with
909 : t_1_unlock( vote_stakes, 0 ). */
910 :
911 : static vacc_t const *
912 : t_1_vacc_query_const( fd_vote_stakes_t * vote_stakes,
913 : ushort fork_id,
914 1237 : fd_pubkey_t const * pubkey ) {
915 1237 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 0 );
916 1237 : return vacc_map_ele_query_const( t_1_cache_map( vote_stakes, cache_idx ), pubkey, NULL, t_1_cache_pool( vote_stakes, cache_idx ) );
917 1237 : }
918 :
919 : /* t_n_vacc_query looks up pubkey in the fork's t-n set, n in 2..5,
920 : and sets *pool to the set's pool. Returns NULL if the set is not
921 : resident or pubkey is not in it. The query does not reorder the
922 : map, which other tiles may be iterating. */
923 : static vacc_t const *
924 : t_n_vacc_query( fd_vote_stakes_t const * vote_stakes,
925 : ushort fork_id,
926 : ulong n,
927 : fd_pubkey_t const * pubkey,
928 485 : vacc_t const ** pool ) {
929 485 : ulong epoch = (ulong)fork_ele( vote_stakes, fork_id )->epoch;
930 485 : ulong back = n-2UL;
931 485 : if( FD_UNLIKELY( epoch<back ) ) return NULL;
932 485 : ulong epoch_idx = (epoch-back) % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
933 485 : if( FD_UNLIKELY( vote_stakes->t_2_epoch[ epoch_idx ]!=epoch-back ) ) return NULL;
934 346 : *pool = t_2_vacc_pool( vote_stakes, epoch_idx );
935 346 : return vacc_map_ele_query_const( t_2_vacc_map( vote_stakes, epoch_idx ), pubkey, NULL, *pool );
936 485 : }
937 :
938 : /* vacc_ele copies vacc to ele_out. vacc_ele_valid adds the fork's
939 : validity for vacc, which must be in the t-2 pool. */
940 :
941 : static void
942 : vacc_ele( vacc_t const * vacc,
943 3101 : fd_vote_stakes_ele_t * ele_out ) {
944 3101 : *ele_out = (fd_vote_stakes_ele_t){
945 3101 : .pubkey = vacc->pubkey,
946 3101 : .node_account = vacc->node_account,
947 3101 : .stake = vacc->stake,
948 3101 : .commission = vacc->commission,
949 3101 : .block_revenue_commission_bps = vacc->block_revenue_commission_bps,
950 3101 : .pending_delegator_rewards = vacc->pending_delegator_rewards,
951 3101 : .inflation_collector = vacc->inflation_collector,
952 3101 : .block_collector = vacc->block_collector,
953 3101 : .rank = vacc->rank,
954 3101 : .location = vacc->location,
955 3101 : };
956 3101 : memcpy( ele_out->bls_key, vacc->bls_key, FD_BLS_PUBKEY_COMPRESSED_SZ );
957 3101 : memcpy( ele_out->bls_key_uncompressed, vacc->bls_key_uncompressed, FD_BLS_PUBKEY_UNCOMPRESSED_SZ );
958 3101 : }
959 :
960 : static void
961 : vacc_ele_valid( fd_vote_stakes_t const * vote_stakes,
962 : ushort fork_id,
963 : vacc_t const * pool,
964 : vacc_t const * vacc,
965 2547 : fd_vote_stakes_ele_t * ele_out ) {
966 2547 : ele_out->is_valid = (uchar)vacc_valid_set_test( valid_set( vote_stakes, fork_id ), vacc_pool_idx( pool, vacc ) );
967 2547 : }
968 :
969 : int
970 : fd_vote_stakes_query_t_1( fd_vote_stakes_t * vote_stakes,
971 : ushort fork_id,
972 : fd_pubkey_t const * pubkey,
973 1237 : fd_vote_stakes_ele_t * ele_out_opt ) {
974 1237 : vacc_t const * vacc = t_1_vacc_query_const( vote_stakes, fork_id, pubkey );
975 1237 : if( FD_LIKELY( vacc && ele_out_opt ) ) vacc_ele( vacc, ele_out_opt );
976 1237 : t_1_unlock( vote_stakes, 0 );
977 1237 : return !!vacc;
978 1237 : }
979 :
980 : int
981 : fd_vote_stakes_query_t_2( fd_vote_stakes_t const * vote_stakes,
982 : ushort fork_id,
983 : fd_pubkey_t const * pubkey,
984 245 : fd_vote_stakes_ele_t * ele_out_opt ) {
985 245 : vacc_t const * pool;
986 245 : vacc_t const * vacc = t_n_vacc_query( vote_stakes, fork_id, 2UL, pubkey, &pool );
987 245 : if( FD_UNLIKELY( !vacc ) ) return 0;
988 236 : if( ele_out_opt ) {
989 236 : vacc_ele( vacc, ele_out_opt );
990 236 : vacc_ele_valid( vote_stakes, fork_id, pool, vacc, ele_out_opt );
991 236 : }
992 236 : return 1;
993 245 : }
994 :
995 : int
996 : fd_vote_stakes_query_t_3( fd_vote_stakes_t const * vote_stakes,
997 : ushort fork_id,
998 : fd_pubkey_t const * pubkey,
999 240 : fd_vote_stakes_ele_t * ele_out_opt ) {
1000 240 : vacc_t const * pool;
1001 240 : vacc_t const * vacc = t_n_vacc_query( vote_stakes, fork_id, 3UL, pubkey, &pool );
1002 240 : if( FD_UNLIKELY( !vacc ) ) return 0;
1003 86 : if( ele_out_opt ) vacc_ele( vacc, ele_out_opt );
1004 86 : return 1;
1005 240 : }
1006 :
1007 : ulong
1008 : fd_vote_stakes_cnt_t_1( fd_vote_stakes_t * vote_stakes,
1009 0 : ushort fork_id ) {
1010 0 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 0 );
1011 0 : ulong cnt = vacc_pool_used( t_1_cache_pool( vote_stakes, cache_idx ) );
1012 0 : t_1_unlock( vote_stakes, 0 );
1013 0 : return cnt;
1014 0 : }
1015 :
1016 : ulong
1017 : fd_vote_stakes_cnt_t_2( fd_vote_stakes_t const * vote_stakes,
1018 0 : ushort fork_id ) {
1019 0 : return vacc_pool_used( t_2_vacc_pool( vote_stakes, (ulong)fork_ele( vote_stakes, fork_id )->epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT ) );
1020 0 : }
1021 :
1022 : ulong
1023 : fd_vote_stakes_total_stake( fd_vote_stakes_t const * vote_stakes,
1024 0 : ulong epoch ) {
1025 0 : ulong idx = epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
1026 0 : if( FD_UNLIKELY( vote_stakes->t_2_epoch[ idx ]!=epoch ) ) return 0UL;
1027 :
1028 0 : vacc_t * pool = t_2_vacc_pool( vote_stakes, idx );
1029 0 : vacc_map_t * map = t_2_vacc_map ( vote_stakes, idx );
1030 :
1031 0 : ulong total = 0UL;
1032 0 : for( vacc_map_iter_t iter = vacc_map_iter_init( map, pool );
1033 0 : !vacc_map_iter_done( iter, map, pool );
1034 0 : iter = vacc_map_iter_next( iter, map, pool ) ) {
1035 0 : total += vacc_map_iter_ele_const( iter, map, pool )->stake;
1036 0 : }
1037 0 : return total;
1038 0 : }
1039 :
1040 : FD_STATIC_ASSERT( FD_VOTE_STAKES_ITER_FOOTPRINT == sizeof(vacc_map_iter_t), iter_footprint );
1041 : FD_STATIC_ASSERT( FD_VOTE_STAKES_ITER_ALIGN == alignof(vacc_map_iter_t), iter_align );
1042 :
1043 : fd_vote_stakes_iter_t *
1044 : fd_vote_stakes_iter_init( fd_vote_stakes_t * vote_stakes,
1045 : ushort fork_id,
1046 : int iter_kind,
1047 1361 : uchar iter_mem[ static FD_VOTE_STAKES_ITER_FOOTPRINT ] ) {
1048 1361 : vacc_map_iter_t iter = {0};
1049 1361 : if( FD_LIKELY( iter_kind==FD_VOTE_STAKES_ITER_T_1 ) ) {
1050 84 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 0 );
1051 84 : iter = vacc_map_iter_init( t_1_cache_map( vote_stakes, cache_idx ), t_1_cache_pool( vote_stakes, cache_idx ) );
1052 84 : t_1_unlock( vote_stakes, 0 );
1053 1277 : } else {
1054 1277 : FD_TEST( iter_kind>=FD_VOTE_STAKES_ITER_T_2 && iter_kind<=FD_VOTE_STAKES_ITER_T_5 );
1055 1277 : ulong fork_epoch = (ulong)fork_ele( vote_stakes, fork_id )->epoch;
1056 1277 : ulong back = (ulong)(iter_kind-FD_VOTE_STAKES_ITER_T_2);
1057 1277 : if( FD_LIKELY( fork_epoch>=back ) ) {
1058 1277 : ulong epoch = fork_epoch-back;
1059 1277 : ulong epoch_idx = epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
1060 1277 : if( FD_LIKELY( vote_stakes->t_2_epoch[ epoch_idx ]==epoch ) ) {
1061 1277 : vacc_t * pool = t_2_vacc_pool( vote_stakes, epoch_idx );
1062 1277 : vacc_map_t * map = t_2_vacc_map ( vote_stakes, epoch_idx );
1063 1277 : iter = vacc_map_iter_init( map, pool );
1064 1277 : }
1065 1277 : }
1066 1277 : }
1067 1361 : memcpy( iter_mem, &iter, sizeof(iter) );
1068 1361 : return (fd_vote_stakes_iter_t *)iter_mem;
1069 1361 : }
1070 :
1071 : int
1072 : fd_vote_stakes_iter_done( fd_vote_stakes_t * vote_stakes,
1073 : ushort fork_id,
1074 : int iter_kind,
1075 3972 : fd_vote_stakes_iter_t * iter ) {
1076 3972 : if( FD_LIKELY( iter_kind==FD_VOTE_STAKES_ITER_T_1 ) ) {
1077 387 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 0 );
1078 387 : int done = vacc_map_iter_done( *(vacc_map_iter_t *)iter, t_1_cache_map( vote_stakes, cache_idx ), t_1_cache_pool( vote_stakes, cache_idx ) );
1079 387 : t_1_unlock( vote_stakes, 0 );
1080 387 : return done;
1081 387 : }
1082 :
1083 3585 : FD_TEST( iter_kind>=FD_VOTE_STAKES_ITER_T_2 && iter_kind<=FD_VOTE_STAKES_ITER_T_5 );
1084 3585 : ulong fork_epoch = (ulong)fork_ele( vote_stakes, fork_id )->epoch;
1085 3585 : ulong back = (ulong)(iter_kind-FD_VOTE_STAKES_ITER_T_2);
1086 3585 : if( FD_UNLIKELY( fork_epoch<back ) ) return 1;
1087 :
1088 3585 : ulong epoch = fork_epoch-back;
1089 3585 : ulong epoch_idx = epoch % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
1090 3585 : if( FD_UNLIKELY( vote_stakes->t_2_epoch[ epoch_idx ]!=epoch ) ) return 1;
1091 :
1092 3585 : vacc_t * pool = t_2_vacc_pool( vote_stakes, epoch_idx );
1093 3585 : vacc_map_t * map = t_2_vacc_map ( vote_stakes, epoch_idx );
1094 3585 : return vacc_map_iter_done( *(vacc_map_iter_t *)iter, map, pool );
1095 3585 : }
1096 :
1097 : static ulong
1098 : iter_set( fd_vote_stakes_t * vote_stakes,
1099 : ushort fork_id,
1100 : int iter_kind,
1101 : vacc_t ** pool,
1102 5228 : vacc_map_t ** map ) {
1103 5228 : if( FD_LIKELY( iter_kind==FD_VOTE_STAKES_ITER_T_1 ) ) {
1104 606 : ulong cache_idx = t_1_lock( vote_stakes, fork_ele( vote_stakes, fork_id )->t_1_idx, 0 );
1105 606 : *pool = t_1_cache_pool( vote_stakes, cache_idx );
1106 606 : *map = t_1_cache_map ( vote_stakes, cache_idx );
1107 606 : return cache_idx;
1108 606 : }
1109 4622 : FD_TEST( iter_kind>=FD_VOTE_STAKES_ITER_T_2 && iter_kind<=FD_VOTE_STAKES_ITER_T_5 );
1110 4622 : ulong epoch_idx = ((ulong)fork_ele( vote_stakes, fork_id )->epoch-(ulong)(iter_kind-FD_VOTE_STAKES_ITER_T_2)) % FD_VOTE_STAKES_EPOCH_CACHE_CNT;
1111 4622 : *pool = t_2_vacc_pool( vote_stakes, epoch_idx );
1112 4622 : *map = t_2_vacc_map ( vote_stakes, epoch_idx );
1113 4622 : return ULONG_MAX;
1114 4622 : }
1115 :
1116 : void
1117 : fd_vote_stakes_iter_next( fd_vote_stakes_t * vote_stakes,
1118 : ushort fork_id,
1119 : int iter_kind,
1120 2615 : fd_vote_stakes_iter_t * iter ) {
1121 2615 : vacc_t * pool;
1122 2615 : vacc_map_t * map;
1123 2615 : ulong cache_idx = iter_set( vote_stakes, fork_id, iter_kind, &pool, &map );
1124 2615 : *(vacc_map_iter_t *)iter = vacc_map_iter_next( *(vacc_map_iter_t *)iter, map, pool );
1125 2615 : if( cache_idx!=ULONG_MAX ) t_1_unlock( vote_stakes, 0 );
1126 2615 : }
1127 :
1128 : void
1129 : fd_vote_stakes_iter_ele( fd_vote_stakes_t * vote_stakes,
1130 : ushort fork_id,
1131 : int iter_kind,
1132 : fd_vote_stakes_iter_t * iter,
1133 2614 : fd_vote_stakes_ele_t * ele_out ) {
1134 2614 : vacc_t * pool;
1135 2614 : vacc_map_t * map;
1136 2614 : ulong cache_idx = iter_set( vote_stakes, fork_id, iter_kind, &pool, &map );
1137 2614 : vacc_t const * vacc = vacc_map_iter_ele_const( *(vacc_map_iter_t *)iter, map, pool );
1138 2614 : vacc_ele( vacc, ele_out );
1139 2614 : if( iter_kind==FD_VOTE_STAKES_ITER_T_2 ) vacc_ele_valid( vote_stakes, fork_id, pool, vacc, ele_out );
1140 2614 : if( cache_idx!=ULONG_MAX ) t_1_unlock( vote_stakes, 0 );
1141 2614 : }
|
