Line data Source code
1 : #include "fd_txncache_shmem.h"
2 : #include "fd_txncache_private.h"
3 :
4 : #define POOL_NAME blockcache_pool
5 0 : #define POOL_T fd_txncache_blockcache_shmem_t
6 : #define POOL_IDX_T ulong
7 0 : #define POOL_NEXT pool.next
8 : #define POOL_IMPL_STYLE 2
9 : #include "../../util/tmpl/fd_pool.c"
10 :
11 : #define MAP_NAME blockhash_map
12 0 : #define MAP_KEY blockhash
13 0 : #define MAP_ELE_T fd_txncache_blockcache_shmem_t
14 : #define MAP_KEY_T fd_hash_t
15 0 : #define MAP_PREV blockhash_map.prev
16 0 : #define MAP_NEXT blockhash_map.next
17 : #define MAP_KEY_EQ(k0,k1) fd_hash_eq( k0, k1 )
18 : #define MAP_KEY_HASH(key,seed) (__extension__({ (void)(seed); fd_ulong_load_8_fast( (key)->uc ); }))
19 : #define MAP_OPTIMIZE_RANDOM_ACCESS_REMOVAL 1
20 : #define MAP_MULTI 1
21 : #define MAP_IMPL_STYLE 2
22 : #include "../../util/tmpl/fd_map_chain.c"
23 :
24 : #define SLIST_NAME root_slist
25 : #define SLIST_ELE_T fd_txncache_blockcache_shmem_t
26 : #define SLIST_IDX_T ulong
27 0 : #define SLIST_NEXT slist.next
28 : #define SLIST_IMPL_STYLE 2
29 : #include "../../util/tmpl/fd_slist.c"
30 :
31 : FD_FN_CONST ulong
32 : fd_txncache_max_txnpages( ulong max_active_slots,
33 0 : ulong max_txn_per_slot ) {
34 : /* The pool must hold the worst case set of simultaneously live
35 : transactions. Entries stay live until their inserting fork's
36 : blockcache is removed, and rooted forks are retained for
37 : FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE further root advances, so up to
38 : max_active_slots forks' insertions are live at once. The worst
39 : case is the boot transient: the snapshot root (holding the full
40 : staged snapshot load) plus every other pool fork full of live
41 : inserts.
42 :
43 : The snapshot transient retains at most 151 slot deltas at
44 : max_txn_per_slot entries each. max_txn_per_slot is 2x the actual
45 : per slot transaction limit, because Agave snapshots contain each
46 : transaction twice (once keyed by signature, once by message hash,
47 : indistinguishably; see FD_PACK_MAX_TXNCACHE_TXN_PER_SLOT). Live
48 : inserts are 1x, so each non-snapshot fork holds at most
49 : max_txn_per_slot/2 entries.
50 :
51 : We count pages, not transactions, so pages might be wasted. The
52 : maximum page wastage occurs when every blockcache has a partially
53 : full tail page, adding max_active_slots-1 pages beyond the global
54 : transaction count bound (one blockcache's tail is already counted
55 : by the ceiling division). */
56 :
57 0 : ulong snapshot_budget = FD_TXNCACHE_MAX_SLOT_DELTAS*max_txn_per_slot;
58 0 : ulong live_budget = (max_active_slots-1UL)*((max_txn_per_slot+1UL)/2UL);
59 0 : return max_active_slots-1UL
60 0 : + 1UL+(snapshot_budget+live_budget-1UL)/FD_TXNCACHE_TXNS_PER_PAGE;
61 0 : }
62 :
63 : FD_FN_CONST ulong
64 : fd_txncache_max_txnpages_per_blockhash( ulong max_active_slots,
65 0 : ulong max_txn_per_slot ) {
66 : /* The maximum number of transaction pages we might need to store all
67 : the transactions that could be seen in a blockhash.
68 :
69 : In the worst case, every transaction in every live bank refers to
70 : the same blockhash, but a blockhash can never hold more pages than
71 : exist in the pool. */
72 :
73 0 : ulong max_txnpages = fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot );
74 0 : ulong result = 1UL+(max_txn_per_slot*max_active_slots)/FD_TXNCACHE_TXNS_PER_PAGE;
75 0 : return fd_ulong_min( result, max_txnpages );
76 0 : }
77 :
78 : FD_FN_CONST ulong
79 0 : fd_txncache_shmem_align( void ) {
80 0 : return FD_TXNCACHE_SHMEM_ALIGN;
81 0 : }
82 :
83 : ulong
84 : fd_txncache_shmem_footprint( ulong max_live_slots,
85 : ulong max_txn_per_slot,
86 0 : ulong cache_footprint ) {
87 0 : if( FD_UNLIKELY( !cache_footprint ) ) return 0UL;
88 0 : if( FD_UNLIKELY( max_live_slots<1UL ) ) return 0UL;
89 0 : if( FD_UNLIKELY( max_txn_per_slot<1UL ) ) return 0UL;
90 :
91 0 : ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots;
92 0 : ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots );
93 0 : ulong bucket_cnt = fd_txncache_bucket_cnt( max_txn_per_slot );
94 :
95 0 : ulong _txnpages_max = fd_txncache_max_txnpages( max_active_slots, max_txn_per_slot );
96 0 : if( FD_UNLIKELY( _txnpages_max>FD_TXNCACHE_MAX_TXNPAGES ) )
97 0 : FD_LOG_ERR(( "txncache needs %lu txnpages but at most %lu are addressable (max_live_slots=%lu max_txn_per_slot=%lu)",
98 0 : _txnpages_max, FD_TXNCACHE_MAX_TXNPAGES, max_live_slots, max_txn_per_slot ));
99 0 : ulong frames_max = fd_ulong_min( 1UL+(cache_footprint-1UL)/sizeof(fd_txncache_txnpage_t), _txnpages_max );
100 :
101 0 : ulong _descends_footprint = descends_set_footprint( max_active_slots );
102 0 : if( FD_UNLIKELY( !_descends_footprint ) ) return 0UL;
103 :
104 0 : ulong l;
105 0 : l = FD_LAYOUT_INIT;
106 0 : l = FD_LAYOUT_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) );
107 0 : l = FD_LAYOUT_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) );
108 0 : l = FD_LAYOUT_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) );
109 0 : l = FD_LAYOUT_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) ); /* blockcache->heads */
110 0 : l = FD_LAYOUT_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint ); /* blockcache->descends */
111 0 : l = FD_LAYOUT_APPEND( l, alignof(uint), _txnpages_max*sizeof(uint) ); /* txnpages_free */
112 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_page_meta_t), _txnpages_max*sizeof(fd_txncache_page_meta_t) ); /* page_meta */
113 0 : l = FD_LAYOUT_APPEND( l, alignof(uint), frames_max*sizeof(uint) ); /* frame_page */
114 0 : l = FD_LAYOUT_APPEND( l, alignof(uint), frames_max*sizeof(uint) ); /* frames_free */
115 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), frames_max*sizeof(fd_txncache_txnpage_t) ); /* txnpages */
116 0 : l = FD_LAYOUT_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) ); /* scratchpad heads for purge stale */
117 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); /* scratchpad txnpage for purge stale */
118 0 : l = FD_LAYOUT_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) ); /* compaction input */
119 0 : return FD_LAYOUT_FINI( l, FD_TXNCACHE_SHMEM_ALIGN );
120 0 : }
121 :
122 : void *
123 : fd_txncache_shmem_new( void * shmem,
124 : ulong max_live_slots,
125 : ulong max_txn_per_slot,
126 : ulong cache_footprint,
127 0 : ulong seed ) {
128 0 : if( FD_UNLIKELY( !shmem ) ) {
129 0 : FD_LOG_WARNING(( "NULL shmem" ));
130 0 : return NULL;
131 0 : }
132 :
133 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shmem, fd_txncache_shmem_align() ) ) ) {
134 0 : FD_LOG_WARNING(( "misaligned shmem" ));
135 0 : return NULL;
136 0 : }
137 :
138 0 : if( FD_UNLIKELY( !cache_footprint ) ) return NULL;
139 0 : if( FD_UNLIKELY( !max_live_slots ) ) return NULL;
140 0 : if( FD_UNLIKELY( !max_txn_per_slot ) ) return NULL;
141 :
142 0 : ulong max_active_slots = FD_TXNCACHE_MAX_BLOCKHASH_DISTANCE+max_live_slots;
143 0 : ulong blockhash_map_chains = fd_ulong_pow2_up( 2UL*max_active_slots );
144 0 : ulong bucket_cnt = fd_txncache_bucket_cnt( max_txn_per_slot );
145 :
146 0 : ulong _txnpages_max = fd_txncache_max_txnpages ( max_active_slots, max_txn_per_slot );
147 0 : ulong _txnpages_per_blockhash_max = fd_txncache_max_txnpages_per_blockhash( max_active_slots, max_txn_per_slot );
148 0 : if( FD_UNLIKELY( _txnpages_max>FD_TXNCACHE_MAX_TXNPAGES ) )
149 0 : FD_LOG_ERR(( "txncache needs %lu txnpages but at most %lu are addressable (max_live_slots=%lu max_txn_per_slot=%lu)",
150 0 : _txnpages_max, FD_TXNCACHE_MAX_TXNPAGES, max_live_slots, max_txn_per_slot ));
151 0 : ulong frames_max = fd_ulong_min( 1UL+(cache_footprint-1UL)/sizeof(fd_txncache_txnpage_t), _txnpages_max );
152 :
153 0 : ulong _descends_footprint = descends_set_footprint( max_active_slots );
154 0 : if( FD_UNLIKELY( !_descends_footprint ) ) return NULL;
155 :
156 0 : FD_SCRATCH_ALLOC_INIT( l, shmem );
157 0 : fd_txncache_shmem_t * tc = FD_SCRATCH_ALLOC_APPEND( l, FD_TXNCACHE_SHMEM_ALIGN, sizeof(fd_txncache_shmem_t) );
158 0 : void * _blockhash_map = FD_SCRATCH_ALLOC_APPEND( l, blockhash_map_align(), blockhash_map_footprint( blockhash_map_chains ) );
159 0 : void * _blockcache_pool = FD_SCRATCH_ALLOC_APPEND( l, blockcache_pool_align(), blockcache_pool_footprint( max_active_slots ) );
160 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), max_active_slots*bucket_cnt*sizeof(uint) );
161 0 : void * _blockcache_descends = FD_SCRATCH_ALLOC_APPEND( l, descends_set_align(), max_active_slots*_descends_footprint );
162 0 : uint * txnpages_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), _txnpages_max*sizeof(uint) );
163 0 : void * _page_meta = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_page_meta_t), _txnpages_max*sizeof(fd_txncache_page_meta_t) );
164 0 : uint * frame_page = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), frames_max*sizeof(uint) );
165 0 : uint * frames_free = FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), frames_max*sizeof(uint) );
166 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), frames_max*sizeof(fd_txncache_txnpage_t) );
167 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(uint), bucket_cnt*sizeof(uint) );
168 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) );
169 0 : FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_txncache_txnpage_t), sizeof(fd_txncache_txnpage_t) );
170 0 : tc->frames_max = frames_max;
171 0 : fd_txncache_page_meta_t * page_meta = _page_meta;
172 :
173 0 : fd_txncache_blockcache_shmem_t * blockcache_pool = blockcache_pool_join( blockcache_pool_new( _blockcache_pool, max_active_slots ) );
174 0 : FD_TEST( blockcache_pool );
175 0 : for( ulong i=0UL; i<max_active_slots; i++ ) blockcache_pool[ i ].frozen = -1;
176 :
177 0 : blockhash_map_t * blockhash_map = blockhash_map_join( blockhash_map_new( _blockhash_map, blockhash_map_chains, 0UL /* seed not used */ ) );
178 0 : FD_TEST( blockhash_map );
179 :
180 0 : for( ulong i=0UL; i<max_active_slots; i++ ) {
181 0 : descends_set_t * descends_set = descends_set_join( descends_set_new( (uchar *)_blockcache_descends + i*_descends_footprint, max_active_slots ) );
182 0 : FD_TEST( descends_set );
183 0 : }
184 :
185 0 : tc->root_cnt = 0UL;
186 0 : FD_TEST( root_slist_join( root_slist_new( tc->root_ll ) ) );
187 :
188 0 : tc->lock->value = 0;
189 0 : fd_rwlock_new( tc->alloc_lock );
190 :
191 0 : tc->txn_per_slot_max = max_txn_per_slot;
192 0 : tc->active_slots_max = max_active_slots;
193 0 : tc->bucket_cnt = bucket_cnt;
194 0 : tc->txnpages_per_blockhash_max = _txnpages_per_blockhash_max;
195 0 : tc->txnpages_max = _txnpages_max;
196 :
197 0 : tc->blockcache_generation = 0U;
198 0 : tc->txnpages_free_cnt = _txnpages_max;
199 0 : tc->frames_free_cnt = frames_max;
200 0 : tc->evict_next = 0UL;
201 0 : tc->release_seq = 0UL;
202 :
203 0 : tc->page_miss_cnt = 0UL;
204 0 : tc->page_spilled_cnt = 0UL;
205 0 : tc->page_spill_read_cnt = 0UL;
206 0 : tc->frame_evicted_cnt = 0UL;
207 0 : tc->spill_ticks = 0UL;
208 0 : tc->lock_read_wait_ticks = 0UL;
209 0 : tc->lock_write_wait_ticks = 0UL;
210 0 : for( ulong i=0UL; i<_txnpages_max; i++ ) {
211 0 : txnpages_free[ i ] = (uint)i;
212 0 : page_meta[ i ].frame = UINT_MAX;
213 0 : page_meta[ i ].dirty = 0U;
214 0 : page_meta[ i ].disk_valid = 0;
215 0 : page_meta[ i ].next = UINT_MAX;
216 0 : page_meta[ i ].ref = 0U;
217 0 : page_meta[ i ].io = 0U;
218 0 : }
219 0 : for( ulong i=0UL; i<frames_max; i++ ) {
220 0 : frame_page[ i ] = UINT_MAX;
221 0 : frames_free[ i ] = (uint)i;
222 0 : }
223 :
224 0 : tc->seed = seed;
225 :
226 0 : FD_COMPILER_MFENCE();
227 0 : FD_VOLATILE( tc->magic ) = FD_TXNCACHE_SHMEM_MAGIC;
228 0 : FD_COMPILER_MFENCE();
229 :
230 0 : return (void *)tc;
231 0 : }
232 :
233 : fd_txncache_shmem_t *
234 0 : fd_txncache_shmem_join( void * shtc ) {
235 0 : if( FD_UNLIKELY( !shtc ) ) {
236 0 : FD_LOG_WARNING(( "NULL shtc" ));
237 0 : return NULL;
238 0 : }
239 :
240 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)shtc, fd_txncache_shmem_align() ) ) ) {
241 0 : FD_LOG_WARNING(( "misaligned shtc" ));
242 0 : return NULL;
243 0 : }
244 :
245 0 : fd_txncache_shmem_t * tc = (fd_txncache_shmem_t *)shtc;
246 :
247 0 : if( FD_UNLIKELY( tc->magic!=FD_TXNCACHE_SHMEM_MAGIC ) ) {
248 0 : FD_LOG_WARNING(( "bad magic" ));
249 0 : return NULL;
250 0 : }
251 :
252 0 : return tc;
253 0 : }
|
