Line data Source code
1 : #include "fd_topob.h"
2 : #include "../net/fd_net_tile.h"
3 : #include "../waker/fd_waker.h"
4 : #include "../sleep/fd_sleep.h"
5 :
6 : #include "../../util/pod/fd_pod_format.h"
7 :
8 : fd_topo_t *
9 : fd_topob_new( void * mem,
10 0 : char const * app_name ) {
11 0 : fd_topo_t * topo = (fd_topo_t *)mem;
12 :
13 0 : if( FD_UNLIKELY( !topo ) ) {
14 0 : FD_LOG_WARNING( ( "NULL topo" ) );
15 0 : return NULL;
16 0 : }
17 :
18 0 : if( FD_UNLIKELY( !fd_ulong_is_aligned( (ulong)topo, alignof(fd_topo_t) ) ) ) {
19 0 : FD_LOG_WARNING( ( "misaligned topo" ) );
20 0 : return NULL;
21 0 : }
22 :
23 0 : fd_memset( topo, 0, sizeof(fd_topo_t) );
24 :
25 0 : FD_TEST( fd_pod_new( topo->props, sizeof(topo->props) ) );
26 :
27 0 : if( FD_UNLIKELY( strlen( app_name )>=sizeof(topo->app_name) ) ) FD_LOG_ERR(( "app_name too long: %s", app_name ));
28 0 : fd_cstr_ncpy( topo->app_name, app_name, sizeof(topo->app_name) );
29 :
30 0 : topo->max_page_size = FD_SHMEM_GIGANTIC_PAGE_SZ;
31 0 : topo->gigantic_page_threshold = 4 * FD_SHMEM_HUGE_PAGE_SZ;
32 :
33 0 : topo->sleep_obj_id = ULONG_MAX;
34 :
35 0 : topo->blocklist_cores_cnt = 0;
36 :
37 0 : return topo;
38 0 : }
39 :
40 : fd_topo_wksp_t *
41 : fd_topob_wksp( fd_topo_t * topo,
42 0 : char const * name ) {
43 0 : if( FD_UNLIKELY( !topo || !name || !strlen( name ) ) ) FD_LOG_ERR(( "NULL args" ));
44 0 : if( FD_UNLIKELY( strlen( name )>=sizeof(topo->workspaces[ topo->wksp_cnt ].name ) ) ) FD_LOG_ERR(( "wksp name too long: %s", name ));
45 0 : if( FD_UNLIKELY( topo->wksp_cnt>=FD_TOPO_MAX_WKSPS ) ) FD_LOG_ERR(( "too many workspaces" ));
46 :
47 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ topo->wksp_cnt ];
48 0 : strncpy( wksp->name, name, sizeof(wksp->name) );
49 0 : wksp->id = topo->wksp_cnt;
50 0 : wksp->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_REGULAR;
51 0 : topo->wksp_cnt++;
52 0 : return wksp;
53 0 : }
54 :
55 : fd_topo_obj_t *
56 : fd_topob_obj( fd_topo_t * topo,
57 : char const * obj_name,
58 0 : char const * wksp_name ) {
59 0 : if( FD_UNLIKELY( !topo || !obj_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
60 0 : if( FD_UNLIKELY( strlen( obj_name )>=sizeof(topo->objs[ topo->obj_cnt ].name ) ) ) FD_LOG_ERR(( "obj name too long: %s", obj_name ));
61 0 : if( FD_UNLIKELY( topo->obj_cnt>=FD_TOPO_MAX_OBJS ) ) FD_LOG_ERR(( "too many objects" ));
62 :
63 0 : ulong wksp_id = fd_topo_find_wksp( topo, wksp_name );
64 0 : if( FD_UNLIKELY( wksp_id==ULONG_MAX ) ) FD_LOG_ERR(( "workspace not found: %s", wksp_name ));
65 :
66 0 : fd_topo_obj_t * obj = &topo->objs[ topo->obj_cnt ];
67 0 : memset( obj, 0, sizeof(fd_topo_obj_t) );
68 0 : strncpy( obj->name, obj_name, sizeof(obj->name) );
69 0 : obj->id = topo->obj_cnt;
70 0 : obj->wksp_id = wksp_id;
71 0 : obj->label_idx = ULONG_MAX;
72 0 : topo->obj_cnt++;
73 :
74 0 : return obj;
75 0 : }
76 :
77 : fd_topo_obj_t *
78 : fd_topob_obj_named( fd_topo_t * topo,
79 : char const * obj_type,
80 : char const * wksp_name,
81 0 : char const * label ) {
82 0 : if( FD_UNLIKELY( !label ) ) FD_LOG_ERR(( "NULL args" ));
83 0 : if( FD_UNLIKELY( strlen( label )>=sizeof(topo->objs[ topo->obj_cnt ].label ) ) ) FD_LOG_ERR(( "obj label too long: %s", label ));
84 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, obj_type, wksp_name );
85 0 : if( FD_UNLIKELY( !obj ) ) return NULL;
86 :
87 0 : fd_cstr_ncpy( obj->label, label, sizeof(obj->label) );
88 0 : obj->label_idx = fd_topo_obj_cnt( topo, obj_type, label );
89 :
90 0 : return obj;
91 0 : }
92 :
93 : fd_topo_link_t *
94 : fd_topob_link( fd_topo_t * topo,
95 : char const * link_name,
96 : char const * wksp_name,
97 : ulong depth,
98 : ulong mtu,
99 0 : ulong burst ) {
100 0 : if( FD_UNLIKELY( !topo || !link_name || !wksp_name ) ) FD_LOG_ERR(( "NULL args" ));
101 0 : if( FD_UNLIKELY( strlen( link_name )>=sizeof(topo->links[ topo->link_cnt ].name ) ) ) FD_LOG_ERR(( "link name too long: %s", link_name ));
102 0 : if( FD_UNLIKELY( topo->link_cnt>=FD_TOPO_MAX_LINKS ) ) FD_LOG_ERR(( "too many links" ));
103 :
104 0 : ulong kind_id = 0UL;
105 0 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
106 0 : if( !strcmp( topo->links[ i ].name, link_name ) ) kind_id++;
107 0 : }
108 :
109 0 : fd_topo_link_t * link = &topo->links[ topo->link_cnt ];
110 0 : strncpy( link->name, link_name, sizeof(link->name) );
111 0 : link->id = topo->link_cnt;
112 0 : link->kind_id = kind_id;
113 0 : link->depth = depth;
114 0 : link->mtu = mtu;
115 0 : link->burst = burst;
116 :
117 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, "mcache", wksp_name );
118 0 : link->mcache_obj_id = obj->id;
119 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
120 :
121 0 : if( mtu ) {
122 0 : obj = fd_topob_obj( topo, "dcache", wksp_name );
123 0 : link->dcache_obj_id = obj->id;
124 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, depth, "obj.%lu.depth", obj->id ) );
125 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, burst, "obj.%lu.burst", obj->id ) );
126 0 : FD_TEST( fd_pod_insertf_ulong( topo->props, mtu, "obj.%lu.mtu", obj->id ) );
127 0 : }
128 0 : topo->link_cnt++;
129 :
130 0 : return link;
131 0 : }
132 :
133 : void
134 : fd_topob_tile_uses( fd_topo_t * topo,
135 : fd_topo_tile_t * tile,
136 : fd_topo_obj_t const * obj,
137 0 : int mode ) {
138 0 : (void)topo;
139 :
140 : /* Links can share an object (e.g. net rx links share the UMEM
141 : dcache), keep one entry with the strongest mode */
142 0 : for( ulong i=0UL; i<tile->uses_obj_cnt; i++ ) {
143 0 : if( FD_UNLIKELY( tile->uses_obj_id[ i ]==obj->id ) ) {
144 0 : tile->uses_obj_mode[ i ] = fd_int_max( tile->uses_obj_mode[ i ], mode );
145 0 : return;
146 0 : }
147 0 : }
148 :
149 0 : if( FD_UNLIKELY( tile->uses_obj_cnt>=FD_TOPO_MAX_TILE_OBJS ) ) FD_LOG_ERR(( "tile `%s` uses too many objects", tile->name ));
150 :
151 0 : tile->uses_obj_id[ tile->uses_obj_cnt ] = obj->id;
152 0 : tile->uses_obj_mode[ tile->uses_obj_cnt ] = mode;
153 0 : tile->uses_obj_cnt++;
154 0 : }
155 :
156 : fd_topo_tile_t *
157 : fd_topob_tile( fd_topo_t * topo,
158 : char const * tile_name,
159 : char const * tile_wksp,
160 : char const * metrics_wksp,
161 : ulong cpu_idx,
162 : int uses_id_keyswitch,
163 : int uses_av_keyswitch,
164 0 : int is_waker_client ) {
165 :
166 0 : if( FD_UNLIKELY( !topo || !tile_name || !tile_wksp || !metrics_wksp ) ) FD_LOG_ERR(( "NULL args" ));
167 0 : if( FD_UNLIKELY( strlen( tile_name )>=sizeof(topo->tiles[ topo->tile_cnt ].name ) ) ) FD_LOG_ERR(( "tile name too long: %s", tile_name ));
168 0 : if( FD_UNLIKELY( topo->tile_cnt>=FD_TOPO_MAX_TILES ) ) FD_LOG_ERR(( "too many tiles %lu", topo->tile_cnt ));
169 :
170 0 : ulong kind_id = 0UL;
171 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
172 0 : if( !strcmp( topo->tiles[ i ].name, tile_name ) ) kind_id++;
173 0 : }
174 :
175 0 : fd_topo_tile_t * tile = &topo->tiles[ topo->tile_cnt ];
176 0 : strncpy( tile->name, tile_name, sizeof(tile->name) );
177 0 : tile->id = topo->tile_cnt;
178 0 : tile->kind_id = kind_id;
179 0 : tile->cpu_idx = fd_ulong_if( cpu_idx<ULONG_MAX, cpu_idx & ~FD_TOPOB_CPU_SHARED, ULONG_MAX );
180 0 : tile->in_cnt = 0UL;
181 0 : tile->out_cnt = 0UL;
182 0 : tile->event_link_id = ULONG_MAX;
183 0 : tile->uses_obj_cnt = 0UL;
184 0 : tile->is_waker_client = is_waker_client;
185 0 : tile->floats = cpu_idx<ULONG_MAX && !!(cpu_idx & FD_TOPOB_CPU_SHARED);
186 0 : tile->waker_client_idx = ULONG_MAX;
187 0 : tile->waker_fseq_obj_id = ULONG_MAX;
188 0 : tile->sleep_eventfd = 0;
189 :
190 0 : fd_topo_obj_t * tile_obj = fd_topob_obj( topo, "tile", tile_wksp );
191 0 : tile->tile_obj_id = tile_obj->id;
192 0 : fd_topob_tile_uses( topo, tile, tile_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
193 :
194 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, "metrics", metrics_wksp );
195 0 : tile->metrics_obj_id = obj->id;
196 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
197 :
198 0 : if( FD_LIKELY( uses_id_keyswitch ) ) {
199 0 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
200 0 : tile->id_keyswitch_obj_id = obj->id;
201 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
202 0 : } else {
203 0 : tile->id_keyswitch_obj_id = ULONG_MAX;
204 0 : }
205 :
206 0 : if( FD_UNLIKELY( uses_av_keyswitch ) ) {
207 0 : obj = fd_topob_obj( topo, "keyswitch", tile_wksp );
208 0 : tile->av_keyswitch_obj_id = obj->id;
209 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
210 0 : } else {
211 0 : tile->av_keyswitch_obj_id = ULONG_MAX;
212 0 : }
213 :
214 0 : topo->tile_cnt++;
215 0 : return tile;
216 0 : }
217 :
218 : void
219 0 : fd_topob_waker( fd_topo_t * topo ) {
220 0 : ulong waker_client_cnt = 0UL;
221 0 : fd_topo_tile_t * waker_tile = NULL;
222 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
223 0 : fd_topo_tile_t * client = &topo->tiles[ i ];
224 0 : if( FD_LIKELY( !client->is_waker_client ) ) continue;
225 0 : if( FD_UNLIKELY( !waker_tile ) ) {
226 0 : char const * metrics_wksp = topo->workspaces[ topo->objs[ client->metrics_obj_id ].wksp_id ].name;
227 0 : fd_topob_wksp( topo, "waker" );
228 0 : fd_topob_wksp( topo, "waker_ready" );
229 0 : waker_tile = fd_topob_tile( topo, "waker", "waker", metrics_wksp, ULONG_MAX, 0, 0, 0 );
230 0 : }
231 0 : fd_topo_obj_t * fseq_obj = fd_topob_obj( topo, "fseq", "waker_ready" );
232 0 : fd_topob_tile_uses( topo, client, fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
233 0 : fd_topob_tile_uses( topo, waker_tile, fseq_obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
234 0 : client->waker_client_idx = waker_client_cnt++;
235 0 : client->waker_fseq_obj_id = fseq_obj->id;
236 0 : }
237 0 : FD_TEST( waker_client_cnt<=FD_WAKER_CLIENT_MAX );
238 0 : }
239 :
240 : void
241 : fd_topob_sleep( fd_topo_t * topo,
242 : char const * metrics_wksp,
243 0 : ulong mwaitx_cpu_idx ) {
244 0 : fd_topob_wksp( topo, "mwaitx" );
245 0 : fd_topob_wksp( topo, "sleep" );
246 0 : topo->sleep_obj_id = fd_topob_obj( topo, "sleep", "sleep" )->id;
247 0 : fd_topob_tile( topo, "mwaitx", "mwaitx", metrics_wksp, mwaitx_cpu_idx, 0, 0, 0 );
248 0 : }
249 :
250 : void
251 0 : fd_topob_sleep_finish( fd_topo_t * topo ) {
252 0 : if( FD_LIKELY( topo->sleep_obj_id==ULONG_MAX ) ) return;
253 :
254 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
255 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
256 0 : int pinned = !!( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
257 0 : tile->floats = tile->cpu_idx!=ULONG_MAX && !pinned;
258 0 : }
259 :
260 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
261 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
262 0 : if( FD_LIKELY( !tile->floats ) ) continue;
263 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
264 0 : if( FD_UNLIKELY( topo->tiles[ j ].cpu_idx==tile->cpu_idx && !topo->tiles[ j ].floats ) ) {
265 0 : tile->floats = 0;
266 0 : break;
267 0 : }
268 0 : }
269 0 : }
270 :
271 0 : FD_TEST( topo->tile_cnt<=FD_SLEEP_TILE_MAX );
272 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
273 0 : fd_topob_tile_uses( topo, &topo->tiles[ i ], &topo->objs[ topo->sleep_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
274 0 : }
275 0 : }
276 :
277 : void
278 : fd_topob_tile_in( fd_topo_t * topo,
279 : char const * tile_name,
280 : ulong tile_kind_id,
281 : char const * fseq_wksp,
282 : char const * link_name,
283 : ulong link_kind_id,
284 : int reliable,
285 0 : int polled ) {
286 0 : if( FD_UNLIKELY( !topo || !tile_name || !fseq_wksp || !link_name ) ) FD_LOG_ERR(( "NULL args" ));
287 :
288 0 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
289 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
290 0 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
291 :
292 0 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
293 0 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
294 0 : fd_topo_link_t * link = &topo->links[ link_id ];
295 :
296 0 : if( FD_UNLIKELY( tile->in_cnt>=FD_TOPO_MAX_TILE_IN_LINKS ) ) FD_LOG_ERR(( "too many in links: %s:%lu", tile_name, tile_kind_id ) );
297 0 : tile->in_link_id[ tile->in_cnt ] = link->id;
298 0 : tile->in_link_reliable[ tile->in_cnt ] = reliable;
299 0 : tile->in_link_poll[ tile->in_cnt ] = polled;
300 0 : fd_topo_obj_t * obj = fd_topob_obj( topo, "fseq", fseq_wksp );
301 0 : fd_topob_tile_uses( topo, tile, obj, FD_SHMEM_JOIN_MODE_READ_WRITE );
302 0 : tile->in_link_fseq_obj_id[ tile->in_cnt ] = obj->id;
303 0 : tile->in_cnt++;
304 :
305 0 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
306 0 : if( FD_LIKELY( link->mtu ) ) {
307 0 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_ONLY );
308 0 : }
309 0 : }
310 :
311 : void
312 : fd_topob_tile_out( fd_topo_t * topo,
313 : char const * tile_name,
314 : ulong tile_kind_id,
315 : char const * link_name,
316 0 : ulong link_kind_id ) {
317 0 : ulong tile_id = fd_topo_find_tile( topo, tile_name, tile_kind_id );
318 0 : if( FD_UNLIKELY( tile_id==ULONG_MAX ) ) FD_LOG_ERR(( "tile not found: %s:%lu", tile_name, tile_kind_id ));
319 0 : fd_topo_tile_t * tile = &topo->tiles[ tile_id ];
320 :
321 0 : ulong link_id = fd_topo_find_link( topo, link_name, link_kind_id );
322 0 : if( FD_UNLIKELY( link_id==ULONG_MAX ) ) FD_LOG_ERR(( "link not found: %s:%lu", link_name, link_kind_id ));
323 0 : fd_topo_link_t * link = &topo->links[ link_id ];
324 :
325 0 : if( FD_UNLIKELY( tile->out_cnt>=FD_TOPO_MAX_TILE_OUT_LINKS ) ) FD_LOG_ERR(( "too many out links: %s", tile_name ));
326 0 : tile->out_link_id[ tile->out_cnt ] = link->id;
327 0 : tile->out_cnt++;
328 :
329 0 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->mcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
330 0 : if( FD_LIKELY( link->mtu ) ) {
331 0 : fd_topob_tile_uses( topo, tile, &topo->objs[ link->dcache_obj_id ], FD_SHMEM_JOIN_MODE_READ_WRITE );
332 0 : }
333 0 : }
334 :
335 : static void
336 0 : validate( fd_topo_t const * topo ) {
337 : /* Objects have valid wksp_ids */
338 0 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
339 0 : if( FD_UNLIKELY( topo->objs[ i ].wksp_id>=topo->wksp_cnt ) )
340 0 : FD_LOG_ERR(( "invalid workspace id %lu", topo->objs[ i ].wksp_id ));
341 0 : }
342 :
343 : /* Tile ins are valid */
344 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
345 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
346 0 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ]>=topo->link_cnt ) )
347 0 : FD_LOG_ERR(( "tile %lu (%s) has invalid in link %lu", i, topo->tiles[ i ].name, topo->tiles[ i ].in_link_id[ j ] ));
348 0 : }
349 0 : }
350 :
351 : /* Tile does not have duplicated ins */
352 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
353 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
354 0 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
355 0 : if( FD_UNLIKELY( j==k ) ) continue;
356 0 : if( FD_UNLIKELY( topo->tiles[ i ].in_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
357 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated in link %lu (%s)", i, topo->tiles[ i ].name,
358 0 : topo->tiles[ i ].in_link_id[ j ], topo->links[ topo->tiles[ i ].in_link_id[ j ] ].name ));
359 0 : }
360 0 : }
361 0 : }
362 :
363 : /* Tile does not have duplicated outs */
364 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
365 0 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
366 0 : for( ulong k=0UL; k<topo->tiles[ i ].out_cnt; k++ ) {
367 0 : if( FD_UNLIKELY( j==k ) ) continue;
368 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].out_link_id[ k ] ) )
369 0 : FD_LOG_ERR(( "tile %lu (%s) has duplicated out link %lu (%s)", i, topo->tiles[ i ].name,
370 0 : topo->tiles[ i ].out_link_id[ j ], topo->links[ topo->tiles[ i ].out_link_id[ j ] ].name ));
371 0 : }
372 0 : }
373 0 : }
374 :
375 : /* Tile outs are different than ins */
376 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
377 0 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
378 0 : for( ulong k=0UL; k<topo->tiles[ i ].in_cnt; k++ ) {
379 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] == topo->tiles[ i ].in_link_id[ k ] ) )
380 0 : FD_LOG_ERR(( "tile %lu has out link %lu same as in", i, topo->tiles[ i ].out_link_id[ j ] ));
381 0 : }
382 0 : }
383 0 : }
384 :
385 : /* Non polling tile ins are also not reliable */
386 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
387 0 : for( ulong j=0UL; j<topo->tiles[ i ].in_cnt; j++ ) {
388 0 : if( FD_UNLIKELY( !topo->tiles[ i ].in_link_poll[ j ] && topo->tiles[ i ].in_link_reliable[ j ] ) )
389 0 : FD_LOG_ERR(( "tile %lu has in link %lu which is not polled but reliable", i, topo->tiles[ i ].in_link_id[ j ] ));
390 0 : }
391 0 : }
392 :
393 : /* Tile outs are valid */
394 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
395 0 : for( ulong j=0UL; j<topo->tiles[ i ].out_cnt; j++ ) {
396 0 : if( FD_UNLIKELY( topo->tiles[ i ].out_link_id[ j ] >= topo->link_cnt ) )
397 0 : FD_LOG_ERR(( "tile %lu has invalid out link %lu", i, topo->tiles[ i ].out_link_id[ j ] ));
398 0 : }
399 0 : }
400 :
401 : /* Waker clients are wired */
402 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
403 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
404 0 : if( FD_UNLIKELY( tile->is_waker_client && tile->waker_client_idx==ULONG_MAX ) )
405 0 : FD_LOG_ERR(( "tile %s:%lu is a waker client but fd_topob_waker was not called", tile->name, tile->kind_id ));
406 0 : }
407 :
408 : /* Floating tiles have a CPU: it places their memory and anchors
409 : their affinity mask */
410 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
411 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
412 0 : if( FD_UNLIKELY( tile->floats && tile->cpu_idx>=FD_TILE_MAX ) )
413 0 : FD_LOG_ERR(( "tile %s:%lu floats but has no CPU", tile->name, tile->kind_id ));
414 0 : }
415 :
416 : /* Workspace names are unique */
417 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
418 0 : for( ulong j=0UL; j<topo->wksp_cnt; j++ ) {
419 0 : if( FD_UNLIKELY( i==j ) ) continue;
420 0 : if( FD_UNLIKELY( !strcmp( topo->workspaces[ i ].name, topo->workspaces[ j ].name ) ) )
421 0 : FD_LOG_ERR(( "duplicate workspace name %s", topo->workspaces[ i ].name ));
422 0 : }
423 0 : }
424 :
425 : /* Each workspace is identified correctly */
426 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
427 0 : if( FD_UNLIKELY( topo->workspaces[ i ].id != i ) )
428 0 : FD_LOG_ERR(( "workspace %lu has id %lu", i, topo->workspaces[ i ].id ));
429 0 : }
430 :
431 : /* Each link has exactly one producer */
432 0 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
433 0 : ulong producer_cnt = 0;
434 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
435 0 : for( ulong k=0UL; k<topo->tiles[ j ].out_cnt; k++ ) {
436 0 : if( topo->tiles[ j ].out_link_id[ k ]==i ) producer_cnt++;
437 0 : }
438 0 : }
439 0 : if( FD_UNLIKELY( producer_cnt>1UL || ( producer_cnt==0UL && !topo->links[ i ].permit_no_producers ) ) )
440 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has %lu producers", i, topo->links[ i ].name, topo->links[ i ].kind_id, producer_cnt ));
441 0 : }
442 :
443 : /* Each link has at least one consumer */
444 0 : for( ulong i=0UL; i<topo->link_cnt; i++ ) {
445 0 : ulong cnt = fd_topo_link_consumer_cnt( topo, &topo->links[ i ] );
446 0 : if( FD_UNLIKELY( cnt < 1UL && !topo->links[ i ].permit_no_consumers ) ) {
447 0 : FD_LOG_ERR(( "link %lu (%s:%lu) has 0 consumers", i, topo->links[ i ].name, topo->links[ i ].kind_id ));
448 0 : }
449 0 : }
450 0 : }
451 :
452 : ulong
453 : fd_numa_node_idx( ulong cpu_idx );
454 :
455 : static void
456 0 : initialize_numa_assignments( fd_topo_t * topo ) {
457 : /* Assign workspaces to NUMA nodes. The heuristic here is pretty
458 : simple for now: workspaces go on the NUMA node of the first
459 : tile which maps the largest object in the workspace. */
460 :
461 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
462 0 : ulong max_footprint = 0UL;
463 0 : ulong max_obj = ULONG_MAX;
464 :
465 0 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
466 0 : fd_topo_obj_t * obj = &topo->objs[ j ];
467 0 : if( obj->wksp_id!=i ) continue;
468 0 : if( FD_UNLIKELY( !obj->footprint ) ) FD_LOG_ERR(( "obj %lu (%s) has invalid parameters", j, obj->name ));
469 :
470 0 : if( FD_UNLIKELY( !max_footprint || obj->footprint>max_footprint ) ) {
471 0 : max_footprint = obj->footprint;
472 0 : max_obj = j;
473 0 : }
474 0 : }
475 :
476 0 : if( FD_UNLIKELY( max_obj==ULONG_MAX ) ) FD_LOG_ERR(( "no object found for workspace %s", topo->workspaces[ i ].name ));
477 :
478 0 : int found_strict = 0;
479 0 : int found_lazy = 0;
480 0 : int found_assigned = 0;
481 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
482 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
483 0 : if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
484 0 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
485 0 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
486 0 : found_strict = 1;
487 0 : found_lazy = 1;
488 0 : found_assigned = 1;
489 0 : break;
490 0 : } else if( FD_UNLIKELY( tile->tile_obj_id==max_obj && tile->cpu_idx>=FD_TILE_MAX ) ) {
491 0 : topo->workspaces[ i ].numa_idx = 0;
492 0 : found_lazy = 1;
493 0 : break;
494 0 : }
495 0 : }
496 :
497 0 : if( FD_LIKELY( !found_strict ) ) {
498 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
499 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
500 0 : for( ulong k=0UL; k<tile->uses_obj_cnt; k++ ) {
501 0 : if( FD_LIKELY( tile->uses_obj_id[ k ]==max_obj && tile->cpu_idx<FD_TILE_MAX ) ) {
502 0 : topo->workspaces[ i ].numa_idx = fd_numa_node_idx( tile->cpu_idx );
503 0 : FD_TEST( topo->workspaces[ i ].numa_idx!=ULONG_MAX );
504 0 : found_lazy = 1;
505 0 : found_assigned = 1;
506 0 : break;
507 0 : } else if( FD_UNLIKELY( tile->uses_obj_id[ k ]==max_obj ) && tile->cpu_idx>=FD_TILE_MAX ) {
508 0 : topo->workspaces[ i ].numa_idx = 0;
509 0 : found_lazy = 1;
510 : /* Don't break, keep looking -- a tile with a CPU assignment
511 : might also use object in which case we want to use that
512 : NUMA node. */
513 0 : }
514 0 : }
515 :
516 0 : if( FD_UNLIKELY( found_assigned ) ) break;
517 0 : }
518 0 : }
519 :
520 0 : if( FD_UNLIKELY( !found_lazy ) ) FD_LOG_ERR(( "no tile uses object %s for workspace %s", topo->objs[ max_obj ].name, topo->workspaces[ i ].name ));
521 0 : }
522 0 : }
523 :
524 : void
525 : fd_topob_finish( fd_topo_t * topo,
526 0 : fd_topo_obj_callbacks_t ** callbacks ) {
527 0 : sock_lo_set_affinity( topo );
528 :
529 0 : for( ulong z=0UL; z<topo->tile_cnt; z++ ) {
530 0 : fd_topo_tile_t * tile = &topo->tiles[ z ];
531 :
532 0 : ulong in_cnt = 0UL;
533 0 : for( ulong i=0UL; i<tile->in_cnt; i++ ) {
534 0 : if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;
535 0 : in_cnt++;
536 0 : }
537 :
538 0 : FD_TEST( !fd_pod_replacef_ulong( topo->props, in_cnt, "obj.%lu.in_cnt", tile->metrics_obj_id ) );
539 0 : }
540 :
541 0 : for( ulong i=0UL; i<topo->wksp_cnt; i++ ) {
542 0 : fd_topo_wksp_t * wksp = &topo->workspaces[ i ];
543 :
544 0 : ulong loose_sz = 0UL;
545 0 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
546 0 : fd_topo_obj_t * obj = &topo->objs[ j ];
547 0 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
548 :
549 0 : fd_topo_obj_callbacks_t * cb = NULL;
550 0 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
551 0 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
552 0 : cb = callbacks[ i ];
553 0 : break;
554 0 : }
555 0 : }
556 0 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
557 :
558 0 : if( FD_UNLIKELY( cb->loose ) ) loose_sz += cb->loose( topo, obj );
559 0 : }
560 :
561 0 : ulong part_max = wksp->part_max;
562 0 : if( !part_max ) part_max = (loose_sz / (64UL << 10)); /* alloc + residual padding */
563 0 : part_max += 3; /* for initial alignment */
564 0 : ulong offset = fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
565 :
566 0 : for( ulong j=0UL; j<topo->obj_cnt; j++ ) {
567 0 : fd_topo_obj_t * obj = &topo->objs[ j ];
568 0 : if( FD_UNLIKELY( obj->wksp_id!=wksp->id ) ) continue;
569 :
570 0 : fd_topo_obj_callbacks_t * cb = NULL;
571 0 : for( ulong i=0UL; callbacks[ i ]; i++ ) {
572 0 : if( FD_UNLIKELY( !strcmp( callbacks[ i ]->name, obj->name ) ) ) {
573 0 : cb = callbacks[ i ];
574 0 : break;
575 0 : }
576 0 : }
577 0 : if( FD_UNLIKELY( !cb ) ) FD_LOG_ERR(( "no callbacks for object %s", obj->name ));
578 :
579 0 : ulong align_ = cb->align( topo, obj );
580 0 : if( FD_UNLIKELY( !fd_ulong_is_pow2( align_ ) ) ) FD_LOG_ERR(( "Return value of fdctl_obj_align(%s,%lu) is not a power of 2", obj->name, obj->id ));
581 0 : offset = fd_ulong_align_up( offset, align_ );
582 0 : obj->offset = offset;
583 0 : obj->footprint = cb->footprint( topo, obj );
584 0 : if( FD_UNLIKELY( 0!=strcmp( obj->name, "tile" ) && (!obj->footprint || obj->footprint>LONG_MAX) ) ) {
585 0 : FD_LOG_ERR(( "fdctl_obj_footprint(%s,%lu) failed", obj->name, obj->id ));
586 0 : }
587 0 : offset += obj->footprint;
588 0 : }
589 :
590 0 : ulong footprint = fd_ulong_align_up( offset, fd_topo_workspace_align() );
591 :
592 0 : part_max = fd_ulong_max( part_max, wksp->min_part_max );
593 0 : loose_sz = fd_ulong_max( loose_sz, wksp->min_loose_sz );
594 :
595 : /* Compute footprint for a workspace that can store our footprint,
596 : with an extra align of padding in case gaddr_lo is not aligned. */
597 0 : ulong total_wksp_footprint = fd_wksp_footprint( part_max, footprint + fd_topo_workspace_align() + loose_sz );
598 :
599 0 : ulong page_sz = topo->max_page_size;
600 0 : if( total_wksp_footprint < topo->gigantic_page_threshold ) page_sz = FD_SHMEM_HUGE_PAGE_SZ;
601 0 : if( FD_UNLIKELY( page_sz!=FD_SHMEM_HUGE_PAGE_SZ && page_sz!=FD_SHMEM_GIGANTIC_PAGE_SZ ) ) FD_LOG_ERR(( "invalid page_sz" ));
602 :
603 0 : ulong wksp_aligned_footprint = fd_ulong_align_up( total_wksp_footprint, page_sz );
604 :
605 : /* Give any leftover space in the underlying shared memory to the
606 : data region of the workspace, since we might as well use it. */
607 0 : wksp->part_max = part_max;
608 0 : wksp->known_footprint = footprint;
609 0 : wksp->total_footprint = wksp_aligned_footprint - fd_ulong_align_up( fd_wksp_private_data_off( part_max ), fd_topo_workspace_align() );
610 0 : wksp->page_sz = page_sz;
611 0 : wksp->page_cnt = wksp_aligned_footprint / page_sz;
612 0 : }
613 :
614 : /* Fingerprint the computed layout so attaching processes can detect
615 : offset divergence from a different build or config. */
616 0 : ulong layout_hash = 0UL;
617 0 : for( ulong i=0UL; i<topo->obj_cnt; i++ ) {
618 0 : fd_topo_obj_t const * obj = &topo->objs[ i ];
619 0 : layout_hash = fd_hash( layout_hash, obj->name, strlen( obj->name ) );
620 0 : layout_hash = fd_hash( layout_hash, topo->workspaces[ obj->wksp_id ].name, strlen( topo->workspaces[ obj->wksp_id ].name ) );
621 0 : layout_hash = fd_hash( layout_hash, &obj->offset, sizeof(obj->offset) );
622 0 : layout_hash = fd_hash( layout_hash, &obj->footprint, sizeof(obj->footprint) );
623 0 : }
624 0 : topo->layout_hash = layout_hash;
625 :
626 0 : initialize_numa_assignments( topo );
627 :
628 0 : fd_topob_validate_cpu_overlaps( topo );
629 0 : validate( topo );
630 0 : }
|
