Line data Source code
1 : #include "fd_topob_layout.h"
2 : #include "../net/fd_net_tile.h"
3 : #include "../../util/tile/fd_tile_private.h" /* fd_tile_private_sibling_idx */
4 :
5 : #include <ctype.h>
6 :
7 : #define SET_NAME cpu_bv
8 : #define SET_MAX FD_TILE_MAX
9 : #include "../../util/tmpl/fd_set.c"
10 :
11 : #define ALWAYS FD_TOPOB_TILE_ALWAYS
12 : #define STARTUP FD_TOPOB_TILE_STARTUP
13 : #define POSTSTART FD_TOPOB_TILE_POSTSTART
14 : #define FLOATING FD_TOPOB_TILE_FLOATING
15 : #define CRITICAL FD_TOPOB_TILE_SMT_CRITICAL
16 : #define DEDICATED FD_TOPOB_TILE_DEDICATED
17 : #define THRUPUT FD_TOPOB_TILE_THROUGHPUT
18 : #define PIN_EFF FD_TOPOB_TILE_PIN_EFFICIENT
19 :
20 : /* Per-tile classes, keyed by tile name. Auto layout places each
21 : phase in turn (ALWAYS, then STARTUP, then POSTSTART), walking rows
22 : top to bottom, so within a phase earlier rows get earlier CPUs in
23 : the NUMA/sibling ordering. Reordering rows changes layouts. */
24 :
25 : static struct { char const * name; uint flags; } const TILE_CLASS[] = {
26 : { "backt", ALWAYS },
27 : { "benchg", ALWAYS|THRUPUT },
28 : { "benchs", ALWAYS|THRUPUT },
29 : { "net", ALWAYS },
30 : { "mlx5", ALWAYS },
31 : { "sock", ALWAYS|PIN_EFF },
32 : { "quic", ALWAYS },
33 : { "bundle", ALWAYS },
34 : { "verify", ALWAYS|THRUPUT },
35 : { "dedup", ALWAYS },
36 : { "resolv", ALWAYS },
37 : { "pack", ALWAYS|CRITICAL },
38 : { "sign", ALWAYS },
39 : { "shred", ALWAYS },
40 : { "event", ALWAYS },
41 : { "gui", ALWAYS },
42 : { "rpc", ALWAYS },
43 : { "gossvf", ALWAYS },
44 : { "gossip", ALWAYS },
45 : { "rotor", ALWAYS },
46 : { "rserve", ALWAYS },
47 : { "replay", ALWAYS|THRUPUT },
48 : { "votor", ALWAYS },
49 : { "pktgen", ALWAYS },
50 : { "mwaitx", ALWAYS|PIN_EFF },
51 :
52 : /* Must shut down after snapshot load */
53 : { "snapct", STARTUP },
54 : { "snapld", STARTUP },
55 : { "snapdc", STARTUP|THRUPUT },
56 : { "snapin", STARTUP|THRUPUT },
57 :
58 : /* Must sleep until snapshot load finishes */
59 : { "accdb", POSTSTART|THRUPUT },
60 : { "execle", POSTSTART|THRUPUT },
61 : { "motor", POSTSTART|CRITICAL },
62 : { "execrp", POSTSTART|DEDICATED|THRUPUT },
63 : { "solcap", POSTSTART|PIN_EFF },
64 :
65 : /* Yield to the kernel scheduler */
66 : { "netlnk", FLOATING },
67 : { "metric", FLOATING },
68 : { "diag", FLOATING },
69 : { "waker", FLOATING },
70 : { "bencho", FLOATING },
71 : { "l4swap", FLOATING },
72 : { "genesi", FLOATING },
73 : { "ipecho", FLOATING },
74 : { "admin", FLOATING },
75 : { "snapsv", FLOATING|PIN_EFF },
76 : { "snapmk", FLOATING },
77 : { "snapzp", FLOATING },
78 : { "snaprd", FLOATING },
79 : };
80 :
81 : #undef ALWAYS
82 : #undef STARTUP
83 : #undef POSTSTART
84 : #undef FLOATING
85 : #undef CRITICAL
86 : #undef DEDICATED
87 : #undef THRUPUT
88 : #undef PIN_EFF
89 :
90 0 : #define TILE_CLASS_CNT (sizeof(TILE_CLASS)/sizeof(TILE_CLASS[0]))
91 :
92 : uint
93 0 : fd_topob_tile_class( char const * name ) {
94 0 : for( ulong i=0UL; i<TILE_CLASS_CNT; i++ ) {
95 0 : if( !strcmp( name, TILE_CLASS[ i ].name ) ) return TILE_CLASS[ i ].flags;
96 0 : }
97 0 : return 0U;
98 0 : }
99 :
100 : /* The pinned tiles that park and wake each other the most, in
101 : placement priority. A wake across an L3 domain costs more than one
102 : inside it, so these are packed into as few domains as possible,
103 : mwaitx first. Floating tiles are not listed: their cpu_idx only
104 : picks the NUMA node of the pool they run over, not an L3. */
105 :
106 : static char const * HOT_TILES[] = {
107 : "mwaitx",
108 : "replay",
109 : "execrp",
110 : "mlx5",
111 : NULL
112 : };
113 :
114 :
115 : static int
116 : tile_name_in( char const * name,
117 0 : char const * const * names ) {
118 0 : for( char const * const * p = names; *p; p++ ) {
119 0 : if( !strcmp( name, *p ) ) return 1;
120 0 : }
121 0 : return 0;
122 0 : }
123 :
124 : static int
125 0 : tile_is_floating( fd_topo_tile_t const * tile ) {
126 : /* RX only loopback sock copies its associated net tile's CPU affinity,
127 : so no extra CPU is needed for pinning. */
128 0 : return !!( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_FLOATING ) || sock_lo_net_tile_id( tile )!=ULONG_MAX;
129 0 : }
130 :
131 : int
132 0 : fd_topob_tile_priority_type( fd_topo_tile_t const * tile ) {
133 0 : uint cls = fd_topob_tile_class( tile->name );
134 0 : if( !cls || tile_is_floating( tile ) ) return FD_TOPOB_PRIORITY_FLOATING;
135 0 : if( cls & FD_TOPOB_TILE_STARTUP ) return FD_TOPOB_PRIORITY_STARTUP;
136 0 : if( cls & FD_TOPOB_TILE_SMT_CRITICAL ) return FD_TOPOB_PRIORITY_CRITICAL;
137 0 : return FD_TOPOB_PRIORITY_NORMAL;
138 0 : }
139 :
140 : FD_STATIC_ASSERT( FD_TILE_MAX<FD_TOPOB_CPU_SHARED, update_tile_to_cpu_type );
141 :
142 : ulong
143 : fd_topob_parse_affinity_cstr( char const * cstr,
144 : ushort * tile_to_cpu,
145 : int allow_repeats,
146 0 : int allow_shared ) {
147 0 : if( !cstr ) return 0UL;
148 0 : ulong cnt = 0UL;
149 :
150 0 : cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
151 0 : cpu_bv_new( cpu_assigned );
152 :
153 0 : char const * p = cstr;
154 0 : for(;;) {
155 :
156 0 : while( fd_isspace( (int)p[0] ) ) p++;
157 :
158 0 : if( p[0]=='f' ) {
159 0 : p++;
160 :
161 0 : ulong float_cnt;
162 :
163 0 : while( fd_isspace( (int)p[0] ) ) p++;
164 0 : if ( p[0]==',' ) float_cnt = 1UL, p++;
165 0 : else if( p[0]=='\0' ) float_cnt = 1UL;
166 0 : else if( !fd_isdigit( (int)p[0] ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (malformed float count)" ));
167 0 : else {
168 0 : float_cnt = fd_cstr_to_ulong( p );
169 0 : if( FD_UNLIKELY( !float_cnt ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count)" ));
170 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
171 0 : while( fd_isspace( (int)p[0] ) ) p++;
172 0 : if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad float count delimiter)" ));
173 0 : if( p[0]==',' ) p++;
174 0 : }
175 :
176 0 : do {
177 0 : if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
178 0 : tile_to_cpu[ cnt++ ] = (ushort)65535;
179 0 : } while( --float_cnt );
180 :
181 0 : continue;
182 0 : }
183 :
184 0 : ulong shared = 0UL;
185 0 : if( p[0]=='s' ) {
186 0 : if( FD_UNLIKELY( !allow_shared ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (shared cpus not supported here)" ));
187 0 : p++; shared = FD_TOPOB_CPU_SHARED;
188 0 : }
189 :
190 0 : if( !fd_isdigit( (int)p[0] ) ) {
191 0 : if( FD_UNLIKELY( shared || p[0]!='\0' ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range lo not a cpu)" ));
192 0 : break;
193 0 : }
194 0 : ulong cpu0 = fd_cstr_to_ulong( p );
195 0 : ulong cpu1 = cpu0;
196 0 : ulong stride = 1UL;
197 0 : int rev = 0; /* Nh lists N first */
198 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
199 0 : while( fd_isspace( (int)p[0] ) ) p++;
200 0 : if( p[0]=='-' ) {
201 0 : p++;
202 0 : while( fd_isspace( (int)p[0] ) ) p++;
203 0 : if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (range hi not a cpu)" ));
204 0 : cpu1 = fd_cstr_to_ulong( p );
205 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
206 0 : while( fd_isspace( (int)p[0] ) ) p++;
207 0 : if( p[0]=='/' || p[0]==':' ) {
208 0 : p++;
209 0 : while( fd_isspace( (int)p[0] ) ) p++;
210 0 : if( FD_UNLIKELY( !fd_isdigit( (int)p[0] ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (stride not an int)" ));
211 0 : stride = fd_cstr_to_ulong( p );
212 0 : p++; while( fd_isdigit( (int)p[0] ) ) p++;
213 0 : }
214 0 : }
215 0 : else if( p[0]=='h' ) {
216 0 : p++;
217 0 : ulong sibling = fd_tile_private_sibling_idx( cpu0 );
218 0 : if( sibling!=ULONG_MAX ) {
219 0 : rev = sibling<cpu0;
220 0 : cpu1 = fd_ulong_max( cpu0, sibling );
221 0 : cpu0 = fd_ulong_min( cpu0, sibling );
222 0 : stride = cpu1-cpu0;
223 0 : }
224 0 : }
225 0 : while( fd_isspace( (int)p[0] ) ) p++;
226 0 : if( FD_UNLIKELY( !( p[0]==',' || p[0]=='\0' ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (bad range delimiter)" ));
227 0 : if( p[0]==',' ) p++;
228 0 : cpu1++;
229 0 : if( FD_UNLIKELY( cpu1<=cpu0 ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid range)" ));
230 0 : if( FD_UNLIKELY( !stride ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (invalid stride)" ));
231 :
232 0 : for( ulong cpu=cpu0; cpu<cpu1; cpu+=stride ) {
233 0 : if( FD_UNLIKELY( cnt>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: too many affinity entries" ));
234 0 : if( FD_UNLIKELY( cpu>=FD_TILE_MAX ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (cpu index too large)" )); /* cpu_assigned holds FD_TILE_MAX */
235 0 : if( FD_UNLIKELY( !allow_repeats && !shared && cpu_bv_test( cpu_assigned, cpu ) ) ) FD_LOG_ERR(( "fd_topob: malformed affinity string (repeated cpu)" ));
236 0 : tile_to_cpu[ cnt++ ] = (ushort)(cpu | shared);
237 0 : if( !shared ) cpu_bv_insert( cpu_assigned, cpu );
238 0 : }
239 0 : if( rev ) { ushort t = tile_to_cpu[ cnt-2UL ]; tile_to_cpu[ cnt-2UL ] = tile_to_cpu[ cnt-1UL ]; tile_to_cpu[ cnt-1UL ] = t; }
240 0 : }
241 :
242 0 : return cnt;
243 0 : }
244 :
245 0 : #define FD_TOPOB_LIVE_ALWAYS (1)
246 0 : #define FD_TOPOB_LIVE_STARTUP (2)
247 0 : #define FD_TOPOB_LIVE_POSTSTART (3)
248 :
249 : static int
250 0 : fd_topob_tile_live_phase( fd_topo_tile_t const * tile ) {
251 0 : uint cls = fd_topob_tile_class( tile->name );
252 0 : if( ( cls & FD_TOPOB_TILE_STARTUP ) || tile->allow_shutdown ) return FD_TOPOB_LIVE_STARTUP;
253 0 : if( cls & FD_TOPOB_TILE_POSTSTART ) return FD_TOPOB_LIVE_POSTSTART;
254 0 : return FD_TOPOB_LIVE_ALWAYS;
255 0 : }
256 :
257 : int
258 : fd_topob_cpu_overlap_allowed( fd_topo_tile_t const * a,
259 : fd_topo_tile_t const * b,
260 0 : int floats_share ) {
261 0 : if( sock_lo_net_tile_id( a )==b->id || sock_lo_net_tile_id( b )==a->id ) return 1;
262 0 : if( floats_share && a->floats && b->floats ) return 1;
263 :
264 0 : int a_phase = fd_topob_tile_live_phase( a );
265 0 : int b_phase = fd_topob_tile_live_phase( b );
266 :
267 0 : return ( a_phase==FD_TOPOB_LIVE_STARTUP && b_phase==FD_TOPOB_LIVE_POSTSTART ) ||
268 0 : ( a_phase==FD_TOPOB_LIVE_POSTSTART && b_phase==FD_TOPOB_LIVE_STARTUP );
269 0 : }
270 :
271 : void
272 0 : fd_topob_validate_cpu_overlaps( fd_topo_t const * topo ) {
273 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
274 0 : fd_topo_tile_t const * a = &topo->tiles[ i ];
275 0 : if( a->cpu_idx>=FD_TILE_MAX ) continue;
276 :
277 0 : for( ulong j=i+1UL; j<topo->tile_cnt; j++ ) {
278 0 : fd_topo_tile_t const * b = &topo->tiles[ j ];
279 0 : if( b->cpu_idx!=a->cpu_idx ) continue;
280 0 : if( FD_LIKELY( fd_topob_cpu_overlap_allowed( a, b, 1 ) ) ) continue;
281 :
282 0 : FD_LOG_ERR(( "tile `%s:%lu` and tile `%s:%lu` are both assigned to CPU %lu and may try to run at the same time",
283 0 : a->name, a->kind_id, b->name, b->kind_id, a->cpu_idx ));
284 0 : }
285 0 : }
286 0 : }
287 :
288 : static char const *
289 : offline_note( fd_topo_cpus_t const * cpus,
290 0 : char buf[ static 128 ] ) {
291 0 : ulong offline_cnt = 0UL;
292 0 : ulong core_cnt = 0UL;
293 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
294 0 : fd_topo_cpu_t const * cpu = &cpus->cpu[ i ];
295 0 : offline_cnt += (ulong)!cpu->online;
296 0 : core_cnt += (ulong)( cpu->online && ( cpu->sibling==ULONG_MAX || cpu->sibling>i ) );
297 0 : }
298 0 : if( FD_LIKELY( !offline_cnt ) ) return "";
299 0 : return fd_cstr_printf( buf, 128UL, NULL, " (%lu CPUs are offline, leaving %lu online physical cores; reduce tile counts)", offline_cnt, core_cnt );
300 0 : }
301 :
302 : /* iso tier test: 3 wants an isolated CPU with a non-isolated
303 : sibling, 2 one whose free threads are all isolated, 1 an isolated
304 : CPU, -1 a non-isolated one, 0 any */
305 :
306 : static int
307 : iso_mismatch( fd_topo_cpus_t const * cpus,
308 : ulong cpu_idx,
309 0 : int iso ) {
310 0 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_idx ];
311 0 : if( iso==3 ) return !cpu->isolated || cpu->sibling==ULONG_MAX || cpus->cpu[ cpu->sibling ].isolated;
312 0 : if( iso==2 ) return !cpu->isolated || ( cpu->sibling!=ULONG_MAX && !cpus->cpu[ cpu->sibling ].isolated );
313 0 : return iso && (iso>0)!=!!cpu->isolated;
314 0 : }
315 :
316 : /* Free CPUs by core type: both threads free (NN, NI, II by isolated
317 : thread count), sibling absent (N1, I1), sibling taken (NH, IH) */
318 :
319 0 : #define CT_NN 0
320 0 : #define CT_NI 1
321 0 : #define CT_II 2
322 0 : #define CT_N1 3
323 0 : #define CT_I1 4
324 0 : #define CT_NH 5
325 0 : #define CT_IH 6
326 :
327 : /* Counts core types as if cpu_id (and its sibling if whole_core) were
328 : taken, cpu_id ULONG_MAX for none */
329 :
330 : static void
331 : core_type_cnt( fd_topo_cpus_t const * cpus,
332 : cpu_bv_t const cpu_assigned[ static cpu_bv_word_cnt ],
333 : ulong cpu_id,
334 : int whole_core,
335 0 : ulong cnt[ static 7 ] ) {
336 0 : cpu_bv_t taken[ cpu_bv_word_cnt ];
337 0 : cpu_bv_copy( taken, cpu_assigned );
338 0 : if( cpu_id!=ULONG_MAX ) {
339 0 : cpu_bv_insert( taken, cpu_id );
340 0 : if( whole_core && cpus->cpu[ cpu_id ].sibling!=ULONG_MAX ) cpu_bv_insert( taken, cpus->cpu[ cpu_id ].sibling );
341 0 : }
342 :
343 0 : for( ulong t=0UL; t<7UL; t++ ) cnt[ t ] = 0UL;
344 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
345 0 : if( cpu_bv_test( taken, i ) ) continue; /* offline CPUs are taken */
346 0 : ulong s = cpus->cpu[ i ].sibling;
347 0 : ulong iso = (ulong)!!cpus->cpu[ i ].isolated;
348 0 : if( s==ULONG_MAX ) cnt[ CT_N1+iso ]++;
349 0 : else if( cpu_bv_test( taken, s ) ) cnt[ CT_NH+iso ]++;
350 0 : else if( s>i ) cnt[ CT_NN+iso+(ulong)!!cpus->cpu[ s ].isolated ]++;
351 0 : }
352 0 : }
353 :
354 : /* Exact test that the tiles left still have CPUs. dem[ phase*3+kind ]
355 : counts them by phase (always, startup, post-start) and kind
356 : (critical: whole core, pinned: any CPU, floater: non-isolated CPU).
357 : Each of the left critical tiles tries each core type, always ones
358 : first; a post-start one lends its core's threads (qi isolated, qn
359 : not) to startup tiles, which never run beside it. */
360 :
361 : static int
362 : plan_fits( ulong cnt[ static 7 ],
363 : ulong const dem[ static 9 ],
364 : ulong left,
365 : ulong qi,
366 0 : ulong qn ) {
367 0 : static uchar const core_ty [ 5 ] = { CT_I1, CT_II, CT_N1, CT_NI, CT_NN };
368 0 : static uchar const core_iso[ 5 ] = { 0, 1, 2, 0, 1 }; /* threads of CT_NN..CT_I1 */
369 0 : static uchar const core_non[ 5 ] = { 2, 1, 0, 1, 0 };
370 0 : if( left ) {
371 0 : int post = left<=dem[ 6 ];
372 0 : for( ulong t=0UL; t<5UL; t++ ) {
373 0 : ulong ty = core_ty[ t ];
374 0 : if( !cnt[ ty ] ) continue;
375 0 : cnt[ ty ]--;
376 0 : int ok = plan_fits( cnt, dem, left-1UL, qi+(post ? core_iso[ ty ] : 0UL), qn+(post ? core_non[ ty ] : 0UL) );
377 0 : cnt[ ty ]++;
378 0 : if( ok ) return 1;
379 0 : }
380 0 : return 0;
381 0 : }
382 0 : ulong i = cnt[ CT_NI ] + 2UL*cnt[ CT_II ] + cnt[ CT_I1 ] + cnt[ CT_IH ];
383 0 : ulong n = cnt[ CT_NI ] + 2UL*cnt[ CT_NN ] + cnt[ CT_N1 ] + cnt[ CT_NH ];
384 0 : return i+n >=dem[ 1 ]+dem[ 2 ]+dem[ 7 ]+dem[ 8 ] && n >=dem[ 2 ]+dem[ 8 ] &&
385 0 : i+n+qi+qn >=dem[ 1 ]+dem[ 2 ]+dem[ 4 ]+dem[ 5 ] && n+qn>=dem[ 2 ]+dem[ 5 ]+fd_ulong_sat_sub( dem[ 1 ], i );
386 0 : }
387 :
388 : /* First free CPU from cpu_idx in tier iso (whole_core: with a free
389 : sibling) that keeps the plan if plan is set. Tier 2, and a
390 : dedicated or floater whole core idling an isolated sibling, also
391 : need the isolated slots left to cover the pinned tiles left. */
392 :
393 : static ulong
394 : auto_tile_cpu_scan( fd_topo_cpus_t const * cpus,
395 : cpu_bv_t const cpu_assigned[ static cpu_bv_word_cnt ],
396 : ushort const cpu_ordering[ static FD_TILE_MAX ],
397 : ulong cpu_idx,
398 : int iso,
399 : int whole_core,
400 : int is_dedicated,
401 : _Bool skip_ht_pairs,
402 : int plan,
403 0 : ulong const dem[ static 9 ] ) {
404 0 : for( ulong j=cpu_idx; j<cpus->cpu_cnt; j++ ) {
405 0 : ulong cpu_id = cpu_ordering[ j ];
406 0 : ulong sibling = cpus->cpu[ cpu_id ].sibling;
407 0 : if( cpu_bv_test( cpu_assigned, cpu_id ) || iso_mismatch( cpus, cpu_id, iso ) ) continue;
408 0 : if( whole_core && sibling!=ULONG_MAX && cpu_bv_test( cpu_assigned, sibling ) ) continue;
409 0 : if( !plan ) return cpu_id;
410 :
411 0 : ulong cnt[ 7 ];
412 0 : core_type_cnt( cpus, cpu_assigned, cpu_id, whole_core, cnt );
413 0 : if( !plan_fits( cnt, dem, dem[ 0 ]+dem[ 6 ], 0UL, 0UL ) ) continue;
414 0 : int guard = iso==2 || ( ( is_dedicated || iso<0 ) && whole_core && sibling!=ULONG_MAX && cpus->cpu[ sibling ].isolated );
415 0 : if( !guard ) return cpu_id;
416 0 : ulong iso_slots = skip_ht_pairs ? cnt[ CT_NI ]+cnt[ CT_II ]+cnt[ CT_I1 ] : cnt[ CT_NI ]+2UL*cnt[ CT_II ]+cnt[ CT_I1 ]+cnt[ CT_IH ];
417 0 : if( iso_slots>=dem[ 0 ]+dem[ 1 ]+fd_ulong_max( dem[ 3 ]+dem[ 4 ], dem[ 6 ]+dem[ 7 ] ) ) return cpu_id;
418 0 : }
419 0 : return ULONG_MAX;
420 0 : }
421 :
422 : /* In efficient mode pinned tiles take isolated CPUs first and floaters
423 : never do. While the tiles left fit, a CPU is only taken if they
424 : still fit after it, and the optional isolated preference and
425 : dedicated whole core relax rather than abort. Pinned non-critical
426 : tiles first skip all-isolated cores the critical tiles left need. */
427 :
428 : static void
429 : auto_tile_cpu( fd_topo_tile_t * tile,
430 : fd_topo_cpus_t * cpus,
431 : ulong * cpu_idx_p,
432 : cpu_bv_t cpu_assigned[ static cpu_bv_word_cnt ],
433 : ushort const cpu_ordering[ static FD_TILE_MAX ],
434 : _Bool skip_ht_pairs,
435 : ulong * spare_siblings,
436 : int efficient,
437 : ulong dem[ static 9 ],
438 0 : ulong phase ) {
439 0 : ulong cpu_idx = *cpu_idx_p;
440 :
441 0 : ulong cpu_cnt = cpus->cpu_cnt;
442 0 : while( cpu_idx<cpu_cnt && cpu_bv_test( cpu_assigned, cpu_ordering[ cpu_idx ] ) ) cpu_idx++;
443 :
444 : /* Certain tiles are latency and throughput critical and
445 : should not get a HT pair assigned. */
446 0 : uint cls = fd_topob_tile_class( tile->name );
447 0 : int crit = !!( cls & FD_TOPOB_TILE_SMT_CRITICAL );
448 0 : int pinned = !!( cls & FD_TOPOB_TILE_EFFICIENT_PINNED );
449 0 : ulong cnt[ 7 ];
450 0 : core_type_cnt( cpus, cpu_assigned, ULONG_MAX, 0, cnt );
451 0 : int plan = efficient && plan_fits( cnt, dem, dem[ 0 ]+dem[ 6 ], 0UL, 0UL );
452 0 : dem[ phase*3UL + ( crit ? 0UL : pinned ? 1UL : 2UL ) ]--;
453 0 : int iso_hi = efficient ? ( pinned ? 1+( crit && plan ) : -1 ) : 0;
454 0 : int iso_lo = efficient && !pinned ? -1 : 0;
455 0 : ulong crit_cores = cnt[ CT_II ]+cnt[ CT_I1 ];
456 0 : if( plan && pinned && !crit && crit_cores && crit_cores<=dem[ 0 ]+dem[ 3 ]+dem[ 6 ] &&
457 0 : cnt[ CT_NI ]+cnt[ CT_IH ]>=dem[ 1 ]+fd_ulong_max( dem[ 4 ], dem[ 7 ] )+1UL ) iso_hi = 3;
458 :
459 0 : ulong cpu_id = ULONG_MAX;
460 0 : int whole_core = 0;
461 0 : int tier_free = 0;
462 0 : for( int iso=iso_hi; iso>=iso_lo && cpu_id==ULONG_MAX; iso-=1+( iso==3 ) ) {
463 0 : ulong first = cpu_idx;
464 0 : while( first<cpu_cnt && ( cpu_bv_test( cpu_assigned, cpu_ordering[ first ] ) || iso_mismatch( cpus, cpu_ordering[ first ], iso ) ) ) first++;
465 0 : tier_free = first<cpu_cnt;
466 0 : if( !tier_free ) continue;
467 :
468 0 : int is_dedicated = !crit && *spare_siblings && ( cls & FD_TOPOB_TILE_DEDICATED ) && cpus->cpu[ cpu_ordering[ first ] ].sibling!=ULONG_MAX;
469 0 : whole_core = crit || is_dedicated || skip_ht_pairs;
470 0 : if( whole_core ) {
471 0 : cpu_id = auto_tile_cpu_scan( cpus, cpu_assigned, cpu_ordering, cpu_idx, iso, 1, is_dedicated, skip_ht_pairs, plan, dem );
472 0 : if( cpu_id!=ULONG_MAX ) { *spare_siblings -= (ulong)is_dedicated; break; }
473 0 : if( !is_dedicated ) continue;
474 0 : *spare_siblings = 0UL; /* spare logical CPUs can be half cores */
475 0 : whole_core = 0;
476 0 : }
477 0 : cpu_id = auto_tile_cpu_scan( cpus, cpu_assigned, cpu_ordering, cpu_idx, iso, 0, 0, skip_ht_pairs, plan, dem );
478 0 : }
479 0 : if( FD_UNLIKELY( cpu_id==ULONG_MAX ) ) {
480 0 : char note[ 128 ];
481 0 : if( tier_free ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned%s", tile->name, tile->kind_id, offline_note( cpus, note ) ));
482 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
483 0 : iso_lo ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
484 0 : }
485 :
486 0 : cpu_bv_insert( cpu_assigned, cpu_id );
487 0 : if( whole_core && cpus->cpu[ cpu_id ].sibling!=ULONG_MAX ) cpu_bv_insert( cpu_assigned, cpus->cpu[ cpu_id ].sibling );
488 0 : tile->cpu_idx = cpu_id;
489 :
490 0 : *cpu_idx_p = cpu_idx;
491 0 : }
492 :
493 : static ulong
494 : l3_free_core_cnt( fd_topo_cpus_t const * cpus,
495 : cpu_bv_t const cpu_assigned[ static cpu_bv_word_cnt ],
496 : ulong l3_idx,
497 0 : int iso ) {
498 0 : ulong cnt = 0UL;
499 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
500 0 : fd_topo_cpu_t const * cpu = &cpus->cpu[ i ];
501 0 : if( cpu->l3_idx!=l3_idx || cpu_bv_test( cpu_assigned, i ) || iso_mismatch( cpus, i, iso ) ) continue;
502 0 : if( cpu->sibling!=ULONG_MAX && ( cpu_bv_test( cpu_assigned, cpu->sibling ) ||
503 0 : ( cpu->sibling<i && !iso_mismatch( cpus, cpu->sibling, iso ) ) ) ) continue; /* core counted once */
504 0 : cnt++;
505 0 : }
506 0 : return cnt;
507 0 : }
508 :
509 : /* Place a hot tile on the first free physical core of L3 domain
510 : *hot_l3, sibling left empty. When it is full, the domain with the
511 : most free cores becomes the hot one. The l3_layout gate leaves a
512 : whole core for every tile, so this never needs the plan. */
513 :
514 : static void
515 : auto_tile_cpu_l3( fd_topo_tile_t * tile,
516 : fd_topo_cpus_t * cpus,
517 : ulong * hot_l3,
518 : cpu_bv_t cpu_assigned[ static cpu_bv_word_cnt ],
519 : ushort const cpu_ordering[ static FD_TILE_MAX ],
520 0 : ulong dem[ static 9 ] ) {
521 0 : uint cls = fd_topob_tile_class( tile->name );
522 0 : int pinned = !!( cls & FD_TOPOB_TILE_EFFICIENT_PINNED );
523 0 : dem[ ( ( cls & FD_TOPOB_TILE_POSTSTART ) ? 6UL : 0UL ) + ( pinned ? 1UL : 2UL ) ]--; /* hot tiles are not critical */
524 0 : int iso = pinned ? 1 : -1;
525 0 : retry:
526 0 : if( *hot_l3==ULONG_MAX || !l3_free_core_cnt( cpus, cpu_assigned, *hot_l3, iso ) ) {
527 0 : ulong best_cnt = 0UL;
528 0 : for( ulong l3=0UL; l3<cpus->l3_cnt; l3++ ) {
529 0 : ulong cnt = l3_free_core_cnt( cpus, cpu_assigned, l3, iso );
530 0 : if( cnt>best_cnt ) { best_cnt = cnt; *hot_l3 = l3; }
531 0 : }
532 0 : if( !best_cnt && iso>0 ) { iso = 0; goto retry; }
533 0 : char note[ 128 ];
534 0 : if( FD_UNLIKELY( !best_cnt ) ) FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned or have a HT pair assigned%s", tile->name, tile->kind_id, offline_note( cpus, note ) ));
535 0 : }
536 :
537 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
538 0 : ulong cpu_idx = cpu_ordering[ i ];
539 0 : fd_topo_cpu_t const * cpu = &cpus->cpu[ cpu_idx ];
540 0 : if( cpu->l3_idx!=*hot_l3 || cpu_bv_test( cpu_assigned, cpu_idx ) || iso_mismatch( cpus, cpu_idx, iso ) ) continue;
541 0 : if( cpu->sibling!=ULONG_MAX && cpu_bv_test( cpu_assigned, cpu->sibling ) ) continue;
542 0 : cpu_bv_insert( cpu_assigned, cpu_idx );
543 0 : if( cpu->sibling!=ULONG_MAX ) cpu_bv_insert( cpu_assigned, cpu->sibling );
544 0 : tile->cpu_idx = cpu_idx;
545 0 : return;
546 0 : }
547 0 : FD_LOG_ERR(( "auto layout found no free core in L3 %lu for tile `%s:%lu`", *hot_l3, tile->name, tile->kind_id ));
548 0 : }
549 :
550 : #undef CT_NN
551 : #undef CT_NI
552 : #undef CT_II
553 : #undef CT_N1
554 : #undef CT_I1
555 : #undef CT_NH
556 : #undef CT_IH
557 :
558 : static ulong
559 : name_idx( char const * name,
560 0 : char const * const * names ) {
561 0 : for( ulong i=0UL; names[ i ]; i++ ) if( !strcmp( name, names[ i ] ) ) return i;
562 0 : return ULONG_MAX;
563 0 : }
564 :
565 : /* Whole core rank of a steady tile, lower first, tier in bits 24+ */
566 :
567 0 : #define SMT_TIER_COLD (7UL)
568 :
569 : static ulong
570 : smt_rank( fd_topo_tile_t const * tile,
571 : ulong execrp_floor,
572 0 : int replay_first ) {
573 0 : static char const * const T4[] = { "votor", "shred", "net", "mlx5", "sock", "accdb", NULL };
574 0 : static char const * const T5[] = { "rotor", NULL };
575 0 : static char const * const T6[] = { "execle", "dedup", "sign", "resolv", "quic", "verify", "bundle", "shred", "net", "mlx5", "sock", NULL };
576 0 : static char const * const T7[] = { "gui", "rpc", "event", "rserve", "gossip", "gossvf", NULL };
577 :
578 0 : char const * name = tile->name;
579 0 : ulong k = tile->kind_id;
580 0 : if( fd_topob_tile_class( name ) & FD_TOPOB_TILE_SMT_CRITICAL ) return k;
581 0 : if( !strcmp( name, "replay" ) ) return replay_first ? 1UL<<24 : 2UL<<24;
582 0 : if( !strcmp( name, "execrp" ) ) return ( k>=execrp_floor ? 3UL<<24 : replay_first ? 2UL<<24 : 1UL<<24 ) | k;
583 0 : if( !strcmp( name, "sign" ) && k ) return (SMT_TIER_COLD<<24) | (4095UL<<12) | k;
584 0 : ulong i;
585 0 : if( !k && (i=name_idx( name, T4 ))!=ULONG_MAX ) return (4UL<<24) | (i<<12);
586 0 : if( (i=name_idx( name, T5 ))!=ULONG_MAX ) return (5UL<<24) | (i<<12) | k;
587 0 : if( (i=name_idx( name, T6 ))!=ULONG_MAX ) return (6UL<<24) | (i<<12) | k;
588 0 : if( (i=name_idx( name, T7 ))!=ULONG_MAX ) return (SMT_TIER_COLD<<24) | (i<<12) | k;
589 0 : return (6UL<<24) | (4094UL<<12) | k;
590 0 : }
591 :
592 0 : #define SMT_WHOLE (0)
593 0 : #define SMT_HALF (1)
594 0 : #define SMT_PAIR (2)
595 :
596 : /* Shares only as many cores as needed, lowest ranked tiles first.
597 : Returns 0 if the tiles do not fit or a critical tile would pair. */
598 :
599 : static int
600 : smt_plan( fd_topo_t const * topo,
601 : fd_topo_cpus_t const * cpus,
602 : cpu_bv_t const assigned[ static cpu_bv_word_cnt ],
603 : ulong const * tile_idx,
604 : ulong cnt,
605 : ulong execrp_floor,
606 : int efficient,
607 : uchar * mode,
608 0 : ulong * partner ) {
609 0 : static char const * const AVX[] = { "verify", "sign", "shred", "benchg", NULL };
610 :
611 0 : ulong p2 = 0UL, p1 = 0UL, half = 0UL;
612 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
613 0 : if( cpu_bv_test( assigned, i ) ) continue;
614 0 : ulong s = cpus->cpu[ i ].sibling;
615 0 : if( s==ULONG_MAX ) p1++;
616 0 : else if( !cpu_bv_test( assigned, s ) ) p2 += (ulong)( s>i );
617 0 : else half++;
618 0 : }
619 0 : ulong h = fd_ulong_min( half, fd_ulong_sat_sub( cnt, p2+p1 ) );
620 0 : ulong d = fd_ulong_sat_sub( cnt, p2+p1+half );
621 0 : if( FD_UNLIKELY( d>p2 ) ) return 0;
622 0 : ulong whole = cnt - h - 2UL*d;
623 :
624 0 : ulong top = 0UL;
625 0 : for( ulong i=0UL; i<cnt; i++ ) {
626 0 : fd_topo_tile_t const * tile = &topo->tiles[ tile_idx[ i ] ];
627 0 : top += (ulong)( ( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_SMT_CRITICAL ) || !strcmp( tile->name, "replay" ) ||
628 0 : ( !strcmp( tile->name, "execrp" ) && tile->kind_id<execrp_floor ) );
629 0 : }
630 0 : int replay_first = whole>=top;
631 :
632 0 : ulong order[ FD_TOPO_MAX_TILES ];
633 0 : ulong key [ FD_TOPO_MAX_TILES ];
634 0 : FD_TEST( cnt<=FD_TOPO_MAX_TILES );
635 0 : for( ulong i=0UL; i<cnt; i++ ) {
636 0 : ulong k = smt_rank( &topo->tiles[ tile_idx[ i ] ], execrp_floor, replay_first );
637 0 : ulong j = i;
638 0 : for( ; j && key[ j-1UL ]>k; j-- ) { key[ j ] = key[ j-1UL ]; order[ j ] = order[ j-1UL ]; }
639 0 : key[ j ] = k; order[ j ] = i;
640 0 : }
641 0 : for( ulong j=whole; j<cnt; j++ ) {
642 0 : if( FD_UNLIKELY( !(key[ j ]>>24) ) ) return 0;
643 0 : }
644 :
645 0 : for( ulong j=0UL; j<cnt; j++ ) {
646 0 : mode [ order[ j ] ] = (uchar)( j<whole ? SMT_WHOLE : j<whole+h ? SMT_HALF : SMT_PAIR );
647 0 : partner[ order[ j ] ] = ULONG_MAX;
648 0 : }
649 :
650 0 : for( ulong pass=0UL; pass<6UL; pass++ ) {
651 0 : for( ulong a=whole+h; a<cnt; a++ ) {
652 0 : if( partner[ order[ a ] ]!=ULONG_MAX ) continue;
653 0 : fd_topo_tile_t const * ta = &topo->tiles[ tile_idx[ order[ a ] ] ];
654 0 : for( ulong b=cnt-1UL; b>a; b-- ) {
655 0 : if( partner[ order[ b ] ]!=ULONG_MAX ) continue;
656 0 : fd_topo_tile_t const * tb = &topo->tiles[ tile_idx[ order[ b ] ] ];
657 0 : int hot_a = (key[ a ]>>24)<=4UL && !( fd_topob_tile_class( ta->name ) & FD_TOPOB_TILE_DEDICATED );
658 0 : int cold_a = (key[ a ]>>24)==SMT_TIER_COLD;
659 0 : int cold_b = (key[ b ]>>24)==SMT_TIER_COLD;
660 0 : int same = !strcmp( ta->name, tb->name );
661 0 : int avx = !same && name_idx( ta->name, AVX )!=ULONG_MAX && name_idx( tb->name, AVX )!=ULONG_MAX;
662 0 : int pin_a = !!( fd_topob_tile_class( ta->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
663 0 : int pin_b = !!( fd_topob_tile_class( tb->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
664 0 : int strict = !avx && ( !efficient || pin_a==pin_b );
665 0 : int ok;
666 0 : switch( pass ) { /* a hot tile that must share gets a cold sibling */
667 0 : case 0UL: ok = strict && hot_a && cold_b; break;
668 0 : case 1UL: ok = strict && cold_a && cold_b; break;
669 0 : case 2UL: ok = strict && same; break;
670 0 : case 3UL: ok = strict && cold_b; break;
671 0 : case 4UL: ok = strict; break;
672 0 : default: ok = 1; break;
673 0 : }
674 0 : if( !ok ) continue;
675 0 : if( FD_UNLIKELY( pass==5UL && avx ) ) FD_LOG_WARNING(( "auto layout is short of physical cores and puts AVX heavy tile `%s:%lu` on the hyperthread sibling of tile `%s:%lu`",
676 0 : tb->name, tb->kind_id, ta->name, ta->kind_id ));
677 0 : partner[ order[ a ] ] = order[ b ];
678 0 : partner[ order[ b ] ] = order[ a ];
679 0 : break;
680 0 : }
681 0 : }
682 0 : }
683 0 : return 1;
684 0 : }
685 :
686 : /* Places plan entries [lo,hi) in CPU order */
687 :
688 : static void
689 : smt_place( fd_topo_t * topo,
690 : fd_topo_cpus_t const * cpus,
691 : cpu_bv_t assigned[ static cpu_bv_word_cnt ],
692 : ushort const cpu_ordering[ static FD_TILE_MAX ],
693 : ulong const * tile_idx,
694 : ulong lo,
695 : ulong hi,
696 : ulong cnt,
697 : uchar * mode,
698 : ulong * partner,
699 0 : int efficient ) {
700 0 : ulong p2_free = 0UL;
701 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
702 0 : ulong s = cpus->cpu[ i ].sibling;
703 0 : p2_free += (ulong)( s!=ULONG_MAX && s>i && !cpu_bv_test( assigned, i ) && !cpu_bv_test( assigned, s ) );
704 0 : }
705 0 : ulong pairs_left = 0UL;
706 0 : for( ulong i=0UL; i<cnt; i++ ) pairs_left += (ulong)( mode[ i ]==SMT_PAIR && partner[ i ]>i && partner[ i ]!=ULONG_MAX &&
707 0 : topo->tiles[ tile_idx[ i ] ].cpu_idx==ULONG_MAX );
708 :
709 0 : for( ulong i=lo; i<hi; i++ ) {
710 0 : fd_topo_tile_t * tile = &topo->tiles[ tile_idx[ i ] ];
711 0 : if( tile->cpu_idx!=ULONG_MAX ) continue;
712 0 : int avoid = efficient && !( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
713 0 : int pair = mode[ i ]==SMT_PAIR && partner[ i ]!=ULONG_MAX;
714 0 : fd_topo_tile_t * mate = pair ? &topo->tiles[ tile_idx[ partner[ i ] ] ] : NULL;
715 0 : int mate_avoid = pair && efficient && !( fd_topob_tile_class( mate->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
716 :
717 0 : ulong pick = ULONG_MAX;
718 0 : ulong any = ULONG_MAX;
719 0 : for( ulong j=0UL; j<cpus->cpu_cnt && pick==ULONG_MAX; j++ ) {
720 0 : ulong c = cpu_ordering[ j ];
721 0 : fd_topo_cpu_t const * cpu = &cpus->cpu[ c ];
722 0 : if( cpu_bv_test( assigned, c ) || ( avoid && cpu->isolated ) ) continue;
723 0 : if( any==ULONG_MAX ) any = c;
724 0 : ulong s = cpu->sibling;
725 0 : int sib_free = s!=ULONG_MAX && !cpu_bv_test( assigned, s );
726 0 : int ok;
727 0 : if( pair ) ok = sib_free && !( mate_avoid && cpus->cpu[ s ].isolated );
728 0 : else if( mode[ i ]==SMT_HALF ) ok = s!=ULONG_MAX && !sib_free;
729 0 : else ok = s==ULONG_MAX || ( sib_free && p2_free>pairs_left );
730 0 : if( ok ) pick = c;
731 0 : }
732 :
733 0 : if( FD_UNLIKELY( pick==ULONG_MAX ) ) {
734 0 : if( FD_UNLIKELY( any==ULONG_MAX ) ) {
735 0 : char note[ 128 ];
736 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
737 0 : avoid ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
738 0 : }
739 0 : if( pair ) { /* the mate is placed on its own */
740 0 : mode [ partner[ i ] ] = SMT_HALF;
741 0 : partner[ partner[ i ] ] = ULONG_MAX;
742 0 : pairs_left--;
743 0 : }
744 0 : pick = any;
745 0 : tile->cpu_idx = pick;
746 0 : cpu_bv_insert( assigned, pick );
747 0 : ulong s = cpus->cpu[ pick ].sibling;
748 0 : if( s!=ULONG_MAX && !cpu_bv_test( assigned, s ) ) p2_free--;
749 0 : continue;
750 0 : }
751 :
752 0 : ulong s = cpus->cpu[ pick ].sibling;
753 0 : tile->cpu_idx = pick;
754 0 : cpu_bv_insert( assigned, pick );
755 0 : if( mode[ i ]==SMT_HALF || s==ULONG_MAX ) continue;
756 0 : cpu_bv_insert( assigned, s );
757 0 : p2_free--;
758 0 : if( pair ) { mate->cpu_idx = s; pairs_left--; }
759 0 : }
760 0 : }
761 :
762 : /* Whole cores free during snapshot load for startup tiles, snapin
763 : first, preferring execrp's cores */
764 :
765 : static void
766 : auto_startup_cpus( fd_topo_t * topo,
767 : fd_topo_cpus_t const * cpus,
768 : cpu_bv_t snap[ static cpu_bv_word_cnt ],
769 : ushort const cpu_ordering[ static FD_TILE_MAX ],
770 0 : int efficient ) {
771 0 : static char const * const ORDER[] = { "snapin", "snapdc", "snapld", "snapct", NULL };
772 :
773 0 : ulong idx[ FD_TOPO_MAX_TILES ];
774 0 : ulong cnt = 0UL;
775 0 : for( ulong r=0UL; r<4UL+TILE_CLASS_CNT; r++ ) {
776 0 : char const * name = r<4UL ? ORDER[ r ] : TILE_CLASS[ r-4UL ].name;
777 0 : if( r>=4UL && ( !( TILE_CLASS[ r-4UL ].flags & FD_TOPOB_TILE_STARTUP ) || name_idx( name, ORDER )!=ULONG_MAX ) ) continue;
778 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
779 0 : if( tile_is_floating( &topo->tiles[ j ] ) || strcmp( topo->tiles[ j ].name, name ) ) continue;
780 0 : idx[ cnt++ ] = j;
781 0 : }
782 0 : }
783 0 : if( !cnt ) return;
784 :
785 0 : cpu_bv_t banned[ cpu_bv_word_cnt ]; cpu_bv_new( banned );
786 0 : cpu_bv_t execrp[ cpu_bv_word_cnt ]; cpu_bv_new( execrp );
787 0 : for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) cpu_bv_insert( banned, topo->blocklist_cores_cpu_idx[ i ] );
788 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) if( !cpus->cpu[ i ].online ) cpu_bv_insert( banned, i );
789 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
790 0 : fd_topo_tile_t const * tile = &topo->tiles[ j ];
791 0 : ulong c = tile->cpu_idx;
792 0 : if( c>=cpus->cpu_cnt ) continue;
793 0 : uint cls = fd_topob_tile_class( tile->name );
794 0 : ulong s = cpus->cpu[ c ].sibling;
795 0 : if( !strcmp( tile->name, "execrp" ) ) { cpu_bv_insert( execrp, c ); if( s!=ULONG_MAX ) cpu_bv_insert( execrp, s ); }
796 0 : if( !( cls & FD_TOPOB_TILE_ALWAYS ) ) continue;
797 0 : cpu_bv_insert( banned, c );
798 0 : if( ( cls & FD_TOPOB_TILE_SMT_CRITICAL ) && s!=ULONG_MAX ) cpu_bv_insert( banned, s );
799 0 : }
800 :
801 0 : ulong free_cnt = 0UL;
802 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_cnt += (ulong)!cpu_bv_test( snap, i );
803 0 : ulong spare = fd_ulong_sat_sub( free_cnt, cnt ); /* siblings that can idle */
804 :
805 0 : for( ulong k=0UL; k<cnt; k++ ) {
806 0 : fd_topo_tile_t * tile = &topo->tiles[ idx[ k ] ];
807 0 : int avoid = efficient && !( fd_topob_tile_class( tile->name ) & FD_TOPOB_TILE_EFFICIENT_PINNED );
808 0 : ulong pick = ULONG_MAX;
809 0 : for( ulong pass=0UL; pass<4UL && pick==ULONG_MAX; pass++ ) {
810 0 : for( ulong j=0UL; j<cpus->cpu_cnt && pick==ULONG_MAX; j++ ) {
811 0 : ulong c = cpu_ordering[ j ];
812 0 : ulong s = cpus->cpu[ c ].sibling;
813 0 : if( avoid && cpus->cpu[ c ].isolated ) continue;
814 0 : switch( pass ) {
815 0 : case 0UL:
816 0 : case 1UL: if( cpu_bv_test( snap, c ) || ( s!=ULONG_MAX && ( cpu_bv_test( snap, s ) || !spare ) ) || ( !pass && !cpu_bv_test( execrp, c ) ) ) continue; break;
817 0 : case 2UL: if( cpu_bv_test( snap, c ) ) continue; break;
818 0 : default: if( cpu_bv_test( banned, c ) ) continue; break;
819 0 : }
820 0 : pick = c;
821 0 : if( pass<2UL && s!=ULONG_MAX ) { cpu_bv_insert( snap, s ); spare--; }
822 0 : }
823 0 : }
824 0 : if( FD_UNLIKELY( pick==ULONG_MAX ) ) {
825 0 : char note[ 128 ];
826 0 : FD_LOG_ERR(( "auto layout cannot set affinity for tile `%s:%lu` because all the CPUs are already assigned%s%s", tile->name, tile->kind_id,
827 0 : avoid ? " or isolated by isolcpus=" : "", offline_note( cpus, note ) ));
828 0 : }
829 0 : tile->cpu_idx = pick;
830 0 : cpu_bv_insert( snap, pick );
831 0 : cpu_bv_insert( banned, pick );
832 0 : }
833 0 : }
834 :
835 : void
836 : fd_topob_auto_layout_cpus( fd_topo_t * topo,
837 0 : fd_topo_cpus_t * cpus ) {
838 : /* See the strategy in fd_topob_layout.h */
839 :
840 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
841 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
842 0 : tile->cpu_idx = ULONG_MAX;
843 0 : }
844 :
845 0 : ushort cpu_ordering[ FD_TILE_MAX ] = {0};
846 0 : cpu_bv_t pairs_assigned[ cpu_bv_word_cnt ]; cpu_bv_new( pairs_assigned );
847 0 : FD_STATIC_ASSERT( FD_TILE_MAX<=USHORT_MAX, layout );
848 :
849 0 : ulong next_cpu_idx = 0UL;
850 0 : for( ulong i=0UL; i<cpus->numa_node_cnt; i++ ) {
851 0 : for( ulong j=0UL; j<cpus->cpu_cnt; j++ ) {
852 0 : fd_topo_cpu_t * cpu = &cpus->cpu[ j ];
853 :
854 0 : if( FD_UNLIKELY( cpu_bv_test( pairs_assigned, j ) || cpu->numa_node!=i ) ) continue;
855 :
856 0 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
857 0 : cpu_ordering[ next_cpu_idx++ ] = (ushort)j;
858 :
859 0 : if( FD_UNLIKELY( cpu->sibling!=ULONG_MAX ) ) {
860 : /* If the CPU has a HT pair, place it immediately after so they
861 : are sequentially assigned. */
862 0 : FD_TEST( next_cpu_idx<FD_TILE_MAX );
863 0 : cpu_ordering[ next_cpu_idx++ ] = (ushort)cpu->sibling;
864 0 : cpu_bv_insert( pairs_assigned, cpu->sibling );
865 0 : }
866 0 : }
867 0 : }
868 :
869 0 : FD_TEST( next_cpu_idx==cpus->cpu_cnt );
870 :
871 : /* excluded cpus are simply considered already assigned */
872 0 : cpu_bv_t cpu_assigned[ cpu_bv_word_cnt ];
873 0 : cpu_bv_new( cpu_assigned );
874 0 : for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) {
875 0 : FD_TEST( topo->blocklist_cores_cpu_idx[ i ]<FD_TILE_MAX );
876 0 : cpu_bv_insert( cpu_assigned, topo->blocklist_cores_cpu_idx[ i ] );
877 0 : }
878 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) if( !cpus->cpu[ i ].online ) cpu_bv_insert( cpu_assigned, i );
879 :
880 : /* Compute total number of available physical cores */
881 0 : ulong available_physical = 0UL;
882 0 : ulong available_floater = 0UL; /* no isolated thread */
883 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
884 0 : ulong sibling = cpus->cpu[ i ].sibling;
885 0 : if( !cpu_bv_test( cpu_assigned, i ) &&
886 0 : !cpu_bv_test( pairs_assigned, i ) &&
887 0 : ( sibling==ULONG_MAX ||
888 0 : !cpu_bv_test( cpu_assigned, sibling ) ) ) {
889 0 : available_physical++;
890 0 : available_floater += (ulong)( !cpus->cpu[ i ].isolated && ( sibling==ULONG_MAX || !cpus->cpu[ sibling ].isolated ) );
891 0 : }
892 0 : }
893 :
894 : /* Tiles that need assignment, dem[ phase*3+kind ] by phase (always,
895 : startup, post-start) and kind (SMT critical, other pinned,
896 : efficient mode floater, which skips isolated CPUs) */
897 0 : ulong dem[ 9 ] = {0};
898 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
899 0 : fd_topo_tile_t const * tile = &topo->tiles[ j ];
900 0 : uint cls = fd_topob_tile_class( tile->name );
901 0 : if( !cls || tile_is_floating( tile ) ) continue;
902 0 : ulong phase = ( cls & FD_TOPOB_TILE_STARTUP ) ? 1UL : ( cls & FD_TOPOB_TILE_POSTSTART ) ? 2UL : 0UL;
903 0 : ulong kind = ( cls & FD_TOPOB_TILE_SMT_CRITICAL ) ? 0UL : ( cls & FD_TOPOB_TILE_EFFICIENT_PINNED ) ? 1UL : 2UL;
904 0 : dem[ phase*3UL+kind ]++;
905 0 : }
906 0 : FD_TEST( !dem[ 3 ] ); /* plan_fits has no SMT critical startup tile */
907 0 : ulong always_tiles_to_assign = dem[ 0 ]+dem[ 1 ]+dem[ 2 ];
908 0 : ulong startup_tiles_to_assign = dem[ 3 ]+dem[ 4 ]+dem[ 5 ];
909 0 : ulong post_start_tiles_to_assign = dem[ 6 ]+dem[ 7 ]+dem[ 8 ];
910 0 : ulong tiles_to_assign = always_tiles_to_assign +
911 0 : fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign );
912 0 : ulong floaters_to_assign = dem[ 2 ] + fd_ulong_max( dem[ 5 ], dem[ 8 ] );
913 :
914 : /* L3 packing gives the hot post-start tiles their own cores instead
915 : of sharing with startup tiles, so it needs that many more.
916 : Performance mode, single-L3 hosts, hosts with a partial cache
917 : topology and hosts too small for it keep the sequential layout. */
918 0 : ulong hot_post_start_tiles = 0UL;
919 0 : ulong hot_post_start_floaters = 0UL;
920 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
921 0 : fd_topo_tile_t const * tile = &topo->tiles[ j ];
922 0 : if( tile_is_floating( tile ) ) continue;
923 0 : uint cls = fd_topob_tile_class( tile->name );
924 0 : int hot = ( cls & FD_TOPOB_TILE_POSTSTART ) && tile_name_in( tile->name, HOT_TILES );
925 0 : hot_post_start_tiles += (ulong)hot;
926 0 : hot_post_start_floaters += (ulong)( hot && !( cls & FD_TOPOB_TILE_EFFICIENT_PINNED ) );
927 0 : }
928 0 : ulong l3_tiles_to_assign = always_tiles_to_assign + hot_post_start_tiles +
929 0 : fd_ulong_max( startup_tiles_to_assign, post_start_tiles_to_assign-hot_post_start_tiles );
930 0 : ulong l3_floaters_to_assign = dem[ 2 ] + hot_post_start_floaters +
931 0 : fd_ulong_max( dem[ 5 ], dem[ 8 ]-hot_post_start_floaters );
932 0 : int efficient = topo->sleep_obj_id!=ULONG_MAX;
933 0 : int l3_layout = efficient && fd_topo_cpus_l3_complete( cpus ) &&
934 0 : available_physical>=l3_tiles_to_assign && available_floater>=l3_floaters_to_assign;
935 :
936 : /* If we have enough physical cores (excluding HT siblings) for all
937 : tiles that need assignment, exclude HT siblings so that no tile
938 : gets scheduled on a hyperthread pair. */
939 0 : _Bool skip_ht_pairs = available_physical>=tiles_to_assign && ( !efficient || available_floater>=floaters_to_assign );
940 :
941 : /* Otherwise spend the spare logical CPUs on whole cores for the
942 : dedicated tiles. Each costs its sibling, like a critical tile.
943 : The L3 layout already places them on whole cores. */
944 0 : ulong spare_siblings = 0UL;
945 0 : if( !skip_ht_pairs && !l3_layout ) {
946 0 : ulong free_logical = 0UL;
947 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_logical += !cpu_bv_test( cpu_assigned, i );
948 0 : ulong critical_cnt = 0UL;
949 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) critical_cnt += (ulong)!!( fd_topob_tile_class( topo->tiles[ j ].name ) & FD_TOPOB_TILE_SMT_CRITICAL );
950 0 : spare_siblings = fd_ulong_sat_sub( free_logical, tiles_to_assign+critical_cnt );
951 0 : }
952 :
953 : /* Short of physical cores: share only as many as needed, by rank */
954 0 : ulong steady_idx [ FD_TOPO_MAX_TILES ];
955 0 : uchar steady_mode [ FD_TOPO_MAX_TILES ];
956 0 : ulong steady_partner[ FD_TOPO_MAX_TILES ];
957 0 : ulong steady_cnt = 0UL;
958 0 : ulong always_cnt = 0UL;
959 0 : int graded = !skip_ht_pairs && !l3_layout;
960 : /* Efficient mode floaters avoid isolcpus=, which the plan ignores */
961 0 : for( ulong i=0UL; i<cpus->cpu_cnt && graded && efficient; i++ ) graded = cpu_bv_test( cpu_assigned, i ) || !cpus->cpu[ i ].isolated;
962 0 : cpu_bv_t numa_fence[ cpu_bv_word_cnt ]; cpu_bv_new( numa_fence );
963 :
964 : /* Pack, motor, accdb and execle go first into one L3 with the cores the
965 : plan gives them, pairing execle to fit and spilling the highest to
966 : the next L3. The rest is planned around them, or without them if
967 : it then does not fit. */
968 0 : static char const * const GROUP[] = { "pack", "motor", "accdb", "execle", NULL };
969 0 : int group = !efficient && fd_topo_cpus_l3_complete( cpus );
970 0 : cpu_bv_t base [ cpu_bv_word_cnt ]; cpu_bv_copy( base, cpu_assigned );
971 0 : cpu_bv_t group_post[ cpu_bv_word_cnt ]; cpu_bv_new( group_post ); /* free during snapshot load */
972 0 : for( ulong attempt=0UL;; attempt++ ) {
973 0 : if( graded ) {
974 0 : steady_cnt = 0UL;
975 0 : for( ulong r=0UL; r<TILE_CLASS_CNT; r++ ) {
976 0 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_ALWAYS ) ) continue;
977 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
978 0 : if( tile_is_floating( &topo->tiles[ j ] ) || topo->tiles[ j ].cpu_idx!=ULONG_MAX || strcmp( topo->tiles[ j ].name, TILE_CLASS[ r ].name ) ) continue;
979 0 : steady_idx[ steady_cnt++ ] = j;
980 0 : }
981 0 : }
982 0 : always_cnt = steady_cnt;
983 0 : for( ulong r=0UL; r<TILE_CLASS_CNT; r++ ) {
984 0 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_POSTSTART ) ) continue;
985 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
986 0 : if( topo->tiles[ j ].cpu_idx!=ULONG_MAX || strcmp( topo->tiles[ j ].name, TILE_CLASS[ r ].name ) ) continue;
987 0 : steady_idx[ steady_cnt++ ] = j;
988 0 : }
989 0 : }
990 : /* Fill the first NUMA nodes that fit before crossing to the next */
991 0 : ulong need = steady_cnt;
992 0 : for( ulong i=0UL; i<steady_cnt; i++ ) need += (ulong)!!( fd_topob_tile_class( topo->tiles[ steady_idx[ i ] ].name ) & FD_TOPOB_TILE_SMT_CRITICAL );
993 0 : ulong node_cnt = 0UL;
994 0 : ulong free_cnt = 0UL;
995 0 : for( ; node_cnt<cpus->numa_node_cnt && free_cnt<need; node_cnt++ ) {
996 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) free_cnt += (ulong)( cpus->cpu[ i ].numa_node==node_cnt && !cpu_bv_test( cpu_assigned, i ) );
997 0 : }
998 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
999 0 : if( cpus->cpu[ i ].numa_node>=node_cnt && !cpu_bv_test( cpu_assigned, i ) ) cpu_bv_insert( numa_fence, i );
1000 0 : }
1001 0 : cpu_bv_union( cpu_assigned, cpu_assigned, numa_fence );
1002 0 : graded = smt_plan( topo, cpus, cpu_assigned, steady_idx, steady_cnt, spare_siblings, efficient, steady_mode, steady_partner );
1003 0 : if( !graded ) {
1004 0 : cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
1005 0 : cpu_bv_new( numa_fence );
1006 0 : graded = smt_plan( topo, cpus, cpu_assigned, steady_idx, steady_cnt, spare_siblings, efficient, steady_mode, steady_partner );
1007 0 : }
1008 0 : }
1009 0 : if( attempt==1UL && !graded ) {
1010 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) topo->tiles[ j ].cpu_idx = ULONG_MAX;
1011 0 : cpu_bv_copy( cpu_assigned, base );
1012 0 : cpu_bv_new( numa_fence );
1013 0 : cpu_bv_new( group_post );
1014 0 : graded = 1;
1015 0 : continue;
1016 0 : }
1017 0 : if( attempt || !group || !( graded || skip_ht_pairs ) ) break;
1018 :
1019 0 : ulong lead = 0UL, ex = 0UL, ew = 0UL; /* cores of pack, motor and accdb, execle, execle on whole cores */
1020 0 : for( ulong i=0UL; i<( graded ? steady_cnt : topo->tile_cnt ); i++ ) {
1021 0 : ulong g = name_idx( topo->tiles[ graded ? steady_idx[ i ] : i ].name, GROUP );
1022 0 : lead += (ulong)( g<3UL );
1023 0 : ex += (ulong)( g==3UL );
1024 0 : ew += (ulong)( g==3UL && ( !graded || steady_mode[ i ]==SMT_WHOLE ) );
1025 0 : }
1026 : /* First L3 in CPU order that fits, else fits execle paired, else the largest */
1027 0 : ulong l3 = ULONG_MAX, l3_cores = 0UL, best = 0UL;
1028 0 : for( ulong i=0UL; i<cpus->cpu_cnt; i++ ) {
1029 0 : ulong k = cpus->cpu[ cpu_ordering[ i ] ].l3_idx;
1030 0 : if( k==ULONG_MAX || ( i && k==cpus->cpu[ cpu_ordering[ i-1UL ] ].l3_idx ) ) continue;
1031 0 : ulong c = l3_free_core_cnt( cpus, cpu_assigned, k, 0 );
1032 0 : ulong score = c>=lead+ew+(ex-ew+1UL)/2UL ? ULONG_MAX : c>=lead+(ex+1UL)/2UL ? ULONG_MAX-1UL : c;
1033 0 : if( score>best ) { best = score; l3 = k; l3_cores = c; }
1034 0 : }
1035 0 : while( ew && lead+ew+(ex-ew+1UL)/2UL>l3_cores ) ew--; /* the lowest execle stay whole */
1036 0 : ulong e = 0UL;
1037 0 : for( ulong g=0UL; GROUP[ g ] && l3!=ULONG_MAX; g++ ) {
1038 0 : for( ulong j=0UL; j<topo->tile_cnt && l3!=ULONG_MAX; j++ ) {
1039 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1040 0 : if( tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, GROUP[ g ] ) ) continue;
1041 0 : ulong c = ULONG_MAX;
1042 0 : for( ulong d=0UL; d<cpus->l3_cnt && c==ULONG_MAX; d++ ) {
1043 0 : for( ulong i=0UL; i<cpus->cpu_cnt && c==ULONG_MAX; i++ ) {
1044 0 : ulong k = cpu_ordering[ i ];
1045 0 : ulong s = cpus->cpu[ k ].sibling;
1046 0 : if( cpus->cpu[ k ].l3_idx==l3 && !cpu_bv_test( cpu_assigned, k ) && ( s==ULONG_MAX || !cpu_bv_test( cpu_assigned, s ) ) ) c = k;
1047 0 : }
1048 0 : if( c==ULONG_MAX ) l3 = (l3+1UL)%cpus->l3_cnt;
1049 0 : }
1050 0 : if( c==ULONG_MAX ) { l3 = ULONG_MAX; break; } /* the rest is placed as usual */
1051 0 : ulong s = cpus->cpu[ c ].sibling;
1052 0 : ulong m = ULONG_MAX; /* tile on the sibling, which goes to the rest if s is cleared */
1053 0 : if( g<3UL ) {
1054 0 : for( ulong i=0UL; i<steady_cnt && graded; i++ ) { /* as planned */
1055 0 : if( steady_idx[ i ]!=j ) continue;
1056 0 : if( steady_mode[ i ]==SMT_PAIR && steady_partner[ i ]!=ULONG_MAX ) m = steady_idx[ steady_partner[ i ] ];
1057 0 : if( steady_mode[ i ]==SMT_HALF ) s = ULONG_MAX;
1058 0 : }
1059 0 : } else if( e++>=ew ) { /* paired with the next execle */
1060 0 : for( ulong k=j+1UL; k<topo->tile_cnt && m==ULONG_MAX; k++ ) if( topo->tiles[ k ].cpu_idx==ULONG_MAX && !strcmp( topo->tiles[ k ].name, "execle" ) ) m = k;
1061 0 : if( m==ULONG_MAX || s==ULONG_MAX ) s = m = ULONG_MAX;
1062 0 : e += (ulong)( m!=ULONG_MAX );
1063 0 : }
1064 0 : tile->cpu_idx = c;
1065 0 : cpu_bv_insert( cpu_assigned, c );
1066 0 : if( g ) cpu_bv_insert( group_post, c );
1067 0 : if( s==ULONG_MAX ) continue;
1068 0 : cpu_bv_insert( cpu_assigned, s );
1069 0 : if( m!=ULONG_MAX ) topo->tiles[ m ].cpu_idx = s;
1070 0 : if( g && ( m==ULONG_MAX || ( fd_topob_tile_class( topo->tiles[ m ].name ) & FD_TOPOB_TILE_POSTSTART ) ) ) cpu_bv_insert( group_post, s );
1071 0 : }
1072 0 : }
1073 0 : if( !graded ) break;
1074 0 : cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
1075 0 : cpu_bv_new( numa_fence );
1076 0 : }
1077 :
1078 0 : if( FD_UNLIKELY( l3_layout ) ) {
1079 0 : ulong hot_l3 = ULONG_MAX;
1080 0 : for( char const ** p = HOT_TILES; *p; p++ ) {
1081 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1082 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1083 0 : if( tile_is_floating( tile ) || strcmp( tile->name, *p ) ) continue;
1084 0 : auto_tile_cpu_l3( tile, cpus, &hot_l3, cpu_assigned, cpu_ordering, dem );
1085 0 : }
1086 0 : }
1087 0 : }
1088 :
1089 : /* First, assign always-on tiles */
1090 0 : ulong cpu_idx = 0UL;
1091 0 : if( graded ) smt_place( topo, cpus, cpu_assigned, cpu_ordering, steady_idx, 0UL, always_cnt, steady_cnt, steady_mode, steady_partner, efficient );
1092 0 : for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
1093 0 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_ALWAYS ) ) continue;
1094 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1095 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1096 0 : if( tile_is_floating( tile ) || tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
1097 0 : auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem, 0UL );
1098 0 : }
1099 0 : }
1100 0 : ulong cpu_idx_startup = cpu_bv_is_null( group_post ) ? cpu_idx : 0UL; /* may reuse group post-start cores */
1101 0 : cpu_bv_t cpu_assigned_startup[ cpu_bv_word_cnt ];
1102 0 : cpu_bv_subtract( cpu_assigned_startup, cpu_assigned, numa_fence );
1103 0 : cpu_bv_subtract( cpu_assigned_startup, cpu_assigned_startup, group_post );
1104 0 : ulong dem_startup[ 9 ] = { 0UL, 0UL, 0UL, dem[ 3 ], dem[ 4 ], dem[ 5 ], 0UL, 0UL, 0UL };
1105 0 : dem[ 3 ] = dem[ 4 ] = dem[ 5 ] = 0UL;
1106 :
1107 : /* Separately assign startup and post-start tiles */
1108 0 : for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
1109 0 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_STARTUP ) ) continue;
1110 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1111 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1112 0 : if( tile_is_floating( tile ) || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
1113 0 : auto_tile_cpu( tile, cpus, &cpu_idx_startup, cpu_assigned_startup, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem_startup, 1UL );
1114 0 : }
1115 0 : }
1116 0 : if( graded ) smt_place( topo, cpus, cpu_assigned, cpu_ordering, steady_idx, always_cnt, steady_cnt, steady_cnt, steady_mode, steady_partner, efficient );
1117 0 : if( graded ) {
1118 0 : cpu_bv_subtract( cpu_assigned, cpu_assigned, numa_fence );
1119 0 : auto_startup_cpus( topo, cpus, cpu_assigned_startup, cpu_ordering, efficient );
1120 0 : }
1121 0 : for( ulong r=0UL; r<TILE_CLASS_CNT && !graded; r++ ) {
1122 0 : if( !( TILE_CLASS[ r ].flags & FD_TOPOB_TILE_POSTSTART ) ) continue;
1123 0 : for( ulong j=0UL; j<topo->tile_cnt; j++ ) {
1124 0 : fd_topo_tile_t * tile = &topo->tiles[ j ];
1125 0 : if( tile->cpu_idx!=ULONG_MAX || strcmp( tile->name, TILE_CLASS[ r ].name ) ) continue;
1126 0 : auto_tile_cpu( tile, cpus, &cpu_idx, cpu_assigned, cpu_ordering, skip_ht_pairs, &spare_siblings, efficient, dem, 2UL );
1127 0 : }
1128 0 : }
1129 :
1130 : /* Make sure all the tiles we haven't set are supposed to be floating. */
1131 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
1132 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
1133 0 : if( tile->cpu_idx!=ULONG_MAX ) continue;
1134 :
1135 0 : if( FD_UNLIKELY( !tile_is_floating( tile ) ) ) FD_LOG_WARNING(( "auto layout cannot affine tile `%s:%lu` because it is unknown. Leaving it floating", tile->name, tile->kind_id ));
1136 0 : }
1137 :
1138 0 : }
1139 :
1140 : void
1141 0 : fd_topob_auto_layout( fd_topo_t * topo ) {
1142 0 : fd_topo_cpus_t cpus[1];
1143 0 : fd_topo_cpus_init( cpus );
1144 0 : fd_topob_auto_layout_cpus( topo, cpus );
1145 0 : }
|
