Line data Source code
1 : #include "fd_cpu_isolation.h"
2 :
3 : #include "../../../../disco/topo/fd_topob.h"
4 :
5 : #include <ctype.h>
6 : #include <errno.h>
7 : #include <fcntl.h>
8 : #include <string.h>
9 : #include <sys/utsname.h>
10 : #include <unistd.h>
11 :
12 : fd_cpuset_t *
13 : fd_cpu_isolation_tile_cpus( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ],
14 0 : fd_topo_t const * topo ) {
15 0 : fd_cpuset_new( cpuset );
16 0 : ulong cpu_cnt = fd_shmem_cpu_cnt();
17 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
18 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
19 0 : if( tile->cpu_idx<cpu_cnt ) fd_cpuset_insert( cpuset, tile->cpu_idx );
20 0 : }
21 0 : return cpuset;
22 0 : }
23 :
24 : fd_cpuset_t *
25 0 : fd_cpu_isolation_online_cpus( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
26 0 : if( FD_UNLIKELY( !fd_cpu_isolation_read_list( "/sys/devices/system/cpu/online", cpuset ) ||
27 0 : !fd_cpuset_cnt( cpuset ) ) )
28 0 : FD_LOG_ERR(( "could not determine online host CPUs" ));
29 0 : return cpuset;
30 0 : }
31 :
32 : int
33 : fd_cpu_isolation_parse_list( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ],
34 0 : char const * list ) {
35 0 : fd_cpuset_new( cpuset );
36 :
37 0 : char const * p = list;
38 0 : while( isspace( (uchar)*p ) ) p++;
39 0 : if( FD_UNLIKELY( !*p ) ) return 1; /* empty set */
40 :
41 : /* /sys/devices/system/cpu/nohz_full contains the literal string
42 : "(null)" on kernels with CONFIG_NO_HZ_FULL but no nohz_full= boot
43 : parameter. */
44 0 : if( FD_UNLIKELY( !strncmp( p, "(null)", 6UL ) ) ) return 1;
45 :
46 0 : for(;;) {
47 0 : if( FD_UNLIKELY( !isdigit( (uchar)*p ) ) ) return 0;
48 0 : ulong lo = 0UL;
49 0 : while( isdigit( (uchar)*p ) ) { lo = lo*10UL + (ulong)(*p-'0'); if( FD_UNLIKELY( lo>FD_TILE_MAX*2UL ) ) return 0; p++; }
50 0 : ulong hi = lo;
51 0 : if( FD_UNLIKELY( *p=='-' ) ) {
52 0 : p++;
53 0 : if( FD_UNLIKELY( !isdigit( (uchar)*p ) ) ) return 0;
54 0 : hi = 0UL;
55 0 : while( isdigit( (uchar)*p ) ) { hi = hi*10UL + (ulong)(*p-'0'); if( FD_UNLIKELY( hi>FD_TILE_MAX*2UL ) ) return 0; p++; }
56 0 : }
57 0 : if( FD_UNLIKELY( hi<lo ) ) return 0;
58 0 : for( ulong cpu=lo; cpu<=hi; cpu++ ) {
59 0 : if( FD_LIKELY( cpu<FD_TILE_MAX ) ) fd_cpuset_insert( cpuset, cpu );
60 0 : }
61 :
62 0 : if( FD_LIKELY( *p==',' ) ) { p++; continue; }
63 0 : while( isspace( (uchar)*p ) ) p++;
64 0 : if( FD_LIKELY( !*p ) ) return 1;
65 0 : return 0;
66 0 : }
67 0 : }
68 :
69 : char *
70 : fd_cpu_isolation_format_list( char * buf,
71 : ulong buf_sz,
72 0 : fd_cpuset_t const * cpuset ) {
73 0 : char * p = buf;
74 0 : char * end = buf+buf_sz-1UL;
75 0 : int first = 1;
76 :
77 0 : ulong cpu = 0UL;
78 0 : while( cpu<FD_TILE_MAX ) {
79 0 : if( FD_LIKELY( !fd_cpuset_test( cpuset, cpu ) ) ) { cpu++; continue; }
80 0 : ulong lo = cpu;
81 0 : while( cpu+1UL<FD_TILE_MAX && fd_cpuset_test( cpuset, cpu+1UL ) ) cpu++;
82 0 : ulong hi = cpu;
83 0 : cpu++;
84 :
85 0 : if( FD_UNLIKELY( (ulong)(end-p)<48UL ) ) break; /* can't overflow with LIST_MAX, defensive */
86 0 : if( FD_LIKELY( !first ) ) *p++ = ',';
87 0 : first = 0;
88 0 : p = fd_cstr_append_ulong_as_text( p, 0, 0, lo, fd_ulong_base10_dig_cnt( lo ) );
89 0 : if( FD_UNLIKELY( hi>lo ) ) {
90 0 : *p++ = '-';
91 0 : p = fd_cstr_append_ulong_as_text( p, 0, 0, hi, fd_ulong_base10_dig_cnt( hi ) );
92 0 : }
93 0 : }
94 0 : *p = '\0';
95 0 : return buf;
96 0 : }
97 :
98 : char const *
99 0 : fd_cpu_isolation_partition_type( fd_topo_t const * topo ) {
100 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) if( topo->tiles[ i ].floats ) return "root";
101 0 : return "isolated";
102 0 : }
103 :
104 : fd_cpuset_t *
105 : fd_cpu_isolation_partition_cpus( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ],
106 0 : fd_topo_t const * topo ) {
107 0 : fd_cpu_isolation_tile_cpus( cpuset, topo );
108 :
109 0 : fd_topo_cpus_t cpus[1];
110 0 : fd_topo_cpus_init( cpus );
111 :
112 0 : FD_CPUSET_DECL( blocklist );
113 0 : for( ulong i=0UL; i<topo->blocklist_cores_cnt; i++ ) fd_cpuset_insert( blocklist, topo->blocklist_cores_cpu_idx[ i ] );
114 0 : FD_CPUSET_DECL( online );
115 0 : fd_cpu_isolation_online_cpus( online );
116 :
117 0 : for( uint pass=0U; pass<2U; pass++ ) {
118 0 : uint cls = pass ? FD_TOPOB_TILE_DEDICATED : FD_TOPOB_TILE_SMT_CRITICAL;
119 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
120 0 : fd_topo_tile_t const * tile = &topo->tiles[ i ];
121 0 : if( FD_LIKELY( !( fd_topob_tile_class( tile->name ) & cls ) ) ) continue;
122 :
123 0 : ulong cpu_idx = tile->cpu_idx;
124 0 : if( FD_UNLIKELY( cpu_idx>=fd_ulong_min( cpus->cpu_cnt, FD_TILE_MAX ) ) ) continue;
125 0 : if( FD_UNLIKELY( tile->floats ) ) continue; /* not alone on that core anyway */
126 :
127 0 : ulong sibling = cpus->cpu[ cpu_idx ].sibling;
128 0 : if( FD_UNLIKELY( sibling==ULONG_MAX || sibling>=FD_TILE_MAX ) ) continue; /* no sibling */
129 0 : if( FD_UNLIKELY( !cpus->cpu[ sibling ].online ) ) continue; /* offline is even better */
130 0 : if( FD_UNLIKELY( fd_cpuset_test( cpuset, sibling ) ) ) continue; /* used by another tile */
131 0 : if( pass ) {
132 0 : FD_CPUSET_DECL( housekeeping );
133 0 : fd_cpuset_subtract( housekeeping, online, cpuset );
134 0 : if( FD_UNLIKELY( fd_cpuset_test( blocklist, sibling ) || fd_cpuset_cnt( housekeeping )<=1UL ) ) continue;
135 0 : }
136 :
137 0 : fd_cpuset_insert( cpuset, sibling );
138 0 : }
139 0 : }
140 0 : return cpuset;
141 0 : }
142 :
143 : fd_cpuset_t *
144 : fd_cpu_isolation_banned_cpus( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ],
145 0 : fd_topo_t const * topo ) {
146 0 : fd_cpu_isolation_partition_cpus( cpuset, topo );
147 :
148 0 : FD_CPUSET_DECL( housekeeping );
149 0 : fd_cpuset_subtract( housekeeping, fd_cpu_isolation_online_cpus( housekeeping ), cpuset );
150 0 : if( FD_LIKELY( fd_cpuset_cnt( housekeeping ) ) ) return cpuset;
151 :
152 0 : FD_LOG_WARNING(( "excluding the idle hyperthread siblings of critical and dedicated tiles would leave no "
153 0 : "housekeeping CPU; excluding only tile CPUs" ));
154 0 : return fd_cpu_isolation_tile_cpus( cpuset, topo );
155 0 : }
156 :
157 : static int
158 0 : cgroup_cpus( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
159 0 : char path[ PATH_MAX ] = {0};
160 0 : char buf[ 4096 ];
161 0 : int fd = open( "/proc/self/cgroup", O_RDONLY );
162 0 : long sz = fd<0 ? -1L : read( fd, buf, sizeof(buf)-1UL );
163 0 : if( FD_LIKELY( fd>=0 ) && FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(/proc/self/cgroup) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
164 0 : if( FD_UNLIKELY( sz<=0L ) ) return 0;
165 0 : buf[ sz ] = '\0';
166 0 : char * line = buf;
167 0 : while( line && strncmp( line, "0::", 3UL ) ) { line = strchr( line, '\n' ); if( line ) line++; }
168 0 : if( FD_UNLIKELY( !line ) ) return 0;
169 0 : char * nl = strchr( line, '\n' ); if( FD_LIKELY( nl ) ) *nl = '\0';
170 0 : if( FD_UNLIKELY( !fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/fs/cgroup%s", line+3UL ) ) ) return 0;
171 :
172 : /* Without the cpuset controller a cgroup inherits its parent's */
173 0 : for(;;) {
174 0 : char file[ PATH_MAX+32UL ];
175 0 : FD_TEST( fd_cstr_printf_check( file, sizeof(file), NULL, "%s/cpuset.cpus.effective", path ) );
176 0 : if( !access( file, R_OK ) && fd_cpu_isolation_read_list( file, cpuset ) && fd_cpuset_cnt( cpuset ) ) return 1;
177 0 : char * slash = strrchr( path, '/' );
178 0 : if( !slash || !strcmp( path, "/sys/fs/cgroup" ) ) return 0;
179 0 : *slash = '\0';
180 0 : }
181 0 : }
182 :
183 : void
184 0 : fd_cpu_isolation_check_cgroup( config_t const * config ) {
185 0 : FD_CPUSET_DECL( cg_allowed );
186 0 : if( FD_LIKELY( cgroup_cpus( cg_allowed ) ) ) {
187 0 : char file[ PATH_MAX ];
188 0 : FD_TEST( fd_cstr_printf_check( file, sizeof(file), NULL, "/sys/fs/cgroup/%s/cpuset.cpus", config->name ) );
189 0 : FD_CPUSET_DECL( instance );
190 0 : if( !access( file, R_OK ) && fd_cpu_isolation_read_list( file, instance ) ) fd_cpuset_union( cg_allowed, cg_allowed, instance );
191 :
192 0 : FD_CPUSET_DECL( outside );
193 0 : fd_cpuset_subtract( outside, fd_cpu_isolation_online_cpus( outside ), cg_allowed );
194 0 : for( ulong i=0UL; i<config->topo.blocklist_cores_cnt; i++ ) {
195 0 : if( config->topo.blocklist_cores_cpu_idx[ i ]<FD_TILE_MAX ) fd_cpuset_insert( outside, config->topo.blocklist_cores_cpu_idx[ i ] );
196 0 : }
197 :
198 0 : static fd_topo_cpus_t cpus[1];
199 0 : fd_topo_cpus_init( cpus );
200 0 : FD_CPUSET_DECL( float_cpu_set );
201 0 : FD_CPUSET_DECL( cpuless_cpu_set );
202 0 : fd_topo_cpus_floating( cpus, &config->topo, cg_allowed, float_cpu_set, cpuless_cpu_set );
203 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
204 0 : fd_topo_tile_t const * tile = &config->topo.tiles[ i ];
205 0 : if( FD_LIKELY( tile->cpu_idx>=FD_TILE_MAX ) ) continue;
206 0 : FD_CPUSET_DECL( run_set ); /* execve_tile's affinity */
207 0 : if( FD_UNLIKELY( tile->floats ) ) {
208 0 : ulong numa_idx = fd_shmem_numa_idx( tile->cpu_idx );
209 0 : for( ulong cpu=0UL; cpu<FD_TILE_MAX; cpu++ )
210 0 : if( fd_cpuset_test( float_cpu_set, cpu ) && fd_shmem_numa_idx( cpu )==numa_idx ) fd_cpuset_insert( run_set, cpu );
211 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( run_set ) ) ) fd_cpuset_copy( run_set, float_cpu_set );
212 0 : }
213 0 : if( FD_LIKELY( !fd_cpuset_cnt( run_set ) ) ) fd_cpuset_insert( run_set, tile->cpu_idx );
214 0 : if( FD_LIKELY( fd_cpuset_cnt( fd_cpuset_intersect( run_set, run_set, cg_allowed ) ) ) ) continue;
215 0 : char allowed_str[ FD_CPU_ISOLATION_LIST_MAX ];
216 0 : char outside_str[ FD_CPU_ISOLATION_LIST_MAX ];
217 0 : FD_LOG_ERR(( "tile %s:%lu is assigned to CPU %lu, but this process may only run on CPUs %s (its cgroup cpuset, e.g. "
218 0 : "systemd AllowedCPUs= or a container limit). Set [layout.blocklist_cores] = \"%s\" or give an "
219 0 : "explicit [layout.affinity] within the allowed CPUs.", tile->name, tile->kind_id, tile->cpu_idx,
220 0 : fd_cpu_isolation_format_list( allowed_str, sizeof(allowed_str), cg_allowed ),
221 0 : fd_cpu_isolation_format_list( outside_str, sizeof(outside_str), outside ) ));
222 0 : }
223 0 : }
224 0 : }
225 :
226 : int
227 : fd_cpu_isolation_parse_mask( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ],
228 0 : char const * mask ) {
229 0 : fd_cpuset_new( cpuset );
230 :
231 0 : char const * end = mask;
232 0 : while( *end && !isspace( (uchar)*end ) ) end++;
233 0 : if( FD_UNLIKELY( end==mask ) ) return 0;
234 :
235 0 : ulong cpu_idx = 0UL;
236 0 : for( char const * p=end; p>mask; ) {
237 0 : p--;
238 0 : if( FD_UNLIKELY( *p==',' ) ) continue;
239 :
240 0 : int digit;
241 0 : if( FD_LIKELY( (*p>='0') & (*p<='9') ) ) digit = *p-'0';
242 0 : else if( FD_LIKELY( (*p>='a') & (*p<='f') ) ) digit = 10+*p-'a';
243 0 : else if( FD_LIKELY( (*p>='A') & (*p<='F') ) ) digit = 10+*p-'A';
244 0 : else return 0;
245 :
246 0 : for( ulong bit=0UL; bit<4UL; bit++ ) {
247 0 : if( FD_UNLIKELY( cpu_idx>=FD_TILE_MAX ) ) return 1;
248 0 : if( FD_UNLIKELY( digit & (1<<bit) ) ) fd_cpuset_insert( cpuset, cpu_idx );
249 0 : cpu_idx++;
250 0 : }
251 0 : }
252 0 : return 1;
253 0 : }
254 :
255 : char *
256 : fd_cpu_isolation_format_mask( char * buf,
257 : ulong buf_sz,
258 0 : fd_cpuset_t const * cpuset ) {
259 0 : ulong cpu_cnt = fd_ulong_max( fd_shmem_cpu_cnt(), 1UL );
260 0 : ulong word_cnt = fd_ulong_max( (cpu_cnt+31UL)/32UL, 1UL );
261 0 : char * p = buf;
262 :
263 0 : FD_TEST( buf_sz>=word_cnt*9UL+1UL );
264 :
265 0 : for( ulong word_rem=word_cnt; word_rem; word_rem-- ) {
266 0 : ulong word_idx = word_rem-1UL;
267 0 : if( FD_UNLIKELY( word_idx!=word_cnt-1UL ) ) *p++ = ',';
268 :
269 0 : for( ulong nib_rem=8UL; nib_rem; nib_rem-- ) {
270 0 : ulong nib_idx = nib_rem-1UL;
271 0 : int digit = 0;
272 0 : for( ulong bit=0UL; bit<4UL; bit++ ) {
273 0 : ulong cpu_idx = word_idx*32UL + nib_idx*4UL + bit;
274 0 : if( FD_LIKELY( cpu_idx<FD_TILE_MAX && fd_cpuset_test( cpuset, cpu_idx ) ) ) digit |= (int)(1UL<<bit);
275 0 : }
276 0 : *p++ = "0123456789abcdef"[ digit ];
277 0 : }
278 0 : }
279 0 : *p = '\0';
280 0 : return buf;
281 0 : }
282 :
283 :
284 : int
285 : fd_cpu_isolation_read_list( char const * path,
286 0 : fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
287 0 : int fd = open( path, O_RDONLY );
288 0 : if( FD_UNLIKELY( fd<0 ) ) {
289 0 : if( FD_LIKELY( errno==ENOENT ) ) return 0;
290 0 : FD_LOG_ERR(( "open(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
291 0 : }
292 :
293 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
294 0 : long list_len = read( fd, list, sizeof(list)-1UL );
295 0 : if( FD_UNLIKELY( list_len<0L ) ) FD_LOG_ERR(( "read(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
296 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
297 0 : list[ list_len ] = '\0';
298 :
299 0 : if( FD_UNLIKELY( !fd_cpu_isolation_parse_list( cpuset, list ) ) )
300 0 : FD_LOG_ERR(( "failed to parse `%s` (\"%s\")", path, list ));
301 0 : return 1;
302 0 : }
303 :
304 : int
305 0 : fd_cpu_isolation_read_wq_mask( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
306 0 : int fd = open( "/sys/devices/virtual/workqueue/cpumask", O_RDONLY );
307 0 : if( FD_UNLIKELY( fd<0 ) ) return 0;
308 :
309 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX+64UL ];
310 0 : long len = read( fd, mask, sizeof(mask)-1UL );
311 0 : int err = close( fd );
312 0 : if( FD_UNLIKELY( len<=0L || err ) ) return 0;
313 0 : mask[ len ] = '\0';
314 0 : return fd_cpu_isolation_parse_mask( cpuset, mask );
315 0 : }
316 :
317 : void
318 : fd_cpu_isolation_warn_wq_change( char const * path,
319 0 : fd_cpuset_t const * before ) {
320 0 : FD_CPUSET_DECL( after );
321 0 : if( FD_UNLIKELY( !fd_cpu_isolation_read_wq_mask( after ) ) ) return;
322 0 : if( FD_LIKELY( fd_cpuset_eq( before, after ) ) ) return;
323 :
324 0 : char old_mask[ FD_CPU_ISOLATION_MASK_MAX ];
325 0 : char new_mask[ FD_CPU_ISOLATION_MASK_MAX ];
326 0 : fd_cpu_isolation_format_mask( old_mask, sizeof(old_mask), before );
327 0 : fd_cpu_isolation_format_mask( new_mask, sizeof(new_mask), after );
328 0 : struct utsname uts;
329 0 : char const * release = uname( &uts ) ? "unknown" : uts.release;
330 :
331 0 : FD_LOG_NOTICE(( "kernel workqueue cpumask changed while writing `%s`: %s -> %s (kernel %s). ", path, old_mask, new_mask, release ));
332 0 : }
|
