Line data Source code
1 : /* The cpuset stage creates a cgroup v2 "isolated" cpuset partition
2 : over the Firedancer tile CPUs, at /sys/fs/cgroup/<name> where <name>
3 : is the instance [name] from the configuration (default "fd1"),
4 : allowing multiple Firedancer instances on one host to own distinct
5 : partitions.
6 :
7 : An isolated partition removes its CPUs from the scheduler domains of
8 : the rest of the system: no load balancing onto them, and processes
9 : outside the cgroup cannot be scheduled (or set affinity) onto them
10 : at all. This is the runtime-configurable equivalent of the
11 : `isolcpus=` boot parameter, and the strongest tool available for
12 : keeping foreign tasks off tile CPUs.
13 :
14 : The partition type follows the topology. When every tile is pinned
15 : to its own CPU the partition is "isolated". When some tiles float
16 : (the kernel schedules them across a shared set of tile CPUs) it is
17 : "root" instead: still exclusive to Firedancer, but the CPUs keep a
18 : scheduler domain, which load balancing needs. An isolated partition
19 : has no scheduler domain, so a floating tile would never move off the
20 : CPU it started on.
21 :
22 : IMPORTANT INTERACTION: once the partition exists, ONLY processes
23 : inside the cgroup may run on tile CPUs. The tile launcher
24 : (execve_tile in run.c) joins the cgroup before setting a fixed
25 : tile's affinity. Setting affinity to a partitioned CPU from
26 : outside the cgroup fails with EINVAL; run `configure fini cpuset`
27 : to remove the partition if that happens with tooling that predates
28 : this stage.
29 :
30 : Notes:
31 : - Requires cgroup v2 (unified hierarchy) with the cpuset controller,
32 : and kernel support for the "isolated" partition type (5.15+ for
33 : root-adjacent partitions; probed at init by the partition write).
34 : - The cgroup directory is created directly under the cgroup root.
35 : systemd tolerates foreign cgroups there, but external cleanup may
36 : remove it; the check stage detects this and reports unconfigured.
37 : - Sibling exclusivity: the partition's CPUs must not appear in any
38 : sibling cgroup's explicit cpuset.cpus. Typical systemd slices
39 : leave cpuset.cpus empty (meaning "parent's effective"), which
40 : does not conflict. If an operator has explicitly assigned tile
41 : CPUs to another cgroup, the partition write fails and init
42 : reports it. */
43 :
44 : #include "configure.h"
45 : #include "fd_cpu_isolation.h"
46 :
47 : #include <errno.h>
48 : #include <fcntl.h>
49 : #include <unistd.h>
50 : #include <sys/stat.h>
51 :
52 0 : #define NAME "cpuset"
53 :
54 : #define CGROUP_ROOT "/sys/fs/cgroup"
55 : #define SUBTREE_CONTROL CGROUP_ROOT "/cgroup.subtree_control"
56 :
57 : /* cgroup_path formats the per-instance cgroup path or one of its files
58 : into buf. file may be NULL for the cgroup directory itself. */
59 :
60 : static char const *
61 : cgroup_path( char buf[ static PATH_MAX ],
62 : config_t const * config,
63 0 : char const * file ) {
64 0 : FD_TEST( fd_cstr_printf_check( buf, PATH_MAX, NULL, CGROUP_ROOT "/%s%s%s",
65 0 : config->name, file ? "/" : "", file ? file : "" ) );
66 0 : return buf;
67 0 : }
68 :
69 : /* write_cstr writes val to the file at path. Returns 1 on success,
70 : 0 on open/write failure with errno set; callers decide severity
71 : (typically FD_LOG_ERR with call-site context). */
72 :
73 : static int
74 : write_cstr( char const * path,
75 0 : char const * val ) {
76 0 : int fd = open( path, O_WRONLY );
77 0 : if( FD_UNLIKELY( fd<0 ) ) return 0;
78 0 : FD_CPUSET_DECL( before );
79 0 : int have_before = fd_cpu_isolation_read_wq_mask( before );
80 0 : ulong val_len = strlen( val );
81 0 : int err = 0;
82 0 : if( FD_UNLIKELY( write( fd, val, val_len )!=(long)val_len ) ) err = errno;
83 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
84 0 : if( FD_UNLIKELY( err ) ) {
85 0 : errno = err; /* preserve write() errno across close() */
86 0 : return 0;
87 0 : }
88 0 : if( have_before ) fd_cpu_isolation_warn_wq_change( path, before );
89 0 : return 1;
90 0 : }
91 :
92 : /* read_cstr reads the first line of path into buf. Returns 1 on
93 : success, 0 if the file does not exist (ENOENT). Any other error is
94 : fatal: these are kernel-provided files, and an unexpected failure
95 : reading them means something is wrong that we should not paper
96 : over. */
97 :
98 : static int
99 : read_cstr( char const * path,
100 : char * buf,
101 0 : ulong buf_sz ) {
102 0 : int fd = open( path, O_RDONLY );
103 0 : if( FD_UNLIKELY( fd<0 ) ) {
104 0 : if( FD_LIKELY( errno==ENOENT ) ) return 0;
105 0 : FD_LOG_ERR(( "open(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
106 0 : }
107 0 : long n = read( fd, buf, buf_sz-1UL );
108 0 : if( FD_UNLIKELY( n<0L ) ) FD_LOG_ERR(( "read(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
109 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
110 0 : buf[ n ] = '\0';
111 0 : char * nl = strchr( buf, '\n' );
112 0 : if( FD_LIKELY( nl ) ) *nl = '\0';
113 0 : return 1;
114 0 : }
115 :
116 : static int
117 0 : enabled( config_t const * config ) {
118 0 : (void)config;
119 : /* cgroup v2 unified hierarchy with the cpuset controller. On v1 or
120 : hybrid systems the unified root has no cgroup.controllers file (or
121 : no cpuset in it) and this stage cannot work. Only genuine absence
122 : disables the stage; any other failure reading the file is fatal
123 : (in read_cstr). */
124 0 : char controllers[ 256 ];
125 0 : if( FD_UNLIKELY( !read_cstr( CGROUP_ROOT "/cgroup.controllers", controllers, sizeof(controllers) ) ) ) return 0;
126 0 : return NULL!=strstr( controllers, "cpuset" );
127 0 : }
128 :
129 : static void
130 : init_perm( fd_cap_chk_t * chk,
131 0 : config_t const * config ) {
132 0 : char path[ PATH_MAX ];
133 0 : static char reason[ PATH_MAX+64UL ];
134 0 : FD_TEST( fd_cstr_printf_check( reason, sizeof(reason), NULL, "create and configure the cgroup `%s`",
135 0 : cgroup_path( path, config, NULL ) ) );
136 0 : fd_cap_chk_root( chk, NAME, reason );
137 0 : }
138 :
139 : static void
140 0 : init( config_t const * config ) {
141 0 : FD_CPUSET_DECL( part_cpus );
142 0 : fd_cpu_isolation_partition_cpus( part_cpus, &config->topo );
143 :
144 0 : char cgroup[ PATH_MAX ]; cgroup_path( cgroup, config, NULL );
145 :
146 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( part_cpus ) ) ) {
147 0 : FD_LOG_WARNING(( "no fixed tile CPUs in topology; not creating `%s`", cgroup ));
148 0 : return;
149 0 : }
150 :
151 0 : FD_CPUSET_DECL( host );
152 0 : fd_cpu_isolation_online_cpus( host );
153 0 : FD_CPUSET_DECL( housekeeping );
154 0 : fd_cpuset_subtract( housekeeping, host, part_cpus );
155 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( housekeeping ) ) )
156 0 : FD_LOG_ERR(( "all host CPUs are assigned to Firedancer tiles; cannot isolate them all (the kernel needs at "
157 0 : "least one housekeeping CPU)" ));
158 :
159 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
160 0 : fd_cpu_isolation_format_list( list, sizeof(list), part_cpus );
161 :
162 : /* The parent must delegate the cpuset controller before a child can
163 : use it. "+cpuset" is idempotent. */
164 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"+cpuset\" > " SUBTREE_CONTROL "`%s", fd_log_style_dim(), fd_log_style_normal() ));
165 0 : if( FD_UNLIKELY( !write_cstr( SUBTREE_CONTROL, "+cpuset" ) ) )
166 0 : FD_LOG_ERR(( "could not enable the cpuset controller in `" SUBTREE_CONTROL "` (%i-%s)", errno, fd_io_strerror( errno ) ));
167 :
168 0 : FD_LOG_NOTICE(( "%sRUN: `mkdir %s`%s", fd_log_style_dim(), cgroup , fd_log_style_normal() ));
169 0 : if( FD_UNLIKELY( mkdir( cgroup, 0755 ) && errno!=EEXIST ) )
170 0 : FD_LOG_ERR(( "mkdir(%s) failed (%i-%s)", cgroup, errno, fd_io_strerror( errno ) ));
171 :
172 0 : char path[ PATH_MAX ];
173 0 : cgroup_path( path, config, "cpuset.cpus" );
174 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > %s`%s", fd_log_style_dim(), list, path , fd_log_style_normal() ));
175 0 : if( FD_UNLIKELY( !write_cstr( path, list ) ) )
176 0 : FD_LOG_ERR(( "write(%s,\"%s\") failed (%i-%s)", path, list, errno, fd_io_strerror( errno ) ));
177 :
178 0 : char const * partition = fd_cpu_isolation_partition_type( &config->topo );
179 0 : cgroup_path( path, config, "cpuset.cpus.partition" );
180 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > %s`%s", fd_log_style_dim(), partition, path , fd_log_style_normal() ));
181 0 : if( FD_UNLIKELY( !write_cstr( path, partition ) ) )
182 0 : FD_LOG_ERR(( "write(%s,\"%s\") failed (%i-%s). The kernel may be too old for %s cpuset "
183 0 : "partitions, or a sibling cgroup may have explicitly claimed one of the CPUs `%s`",
184 0 : path, partition, errno, fd_io_strerror( errno ), partition, list ));
185 0 : }
186 :
187 : static int
188 : fini( config_t const * config,
189 0 : int pre_init ) {
190 0 : (void)pre_init;
191 :
192 0 : char cgroup[ PATH_MAX ]; cgroup_path( cgroup, config, NULL );
193 0 : if( FD_UNLIKELY( access( cgroup, F_OK ) ) ) return 0;
194 :
195 : /* Downgrade to a regular member cgroup first so the CPUs return to
196 : the system scheduler domains even if rmdir fails (e.g. because
197 : Firedancer is still running in it). ENOENT is tolerated here and
198 : below: the access() check above races with concurrent removal (a
199 : second `configure fini`, or systemd cleaning up foreign cgroups),
200 : and someone else deleting the cgroup is this stage's goal state,
201 : not an error. */
202 0 : char path[ PATH_MAX ];
203 0 : cgroup_path( path, config, "cpuset.cpus.partition" );
204 0 : if( FD_UNLIKELY( !write_cstr( path, "member" ) && errno!=ENOENT ) )
205 0 : FD_LOG_ERR(( "write(%s,\"member\") failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
206 :
207 0 : FD_LOG_NOTICE(( "%sRUN: `rmdir %s`%s", fd_log_style_dim(), cgroup , fd_log_style_normal() ));
208 0 : if( FD_UNLIKELY( rmdir( cgroup ) && errno!=ENOENT ) ) {
209 0 : if( FD_LIKELY( errno==EBUSY ) ) {
210 0 : FD_LOG_ERR(( "Removal of the CPU isolation cgroup `%s` failed because processes are still inside it, "
211 0 : "likely because Firedancer is still running. The partition was downgraded so its CPUs are "
212 0 : "returned to the system, but the cgroup itself could not be removed. Stop the validator and "
213 0 : "run `%s configure fini cpuset` again, or remove it manually with `rmdir %s`",
214 0 : cgroup, FD_BINARY_NAME, cgroup ));
215 0 : } else {
216 0 : FD_LOG_ERR(( "rmdir(%s) failed (%i-%s)", cgroup, errno, fd_io_strerror( errno ) ));
217 0 : }
218 0 : }
219 0 : return 1;
220 0 : }
221 :
222 : static configure_result_t
223 : check( config_t const * config,
224 0 : int check_type ) {
225 0 : (void)check_type;
226 :
227 0 : FD_CPUSET_DECL( part_cpus );
228 0 : fd_cpu_isolation_partition_cpus( part_cpus, &config->topo );
229 :
230 0 : char cgroup[ PATH_MAX ]; cgroup_path( cgroup, config, NULL );
231 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( part_cpus ) ) ) {
232 0 : if( FD_UNLIKELY( !access( cgroup, F_OK ) ) ) PARTIALLY_CONFIGURED( "`%s` exists but the topology pins no CPUs", cgroup );
233 0 : CONFIGURE_OK();
234 0 : }
235 0 : if( FD_UNLIKELY( access( cgroup, F_OK ) ) )
236 0 : NOT_CONFIGURED( "`%s` does not exist", cgroup );
237 :
238 0 : char path[ PATH_MAX ];
239 0 : cgroup_path( path, config, "cpuset.cpus" );
240 0 : char cpus[ FD_CPU_ISOLATION_LIST_MAX ];
241 0 : if( FD_UNLIKELY( !read_cstr( path, cpus, sizeof(cpus) ) ) )
242 0 : NOT_CONFIGURED( "`%s` does not exist", path ); /* cgroup removed concurrently */
243 :
244 0 : FD_CPUSET_DECL( current );
245 0 : if( FD_UNLIKELY( !fd_cpu_isolation_parse_list( current, cpus ) ) )
246 0 : FD_LOG_ERR(( "failed to parse `%s` (\"%s\")", path, cpus ));
247 :
248 0 : if( FD_UNLIKELY( !fd_cpuset_eq( current, part_cpus ) ) ) {
249 0 : char expected[ FD_CPU_ISOLATION_LIST_MAX ];
250 0 : fd_cpu_isolation_format_list( expected, sizeof(expected), part_cpus );
251 0 : PARTIALLY_CONFIGURED( "`%s` is \"%s\", expected \"%s\"", path, cpus, expected );
252 0 : }
253 :
254 0 : cgroup_path( path, config, "cpuset.cpus.partition" );
255 0 : char partition[ 64 ];
256 0 : if( FD_UNLIKELY( !read_cstr( path, partition, sizeof(partition) ) ) )
257 0 : NOT_CONFIGURED( "`%s` does not exist", path ); /* cgroup removed concurrently */
258 :
259 : /* An invalid partition reads as e.g. "isolated invalid (...)". */
260 0 : char const * expected = fd_cpu_isolation_partition_type( &config->topo );
261 0 : if( FD_UNLIKELY( strcmp( partition, expected ) ) )
262 0 : PARTIALLY_CONFIGURED( "`%s` is \"%s\", expected \"%s\"", path, partition, expected );
263 :
264 0 : CONFIGURE_OK();
265 0 : }
266 :
267 : configure_stage_t fd_cfg_stage_cpuset = {
268 : .name = NAME,
269 : .always_recreate = 0,
270 : .enabled = enabled,
271 : .init_perm = init_perm,
272 : .fini_perm = init_perm,
273 : .init = init,
274 : .fini = fini,
275 : .check = check,
276 : };
277 :
278 : #undef NAME
279 : #undef CGROUP_ROOT
280 : #undef SUBTREE_CONTROL
|
