Line data Source code
1 : /* The kworkers stage removes Firedancer tile CPUs from the cpumask of
2 : unbound kernel workqueues, so that deferred kernel work (writeback,
3 : vmstat refresh, various driver bottom halves, and notably the
4 : nohz_full remote tick offload) runs on housekeeping CPUs instead of
5 : stealing time from spinning tiles. The idle hyperthread siblings
6 : the cpuset stage isolates are excluded too.
7 :
8 : The kernel exposes this as a global hex mask at
9 : /sys/devices/virtual/workqueue/cpumask. Per-CPU (bound) kworkers
10 : are unaffected: they are idle unless work is queued on their CPU,
11 : which the irq-affinity and cpuset stages minimize.
12 :
13 : The kernel rejects a mask with no online CPU, so like irq-affinity
14 : this stage refuses to run if tiles cover every host CPU.
15 :
16 : fini restores the mask to all host CPUs (the kernel default),
17 : which may over-restore if the operator had their own narrower mask
18 : before init; the previous mask is not persisted anywhere. */
19 :
20 : #include "configure.h"
21 : #include "fd_cpu_isolation.h"
22 :
23 : #include <errno.h>
24 : #include <fcntl.h>
25 : #include <unistd.h>
26 :
27 0 : #define NAME "kworkers"
28 :
29 0 : #define WQ_CPUMASK_PATH "/sys/devices/virtual/workqueue/cpumask"
30 :
31 : /* read_wq_cpumask reads and parses the workqueue cpumask. Fails fast
32 : on any error except the file being absent (which enabled() already
33 : gates on, but guard against removal races anyway): this is a
34 : kernel-provided file with a fixed format, so parse or IO failures
35 : mean something is wrong that we should not paper over. */
36 :
37 : static void
38 0 : read_wq_cpumask( fd_cpuset_t cpuset[ static fd_cpuset_word_cnt ] ) {
39 0 : int fd = open( WQ_CPUMASK_PATH, O_RDONLY );
40 0 : if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
41 :
42 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX+64UL ];
43 0 : long mask_len = read( fd, mask, sizeof(mask)-1UL );
44 0 : if( FD_UNLIKELY( mask_len<=0L ) ) FD_LOG_ERR(( "read(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
45 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
46 0 : mask[ mask_len ] = '\0';
47 :
48 0 : if( FD_UNLIKELY( !fd_cpu_isolation_parse_mask( cpuset, mask ) ) )
49 0 : FD_LOG_ERR(( "failed to parse `" WQ_CPUMASK_PATH "` (\"%s\")", mask ));
50 0 : }
51 :
52 : static void
53 0 : write_wq_cpumask( fd_cpuset_t const * cpuset ) {
54 0 : char mask[ FD_CPU_ISOLATION_MASK_MAX ];
55 0 : fd_cpu_isolation_format_mask( mask, sizeof(mask), cpuset );
56 0 : ulong mask_len = strlen( mask );
57 :
58 0 : FD_LOG_NOTICE(( "%sRUN: `echo \"%s\" > " WQ_CPUMASK_PATH "`%s", fd_log_style_dim(), mask, fd_log_style_normal() ));
59 :
60 0 : int fd = open( WQ_CPUMASK_PATH, O_WRONLY );
61 0 : if( FD_UNLIKELY( fd<0 ) ) FD_LOG_ERR(( "open(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
62 0 : FD_CPUSET_DECL( before );
63 0 : int have_before = fd_cpu_isolation_read_wq_mask( before );
64 0 : if( FD_UNLIKELY( write( fd, mask, mask_len )!=(long)mask_len ) )
65 0 : FD_LOG_ERR(( "write(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
66 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(" WQ_CPUMASK_PATH ") failed (%i-%s)", errno, fd_io_strerror( errno ) ));
67 0 : if( have_before ) fd_cpu_isolation_warn_wq_change( WQ_CPUMASK_PATH, before );
68 0 : }
69 :
70 : static int
71 0 : enabled( config_t const * config ) {
72 0 : (void)config;
73 : /* Not all kernels expose the unbound workqueue mask (CONFIG_SYSFS,
74 : ancient kernels). If absent there is nothing to configure. */
75 0 : return 0==access( WQ_CPUMASK_PATH, F_OK );
76 0 : }
77 :
78 : static void
79 : init_perm( fd_cap_chk_t * chk,
80 0 : config_t const * config FD_PARAM_UNUSED ) {
81 0 : fd_cap_chk_root( chk, NAME, "modify `" WQ_CPUMASK_PATH "`" );
82 0 : }
83 :
84 : static void
85 : fini_perm( fd_cap_chk_t * chk,
86 0 : config_t const * config FD_PARAM_UNUSED ) {
87 0 : fd_cap_chk_root( chk, NAME, "modify `" WQ_CPUMASK_PATH "`" );
88 0 : }
89 :
90 : static void
91 0 : init( config_t const * config ) {
92 0 : FD_CPUSET_DECL( tile_cpus );
93 0 : fd_cpu_isolation_banned_cpus( tile_cpus, &config->topo );
94 :
95 0 : FD_CPUSET_DECL( allowed );
96 0 : fd_cpu_isolation_online_cpus( allowed );
97 0 : fd_cpuset_subtract( allowed, allowed, tile_cpus );
98 :
99 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( allowed ) ) )
100 0 : FD_LOG_ERR(( "all host CPUs are assigned to Firedancer tiles; cannot reserve any CPU for kernel workqueues" ));
101 :
102 0 : write_wq_cpumask( allowed );
103 0 : }
104 :
105 : static int
106 : fini( config_t const * config,
107 0 : int pre_init ) {
108 0 : (void)config; (void)pre_init;
109 :
110 0 : FD_CPUSET_DECL( all );
111 0 : ulong cpu_cnt = fd_shmem_cpu_cnt();
112 0 : for( ulong cpu=0UL; cpu<cpu_cnt; cpu++ ) fd_cpuset_insert( all, cpu );
113 :
114 0 : FD_CPUSET_DECL( excluded );
115 0 : read_wq_cpumask( excluded );
116 0 : fd_cpuset_subtract( excluded, all, excluded );
117 :
118 0 : write_wq_cpumask( all );
119 0 : return !fd_cpuset_is_null( excluded );
120 0 : }
121 :
122 : static configure_result_t
123 : check( config_t const * config,
124 0 : int check_type ) {
125 0 : (void)check_type;
126 :
127 0 : FD_CPUSET_DECL( current );
128 0 : read_wq_cpumask( current );
129 :
130 0 : FD_CPUSET_DECL( tile_cpus );
131 0 : fd_cpu_isolation_banned_cpus( tile_cpus, &config->topo );
132 :
133 0 : if( FD_UNLIKELY( fd_cpuset_is_null( tile_cpus ) ) ) {
134 0 : FD_CPUSET_DECL( missing );
135 0 : fd_cpu_isolation_online_cpus( missing );
136 0 : fd_cpuset_subtract( missing, missing, current );
137 0 : if( FD_UNLIKELY( !fd_cpuset_is_null( missing ) ) ) {
138 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
139 0 : fd_cpu_isolation_format_list( list, sizeof(list), missing );
140 0 : NOT_CONFIGURED( "kernel workqueue cpumask excludes CPUs %s but no tile CPUs are pinned", list );
141 0 : }
142 0 : CONFIGURE_OK();
143 0 : }
144 :
145 0 : FD_CPUSET_DECL( overlap );
146 0 : fd_cpuset_intersect( overlap, current, tile_cpus );
147 0 : if( FD_UNLIKELY( !fd_cpuset_is_null( overlap ) ) ) {
148 0 : char list[ FD_CPU_ISOLATION_LIST_MAX ];
149 0 : fd_cpu_isolation_format_list( list, sizeof(list), overlap );
150 0 : NOT_CONFIGURED( "kernel workqueue cpumask includes Firedancer tile CPUs or their isolated siblings %s", list );
151 0 : }
152 :
153 0 : CONFIGURE_OK();
154 0 : }
155 :
156 : configure_stage_t fd_cfg_stage_kworkers = {
157 : .name = NAME,
158 : .always_recreate = 0,
159 : .enabled = enabled,
160 : .init_perm = init_perm,
161 : .fini_perm = fini_perm,
162 : .init = init,
163 : .fini = fini,
164 : .check = check,
165 : };
166 :
167 : #undef NAME
168 : #undef WQ_CPUMASK_PATH
|
