Line data Source code
1 : #define _GNU_SOURCE
2 : #include "../../shared/fd_config.h"
3 : #include "../../shared/fd_bootinfo.h"
4 : #include "../../shared/fd_action.h"
5 : #include "../../platform/fd_sys_util.h"
6 : #include "../../../disco/metrics/fd_metrics.h"
7 :
8 : #include <errno.h>
9 : #include <stdio.h>
10 : #include <stdlib.h>
11 : #include <unistd.h>
12 : #include <sys/mman.h>
13 : #include <sys/prctl.h>
14 : #include <sys/wait.h>
15 : #include <linux/capability.h>
16 :
17 : FD_IMPORT_BINARY( fd_flamegraph_template, "src/app/shared_dev/commands/flamegraph_template.html" );
18 : FD_IMPORT_BINARY( fd_flamegraph_script, "src/app/shared_dev/commands/flamegraph.py" );
19 :
20 : static int
21 : memfd_from( char const * name,
22 : uchar const * data,
23 0 : ulong data_sz ) {
24 0 : int fd = memfd_create( name, 0U );
25 0 : if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "memfd_create() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
26 0 : if( FD_LIKELY( data_sz ) ) {
27 0 : int err = fd_io_write( fd, data, data_sz, data_sz, &(ulong){0} );
28 0 : if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "write() failed (%i-%s)", err, fd_io_strerror( err ) ));
29 0 : }
30 0 : return fd;
31 0 : }
32 :
33 : static void
34 0 : dump_child_output( int fd ) {
35 0 : if( FD_UNLIKELY( -1==lseek( fd, 0L, SEEK_SET ) ) ) return;
36 0 : char buf[ 4096 ];
37 0 : long sz;
38 0 : while( (sz = read( fd, buf, sizeof(buf) ))>0L ) {
39 0 : ulong written = 0UL;
40 0 : while( written<(ulong)sz ) {
41 0 : long w = write( STDERR_FILENO, buf+written, (ulong)sz-written );
42 0 : if( FD_UNLIKELY( w<=0L ) ) return;
43 0 : written += (ulong)w;
44 0 : }
45 0 : }
46 0 : }
47 :
48 : static int record_pid;
49 :
50 : static void
51 0 : parent_signal( int sig ) {
52 0 : FD_LOG_NOTICE(( "Received signal %s%s%s %s(%s)%s", fd_log_style_bold(), fd_io_strsignal_name( sig ), fd_log_style_normal(), fd_log_style_dim(), fd_io_strsignal_desc( sig ), fd_log_style_normal() ));
53 0 : if( FD_LIKELY( record_pid ) ) {
54 0 : if( FD_UNLIKELY( -1==kill( record_pid, SIGINT ) ) ) FD_LOG_ERR(( "kill() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
55 0 : }
56 0 : }
57 :
58 : static void
59 0 : install_parent_signals( void ) {
60 0 : struct sigaction sa = {
61 0 : .sa_handler = parent_signal,
62 0 : .sa_flags = 0,
63 0 : };
64 0 : if( FD_UNLIKELY( sigaction( SIGTERM, &sa, NULL ) ) )
65 0 : FD_LOG_ERR(( "sigaction(SIGTERM) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
66 0 : if( FD_UNLIKELY( sigaction( SIGINT, &sa, NULL ) ) )
67 0 : FD_LOG_ERR(( "sigaction(SIGINT) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
68 0 : }
69 :
70 : void
71 : flame_cmd_perm( args_t * args FD_PARAM_UNUSED,
72 : fd_cap_chk_t * chk,
73 0 : config_t const * config FD_PARAM_UNUSED ) {
74 0 : fd_cap_chk_root( chk, "flame", "read system performance counters with `/usr/bin/perf`" );
75 0 : }
76 :
77 : void
78 : flame_cmd_args( int * pargc,
79 : char *** pargv,
80 0 : args_t * args ) {
81 0 : args->flame.freq = fd_env_strip_cmdline_uint( pargc, pargv, "--freq", NULL, 997U );
82 0 : if( FD_UNLIKELY( !args->flame.freq ) ) FD_LOG_ERR(( "--freq must be non-zero" ));
83 :
84 0 : if( FD_UNLIKELY( !*pargc ) ) {
85 0 : strncpy( args->flame.name, "all", sizeof( args->flame.name ) - 1 );
86 0 : return;
87 0 : }
88 0 : strncpy( args->flame.name, **pargv, sizeof( args->flame.name ) - 1 );
89 :
90 0 : (*pargc)--;
91 0 : (*pargv)++;
92 0 : }
93 :
94 : void
95 : flame_cmd_fn( args_t * args,
96 0 : config_t * config ) {
97 0 : install_parent_signals();
98 :
99 0 : fd_bootinfo_adopt( config );
100 0 : fd_bootinfo_check_layout( config );
101 0 : fd_topo_join_workspaces( &config->topo, FD_SHMEM_JOIN_MODE_READ_ONLY, FD_TOPO_CORE_DUMP_LEVEL_DISABLED );
102 0 : fd_topo_fill( &config->topo );
103 :
104 0 : ulong tile_cnt = 0UL;
105 0 : ulong tile_idxs[ 128UL ];
106 :
107 0 : int whole_process = 0;
108 0 : if( FD_UNLIKELY( !strcmp( "all", args->flame.name ) ) ) {
109 0 : FD_TEST( config->topo.tile_cnt<sizeof(tile_idxs)/sizeof(tile_idxs[0]) );
110 0 : for( ulong i=0UL; i<config->topo.tile_cnt; i++ ) {
111 0 : tile_idxs[ tile_cnt ] = i;
112 0 : tile_cnt++;
113 0 : }
114 0 : } else {
115 0 : char * sep = strchr( args->flame.name, ':' );
116 :
117 0 : ulong tile_idx;
118 0 : if( FD_UNLIKELY( !sep ) ) {
119 0 : tile_idx = fd_topo_find_tile( &config->topo, args->flame.name, 0UL );
120 0 : } else {
121 0 : char * endptr;
122 0 : *sep = '\0';
123 0 : ulong kind_id = strtoul( sep+1, &endptr, 10 );
124 0 : if( FD_UNLIKELY( *endptr!='\0' || kind_id==ULONG_MAX ) ) FD_LOG_ERR(( "invalid tile kind id provided `%s`", sep+1 ));
125 0 : tile_idx = fd_topo_find_tile( &config->topo, args->flame.name, kind_id );
126 0 : }
127 :
128 0 : if( FD_UNLIKELY( tile_idx==ULONG_MAX ) ) FD_LOG_ERR(( "tile %s%s%s not found", fd_log_style_bold(), args->flame.name, fd_log_style_normal() ));
129 0 : tile_idxs[ 0 ] = tile_idx;
130 0 : tile_cnt = 1UL;
131 0 : }
132 :
133 0 : char threads[ 4096 ] = {0};
134 0 : ulong len = 0UL;
135 0 : for( ulong i=0UL; i<tile_cnt; i++ ) {
136 0 : ulong tid = fd_metrics_tile( config->topo.tiles[ tile_idxs[ i ] ].metrics )[ FD_METRICS_GAUGE_TILE_TID_OFF ];
137 0 : ulong pid = fd_metrics_tile( config->topo.tiles[ tile_idxs[ i ] ].metrics )[ FD_METRICS_GAUGE_TILE_PID_OFF ];
138 :
139 0 : FD_TEST( pid<=INT_MAX );
140 0 : if( FD_UNLIKELY( -1==kill( (int)tid, 0 ) ) ) {
141 0 : if( FD_LIKELY( config->topo.tiles[ tile_idxs[ i ] ].allow_shutdown ) ) continue;
142 :
143 0 : if( FD_UNLIKELY( errno==ESRCH ) ) FD_LOG_ERR(( "tile %s%s:%lu%s is not running", fd_log_style_bold(), config->topo.tiles[ tile_idxs[ i ] ].name, config->topo.tiles[ tile_idxs[ i ] ].kind_id, fd_log_style_normal() ));
144 0 : else FD_LOG_ERR(( "kill() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
145 0 : }
146 :
147 0 : ulong arg_len;
148 0 : FD_TEST( fd_cstr_printf_check( threads+len, sizeof(threads)-len, &arg_len, "%lu", fd_ulong_if( whole_process, pid, tid ) ) );
149 0 : len += arg_len;
150 :
151 0 : if( FD_LIKELY( i!=tile_cnt-1UL ) ) {
152 0 : FD_TEST( fd_cstr_printf_check( threads+len, sizeof(threads)-len, NULL, "," ) );
153 0 : len += 1UL;
154 0 : }
155 0 : }
156 0 : FD_TEST( len<sizeof(threads) );
157 :
158 0 : char freq[ 16UL ];
159 0 : FD_TEST( fd_cstr_printf_check( freq, sizeof(freq), NULL, "%u", args->flame.freq ) );
160 :
161 0 : FD_LOG_NOTICE(( "%s/usr/bin/perf record -e task-clock --off-cpu -g -F %s -%c %s && /usr/bin/perf script report flamegraph%s", fd_log_style_bold(), freq, fd_char_if( whole_process, 'p', 't' ), threads, fd_log_style_normal() ));
162 :
163 0 : int record_out = memfd_from( "perf_record_output", NULL, 0UL );
164 :
165 0 : record_pid = fork();
166 0 : if( FD_UNLIKELY( -1==record_pid ) ) FD_LOG_ERR(( "fork() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
167 0 : if( FD_LIKELY( !record_pid ) ) {
168 : /* Silence perf unless it fails */
169 0 : if( FD_UNLIKELY( -1==dup2( record_out, STDOUT_FILENO ) || -1==dup2( record_out, STDERR_FILENO ) ) ) FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
170 0 : char * perf_args[ 13 ] = {
171 0 : "/usr/bin/perf",
172 0 : "record",
173 0 : "-e",
174 0 : "task-clock",
175 0 : "--off-cpu",
176 0 : "-g",
177 0 : "-F",
178 0 : freq,
179 0 : whole_process ? "-p" : "-t",
180 0 : threads,
181 0 : NULL,
182 0 : };
183 0 : if( FD_UNLIKELY( -1==execve( "/usr/bin/perf", (char * const *)perf_args, NULL ) ) ) FD_LOG_ERR(( "execve() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
184 0 : }
185 :
186 0 : FD_LOG_NOTICE(( "perf collection running, %sCtrl+C%s to stop", fd_log_style_bold(), fd_log_style_normal() ));
187 :
188 0 : for(;;) {
189 0 : int wstatus;
190 0 : int exited_pid = waitpid( -1, &wstatus, 0 );
191 0 : if( FD_UNLIKELY( -1==exited_pid ) ) {
192 0 : if( FD_LIKELY( errno==EAGAIN || errno==EINTR ) ) continue;
193 0 : FD_LOG_ERR(( "waitpid() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
194 0 : }
195 :
196 0 : int graceful_exit = !WIFEXITED( wstatus ) && WTERMSIG( wstatus )==SIGINT;
197 0 : if( FD_UNLIKELY( !graceful_exit ) ) {
198 0 : dump_child_output( record_out );
199 0 : if( FD_UNLIKELY( !WIFEXITED( wstatus ) ) ) FD_LOG_ERR(( "%sperf record%s exited unexpectedly with signal %s%s%s %s(%s)%s", fd_log_style_bold(), fd_log_style_normal(), fd_log_style_bold(), fd_io_strsignal_name( WTERMSIG( wstatus ) ), fd_log_style_normal(), fd_log_style_dim(), fd_io_strsignal_desc( WTERMSIG( wstatus ) ), fd_log_style_normal() ));
200 0 : if( FD_UNLIKELY( WEXITSTATUS( wstatus ) ) ) FD_LOG_ERR(( "%sperf record%s exited unexpectedly with code %d", fd_log_style_bold(), fd_log_style_normal(), WEXITSTATUS( wstatus ) ));
201 0 : }
202 0 : break;
203 0 : }
204 :
205 0 : int template_fd = memfd_from( "flamegraph_template", fd_flamegraph_template, fd_flamegraph_template_sz );
206 0 : int script_fd = memfd_from( "flamegraph_script", fd_flamegraph_script, fd_flamegraph_script_sz );
207 :
208 0 : char template_env[ PATH_MAX ];
209 0 : FD_TEST( fd_cstr_printf_check( template_env, sizeof(template_env), NULL, "FD_FLAMEGRAPH_TEMPLATE=/dev/fd/%d", template_fd ) );
210 0 : char script_path[ PATH_MAX ];
211 0 : FD_TEST( fd_cstr_printf_check( script_path, sizeof(script_path), NULL, "py:/dev/fd/%d", script_fd ) );
212 :
213 0 : int report_out = memfd_from( "perf_report_output", NULL, 0UL );
214 :
215 0 : int report_pid = fork();
216 0 : if( FD_UNLIKELY( -1==report_pid ) ) FD_LOG_ERR(( "fork() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
217 0 : if( FD_LIKELY( !report_pid ) ) {
218 0 : if( FD_UNLIKELY( -1==dup2( report_out, STDOUT_FILENO ) || -1==dup2( report_out, STDERR_FILENO ) ) ) FD_LOG_ERR(( "dup2() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
219 : /* Sandboxed tiles pivot_root into an empty mount namespace. perf
220 : with CAP_SYS_ADMIN setns()es into it to resolve symbols and finds
221 : nothing, so all stacks show as [unknown]. Drop the capability so
222 : perf falls back to the host namespace paths. */
223 0 : if( FD_UNLIKELY( -1==prctl( PR_CAPBSET_DROP, CAP_SYS_ADMIN, 0, 0, 0 ) ) ) FD_LOG_ERR(( "prctl(PR_CAPBSET_DROP) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
224 0 : char * args[ 5 ] = {
225 0 : "/usr/bin/perf",
226 0 : "script",
227 0 : "-s",
228 0 : script_path,
229 0 : NULL,
230 0 : };
231 0 : char * envp[ 2 ] = { template_env, NULL };
232 0 : if( FD_UNLIKELY( -1==execve( "/usr/bin/perf", (char * const *)args, envp ) ) ) FD_LOG_ERR(( "execve() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
233 0 : }
234 :
235 0 : for(;;) {
236 0 : int wstatus;
237 0 : int exited_pid = waitpid( -1, &wstatus, 0 );
238 0 : if( FD_UNLIKELY( -1==exited_pid ) ) {
239 0 : if( FD_LIKELY( errno==EAGAIN || errno==EINTR ) ) continue;
240 0 : FD_LOG_ERR(( "waitpid() failed (%d-%s)", errno, fd_io_strerror( errno ) ));
241 0 : }
242 :
243 0 : if( FD_UNLIKELY( !WIFEXITED( wstatus ) || WEXITSTATUS( wstatus ) ) ) dump_child_output( report_out );
244 0 : if( FD_UNLIKELY( !WIFEXITED( wstatus ) ) ) FD_LOG_ERR(( "%sperf report%s exited unexpectedly with signal %s%s%s %s(%s)%s", fd_log_style_bold(), fd_log_style_normal(), fd_log_style_bold(), fd_io_strsignal_name( WTERMSIG( wstatus ) ), fd_log_style_normal(), fd_log_style_dim(), fd_io_strsignal_desc( WTERMSIG( wstatus ) ), fd_log_style_normal() ));
245 0 : if( FD_UNLIKELY( WEXITSTATUS( wstatus ) ) ) FD_LOG_ERR(( "%sperf report%s exited unexpectedly with code %d", fd_log_style_bold(), fd_log_style_normal(), WEXITSTATUS( wstatus ) ));
246 0 : break;
247 0 : }
248 :
249 0 : char cwd[ PATH_MAX ];
250 0 : if( FD_LIKELY( getcwd( cwd, sizeof(cwd) ) ) ) FD_LOG_NOTICE(( "flamegraph written to %s%s/flamegraph.html%s", fd_log_style_bold(), cwd, fd_log_style_normal() ));
251 :
252 0 : fd_sys_util_exit_group( 0 );
253 0 : }
254 :
255 : static void
256 0 : flame_args_help( fd_action_help_t * help ) {
257 0 : fd_action_help_arg( help, "--freq", "<hz>", "Sampling frequency in Hz (default 997)" );
258 0 : }
259 :
260 : action_t fd_action_flame = {
261 : .name = "flame",
262 : .args = flame_cmd_args,
263 : .fn = flame_cmd_fn,
264 : .perm = flame_cmd_perm,
265 : .description = "Capture a perf flamegraph",
266 : .usage = "flame [all|tile|tile:idx] [--freq <hz>]",
267 : .args_help = flame_args_help,
268 : .is_diagnostic = 1
269 : };
|
