Line data Source code
1 : #define _GNU_SOURCE
2 : #include "fd_topo.h"
3 : #include "fd_cpu_topo.h"
4 :
5 : #include "../waker/fd_waker.h"
6 : #include "../sleep/fd_sleep.h"
7 : #include "../metrics/fd_metrics.h"
8 : #include "../events/fd_event_report.h"
9 : #include "../../util/tile/fd_tile_private.h"
10 :
11 : #include <unistd.h>
12 : #include <errno.h>
13 : #include <fcntl.h>
14 : #include <limits.h>
15 : #include <pthread.h>
16 : #include <linux/futex.h>
17 : #include <sys/resource.h>
18 : #include <sys/prctl.h>
19 : #include <sys/stat.h>
20 : #include <sys/mman.h>
21 : #include <net/if.h>
22 :
23 : static void
24 : initialize_logging( char const * tile_name,
25 : ulong tile_kind_id,
26 0 : ulong tid ) {
27 0 : fd_log_cpu_set( NULL );
28 0 : fd_log_private_tid_set( tid );
29 0 : char thread_name[ 20 ];
30 0 : FD_TEST( fd_cstr_printf_check( thread_name, sizeof( thread_name ), NULL, "%s:%lu", tile_name, tile_kind_id ) );
31 0 : fd_log_thread_set( thread_name );
32 0 : fd_log_private_stack_discover( FD_TILE_PRIVATE_STACK_SZ,
33 0 : &fd_tile_private_stack0, &fd_tile_private_stack1 );
34 0 : FD_LOG_INFO(( "booting tile %s pid:%lu tid:%lu", thread_name, fd_log_group_id(), tid ));
35 :
36 : /* FD_LOG_* calls fd_log_wallclock_cstr, which calls localtime_r. In
37 : glibc, this ends up calling a function called tzset_internal. The
38 : first time tzset_internal is called by a process, it may (and
39 : almost always does) call __tzfile_read, which invokes the openat
40 : syscall, and possibly several others on the time zone file
41 : (typically /etc/localtime) or on a file in the time zone directory.
42 : This kind of behavior is tricky to sandbox, so the easiest thing to
43 : do is initialize it prior to the sandbox and hope whatever libc is
44 : used behaves like glibc. This only matters when both the logfile
45 : and stderr filters are strict enough so that the immediately prior
46 : FD_LOG_INFO call is a no-op, since otherwise that call would have
47 : taken care of it. */
48 0 : char wallclock[FD_LOG_WALLCLOCK_CSTR_BUF_SZ];
49 0 : fd_log_wallclock_cstr( 0L, wallclock );
50 0 : }
51 :
52 : void
53 : fd_topo_run_tile( fd_topo_t * topo,
54 : fd_topo_tile_t * tile,
55 : int sandbox,
56 : int keep_controlling_terminal,
57 : int core_dump_level,
58 : uint uid,
59 : uint gid,
60 : int allow_fd,
61 0 : fd_topo_run_tile_t * tile_run ) {
62 0 : char thread_name[ 20 ];
63 0 : FD_TEST( fd_cstr_printf_check( thread_name, sizeof( thread_name ), NULL, "%s:%lu", tile->name, tile->kind_id ) );
64 0 : if( FD_UNLIKELY( prctl( PR_SET_NAME, thread_name, 0, 0, 0 ) ) ) FD_LOG_ERR(( "prctl(PR_SET_NAME) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
65 :
66 0 : if( FD_UNLIKELY( topo->sleep_obj_id!=ULONG_MAX ) ) {
67 0 : if( FD_UNLIKELY( -1==prctl( PR_SET_TIMERSLACK, 1UL, 0UL, 0UL, 0UL ) ) ) FD_LOG_ERR(( "prctl(PR_SET_TIMERSLACK) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
68 0 : }
69 :
70 0 : fd_tile_rseq_unregister();
71 :
72 0 : ulong pid = fd_sandbox_getpid(); /* Need to read /proc again.. we got a new PID from clone */
73 0 : ulong tid = fd_sandbox_gettid(); /* Need to read /proc again.. we got a new TID from clone */
74 :
75 0 : initialize_logging( tile->name, tile->kind_id, tid );
76 :
77 : /* preload shared memory before sandboxing, so it is already mapped */
78 0 : fd_topo_join_tile_workspaces( topo, tile, core_dump_level );
79 0 : fd_topo_mlock_code();
80 :
81 0 : if( FD_UNLIKELY( tile_run->privileged_init ) )
82 0 : tile_run->privileged_init( topo, tile );
83 :
84 0 : ulong allow_fds_offset = 0UL;
85 0 : int allow_fds[ FD_SANDBOX_ALLOWED_FD_CNT_MAX ];
86 0 : for( ulong i=0UL; i<FD_SANDBOX_ALLOWED_FD_CNT_MAX; i++ ) allow_fds[ i ] = -1;
87 0 : if( FD_LIKELY( -1!=allow_fd ) ) {
88 0 : allow_fds_offset = 1UL;
89 0 : allow_fds[ 0 ] = allow_fd;
90 0 : }
91 0 : ulong allow_fds_cnt = 0UL;
92 0 : if( FD_LIKELY( tile_run->populate_allowed_fds ) ) {
93 0 : allow_fds_cnt = tile_run->populate_allowed_fds( topo,
94 0 : tile,
95 0 : FD_SANDBOX_ALLOWED_FD_CNT_MAX-allow_fds_offset,
96 0 : allow_fds+allow_fds_offset );
97 0 : }
98 :
99 :
100 0 : struct sock_filter seccomp_filter[ 256UL ];
101 0 : ulong seccomp_filter_cnt = 0UL;
102 0 : if( FD_LIKELY( tile_run->populate_allowed_seccomp ) ) {
103 0 : seccomp_filter_cnt = tile_run->populate_allowed_seccomp( topo,
104 0 : tile,
105 0 : sizeof(seccomp_filter)/sizeof(seccomp_filter[ 0 ]),
106 0 : seccomp_filter );
107 0 : }
108 :
109 0 : ulong rlimit_file_cnt = tile_run->rlimit_file_cnt;
110 0 : if( tile_run->rlimit_file_cnt_fn ) {
111 0 : rlimit_file_cnt = tile_run->rlimit_file_cnt_fn( topo, tile );
112 0 : }
113 :
114 0 : if( FD_LIKELY( sandbox ) ) {
115 0 : int dumpable = core_dump_level == FD_TOPO_CORE_DUMP_LEVEL_DISABLED ? 0 : 1;
116 0 : fd_sandbox_enter( uid,
117 0 : gid,
118 0 : tile_run->keep_host_networking,
119 0 : tile_run->allow_connect,
120 0 : tile_run->allow_renameat,
121 0 : keep_controlling_terminal,
122 0 : dumpable,
123 0 : rlimit_file_cnt,
124 0 : tile_run->rlimit_address_space,
125 0 : tile_run->rlimit_data,
126 0 : tile_run->rlimit_nproc,
127 0 : allow_fds_cnt+allow_fds_offset,
128 0 : allow_fds,
129 0 : seccomp_filter_cnt,
130 0 : seccomp_filter );
131 0 : } else {
132 0 : fd_sandbox_switch_uid_gid( uid, gid );
133 0 : }
134 :
135 : /* Now we are sandboxed, join all the tango IPC objects in the workspaces */
136 0 : fd_topo_fill_tile( topo, tile );
137 :
138 0 : FD_TEST( tile->metrics );
139 0 : fd_metrics_register( tile->metrics );
140 0 : fd_event_register( topo, tile );
141 :
142 0 : FD_MGAUGE_SET( TILE, PID, pid );
143 0 : FD_MGAUGE_SET( TILE, TID, tid );
144 0 : FD_MGAUGE_SET( TILE, HEARTBEAT_TIME, (ulong)fd_log_wallclock() );
145 :
146 0 : if( FD_UNLIKELY( tile_run->unprivileged_init ) )
147 0 : tile_run->unprivileged_init( topo, tile );
148 :
149 0 : tile_run->run( topo, tile );
150 0 : if( FD_UNLIKELY( !tile->allow_shutdown ) ) FD_LOG_ERR(( "tile %s:%lu run loop returned", tile->name, tile->kind_id ));
151 :
152 0 : FD_MGAUGE_SET( TILE, STATE, 2UL );
153 0 : }
154 :
155 : typedef struct {
156 : fd_topo_t * topo;
157 : fd_topo_tile_t * tile;
158 : fd_topo_run_tile_t tile_run;
159 : uint uid;
160 : uint gid;
161 : volatile int copied;
162 : void * stack_lo;
163 : void * stack_hi;
164 : } fd_topo_run_thread_args_t;
165 :
166 : static void *
167 0 : run_tile_thread_main( void * _args ) {
168 0 : fd_topo_run_thread_args_t args = *(fd_topo_run_thread_args_t *)_args;
169 0 : FD_COMPILER_MFENCE();
170 0 : ((fd_topo_run_thread_args_t *)_args)->copied = 1;
171 0 : FD_COMPILER_MFENCE();
172 :
173 : /* Prevent fork() from smashing the stack */
174 0 : if( FD_UNLIKELY( madvise( args.stack_lo, (ulong)args.stack_hi - (ulong)args.stack_lo, MADV_DONTFORK ) ) ) {
175 0 : FD_LOG_ERR(( "madvise(stack,MADV_DONTFORK) failed (%i-%s)", errno, fd_io_strerror( errno ) ));
176 0 : }
177 :
178 0 : fd_topo_run_tile( args.topo, args.tile, 0, 1, 1, args.uid, args.gid, -1, &args.tile_run );
179 0 : FD_TEST( args.tile->allow_shutdown );
180 0 : return NULL;
181 0 : }
182 :
183 : /* fd_topo_tile_stack_join_anon is a variant of fd_topo_tile_stack_join
184 : that acquires private anonymous memory instead of shared pages.
185 :
186 : This is required for fork() to work, as the parent and child process
187 : would otherwise share a stack and corrupt each other. While fork()
188 : is banned in tile user code, some dynamic analysis tools (like MSan)
189 : unfortunately rely on it. */
190 :
191 : FD_FN_UNUSED static void *
192 0 : fd_topo_tile_stack_join_anon( void ) {
193 0 :
194 0 : ulong sz = 2*FD_TILE_PRIVATE_STACK_SZ;
195 0 : int prot = PROT_READ|PROT_WRITE;
196 0 : int flags = MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK;
197 0 :
198 0 : uchar * stack = MAP_FAILED;
199 0 : #if !FD_HAS_ASAN && !FD_HAS_MSAN
200 0 : stack = mmap( NULL, sz, prot, flags|MAP_HUGETLB, -1, 0 );
201 0 : #endif
202 0 :
203 0 : if( stack==MAP_FAILED ) {
204 0 : stack = mmap( NULL, sz, prot, flags, -1, 0 );
205 0 : if( FD_UNLIKELY( stack==MAP_FAILED ) ) {
206 0 : FD_LOG_ERR(( "mmap() for stack failed (%i-%s)", errno, fd_io_strerror( errno ) ));
207 0 : }
208 0 : }
209 0 :
210 0 : /* Create the guard regions in the extra space */
211 0 : void * guard_lo = (void *)( stack - FD_SHMEM_NORMAL_PAGE_SZ );
212 0 : if( FD_UNLIKELY( mmap( guard_lo, FD_SHMEM_NORMAL_PAGE_SZ, PROT_NONE,
213 0 : MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, (off_t)0 )!=guard_lo ) )
214 0 : FD_LOG_ERR(( "mmap(%p) failed (%i-%s)", guard_lo, errno, fd_io_strerror( errno ) ));
215 0 :
216 0 : void * guard_hi = (void *)( stack + FD_TILE_PRIVATE_STACK_SZ );
217 0 : if( FD_UNLIKELY( mmap( guard_hi, FD_SHMEM_NORMAL_PAGE_SZ, PROT_NONE,
218 0 : MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, (off_t)0 )!=guard_hi ) )
219 0 : FD_LOG_ERR(( "mmap(%p) failed (%i-%s)", guard_hi, errno, fd_io_strerror( errno ) ));
220 0 :
221 0 : return stack;
222 0 : }
223 :
224 : void *
225 : fd_topo_tile_stack_join( char const * app_name,
226 : char const * tile_name,
227 0 : ulong tile_kind_id ) {
228 : #if FD_HAS_MSAN
229 : return fd_topo_tile_stack_join_anon();
230 : #endif
231 :
232 0 : char name[ PATH_MAX ];
233 0 : FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_stack_%s%lu", app_name, tile_name, tile_kind_id ) );
234 :
235 0 : int dump = strcmp( tile_name, "sign" ) ? 1 : 0; /* avoid core dumps of sign tile stacks */
236 0 : uchar * stack = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, dump, NULL, NULL, NULL );
237 0 : if( FD_UNLIKELY( !stack ) ) FD_LOG_ERR(( "fd_shmem_join failed" ));
238 :
239 : /* Make space for guard lo and guard hi */
240 0 : if( FD_UNLIKELY( fd_shmem_release( stack, FD_SHMEM_HUGE_PAGE_SZ, 1UL ) ) )
241 0 : FD_LOG_ERR(( "fd_shmem_release (%d-%s)", errno, fd_io_strerror( errno ) ));
242 0 : stack += FD_SHMEM_HUGE_PAGE_SZ;
243 0 : if( FD_UNLIKELY( fd_shmem_release( stack + FD_TILE_PRIVATE_STACK_SZ, FD_SHMEM_HUGE_PAGE_SZ, 1UL ) ) )
244 0 : FD_LOG_ERR(( "fd_shmem_release (%d-%s)", errno, fd_io_strerror( errno ) ));
245 :
246 : /* Create the guard regions in the extra space */
247 0 : void * guard_lo = (void *)(stack - FD_SHMEM_NORMAL_PAGE_SZ );
248 0 : if( FD_UNLIKELY( mmap( guard_lo, FD_SHMEM_NORMAL_PAGE_SZ, PROT_NONE,
249 0 : MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, (off_t)0 )!=guard_lo ) )
250 0 : FD_LOG_ERR(( "mmap failed (%i-%s)", errno, fd_io_strerror( errno ) ));
251 :
252 0 : void * guard_hi = (void *)(stack + FD_TILE_PRIVATE_STACK_SZ);
253 0 : if( FD_UNLIKELY( mmap( guard_hi, FD_SHMEM_NORMAL_PAGE_SZ, PROT_NONE,
254 0 : MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, (off_t)0 )!=guard_hi ) )
255 0 : FD_LOG_ERR(( "mmap failed (%i-%s)", errno, fd_io_strerror( errno ) ));
256 :
257 0 : return stack;
258 0 : }
259 :
260 : static inline void
261 : run_tile_thread( fd_topo_t * topo,
262 : fd_topo_tile_t * tile,
263 : fd_topo_run_tile_t tile_run,
264 : uint uid,
265 : uint gid,
266 : fd_cpuset_t const * float_cpu_set,
267 : fd_cpuset_t const * cpuless_cpu_set,
268 : int floating_priority,
269 0 : fd_topo_run_thread_args_t * args ) {
270 : /* tpool will assign a thread later */
271 0 : if( FD_UNLIKELY( tile_run.for_tpool ) ) return;
272 0 : void * stack = fd_topo_tile_stack_join( topo->app_name, tile->name, tile->kind_id );
273 :
274 0 : pthread_attr_t attr[ 1 ];
275 0 : int err = pthread_attr_init( attr );
276 0 : if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "pthread_attr_init() failed (%i-%s)", err, fd_io_strerror( err ) ));
277 0 : err = pthread_attr_setstack( attr, stack, FD_TILE_PRIVATE_STACK_SZ );
278 0 : if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "pthread_attr_setstack() failed (%i-%s)", err, fd_io_strerror( err ) ));
279 :
280 0 : FD_CPUSET_DECL( cpu_set );
281 0 : if( FD_LIKELY( tile->cpu_idx<65535UL ) ) {
282 : /* set the thread affinity before we clone the new process to ensure
283 : kernel first touch happens on the desired thread. */
284 0 : if( FD_UNLIKELY( tile->floats ) ) {
285 0 : ulong numa_idx = fd_shmem_numa_idx( tile->cpu_idx );
286 0 : for( ulong cpu=0UL; cpu<FD_TILE_MAX; cpu++ )
287 0 : if( fd_cpuset_test( float_cpu_set, cpu ) && fd_shmem_numa_idx( cpu )==numa_idx ) fd_cpuset_insert( cpu_set, cpu );
288 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( cpu_set ) ) ) fd_memcpy( cpu_set, float_cpu_set, fd_cpuset_footprint() );
289 0 : }
290 0 : if( FD_UNLIKELY( !fd_cpuset_cnt( cpu_set ) ) ) fd_cpuset_insert( cpu_set, tile->cpu_idx );
291 0 : if( FD_UNLIKELY( -1==setpriority( PRIO_PROCESS, 0, -19 ) ) ) FD_LOG_ERR(( "setpriority() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
292 0 : } else {
293 0 : fd_cpuset_copy( cpu_set, cpuless_cpu_set );
294 0 : if( FD_UNLIKELY( -1==setpriority( PRIO_PROCESS, 0, floating_priority ) ) ) FD_LOG_ERR(( "setpriority() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
295 0 : }
296 :
297 0 : if( FD_UNLIKELY( fd_cpuset_setaffinity( 0, cpu_set ) ) ) {
298 0 : if( FD_LIKELY( errno==EINVAL ) ) {
299 0 : FD_LOG_ERR(( "Unable to set the thread affinity for tile %s:%lu on cpu %lu. It is likely that the affinity "
300 0 : "you have specified for this tile in [layout.affinity] of your configuration file contains a "
301 0 : "CPU (%lu) which does not exist on this machine.",
302 0 : tile->name, tile->kind_id, tile->cpu_idx, tile->cpu_idx ));
303 0 : } else {
304 0 : FD_LOG_ERR(( "sched_setaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
305 0 : }
306 0 : }
307 :
308 0 : *args = (fd_topo_run_thread_args_t) {
309 0 : .topo = topo,
310 0 : .tile = tile,
311 0 : .tile_run = tile_run,
312 0 : .uid = uid,
313 0 : .gid = gid,
314 0 : .copied = 0,
315 0 : .stack_lo = stack,
316 0 : .stack_hi = (uchar *)stack + FD_TILE_PRIVATE_STACK_SZ
317 0 : };
318 :
319 0 : pthread_t pthread;
320 0 : err = pthread_create( &pthread, attr, run_tile_thread_main, args );
321 0 : if( FD_UNLIKELY( err ) ) FD_LOG_ERR(( "pthread_create() failed (%i-%s)", err, fd_io_strerror( err ) ));
322 0 : }
323 :
324 : static void
325 0 : join_isolation_cgroup( char const * app_name ) {
326 0 : char path[ PATH_MAX ];
327 0 : FD_TEST( fd_cstr_printf_check( path, sizeof(path), NULL, "/sys/fs/cgroup/%s/cgroup.procs", app_name ) );
328 :
329 0 : int fd = open( path, O_WRONLY );
330 0 : if( FD_UNLIKELY( fd<0 ) ) {
331 0 : if( FD_LIKELY( errno==ENOENT ) ) return; /* cpuset stage not configured */
332 0 : FD_LOG_ERR(( "open(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
333 0 : }
334 :
335 0 : char pid[ 32 ];
336 0 : ulong pid_len;
337 0 : FD_TEST( fd_cstr_printf_check( pid, sizeof(pid), &pid_len, "%ld", (long)getpid() ) );
338 0 : if( FD_UNLIKELY( write( fd, pid, pid_len )!=(long)pid_len ) )
339 0 : FD_LOG_ERR(( "write(%s,\"%s\") failed (%i-%s)", path, pid, errno, fd_io_strerror( errno ) ));
340 0 : if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "close(%s) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
341 0 : }
342 :
343 : void
344 : fd_topo_run_single_process( fd_topo_t * topo,
345 : uint uid,
346 : uint gid,
347 0 : fd_topo_run_tile_t (* tile_run )( fd_topo_tile_t const * tile ) ) {
348 0 : join_isolation_cgroup( topo->app_name );
349 :
350 : /* Lock code once before spawning tile threads, so they don't all
351 : queue on mmap_lock behind a cold-cache populate. */
352 0 : fd_topo_mlock_code();
353 :
354 0 : ulong waker_client_cnt = 0UL;
355 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
356 0 : ulong idx = topo->tiles[ i ].waker_client_idx;
357 0 : if( FD_UNLIKELY( idx!=ULONG_MAX ) ) waker_client_cnt = fd_ulong_max( waker_client_cnt, idx+1UL );
358 0 : }
359 0 : fd_waker_install( waker_client_cnt );
360 0 : fd_sleep_eventfd_install( topo );
361 :
362 : /* Save the current affinity, it will be restored after creating any child tiles */
363 0 : FD_CPUSET_DECL( floating_cpu_set );
364 0 : if( FD_UNLIKELY( fd_cpuset_getaffinity( 0, floating_cpu_set ) ) )
365 0 : FD_LOG_ERR(( "sched_getaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
366 :
367 0 : FD_CPUSET_DECL( float_cpu_set );
368 0 : FD_CPUSET_DECL( cpuless_cpu_set );
369 0 : static fd_topo_cpus_t cpus[1];
370 0 : fd_topo_cpus_init( cpus );
371 0 : fd_topo_cpus_floating( cpus, topo, floating_cpu_set, float_cpu_set, cpuless_cpu_set );
372 0 : int any_floats = 0;
373 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) any_floats |= topo->tiles[ i ].floats;
374 :
375 0 : errno = 0;
376 0 : int save_priority = getpriority( PRIO_PROCESS, 0 );
377 0 : if( FD_UNLIKELY( -1==save_priority && errno ) ) FD_LOG_ERR(( "getpriority() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
378 :
379 0 : fd_topo_run_thread_args_t args[ FD_TOPO_MAX_TILES ];
380 :
381 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
382 0 : fd_topo_tile_t * tile = &topo->tiles[ i ];
383 0 : fd_topo_run_tile_t run_tile = tile_run( tile );
384 0 : int floating_priority = ( any_floats && !strcmp( tile->name, "waker" ) ) ? -19 : save_priority; /* the waker delivers floaters' fd readiness: never behind them */
385 0 : run_tile_thread( topo, tile, run_tile, uid, gid, float_cpu_set, cpuless_cpu_set, floating_priority, &args[ i ] );
386 0 : }
387 :
388 0 : for( ulong i=0UL; i<topo->tile_cnt; i++ ) {
389 0 : while( !FD_VOLATILE( args[ i ].copied ) ) FD_SPIN_PAUSE();
390 0 : }
391 :
392 0 : fd_sandbox_switch_uid_gid( uid, gid );
393 :
394 0 : if( FD_UNLIKELY( -1==setpriority( PRIO_PROCESS, 0, save_priority ) ) ) FD_LOG_ERR(( "setpriority() failed (%i-%s)", errno, fd_io_strerror( errno ) ));
395 0 : if( FD_UNLIKELY( fd_cpuset_setaffinity( 0, floating_cpu_set ) ) )
396 0 : FD_LOG_ERR(( "sched_setaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) ));
397 0 : }
|
