Line data Source code
1 : /* syscall API requires _GNU_SOURCE */
2 : #define _GNU_SOURCE
3 : #include "fd_shmem_private.h"
4 : #include "../sanitize/fd_msan.h"
5 : #include <errno.h>
6 : #include <dirent.h>
7 :
8 : /* The below uses the sysfs API added ~2009-Dec. See
9 : https://github.com/torvalds/linux/commit/1830794ae6392ce12d36dbcc5ff52f11298ddab6 */
10 :
11 : /* fd_numa_private_parse_idx parses a cstr of the form
12 : `prefix[0-9]+` into an index. The value is strictly interpreted as
13 : a non-negative base 10 value. Returns -1 if the value could not be
14 : parsed (e.g. s is NULL, s does not have the prefix, s does not
15 : have a base 10 suffix, the value overflows an int representation). */
16 :
17 : FD_FN_PURE static int
18 : fd_numa_private_parse_idx( char const * s,
19 : char const * prefix,
20 1183 : ulong prefix_len ) {
21 1183 : if( FD_UNLIKELY( !s ) ) return -1;
22 1183 : if( FD_UNLIKELY( strncmp( s, prefix, prefix_len ) ) ) return -1;
23 259 : s += prefix_len;
24 :
25 259 : long val = 0L;
26 :
27 259 : char const * t = s;
28 662 : for(;;) {
29 662 : char c = *t;
30 662 : if( !c ) break; /* host dep branch prob */
31 405 : if( FD_UNLIKELY( (c<'0') | (c>'9') ) ) return -1; /* non-digit encountered */
32 403 : val = (long)(c-'0') + 10L*val;
33 403 : if( FD_UNLIKELY( val>(long)INT_MAX ) ) return -1; /* overflow */
34 403 : t++;
35 403 : }
36 257 : if( FD_UNLIKELY( s==t ) ) return -1; /* empty idx */
37 :
38 257 : return (int)val;
39 257 : }
40 :
41 : ulong
42 1 : fd_numa_node_cnt( void ) {
43 :
44 : /* Open sysfs dir containing NUMA config. Abort if this fails. */
45 :
46 1 : char const * path = "/sys/devices/system/node";
47 1 : DIR * dir = opendir( path );
48 1 : if( FD_UNLIKELY( !dir ) ) {
49 0 : FD_LOG_WARNING(( "opendir( \"%s\" ) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
50 0 : return 0UL;
51 0 : }
52 :
53 : /* Scan dir to get number of NUMA nodes. Note that we do not assume
54 : the system indexes numa nodes contiguously (but it almost certainly
55 : does). */
56 :
57 1 : int node_idx_max = INT_MIN;
58 12 : for(;;) {
59 12 : struct dirent * dirent = readdir( dir );
60 12 : if( !dirent ) break;
61 11 : node_idx_max = fd_int_max( fd_numa_private_parse_idx( dirent->d_name, "node", 4UL ), node_idx_max );
62 11 : }
63 :
64 : /* Close dir and return what was found */
65 :
66 1 : if( FD_UNLIKELY( closedir( dir ) ) )
67 0 : FD_LOG_WARNING(( "closedir( \"%s\" ) failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
68 :
69 1 : if( FD_UNLIKELY( node_idx_max<0 ) ) {
70 0 : FD_LOG_WARNING(( "No numa nodes found in \"%s\"", path ));
71 0 : return 0UL;
72 0 : }
73 :
74 1 : return ((ulong)node_idx_max) + 1UL;
75 1 : }
76 :
77 : ulong
78 1 : fd_numa_private_cpu_cnt( char const * path ) {
79 :
80 1 : DIR * dir = opendir( path );
81 1 : if( FD_UNLIKELY( !dir ) ) {
82 0 : FD_LOG_WARNING(( "opendir( \"%s\" ) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
83 0 : return 0UL;
84 0 : }
85 :
86 1 : int cpu_idx_max = -1;
87 1 : int read_error = 0;
88 149 : for(;;) {
89 149 : errno = 0;
90 149 : struct dirent * dirent = readdir( dir );
91 149 : if( !dirent ) {
92 1 : if( FD_UNLIKELY( errno==EINTR ) ) continue;
93 1 : read_error = errno;
94 1 : break;
95 1 : }
96 148 : cpu_idx_max = fd_int_max( fd_numa_private_parse_idx( dirent->d_name, "cpu", 3UL ), cpu_idx_max );
97 148 : }
98 :
99 1 : if( FD_UNLIKELY( closedir( dir ) ) )
100 0 : FD_LOG_WARNING(( "closedir( \"%s\" ) failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
101 :
102 1 : if( FD_UNLIKELY( read_error ) ) {
103 0 : FD_LOG_WARNING(( "readdir( \"%s\" ) failed (%i-%s)", path, read_error, fd_io_strerror( read_error ) ));
104 0 : return 0UL;
105 0 : }
106 :
107 1 : if( FD_UNLIKELY( cpu_idx_max<0 ) ) {
108 0 : FD_LOG_WARNING(( "No CPUs found in \"%s\"", path ));
109 0 : return 0UL;
110 0 : }
111 :
112 1 : return ((ulong)cpu_idx_max) + 1UL;
113 1 : }
114 :
115 : ulong
116 1 : fd_numa_cpu_cnt( void ) {
117 1 : return fd_numa_private_cpu_cnt( "/sys/devices/system/cpu" );
118 1 : }
119 :
120 : ulong
121 128 : fd_numa_node_idx( ulong cpu_idx ) {
122 :
123 : /* Open sysfs dir containing CPU config. Abort if this fails. */
124 :
125 128 : char path[64];
126 128 : DIR * dir = opendir( fd_cstr_printf( path, 64UL, NULL, "/sys/devices/system/cpu/cpu%lu", cpu_idx ) );
127 128 : if( FD_UNLIKELY( !dir ) ) {
128 0 : FD_LOG_WARNING(( "opendir( \"%s\" ) failed (%i-%s)", path, errno, fd_io_strerror( errno ) ));
129 0 : return ULONG_MAX;
130 0 : }
131 :
132 : /* Scan dir for symlink to numa config */
133 :
134 128 : int node_idx = -1;
135 1024 : for(;;) {
136 1024 : struct dirent * dirent = readdir( dir );
137 1024 : if( !dirent ) break;
138 1024 : node_idx = fd_numa_private_parse_idx( dirent->d_name, "node", 4UL );
139 1024 : if( node_idx!=-1 ) break;
140 1024 : }
141 :
142 : /* Close dir and return what was found */
143 :
144 128 : if( FD_UNLIKELY( closedir( dir ) ) )
145 0 : FD_LOG_WARNING(( "closedir( \"%s\" ) failed (%i-%s); attempting to continue", path, errno, fd_io_strerror( errno ) ));
146 :
147 128 : if( FD_UNLIKELY( node_idx<0 ) ) {
148 0 : FD_LOG_WARNING(( "No numa node found in \"%s\"", path ));
149 0 : return ULONG_MAX;
150 0 : }
151 :
152 128 : return (ulong)node_idx;
153 128 : }
154 :
155 : /* FIXME: probably should do a FD_HAS_ASAN switch for the below to use
156 : the appropriate functionality when FD_HAS_ASAN is set (or maybe have
157 : a separate implementation for compiling under FD_HAS_ASAN). */
158 :
159 : #include <unistd.h>
160 : #include <sys/syscall.h>
161 :
162 : /* Note that the LLVM AddressSanitizer (ASan) intercepts all mlock
163 : calls.
164 :
165 : This has an interesting history. These interceptors were first added
166 : in 2012 and are still present in LLVM 14.0.6:
167 :
168 : https://github.com/llvm/llvm-project/commit/71d759d392f03025bcc8b20f060bc5c22e580ea1
169 :
170 : They stub `mlock`, `munlock`, `mlockall`, `munlockall` to no-ops.
171 :
172 : ASan is known to map large amounts (~16TiB) of unbacked pages. This
173 : rules out the use of `mlockall`.
174 :
175 : `mlock` only locks selected pages, therefore should be fine. The
176 : comments in various revisions of these interceptors suggest that
177 : older Linux kernels had a bug that prevented the use of `mlock`.
178 :
179 : However, current Firedancer will use the `move_pages` syscall to
180 : verify whether "allocated" pages are actually backed by DRAM.
181 :
182 : This makes Firedancer and ASan incompatible unless we either
183 :
184 : 1) Remove the `mlock` interceptor upstream, or
185 : 2) Circumvent the interceptor with a raw syscall
186 :
187 : We do option 2 below */
188 :
189 : int
190 : fd_numa_mlock( void const * addr,
191 12 : ulong len ) {
192 12 : return (int)syscall( SYS_mlock, addr, len );
193 12 : }
194 :
195 : int
196 : fd_numa_munlock( void const * addr,
197 0 : ulong len ) {
198 0 : return (int)syscall( SYS_mlock, addr, len );
199 0 : }
200 :
201 : long
202 : fd_numa_get_mempolicy( int * mode,
203 : ulong * nodemask,
204 : ulong maxnode,
205 : void * addr,
206 12 : uint flags ) {
207 12 : long rc = syscall( SYS_get_mempolicy, mode, nodemask, maxnode, addr, flags );
208 12 : if( rc==0 ) {
209 12 : if( mode ) fd_msan_unpoison( mode, sizeof(int) );
210 12 : if( nodemask ) fd_msan_unpoison( nodemask, 8UL*((maxnode+63UL)/64UL) );
211 12 : }
212 12 : return rc;
213 12 : }
214 :
215 : long
216 : fd_numa_set_mempolicy( int mode,
217 : ulong const * nodemask,
218 24 : ulong maxnode ) {
219 24 : return syscall( SYS_set_mempolicy, mode, nodemask, maxnode );
220 24 : }
221 :
222 : long
223 : fd_numa_mbind( void * addr,
224 : ulong len,
225 : int mode,
226 : ulong const * nodemask,
227 : ulong maxnode,
228 12 : uint flags ) {
229 12 : return syscall( SYS_mbind, addr, len, mode, nodemask, maxnode, flags );
230 12 : }
231 :
232 : long
233 : fd_numa_move_pages( int pid,
234 : ulong count,
235 : void ** pages,
236 : int const * nodes,
237 : int * status,
238 12 : int flags ) {
239 12 : long rc = syscall( SYS_move_pages, pid, count, pages, nodes, status, flags );
240 12 : if( rc==0 ) fd_msan_unpoison( status, count*sizeof(int) );
241 12 : return rc;
242 12 : }
|
