Line | Count | Source |
1 | | // Copyright 2011 Google Inc. All Rights Reserved. |
2 | | // |
3 | | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | | // you may not use this file except in compliance with the License. |
5 | | // You may obtain a copy of the License at |
6 | | // |
7 | | // http://www.apache.org/licenses/LICENSE-2.0 |
8 | | // |
9 | | // Unless required by applicable law or agreed to in writing, software |
10 | | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | | // See the License for the specific language governing permissions and |
13 | | // limitations under the License. |
14 | | |
15 | | #include "util.h" |
16 | | |
17 | | #ifdef __CYGWIN__ |
18 | | #include <windows.h> |
19 | | #include <io.h> |
20 | | #elif defined( _WIN32) |
21 | | #include <windows.h> |
22 | | #include <io.h> |
23 | | #include <share.h> |
24 | | #include <direct.h> |
25 | | #endif |
26 | | |
27 | | #include <assert.h> |
28 | | #include <errno.h> |
29 | | #include <fcntl.h> |
30 | | #include <stdarg.h> |
31 | | #include <stdio.h> |
32 | | #include <stdlib.h> |
33 | | #include <string.h> |
34 | | #include <sys/stat.h> |
35 | | #include <sys/types.h> |
36 | | |
37 | | #ifndef _WIN32 |
38 | | #include <unistd.h> |
39 | | #include <sys/time.h> |
40 | | #endif |
41 | | |
42 | | #include <algorithm> |
43 | | #include <vector> |
44 | | |
45 | | #if defined(__APPLE__) || defined(__FreeBSD__) |
46 | | #include <sys/sysctl.h> |
47 | | #elif defined(__SVR4) && defined(__sun) |
48 | | #include <unistd.h> |
49 | | #include <sys/loadavg.h> |
50 | | #elif defined(_AIX) && !defined(__PASE__) |
51 | | #include <libperfstat.h> |
52 | | #elif defined(__linux__) || defined(__GLIBC__) |
53 | | #include <sys/sysinfo.h> |
54 | | #include <fstream> |
55 | | #include <map> |
56 | | #include "string_piece_util.h" |
57 | | #endif |
58 | | |
59 | | #if defined(__FreeBSD__) |
60 | | #include <sys/cpuset.h> |
61 | | #endif |
62 | | |
63 | | #include "edit_distance.h" |
64 | | |
65 | | using namespace std; |
66 | | |
67 | 0 | void Fatal(const char* msg, ...) { |
68 | 0 | va_list ap; |
69 | 0 | fprintf(stderr, "ninja: fatal: "); |
70 | 0 | va_start(ap, msg); |
71 | 0 | vfprintf(stderr, msg, ap); |
72 | 0 | va_end(ap); |
73 | 0 | fprintf(stderr, "\n"); |
74 | | #ifdef _WIN32 |
75 | | // On Windows, some tools may inject extra threads. |
76 | | // exit() may block on locks held by those threads, so forcibly exit. |
77 | | fflush(stderr); |
78 | | fflush(stdout); |
79 | | ExitProcess(1); |
80 | | #else |
81 | 0 | exit(1); |
82 | 0 | #endif |
83 | 0 | } |
84 | | |
85 | 188k | void Warning(const char* msg, va_list ap) { |
86 | 188k | fprintf(stderr, "ninja: warning: "); |
87 | 188k | vfprintf(stderr, msg, ap); |
88 | 188k | fprintf(stderr, "\n"); |
89 | 188k | } |
90 | | |
91 | 188k | void Warning(const char* msg, ...) { |
92 | 188k | va_list ap; |
93 | 188k | va_start(ap, msg); |
94 | 188k | Warning(msg, ap); |
95 | 188k | va_end(ap); |
96 | 188k | } |
97 | | |
98 | 0 | void Error(const char* msg, va_list ap) { |
99 | 0 | fprintf(stderr, "ninja: error: "); |
100 | 0 | vfprintf(stderr, msg, ap); |
101 | 0 | fprintf(stderr, "\n"); |
102 | 0 | } |
103 | | |
104 | 0 | void Error(const char* msg, ...) { |
105 | 0 | va_list ap; |
106 | 0 | va_start(ap, msg); |
107 | 0 | Error(msg, ap); |
108 | 0 | va_end(ap); |
109 | 0 | } |
110 | | |
111 | 0 | void Info(const char* msg, va_list ap) { |
112 | 0 | fprintf(stdout, "ninja: "); |
113 | 0 | vfprintf(stdout, msg, ap); |
114 | 0 | fprintf(stdout, "\n"); |
115 | 0 | } |
116 | | |
117 | 0 | void Info(const char* msg, ...) { |
118 | 0 | va_list ap; |
119 | 0 | va_start(ap, msg); |
120 | 0 | Info(msg, ap); |
121 | 0 | va_end(ap); |
122 | 0 | } |
123 | | |
124 | 1.61M | void CanonicalizePath(string* path, uint64_t* slash_bits) { |
125 | 1.61M | size_t len = path->size(); |
126 | 1.61M | char* str = 0; |
127 | 1.61M | if (len > 0) |
128 | 1.61M | str = &(*path)[0]; |
129 | 1.61M | CanonicalizePath(str, &len, slash_bits); |
130 | 1.61M | path->resize(len); |
131 | 1.61M | } |
132 | | |
133 | 2.80M | static bool IsPathSeparator(char c) { |
134 | | #ifdef _WIN32 |
135 | | return c == '/' || c == '\\'; |
136 | | #else |
137 | 2.80M | return c == '/'; |
138 | 2.80M | #endif |
139 | 2.80M | } |
140 | | |
141 | 1.61M | void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits) { |
142 | | // WARNING: this function is performance-critical; please benchmark |
143 | | // any changes you make to it. |
144 | 1.61M | if (*len == 0) { |
145 | 0 | return; |
146 | 0 | } |
147 | | |
148 | 1.61M | char* start = path; |
149 | 1.61M | char* dst = start; |
150 | 1.61M | char* dst_start = dst; |
151 | 1.61M | const char* src = start; |
152 | 1.61M | const char* end = start + *len; |
153 | 1.61M | const char* src_next; |
154 | | |
155 | | // For absolute paths, skip the leading directory separator |
156 | | // as this one should never be removed from the result. |
157 | 1.61M | if (IsPathSeparator(*src)) { |
158 | | #ifdef _WIN32 |
159 | | // Windows network path starts with // |
160 | | if (src + 2 <= end && IsPathSeparator(src[1])) { |
161 | | src += 2; |
162 | | dst += 2; |
163 | | } else { |
164 | | ++src; |
165 | | ++dst; |
166 | | } |
167 | | #else |
168 | 748k | ++src; |
169 | 748k | ++dst; |
170 | 748k | #endif |
171 | 748k | dst_start = dst; |
172 | 871k | } else { |
173 | | // For relative paths, skip any leading ../ as these are quite common |
174 | | // to reference source files in build plans, and doing this here makes |
175 | | // the loop work below faster in general. |
176 | 872k | while (src + 3 <= end && src[0] == '.' && src[1] == '.' && |
177 | 39.3k | IsPathSeparator(src[2])) { |
178 | 1.54k | src += 3; |
179 | 1.54k | dst += 3; |
180 | 1.54k | } |
181 | 871k | } |
182 | | |
183 | | // Loop over all components of the paths _except_ the last one, in |
184 | | // order to simplify the loop's code and make it faster. |
185 | 1.61M | int component_count = 0; |
186 | 1.61M | char* dst0 = dst; |
187 | 1.85M | for (; src < end; src = src_next) { |
188 | 1.19M | #ifndef _WIN32 |
189 | | // Use memchr() for faster lookups thanks to optimized C library |
190 | | // implementation. `hyperfine canon_perftest` shows a significant |
191 | | // difference (e,g, 484ms vs 437ms). |
192 | 1.19M | const char* next_sep = |
193 | 1.19M | static_cast<const char*>(::memchr(src, '/', end - src)); |
194 | 1.19M | if (!next_sep) { |
195 | | // This is the last component, will be handled out of the loop. |
196 | 960k | break; |
197 | 960k | } |
198 | | #else |
199 | | // Need to check for both '/' and '\\' so do not use memchr(). |
200 | | // Cannot use strpbrk() because end[0] can be \0 or something else! |
201 | | const char* next_sep = src; |
202 | | while (next_sep != end && !IsPathSeparator(*next_sep)) |
203 | | ++next_sep; |
204 | | if (next_sep == end) { |
205 | | // This is the last component, will be handled out of the loop. |
206 | | break; |
207 | | } |
208 | | #endif |
209 | | // Position for next loop iteration. |
210 | 236k | src_next = next_sep + 1; |
211 | | // Length of the component, excluding trailing directory. |
212 | 236k | size_t component_len = next_sep - src; |
213 | | |
214 | 236k | if (component_len <= 2) { |
215 | 120k | if (component_len == 0) { |
216 | 25.7k | continue; // Ignore empty component, e.g. 'foo//bar' -> 'foo/bar'. |
217 | 25.7k | } |
218 | 94.4k | if (src[0] == '.') { |
219 | 26.0k | if (component_len == 1) { |
220 | 20.6k | continue; // Ignore '.' component, e.g. './foo' -> 'foo'. |
221 | 20.6k | } else if (src[1] == '.') { |
222 | | // Process the '..' component if found. Back up if possible. |
223 | 4.39k | if (component_count > 0) { |
224 | | // Move back to start of previous component. |
225 | 4.27k | --component_count; |
226 | 76.1k | while (--dst > dst0 && !IsPathSeparator(dst[-1])) { |
227 | | // nothing to do here, decrement happens before condition check. |
228 | 71.8k | } |
229 | 4.27k | } else { |
230 | 116 | dst[0] = '.'; |
231 | 116 | dst[1] = '.'; |
232 | 116 | dst[2] = src[2]; |
233 | 116 | dst += 3; |
234 | 116 | } |
235 | 4.39k | continue; |
236 | 4.39k | } |
237 | 26.0k | } |
238 | 94.4k | } |
239 | 185k | ++component_count; |
240 | | |
241 | | // Copy or skip component, including trailing directory separator. |
242 | 185k | if (dst != src) { |
243 | 10.0k | ::memmove(dst, src, src_next - src); |
244 | 10.0k | } |
245 | 185k | dst += src_next - src; |
246 | 185k | } |
247 | | |
248 | | // Handling the last component that does not have a trailing separator. |
249 | | // The logic here is _slightly_ different since there is no trailing |
250 | | // directory separator. |
251 | 1.61M | size_t component_len = end - src; |
252 | 1.61M | do { |
253 | 1.61M | if (component_len == 0) |
254 | 659k | break; // Ignore empty component (e.g. 'foo//' -> 'foo/') |
255 | 960k | if (src[0] == '.') { |
256 | 99.7k | if (component_len == 1) |
257 | 13.8k | break; // Ignore trailing '.' (e.g. 'foo/.' -> 'foo/') |
258 | 85.9k | if (component_len == 2 && src[1] == '.') { |
259 | | // Handle '..'. Back up if possible. |
260 | 6.64k | if (component_count > 0) { |
261 | 23.7k | while (--dst > dst0 && !IsPathSeparator(dst[-1])) { |
262 | | // nothing to do here, decrement happens before condition check. |
263 | 21.9k | } |
264 | 4.77k | } else { |
265 | 4.77k | dst[0] = '.'; |
266 | 4.77k | dst[1] = '.'; |
267 | 4.77k | dst += 2; |
268 | | // No separator to add here. |
269 | 4.77k | } |
270 | 6.64k | break; |
271 | 6.64k | } |
272 | 85.9k | } |
273 | | // Skip or copy last component, no trailing separator. |
274 | 939k | if (dst != src) { |
275 | 26.5k | ::memmove(dst, src, component_len); |
276 | 26.5k | } |
277 | 939k | dst += component_len; |
278 | 939k | } while (0); |
279 | | |
280 | | // Remove trailing path separator if any, but keep the initial |
281 | | // path separator(s) if there was one (or two on Windows). |
282 | 1.61M | if (dst > dst_start && IsPathSeparator(dst[-1])) |
283 | 105k | dst--; |
284 | | |
285 | 1.61M | if (dst == start) { |
286 | | // Handle special cases like "aa/.." -> "." |
287 | 13.2k | *dst++ = '.'; |
288 | 13.2k | } |
289 | | |
290 | 1.61M | *len = dst - start; // dst points after the trailing char here. |
291 | | #ifdef _WIN32 |
292 | | uint64_t bits = 0; |
293 | | uint64_t bits_mask = 1; |
294 | | |
295 | | for (char* c = start; c < start + *len; ++c) { |
296 | | switch (*c) { |
297 | | case '\\': |
298 | | bits |= bits_mask; |
299 | | *c = '/'; |
300 | | NINJA_FALLTHROUGH; |
301 | | case '/': |
302 | | bits_mask <<= 1; |
303 | | } |
304 | | } |
305 | | |
306 | | *slash_bits = bits; |
307 | | #else |
308 | 1.61M | *slash_bits = 0; |
309 | 1.61M | #endif |
310 | 1.61M | } |
311 | | |
312 | 0 | static inline bool IsKnownShellSafeCharacter(char ch) { |
313 | 0 | if ('A' <= ch && ch <= 'Z') return true; |
314 | 0 | if ('a' <= ch && ch <= 'z') return true; |
315 | 0 | if ('0' <= ch && ch <= '9') return true; |
316 | | |
317 | 0 | switch (ch) { |
318 | 0 | case '_': |
319 | 0 | case '+': |
320 | 0 | case '-': |
321 | 0 | case '.': |
322 | 0 | case '/': |
323 | 0 | return true; |
324 | 0 | default: |
325 | 0 | return false; |
326 | 0 | } |
327 | 0 | } |
328 | | |
329 | 0 | static inline bool IsKnownWin32SafeCharacter(char ch) { |
330 | 0 | switch (ch) { |
331 | 0 | case ' ': |
332 | 0 | case '"': |
333 | 0 | return false; |
334 | 0 | default: |
335 | 0 | return true; |
336 | 0 | } |
337 | 0 | } |
338 | | |
339 | 0 | static inline bool StringNeedsShellEscaping(const string& input) { |
340 | 0 | for (size_t i = 0; i < input.size(); ++i) { |
341 | 0 | if (!IsKnownShellSafeCharacter(input[i])) return true; |
342 | 0 | } |
343 | 0 | return false; |
344 | 0 | } |
345 | | |
346 | 0 | static inline bool StringNeedsWin32Escaping(const string& input) { |
347 | 0 | for (size_t i = 0; i < input.size(); ++i) { |
348 | 0 | if (!IsKnownWin32SafeCharacter(input[i])) return true; |
349 | 0 | } |
350 | 0 | return false; |
351 | 0 | } |
352 | | |
353 | 0 | void GetShellEscapedString(const string& input, string* result) { |
354 | 0 | assert(result); |
355 | | |
356 | 0 | if (!StringNeedsShellEscaping(input)) { |
357 | 0 | result->append(input); |
358 | 0 | return; |
359 | 0 | } |
360 | | |
361 | 0 | const char kQuote = '\''; |
362 | 0 | const char kEscapeSequence[] = "'\\'"; |
363 | |
|
364 | 0 | result->push_back(kQuote); |
365 | |
|
366 | 0 | string::const_iterator span_begin = input.begin(); |
367 | 0 | for (string::const_iterator it = input.begin(), end = input.end(); it != end; |
368 | 0 | ++it) { |
369 | 0 | if (*it == kQuote) { |
370 | 0 | result->append(span_begin, it); |
371 | 0 | result->append(kEscapeSequence); |
372 | 0 | span_begin = it; |
373 | 0 | } |
374 | 0 | } |
375 | 0 | result->append(span_begin, input.end()); |
376 | 0 | result->push_back(kQuote); |
377 | 0 | } |
378 | | |
379 | | |
380 | 0 | void GetWin32EscapedString(const string& input, string* result) { |
381 | 0 | assert(result); |
382 | 0 | if (!StringNeedsWin32Escaping(input)) { |
383 | 0 | result->append(input); |
384 | 0 | return; |
385 | 0 | } |
386 | | |
387 | 0 | const char kQuote = '"'; |
388 | 0 | const char kBackslash = '\\'; |
389 | |
|
390 | 0 | result->push_back(kQuote); |
391 | 0 | size_t consecutive_backslash_count = 0; |
392 | 0 | string::const_iterator span_begin = input.begin(); |
393 | 0 | for (string::const_iterator it = input.begin(), end = input.end(); it != end; |
394 | 0 | ++it) { |
395 | 0 | switch (*it) { |
396 | 0 | case kBackslash: |
397 | 0 | ++consecutive_backslash_count; |
398 | 0 | break; |
399 | 0 | case kQuote: |
400 | 0 | result->append(span_begin, it); |
401 | 0 | result->append(consecutive_backslash_count + 1, kBackslash); |
402 | 0 | span_begin = it; |
403 | 0 | consecutive_backslash_count = 0; |
404 | 0 | break; |
405 | 0 | default: |
406 | 0 | consecutive_backslash_count = 0; |
407 | 0 | break; |
408 | 0 | } |
409 | 0 | } |
410 | 0 | result->append(span_begin, input.end()); |
411 | 0 | result->append(consecutive_backslash_count, kBackslash); |
412 | 0 | result->push_back(kQuote); |
413 | 0 | } |
414 | | |
415 | 695 | int ReadFile(const string& path, string* contents, string* err) { |
416 | | #ifdef _WIN32 |
417 | | // This makes a ninja run on a set of 1500 manifest files about 4% faster |
418 | | // than using the generic fopen code below. |
419 | | err->clear(); |
420 | | HANDLE f = ::CreateFileA(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, |
421 | | OPEN_EXISTING, FILE_FLAG_SEQUENTIAL_SCAN, NULL); |
422 | | if (f == INVALID_HANDLE_VALUE) { |
423 | | err->assign(GetLastErrorString()); |
424 | | return -ENOENT; |
425 | | } |
426 | | |
427 | | for (;;) { |
428 | | DWORD len; |
429 | | char buf[64 << 10]; |
430 | | if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { |
431 | | err->assign(GetLastErrorString()); |
432 | | contents->clear(); |
433 | | ::CloseHandle(f); |
434 | | return -EIO; |
435 | | } |
436 | | if (len == 0) |
437 | | break; |
438 | | contents->append(buf, len); |
439 | | } |
440 | | ::CloseHandle(f); |
441 | | return 0; |
442 | | #else |
443 | 695 | FILE* f = fopen(path.c_str(), "rb"); |
444 | 695 | if (!f) { |
445 | 0 | err->assign(strerror(errno)); |
446 | 0 | return -errno; |
447 | 0 | } |
448 | | |
449 | 695 | #ifdef __USE_LARGEFILE64 |
450 | 695 | struct stat64 st; |
451 | 695 | if (fstat64(fileno(f), &st) < 0) { |
452 | | #else |
453 | | struct stat st; |
454 | | if (fstat(fileno(f), &st) < 0) { |
455 | | #endif |
456 | 0 | err->assign(strerror(errno)); |
457 | 0 | fclose(f); |
458 | 0 | return -errno; |
459 | 0 | } |
460 | | |
461 | | // +1 is for the resize in ManifestParser::Load |
462 | 695 | contents->reserve(st.st_size + 1); |
463 | | |
464 | 695 | char buf[64 << 10]; |
465 | 695 | size_t len; |
466 | 1.66k | while (!feof(f) && (len = fread(buf, 1, sizeof(buf), f)) > 0) { |
467 | 968 | contents->append(buf, len); |
468 | 968 | } |
469 | 695 | if (ferror(f)) { |
470 | 1 | err->assign(strerror(errno)); // XXX errno? |
471 | 1 | contents->clear(); |
472 | 1 | fclose(f); |
473 | 1 | return -errno; |
474 | 1 | } |
475 | 694 | fclose(f); |
476 | 694 | return 0; |
477 | 695 | #endif |
478 | 695 | } |
479 | | |
480 | 0 | void SetCloseOnExec(int fd) { |
481 | 0 | #ifndef _WIN32 |
482 | 0 | int flags = fcntl(fd, F_GETFD); |
483 | 0 | if (flags < 0) { |
484 | 0 | perror("fcntl(F_GETFD)"); |
485 | 0 | } else { |
486 | 0 | if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) < 0) |
487 | 0 | perror("fcntl(F_SETFD)"); |
488 | 0 | } |
489 | | #else |
490 | | HANDLE hd = (HANDLE) _get_osfhandle(fd); |
491 | | if (! SetHandleInformation(hd, HANDLE_FLAG_INHERIT, 0)) { |
492 | | fprintf(stderr, "SetHandleInformation(): %s", GetLastErrorString().c_str()); |
493 | | } |
494 | | #endif // ! _WIN32 |
495 | 0 | } |
496 | | |
497 | | |
498 | | const char* SpellcheckStringV(const string& text, |
499 | 0 | const vector<const char*>& words) { |
500 | 0 | const bool kAllowReplacements = true; |
501 | 0 | const int kMaxValidEditDistance = 3; |
502 | |
|
503 | 0 | int min_distance = kMaxValidEditDistance + 1; |
504 | 0 | const char* result = NULL; |
505 | 0 | for (vector<const char*>::const_iterator i = words.begin(); |
506 | 0 | i != words.end(); ++i) { |
507 | 0 | int distance = EditDistance(*i, text, kAllowReplacements, |
508 | 0 | kMaxValidEditDistance); |
509 | 0 | if (distance < min_distance) { |
510 | 0 | min_distance = distance; |
511 | 0 | result = *i; |
512 | 0 | } |
513 | 0 | } |
514 | 0 | return result; |
515 | 0 | } |
516 | | |
517 | 0 | const char* SpellcheckString(const char* text, ...) { |
518 | | // Note: This takes a const char* instead of a string& because using |
519 | | // va_start() with a reference parameter is undefined behavior. |
520 | 0 | va_list ap; |
521 | 0 | va_start(ap, text); |
522 | 0 | vector<const char*> words; |
523 | 0 | const char* word; |
524 | 0 | while ((word = va_arg(ap, const char*))) |
525 | 0 | words.push_back(word); |
526 | 0 | va_end(ap); |
527 | 0 | return SpellcheckStringV(text, words); |
528 | 0 | } |
529 | | |
530 | | #ifdef _WIN32 |
531 | | string GetLastErrorString() { |
532 | | DWORD err = GetLastError(); |
533 | | |
534 | | char* msg_buf; |
535 | | FormatMessageA( |
536 | | FORMAT_MESSAGE_ALLOCATE_BUFFER | |
537 | | FORMAT_MESSAGE_FROM_SYSTEM | |
538 | | FORMAT_MESSAGE_IGNORE_INSERTS, |
539 | | NULL, |
540 | | err, |
541 | | MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), |
542 | | (char*)&msg_buf, |
543 | | 0, |
544 | | NULL); |
545 | | |
546 | | if (msg_buf == nullptr) { |
547 | | char fallback_msg[128] = {0}; |
548 | | snprintf(fallback_msg, sizeof(fallback_msg), "GetLastError() = %lu", err); |
549 | | return fallback_msg; |
550 | | } |
551 | | |
552 | | string msg = msg_buf; |
553 | | LocalFree(msg_buf); |
554 | | return msg; |
555 | | } |
556 | | |
557 | | void Win32Fatal(const char* function, const char* hint) { |
558 | | if (hint) { |
559 | | Fatal("%s: %s (%s)", function, GetLastErrorString().c_str(), hint); |
560 | | } else { |
561 | | Fatal("%s: %s", function, GetLastErrorString().c_str()); |
562 | | } |
563 | | } |
564 | | #endif |
565 | | |
566 | 0 | bool islatinalpha(int c) { |
567 | | // isalpha() is locale-dependent. |
568 | 0 | return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); |
569 | 0 | } |
570 | | |
571 | 0 | string StripAnsiEscapeCodes(const string& in) { |
572 | 0 | string stripped; |
573 | 0 | stripped.reserve(in.size()); |
574 | |
|
575 | 0 | for (size_t i = 0; i < in.size(); ++i) { |
576 | 0 | if (in[i] != '\33') { |
577 | | // Not an escape code. |
578 | 0 | stripped.push_back(in[i]); |
579 | 0 | continue; |
580 | 0 | } |
581 | | |
582 | | // Only strip CSIs for now. |
583 | 0 | if (i + 1 >= in.size()) break; |
584 | 0 | if (in[i + 1] != '[') continue; // Not a CSI. |
585 | 0 | i += 2; |
586 | | |
587 | | // Skip everything up to and including the next [a-zA-Z]. |
588 | 0 | while (i < in.size() && !islatinalpha(in[i])) |
589 | 0 | ++i; |
590 | 0 | } |
591 | 0 | return stripped; |
592 | 0 | } |
593 | | |
594 | | #if defined(__linux__) || defined(__GLIBC__) |
595 | 0 | std::pair<int64_t, bool> readCount(const std::string& path) { |
596 | 0 | std::ifstream file(path.c_str()); |
597 | 0 | if (!file.is_open()) |
598 | 0 | return std::make_pair(0, false); |
599 | 0 | int64_t n = 0; |
600 | 0 | file >> n; |
601 | 0 | if (file.good()) |
602 | 0 | return std::make_pair(n, true); |
603 | 0 | return std::make_pair(0, false); |
604 | 0 | } |
605 | | |
606 | | struct MountPoint { |
607 | | int mountId; |
608 | | int parentId; |
609 | | StringPiece deviceId; |
610 | | StringPiece root; |
611 | | StringPiece mountPoint; |
612 | | vector<StringPiece> options; |
613 | | vector<StringPiece> optionalFields; |
614 | | StringPiece fsType; |
615 | | StringPiece mountSource; |
616 | | vector<StringPiece> superOptions; |
617 | 0 | bool parse(const string& line) { |
618 | 0 | vector<StringPiece> pieces = SplitStringPiece(line, ' '); |
619 | 0 | if (pieces.size() < 10) |
620 | 0 | return false; |
621 | 0 | size_t optionalStart = 0; |
622 | 0 | for (size_t i = 6; i < pieces.size(); i++) { |
623 | 0 | if (pieces[i] == "-") { |
624 | 0 | optionalStart = i + 1; |
625 | 0 | break; |
626 | 0 | } |
627 | 0 | } |
628 | 0 | if (optionalStart == 0) |
629 | 0 | return false; |
630 | 0 | if (optionalStart + 3 != pieces.size()) |
631 | 0 | return false; |
632 | 0 | mountId = atoi(pieces[0].AsString().c_str()); |
633 | 0 | parentId = atoi(pieces[1].AsString().c_str()); |
634 | 0 | deviceId = pieces[2]; |
635 | 0 | root = pieces[3]; |
636 | 0 | mountPoint = pieces[4]; |
637 | 0 | options = SplitStringPiece(pieces[5], ','); |
638 | 0 | optionalFields = |
639 | 0 | vector<StringPiece>(&pieces[6], &pieces[optionalStart - 1]); |
640 | 0 | fsType = pieces[optionalStart]; |
641 | 0 | mountSource = pieces[optionalStart + 1]; |
642 | 0 | superOptions = SplitStringPiece(pieces[optionalStart + 2], ','); |
643 | 0 | return true; |
644 | 0 | } |
645 | 0 | string translate(string& path) const { |
646 | | // path must be sub dir of root |
647 | 0 | if (path.compare(0, root.len_, root.str_, root.len_) != 0) { |
648 | 0 | return string(); |
649 | 0 | } |
650 | 0 | path.erase(0, root.len_); |
651 | 0 | if (path == ".." || (path.length() > 2 && path.compare(0, 3, "../") == 0)) { |
652 | 0 | return string(); |
653 | 0 | } |
654 | 0 | return mountPoint.AsString() + "/" + path; |
655 | 0 | } |
656 | | }; |
657 | | |
658 | | struct CGroupSubSys { |
659 | | int id; |
660 | | string name; |
661 | | vector<string> subsystems; |
662 | 0 | bool parse(string& line) { |
663 | 0 | size_t first = line.find(':'); |
664 | 0 | if (first == string::npos) |
665 | 0 | return false; |
666 | 0 | line[first] = '\0'; |
667 | 0 | size_t second = line.find(':', first + 1); |
668 | 0 | if (second == string::npos) |
669 | 0 | return false; |
670 | 0 | line[second] = '\0'; |
671 | 0 | id = atoi(line.c_str()); |
672 | 0 | name = line.substr(second + 1); |
673 | 0 | vector<StringPiece> pieces = |
674 | 0 | SplitStringPiece(StringPiece(line.c_str() + first + 1), ','); |
675 | 0 | for (size_t i = 0; i < pieces.size(); i++) { |
676 | 0 | subsystems.push_back(pieces[i].AsString()); |
677 | 0 | } |
678 | 0 | return true; |
679 | 0 | } |
680 | | }; |
681 | | |
682 | 0 | map<string, string> ParseMountInfo(map<string, CGroupSubSys>& subsystems) { |
683 | 0 | map<string, string> cgroups; |
684 | 0 | ifstream mountinfo("/proc/self/mountinfo"); |
685 | 0 | if (!mountinfo.is_open()) |
686 | 0 | return cgroups; |
687 | 0 | while (!mountinfo.eof()) { |
688 | 0 | string line; |
689 | 0 | getline(mountinfo, line); |
690 | 0 | MountPoint mp; |
691 | 0 | if (!mp.parse(line)) |
692 | 0 | continue; |
693 | 0 | if (mp.fsType == "cgroup") { |
694 | 0 | for (size_t i = 0; i < mp.superOptions.size(); i++) { |
695 | 0 | std::string opt = mp.superOptions[i].AsString(); |
696 | 0 | auto subsys = subsystems.find(opt); |
697 | 0 | if (subsys == subsystems.end()) { |
698 | 0 | continue; |
699 | 0 | } |
700 | 0 | std::string newPath = mp.translate(subsys->second.name); |
701 | 0 | if (!newPath.empty()) { |
702 | 0 | cgroups.emplace(opt, newPath); |
703 | 0 | } |
704 | 0 | } |
705 | 0 | } else if (mp.fsType == "cgroup2") { |
706 | | // Find cgroup2 entry in format "0::/path/to/cgroup" |
707 | 0 | auto subsys = std::find_if(subsystems.begin(), subsystems.end(), |
708 | 0 | [](const auto& sys) { |
709 | 0 | return sys.first == "" && sys.second.id == 0; |
710 | 0 | }); |
711 | 0 | if (subsys == subsystems.end()) { |
712 | 0 | continue; |
713 | 0 | } |
714 | 0 | std::string path = mp.mountPoint.AsString(); |
715 | 0 | if (subsys->second.name != "/") { |
716 | | // Append the relative path for the cgroup to the mount point |
717 | 0 | path.append(subsys->second.name); |
718 | 0 | } |
719 | 0 | cgroups.emplace("cgroup2", path); |
720 | 0 | } |
721 | 0 | } |
722 | 0 | return cgroups; |
723 | 0 | } |
724 | | |
725 | 0 | map<string, CGroupSubSys> ParseSelfCGroup() { |
726 | 0 | map<string, CGroupSubSys> cgroups; |
727 | 0 | ifstream cgroup("/proc/self/cgroup"); |
728 | 0 | if (!cgroup.is_open()) |
729 | 0 | return cgroups; |
730 | 0 | string line; |
731 | 0 | while (!cgroup.eof()) { |
732 | 0 | getline(cgroup, line); |
733 | 0 | CGroupSubSys subsys; |
734 | 0 | if (!subsys.parse(line)) |
735 | 0 | continue; |
736 | 0 | for (size_t i = 0; i < subsys.subsystems.size(); i++) { |
737 | 0 | cgroups.insert(make_pair(subsys.subsystems[i], subsys)); |
738 | 0 | } |
739 | 0 | } |
740 | 0 | return cgroups; |
741 | 0 | } |
742 | | |
743 | 0 | int ParseCgroupV1(std::string& path) { |
744 | 0 | std::pair<int64_t, bool> quota = readCount(path + "/cpu.cfs_quota_us"); |
745 | 0 | if (!quota.second || quota.first == -1) |
746 | 0 | return -1; |
747 | 0 | std::pair<int64_t, bool> period = readCount(path + "/cpu.cfs_period_us"); |
748 | 0 | if (!period.second) |
749 | 0 | return -1; |
750 | 0 | if (period.first == 0) |
751 | 0 | return -1; |
752 | 0 | return quota.first / period.first; |
753 | 0 | } |
754 | | |
755 | 0 | int ParseCgroupV2(std::string& path) { |
756 | | // Read CPU quota from cgroup v2 |
757 | 0 | std::ifstream cpu_max(path + "/cpu.max"); |
758 | 0 | if (!cpu_max.is_open()) { |
759 | 0 | return -1; |
760 | 0 | } |
761 | 0 | std::string max_line; |
762 | 0 | if (!std::getline(cpu_max, max_line) || max_line.empty()) { |
763 | 0 | return -1; |
764 | 0 | } |
765 | | // Format is "quota period" or "max period" |
766 | 0 | size_t space_pos = max_line.find(' '); |
767 | 0 | if (space_pos == string::npos) { |
768 | 0 | return -1; |
769 | 0 | } |
770 | 0 | std::string quota_str = max_line.substr(0, space_pos); |
771 | 0 | std::string period_str = max_line.substr(space_pos + 1); |
772 | 0 | if (quota_str == "max") { |
773 | 0 | return -1; // No CPU limit set |
774 | 0 | } |
775 | | // Convert quota string to integer |
776 | 0 | char* quota_end = nullptr; |
777 | 0 | errno = 0; |
778 | 0 | int64_t quota = strtoll(quota_str.c_str(), "a_end, 10); |
779 | | // Check for conversion errors |
780 | 0 | if (errno == ERANGE || quota_end == quota_str.c_str() || *quota_end != '\0' || |
781 | 0 | quota <= 0) { |
782 | 0 | return -1; |
783 | 0 | } |
784 | | // Convert period string to integer |
785 | 0 | char* period_end = nullptr; |
786 | 0 | errno = 0; |
787 | 0 | int64_t period = strtoll(period_str.c_str(), &period_end, 10); |
788 | | // Check for conversion errors |
789 | 0 | if (errno == ERANGE || period_end == period_str.c_str() || |
790 | 0 | *period_end != '\0' || period <= 0) { |
791 | 0 | return -1; |
792 | 0 | } |
793 | 0 | return quota / period; |
794 | 0 | } |
795 | | |
796 | 0 | int ParseCPUFromCGroup() { |
797 | 0 | auto subsystems = ParseSelfCGroup(); |
798 | 0 | auto cgroups = ParseMountInfo(subsystems); |
799 | | |
800 | | // Prefer cgroup v2 if both v1 and v2 should be present |
801 | 0 | const auto cgroup2 = cgroups.find("cgroup2"); |
802 | 0 | if (cgroup2 != cgroups.end()) { |
803 | 0 | return ParseCgroupV2(cgroup2->second); |
804 | 0 | } |
805 | | |
806 | 0 | const auto cpu = cgroups.find("cpu"); |
807 | 0 | if (cpu != cgroups.end()) { |
808 | 0 | return ParseCgroupV1(cpu->second); |
809 | 0 | } |
810 | 0 | return -1; |
811 | 0 | } |
812 | | #endif |
813 | | |
814 | 0 | int GetProcessorCount() { |
815 | | #ifdef _WIN32 |
816 | | DWORD cpuCount = 0; |
817 | | #ifndef _WIN64 |
818 | | // Need to use GetLogicalProcessorInformationEx to get real core count on |
819 | | // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 |
820 | | DWORD len = 0; |
821 | | if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len) |
822 | | && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { |
823 | | std::vector<char> buf(len); |
824 | | int cores = 0; |
825 | | if (GetLogicalProcessorInformationEx(RelationProcessorCore, |
826 | | reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>( |
827 | | buf.data()), &len)) { |
828 | | for (DWORD i = 0; i < len; ) { |
829 | | auto info = reinterpret_cast<PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX>( |
830 | | buf.data() + i); |
831 | | if (info->Relationship == RelationProcessorCore && |
832 | | info->Processor.GroupCount == 1) { |
833 | | for (KAFFINITY core_mask = info->Processor.GroupMask[0].Mask; |
834 | | core_mask; core_mask >>= 1) { |
835 | | cores += (core_mask & 1); |
836 | | } |
837 | | } |
838 | | i += info->Size; |
839 | | } |
840 | | if (cores != 0) { |
841 | | cpuCount = cores; |
842 | | } |
843 | | } |
844 | | } |
845 | | #endif |
846 | | if (cpuCount == 0) { |
847 | | cpuCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); |
848 | | } |
849 | | JOBOBJECT_CPU_RATE_CONTROL_INFORMATION info; |
850 | | // reference: |
851 | | // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_cpu_rate_control_information |
852 | | if (QueryInformationJobObject(NULL, JobObjectCpuRateControlInformation, &info, |
853 | | sizeof(info), NULL)) { |
854 | | if (info.ControlFlags & (JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | |
855 | | JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP)) { |
856 | | return cpuCount * info.CpuRate / 10000; |
857 | | } |
858 | | } |
859 | | return cpuCount; |
860 | | #else |
861 | 0 | int cgroupCount = -1; |
862 | 0 | int schedCount = -1; |
863 | 0 | #if defined(__linux__) || defined(__GLIBC__) |
864 | 0 | cgroupCount = ParseCPUFromCGroup(); |
865 | 0 | #endif |
866 | | // The number of exposed processors might not represent the actual number of |
867 | | // processors threads can run on. This happens when a CPU set limitation is |
868 | | // active, see https://github.com/ninja-build/ninja/issues/1278 |
869 | | #if defined(__FreeBSD__) |
870 | | cpuset_t mask; |
871 | | CPU_ZERO(&mask); |
872 | | if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(mask), |
873 | | &mask) == 0) { |
874 | | return CPU_COUNT(&mask); |
875 | | } |
876 | | #elif defined(CPU_COUNT) |
877 | | cpu_set_t set; |
878 | 0 | if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) { |
879 | 0 | schedCount = CPU_COUNT(&set); |
880 | 0 | } |
881 | 0 | #endif |
882 | 0 | if (cgroupCount >= 0 && schedCount >= 0) return std::min(cgroupCount, schedCount); |
883 | 0 | if (cgroupCount < 0 && schedCount < 0) |
884 | 0 | return static_cast<int>(sysconf(_SC_NPROCESSORS_ONLN)); |
885 | 0 | return std::max(cgroupCount, schedCount); |
886 | 0 | #endif |
887 | 0 | } |
888 | | |
889 | | #if defined(_WIN32) || defined(__CYGWIN__) |
890 | | static double CalculateProcessorLoad(uint64_t idle_ticks, uint64_t total_ticks) |
891 | | { |
892 | | static uint64_t previous_idle_ticks = 0; |
893 | | static uint64_t previous_total_ticks = 0; |
894 | | static double previous_load = -0.0; |
895 | | |
896 | | uint64_t idle_ticks_since_last_time = idle_ticks - previous_idle_ticks; |
897 | | uint64_t total_ticks_since_last_time = total_ticks - previous_total_ticks; |
898 | | |
899 | | bool first_call = (previous_total_ticks == 0); |
900 | | bool ticks_not_updated_since_last_call = (total_ticks_since_last_time == 0); |
901 | | |
902 | | double load; |
903 | | if (first_call || ticks_not_updated_since_last_call) { |
904 | | load = previous_load; |
905 | | } else { |
906 | | // Calculate load. |
907 | | double idle_to_total_ratio = |
908 | | ((double)idle_ticks_since_last_time) / total_ticks_since_last_time; |
909 | | double load_since_last_call = 1.0 - idle_to_total_ratio; |
910 | | |
911 | | // Filter/smooth result when possible. |
912 | | if(previous_load > 0) { |
913 | | load = 0.9 * previous_load + 0.1 * load_since_last_call; |
914 | | } else { |
915 | | load = load_since_last_call; |
916 | | } |
917 | | } |
918 | | |
919 | | previous_load = load; |
920 | | previous_total_ticks = total_ticks; |
921 | | previous_idle_ticks = idle_ticks; |
922 | | |
923 | | return load; |
924 | | } |
925 | | |
926 | | static uint64_t FileTimeToTickCount(const FILETIME & ft) |
927 | | { |
928 | | uint64_t high = (((uint64_t)(ft.dwHighDateTime)) << 32); |
929 | | uint64_t low = ft.dwLowDateTime; |
930 | | return (high | low); |
931 | | } |
932 | | |
933 | | double GetLoadAverage() { |
934 | | FILETIME idle_time, kernel_time, user_time; |
935 | | BOOL get_system_time_succeeded = |
936 | | GetSystemTimes(&idle_time, &kernel_time, &user_time); |
937 | | |
938 | | double posix_compatible_load; |
939 | | if (get_system_time_succeeded) { |
940 | | uint64_t idle_ticks = FileTimeToTickCount(idle_time); |
941 | | |
942 | | // kernel_time from GetSystemTimes already includes idle_time. |
943 | | uint64_t total_ticks = |
944 | | FileTimeToTickCount(kernel_time) + FileTimeToTickCount(user_time); |
945 | | |
946 | | double processor_load = CalculateProcessorLoad(idle_ticks, total_ticks); |
947 | | posix_compatible_load = processor_load * GetProcessorCount(); |
948 | | |
949 | | } else { |
950 | | posix_compatible_load = -0.0; |
951 | | } |
952 | | |
953 | | return posix_compatible_load; |
954 | | } |
955 | | #elif defined(__PASE__) |
956 | | double GetLoadAverage() { |
957 | | return -0.0f; |
958 | | } |
959 | | #elif defined(_AIX) |
960 | | double GetLoadAverage() { |
961 | | perfstat_cpu_total_t cpu_stats; |
962 | | if (perfstat_cpu_total(NULL, &cpu_stats, sizeof(cpu_stats), 1) < 0) { |
963 | | return -0.0f; |
964 | | } |
965 | | |
966 | | // Calculation taken from comment in libperfstats.h |
967 | | return double(cpu_stats.loadavg[0]) / double(1 << SBITS); |
968 | | } |
969 | | #elif defined(__UCLIBC__) || (defined(__BIONIC__) && __ANDROID_API__ < 29) |
970 | | double GetLoadAverage() { |
971 | | struct sysinfo si; |
972 | | if (sysinfo(&si) != 0) |
973 | | return -0.0f; |
974 | | return 1.0 / (1 << SI_LOAD_SHIFT) * si.loads[0]; |
975 | | } |
976 | | #elif defined(__HAIKU__) |
977 | | double GetLoadAverage() { |
978 | | return -0.0f; |
979 | | } |
980 | | #else |
981 | 0 | double GetLoadAverage() { |
982 | 0 | double loadavg[3] = { 0.0f, 0.0f, 0.0f }; |
983 | 0 | if (getloadavg(loadavg, 3) < 0) { |
984 | | // Maybe we should return an error here or the availability of |
985 | | // getloadavg(3) should be checked when ninja is configured. |
986 | 0 | return -0.0f; |
987 | 0 | } |
988 | 0 | return loadavg[0]; |
989 | 0 | } |
990 | | #endif // _WIN32 |
991 | | |
992 | 0 | std::string GetWorkingDirectory() { |
993 | 0 | std::string ret; |
994 | 0 | char* success = NULL; |
995 | 0 | do { |
996 | 0 | ret.resize(ret.size() + 1024); |
997 | 0 | errno = 0; |
998 | 0 | success = getcwd(&ret[0], ret.size()); |
999 | 0 | } while (!success && errno == ERANGE); |
1000 | 0 | if (!success) { |
1001 | 0 | Fatal("cannot determine working directory: %s", strerror(errno)); |
1002 | 0 | } |
1003 | 0 | ret.resize(strlen(&ret[0])); |
1004 | 0 | return ret; |
1005 | 0 | } |
1006 | | |
1007 | 0 | bool Truncate(const string& path, size_t size, string* err) { |
1008 | | #ifdef _WIN32 |
1009 | | int fh = _sopen(path.c_str(), _O_RDWR | _O_CREAT, _SH_DENYNO, |
1010 | | _S_IREAD | _S_IWRITE); |
1011 | | int success = _chsize(fh, size); |
1012 | | _close(fh); |
1013 | | #else |
1014 | 0 | int success = truncate(path.c_str(), size); |
1015 | 0 | #endif |
1016 | | // Both truncate() and _chsize() return 0 on success and set errno and return |
1017 | | // -1 on failure. |
1018 | 0 | if (success < 0) { |
1019 | 0 | *err = strerror(errno); |
1020 | 0 | return false; |
1021 | 0 | } |
1022 | 0 | return true; |
1023 | 0 | } |
1024 | | |
1025 | 0 | int platformAwareUnlink(const char* filename) { |
1026 | | #ifdef _WIN32 |
1027 | | return _unlink(filename); |
1028 | | #else |
1029 | 0 | return unlink(filename); |
1030 | 0 | #endif |
1031 | 0 | } |