Line | Count | Source |
1 | | /* Simple NUMA library. |
2 | | Copyright (C) 2003,2004,2005,2008 Andi Kleen,SuSE Labs and |
3 | | Cliff Wickman,SGI. |
4 | | |
5 | | libnuma is free software; you can redistribute it and/or |
6 | | modify it under the terms of the GNU Lesser General Public |
7 | | License as published by the Free Software Foundation; version |
8 | | 2.1. |
9 | | |
10 | | libnuma is distributed in the hope that it will be useful, |
11 | | but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
13 | | Lesser General Public License for more details. |
14 | | |
15 | | You should find a copy of v2.1 of the GNU Lesser General Public License |
16 | | somewhere on your Linux system; if not, write to the Free Software |
17 | | Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
18 | | |
19 | | All calls are undefined when numa_available returns an error. */ |
20 | | #define _GNU_SOURCE |
21 | | #include <stdlib.h> |
22 | | #include <stdio.h> |
23 | | #include <unistd.h> |
24 | | #include <string.h> |
25 | | #include <sched.h> |
26 | | #include <dirent.h> |
27 | | #include <errno.h> |
28 | | #include <stdarg.h> |
29 | | #include <ctype.h> |
30 | | #include <assert.h> |
31 | | |
32 | | #include <sys/mman.h> |
33 | | #include <limits.h> |
34 | | |
35 | | #include "config.h" |
36 | | #include "numa.h" |
37 | | #include "numaif.h" |
38 | | #include "numaint.h" |
39 | | #include "util.h" |
40 | | #include "affinity.h" |
41 | | |
42 | | #define WEAK __attribute__((weak)) |
43 | | |
44 | 0 | #define CPU_BUFFER_SIZE 4096 /* This limits you to 32768 CPUs */ |
45 | | |
46 | | /* these are the old (version 1) masks */ |
47 | | nodemask_t numa_no_nodes; |
48 | | nodemask_t numa_all_nodes; |
49 | | /* these are now the default bitmask (pointers to) (version 2) */ |
50 | | struct bitmask *numa_no_nodes_ptr = NULL; |
51 | | struct bitmask *numa_all_nodes_ptr = NULL; |
52 | | struct bitmask *numa_possible_nodes_ptr = NULL; |
53 | | struct bitmask *numa_all_cpus_ptr = NULL; |
54 | | struct bitmask *numa_possible_cpus_ptr = NULL; |
55 | | /* I would prefer to use symbol versioning to create v1 and v2 versions |
56 | | of numa_no_nodes and numa_all_nodes, but the loader does not correctly |
57 | | handle versioning of BSS versus small data items */ |
58 | | |
59 | | struct bitmask *numa_nodes_ptr = NULL; |
60 | | static struct bitmask *numa_memnode_ptr = NULL; |
61 | | static unsigned long *node_cpu_mask_v1[NUMA_NUM_NODES]; |
62 | | static char node_cpu_mask_v1_stale = 1; |
63 | | static struct bitmask **node_cpu_mask_v2; |
64 | | static char node_cpu_mask_v2_stale = 1; |
65 | | |
66 | | WEAK void numa_error(char *where); |
67 | | |
68 | | #ifndef TLS |
69 | | #warning "not threadsafe" |
70 | | #define __thread |
71 | | #endif |
72 | | |
73 | | static __thread int bind_policy = MPOL_BIND; |
74 | | static __thread unsigned int mbind_flags = 0; |
75 | | static int sizes_set=0; |
76 | | static int maxconfigurednode = -1; |
77 | | static int maxconfiguredcpu = -1; |
78 | | static int numprocnode = -1; |
79 | | static int numproccpu = -1; |
80 | | static int nodemask_sz = 0; |
81 | | static int cpumask_sz = 0; |
82 | | |
83 | | static int has_preferred_many = -1; |
84 | | |
85 | | int numa_exit_on_error = 0; |
86 | | int numa_exit_on_warn = 0; |
87 | | int numa_fail_alloc_on_error = 0; |
88 | | static void set_sizes(void); |
89 | | |
90 | | /* |
91 | | * There are two special functions, _init(void) and _fini(void), which |
92 | | * are called automatically by the dynamic loader whenever a library is loaded. |
93 | | * |
94 | | * The v1 library depends upon nodemask_t's of all nodes and no nodes. |
95 | | */ |
96 | | void __attribute__((constructor)) |
97 | | numa_init(void) |
98 | 2 | { |
99 | 2 | int max,i; |
100 | | |
101 | 2 | if (sizes_set) |
102 | 0 | return; |
103 | | |
104 | 2 | set_sizes(); |
105 | | /* numa_all_nodes should represent existing nodes on this system */ |
106 | 2 | max = numa_num_configured_nodes(); |
107 | 4 | for (i = 0; i < max; i++) |
108 | 2 | nodemask_set_compat((nodemask_t *)&numa_all_nodes, i); |
109 | 2 | memset(&numa_no_nodes, 0, sizeof(numa_no_nodes)); |
110 | | |
111 | | /* clear errno */ |
112 | 2 | errno = 0; |
113 | 2 | } |
114 | | |
115 | | static void cleanup_node_cpu_mask_v2(void); |
116 | | |
117 | 0 | #define FREE_AND_ZERO(x) if (x) { \ |
118 | 0 | numa_bitmask_free(x); \ |
119 | 0 | x = NULL; \ |
120 | 0 | } |
121 | | |
122 | | void __attribute__((destructor)) |
123 | | numa_fini(void) |
124 | 0 | { |
125 | 0 | FREE_AND_ZERO(numa_all_cpus_ptr); |
126 | 0 | FREE_AND_ZERO(numa_possible_cpus_ptr); |
127 | 0 | FREE_AND_ZERO(numa_all_nodes_ptr); |
128 | 0 | FREE_AND_ZERO(numa_possible_nodes_ptr); |
129 | 0 | FREE_AND_ZERO(numa_no_nodes_ptr); |
130 | 0 | FREE_AND_ZERO(numa_memnode_ptr); |
131 | 0 | FREE_AND_ZERO(numa_nodes_ptr); |
132 | 0 | cleanup_node_cpu_mask_v2(); |
133 | 0 | numa_distance_cleanup(); |
134 | 0 | } |
135 | | |
136 | | static int numa_find_first(struct bitmask *mask) |
137 | 0 | { |
138 | 0 | int i; |
139 | 0 | for (i = 0; i < mask->size; i++) |
140 | 0 | if (numa_bitmask_isbitset(mask, i)) |
141 | 0 | return i; |
142 | 0 | return -1; |
143 | 0 | } |
144 | | |
145 | | /* |
146 | | * The following bitmask declarations, bitmask_*() routines, and associated |
147 | | * _setbit() and _getbit() routines are: |
148 | | * Copyright (c) 2004_2007 Silicon Graphics, Inc. (SGI) All rights reserved. |
149 | | * SGI publishes it under the terms of the Library GNU General Public License, |
150 | | * v2, as published by the Free Software Foundation. |
151 | | */ |
152 | | static unsigned int |
153 | | _getbit(const struct bitmask *bmp, unsigned int n) |
154 | 130k | { |
155 | 130k | if (n < bmp->size) |
156 | 130k | return (bmp->maskp[n/bitsperlong] >> (n % bitsperlong)) & 1; |
157 | 162 | else |
158 | 162 | return 0; |
159 | 130k | } |
160 | | |
161 | | static void |
162 | | _setbit(struct bitmask *bmp, unsigned int n, unsigned int v) |
163 | 18.3k | { |
164 | 18.3k | if (n < bmp->size) { |
165 | 18.3k | if (v) |
166 | 9.88k | bmp->maskp[n/bitsperlong] |= 1UL << (n % bitsperlong); |
167 | 8.41k | else |
168 | 8.41k | bmp->maskp[n/bitsperlong] &= ~(1UL << (n % bitsperlong)); |
169 | 18.3k | } |
170 | 18.3k | } |
171 | | |
172 | | int |
173 | | numa_bitmask_isbitset(const struct bitmask *bmp, unsigned int i) |
174 | 128k | { |
175 | 128k | return _getbit(bmp, i); |
176 | 128k | } |
177 | | |
178 | | struct bitmask * |
179 | | numa_bitmask_setall(struct bitmask *bmp) |
180 | 0 | { |
181 | 0 | unsigned int i; |
182 | 0 | for (i = 0; i < bmp->size; i++) |
183 | 0 | _setbit(bmp, i, 1); |
184 | 0 | return bmp; |
185 | 0 | } |
186 | | |
187 | | struct bitmask * |
188 | | numa_bitmask_clearall(struct bitmask *bmp) |
189 | 131 | { |
190 | 131 | unsigned int i; |
191 | 8.51k | for (i = 0; i < bmp->size; i++) |
192 | 8.38k | _setbit(bmp, i, 0); |
193 | 131 | return bmp; |
194 | 131 | } |
195 | | |
196 | | struct bitmask * |
197 | | numa_bitmask_setbit(struct bitmask *bmp, unsigned int i) |
198 | 9.88k | { |
199 | 9.88k | _setbit(bmp, i, 1); |
200 | 9.88k | return bmp; |
201 | 9.88k | } |
202 | | |
203 | | struct bitmask * |
204 | | numa_bitmask_clearbit(struct bitmask *bmp, unsigned int i) |
205 | 35 | { |
206 | 35 | _setbit(bmp, i, 0); |
207 | 35 | return bmp; |
208 | 35 | } |
209 | | |
210 | | unsigned int |
211 | | numa_bitmask_nbytes(struct bitmask *bmp) |
212 | 2 | { |
213 | 2 | return longsperbits(bmp->size) * sizeof(unsigned long); |
214 | 2 | } |
215 | | |
216 | | /* where n is the number of bits in the map */ |
217 | | /* This function should not exit on failure, but right now we cannot really |
218 | | recover from this. */ |
219 | | struct bitmask * |
220 | | numa_bitmask_alloc(unsigned int n) |
221 | 1.80k | { |
222 | 1.80k | struct bitmask *bmp; |
223 | | |
224 | 1.80k | if (n < 1) { |
225 | 0 | errno = EINVAL; |
226 | 0 | numa_error("request to allocate mask for invalid number"); |
227 | 0 | return NULL; |
228 | 0 | } |
229 | 1.80k | bmp = malloc(sizeof(*bmp)); |
230 | 1.80k | if (bmp == 0) |
231 | 0 | goto oom; |
232 | 1.80k | bmp->size = n; |
233 | 1.80k | bmp->maskp = calloc(longsperbits(n), sizeof(unsigned long)); |
234 | 1.80k | if (bmp->maskp == 0) { |
235 | 0 | free(bmp); |
236 | 0 | goto oom; |
237 | 0 | } |
238 | 1.80k | return bmp; |
239 | | |
240 | 0 | oom: |
241 | 0 | numa_error("Out of memory allocating bitmask"); |
242 | 0 | exit(1); |
243 | 1.80k | } |
244 | | |
245 | | void |
246 | | numa_bitmask_free(struct bitmask *bmp) |
247 | 1.78k | { |
248 | 1.78k | if (bmp == 0) |
249 | 0 | return; |
250 | 1.78k | free(bmp->maskp); |
251 | 1.78k | bmp->maskp = (unsigned long *)0xdeadcdef; /* double free tripwire */ |
252 | 1.78k | free(bmp); |
253 | 1.78k | return; |
254 | 1.78k | } |
255 | | |
256 | | /* True if two bitmasks are equal */ |
257 | | int |
258 | | numa_bitmask_equal(const struct bitmask *bmp1, const struct bitmask *bmp2) |
259 | 0 | { |
260 | 0 | unsigned int i; |
261 | 0 | for (i = 0; i < bmp1->size || i < bmp2->size; i++) |
262 | 0 | if (_getbit(bmp1, i) != _getbit(bmp2, i)) |
263 | 0 | return 0; |
264 | 0 | return 1; |
265 | 0 | } |
266 | | |
267 | | /* Hamming Weight: number of set bits */ |
268 | | unsigned int numa_bitmask_weight(const struct bitmask *bmp) |
269 | 4 | { |
270 | 4 | unsigned int i; |
271 | 4 | unsigned int w = 0; |
272 | 2.18k | for (i = 0; i < bmp->size; i++) |
273 | 2.17k | if (_getbit(bmp, i)) |
274 | 66 | w++; |
275 | 4 | return w; |
276 | 4 | } |
277 | | |
278 | | /* *****end of bitmask_ routines ************ */ |
279 | | |
280 | | /* Next two can be overwritten by the application for different error handling */ |
281 | | WEAK void numa_error(char *where) |
282 | 0 | { |
283 | 0 | int olde = errno; |
284 | 0 | perror(where); |
285 | 0 | if (numa_exit_on_error) |
286 | 0 | exit(1); |
287 | 0 | errno = olde; |
288 | 0 | } |
289 | | |
290 | | WEAK void numa_warn(int num, char *fmt, ...) |
291 | 1.58k | { |
292 | 1.58k | static unsigned warned; |
293 | 1.58k | va_list ap; |
294 | 1.58k | int olde = errno; |
295 | | |
296 | | /* Give each warning only once */ |
297 | 1.58k | if ((1<<num) & warned) |
298 | 1.57k | return; |
299 | 9 | warned |= (1<<num); |
300 | | |
301 | 9 | va_start(ap,fmt); |
302 | 9 | fprintf(stderr, "libnuma: Warning: "); |
303 | 9 | vfprintf(stderr, fmt, ap); |
304 | 9 | fputc('\n', stderr); |
305 | 9 | va_end(ap); |
306 | | |
307 | 9 | if (numa_exit_on_warn) |
308 | 0 | exit(1); |
309 | | |
310 | 9 | errno = olde; |
311 | 9 | } |
312 | | |
313 | | static void setpol(int policy, struct bitmask *bmp) |
314 | 0 | { |
315 | 0 | if (set_mempolicy(policy, bmp->maskp, bmp->size) < 0) |
316 | 0 | numa_error("set_mempolicy"); |
317 | 0 | } |
318 | | |
319 | | static void getpol(int *oldpolicy, struct bitmask *bmp) |
320 | 0 | { |
321 | 0 | if (get_mempolicy(oldpolicy, bmp->maskp, bmp->size, 0, 0) < 0) |
322 | 0 | numa_error("get_mempolicy"); |
323 | 0 | } |
324 | | |
325 | | static int dombind(void *mem, size_t size, int pol, struct bitmask *bmp) |
326 | 0 | { |
327 | 0 | if (mbind(mem, size, pol, bmp ? bmp->maskp : NULL, bmp ? bmp->size : 0, |
328 | 0 | mbind_flags) < 0) { |
329 | 0 | numa_error("mbind"); |
330 | 0 | return -1; |
331 | 0 | } |
332 | 0 | return 0; |
333 | 0 | } |
334 | | |
335 | | static void *dombind_or_free(void *mem, size_t size, int pol, struct bitmask *bmp) |
336 | 0 | { |
337 | 0 | if (dombind(mem, size, pol, bmp) < 0 && numa_fail_alloc_on_error) { |
338 | 0 | munmap(mem, size); |
339 | 0 | return NULL; |
340 | 0 | } |
341 | 0 | return mem; |
342 | 0 | } |
343 | | |
344 | | /* (undocumented) */ |
345 | | /* gives the wrong answer for hugetlbfs mappings. */ |
346 | | int numa_pagesize(void) |
347 | 0 | { |
348 | 0 | static int pagesize; |
349 | 0 | if (pagesize > 0) |
350 | 0 | return pagesize; |
351 | 0 | pagesize = getpagesize(); |
352 | 0 | return pagesize; |
353 | 0 | } |
354 | | |
355 | | make_internal_alias(numa_pagesize); |
356 | | |
357 | | /* |
358 | | * Find nodes (numa_nodes_ptr), nodes with memory (numa_memnode_ptr) |
359 | | * and the highest numbered existing node (maxconfigurednode). |
360 | | */ |
361 | | static void |
362 | | set_configured_nodes(void) |
363 | 2 | { |
364 | 2 | DIR *d; |
365 | 2 | struct dirent *de; |
366 | | |
367 | 2 | numa_memnode_ptr = numa_allocate_nodemask(); |
368 | 2 | numa_nodes_ptr = numa_allocate_nodemask(); |
369 | 2 | if (!numa_memnode_ptr || !numa_nodes_ptr) |
370 | 0 | return; |
371 | | |
372 | 2 | d = opendir("/sys/devices/system/node"); |
373 | 2 | if (!d) { |
374 | 0 | maxconfigurednode = 0; |
375 | 2 | } else { |
376 | 24 | while ((de = readdir(d)) != NULL) { |
377 | 22 | int nd; |
378 | 22 | if (strncmp(de->d_name, "node", 4)) |
379 | 20 | continue; |
380 | 2 | nd = strtoul(de->d_name+4, NULL, 0); |
381 | 2 | numa_bitmask_setbit(numa_nodes_ptr, nd); |
382 | 2 | numa_bitmask_setbit(numa_memnode_ptr, nd); |
383 | 2 | if (maxconfigurednode < nd) |
384 | 2 | maxconfigurednode = nd; |
385 | 2 | } |
386 | 2 | closedir(d); |
387 | 2 | } |
388 | 2 | } |
389 | | |
390 | | static inline int is_digit(char s) |
391 | 574 | { |
392 | 574 | return (s >= '0' && s <= '9') |
393 | 62 | || (s >= 'a' && s <= 'f') |
394 | 62 | || (s >= 'A' && s <= 'F'); |
395 | 574 | } |
396 | | |
397 | | /* Is string 'pre' a prefix of string 's'? */ |
398 | | static int strprefix(const char *s, const char *pre) |
399 | 112 | { |
400 | 112 | return strncmp(s, pre, strlen(pre)) == 0; |
401 | 112 | } |
402 | | |
403 | | static const char *mask_size_file = "/proc/self/status"; |
404 | | static const char *nodemask_prefix = "Mems_allowed:\t"; |
405 | | /* |
406 | | * (do this the way Paul Jackson's libcpuset does it) |
407 | | * The nodemask values in /proc/self/status are in an |
408 | | * ascii format that uses 9 characters for each 32 bits of mask. |
409 | | * (this could also be used to find the cpumask size) |
410 | | */ |
411 | | static void |
412 | | set_nodemask_size(void) |
413 | 2 | { |
414 | 2 | FILE *fp; |
415 | 2 | char *buf = NULL; |
416 | 2 | char *tmp_buf = NULL; |
417 | 2 | int digit_len = 0; |
418 | 2 | size_t bufsize = 0; |
419 | | |
420 | 2 | if ((fp = fopen(mask_size_file, "r")) == NULL) |
421 | 0 | goto done; |
422 | | |
423 | 114 | while (getline(&buf, &bufsize, fp) > 0) { |
424 | 112 | if (strprefix(buf, nodemask_prefix)) { |
425 | 2 | tmp_buf = buf; |
426 | 2 | tmp_buf += strlen(nodemask_prefix); |
427 | 576 | while (*tmp_buf != '\n' && *tmp_buf != '\0') { |
428 | 574 | if (is_digit(*tmp_buf)) |
429 | 512 | digit_len++; |
430 | 574 | tmp_buf++; |
431 | 574 | } |
432 | 2 | nodemask_sz = digit_len * 4; |
433 | 2 | } |
434 | 112 | } |
435 | 2 | free(buf); |
436 | 2 | fclose(fp); |
437 | 2 | done: |
438 | 2 | if (nodemask_sz == 0) {/* fall back on error */ |
439 | 0 | int pol; |
440 | 0 | unsigned long *mask = NULL, *origmask; |
441 | 0 | nodemask_sz = 16; |
442 | 0 | do { |
443 | 0 | nodemask_sz <<= 1; |
444 | 0 | origmask = mask; |
445 | 0 | mask = realloc(mask, nodemask_sz / 8 + sizeof(unsigned long)); |
446 | 0 | if (!mask) { |
447 | 0 | free(origmask); |
448 | 0 | return; |
449 | 0 | } |
450 | 0 | } while (get_mempolicy(&pol, mask, nodemask_sz, 0, 0) < 0 && errno == EINVAL && |
451 | 0 | nodemask_sz < 4096*8); |
452 | 0 | free(mask); |
453 | 0 | } |
454 | 2 | } |
455 | | |
456 | | /* |
457 | | * Read a mask consisting of a sequence of hexadecimal longs separated by |
458 | | * commas. Order them correctly and return the number of bits set. |
459 | | */ |
460 | | static int |
461 | | read_mask(char *s, struct bitmask *bmp) |
462 | 4 | { |
463 | 4 | char *end = s; |
464 | 4 | int tmplen = (bmp->size + bitsperint - 1) / bitsperint; |
465 | 4 | unsigned int tmp[tmplen]; |
466 | 4 | unsigned int *start = tmp; |
467 | 4 | unsigned int i, n = 0, m = 0; |
468 | | |
469 | 4 | if (!s) |
470 | 0 | return 0; /* shouldn't happen */ |
471 | | |
472 | 4 | i = strtoul(s, &end, 16); |
473 | | |
474 | | /* Skip leading zeros */ |
475 | 66 | while (!i && *end++ == ',') { |
476 | 62 | i = strtoul(end, &end, 16); |
477 | 62 | } |
478 | | |
479 | 4 | if (!i) |
480 | | /* End of string. No mask */ |
481 | 0 | return -1; |
482 | | |
483 | 4 | if (n >= tmplen) |
484 | 0 | return -1; |
485 | 4 | start[n++] = i; |
486 | | /* Read sequence of ints */ |
487 | 4 | while (*end++ == ',') { |
488 | 0 | i = strtoul(end, &end, 16); |
489 | | |
490 | | /* buffer overflow */ |
491 | 0 | if (n >= tmplen) |
492 | 0 | return -1; |
493 | 0 | start[n++] = i; |
494 | 0 | } |
495 | | |
496 | | /* |
497 | | * Invert sequence of ints if necessary since the first int |
498 | | * is the highest and we put it first because we read it first. |
499 | | */ |
500 | 8 | while (n) { |
501 | 4 | int w; |
502 | 4 | unsigned long x = 0; |
503 | | /* read into long values in an endian-safe way */ |
504 | 8 | for (w = 0; n && w < bitsperlong; w += bitsperint) |
505 | 4 | x |= ((unsigned long)start[n-- - 1] << w); |
506 | | |
507 | 4 | bmp->maskp[m++] = x; |
508 | 4 | } |
509 | | /* |
510 | | * Return the number of bits set |
511 | | */ |
512 | 4 | return numa_bitmask_weight(bmp); |
513 | 4 | } |
514 | | |
515 | | /* |
516 | | * Read a processes constraints in terms of nodes and cpus from |
517 | | * /proc/self/status. |
518 | | */ |
519 | | static void |
520 | | set_task_constraints(void) |
521 | 2 | { |
522 | 2 | int hicpu = maxconfiguredcpu; |
523 | 2 | int i; |
524 | 2 | char *buffer = NULL; |
525 | 2 | size_t buflen = 0; |
526 | 2 | FILE *f; |
527 | | |
528 | 2 | numa_all_cpus_ptr = numa_allocate_cpumask(); |
529 | 2 | numa_possible_cpus_ptr = numa_allocate_cpumask(); |
530 | 2 | numa_all_nodes_ptr = numa_allocate_nodemask(); |
531 | 2 | numa_possible_nodes_ptr = numa_allocate_nodemask(); |
532 | 2 | numa_no_nodes_ptr = numa_allocate_nodemask(); |
533 | | |
534 | | // partial leak shouldn't happen because its transient |
535 | 2 | if (!numa_all_cpus_ptr || !numa_possible_cpus_ptr || |
536 | 2 | !numa_all_nodes_ptr || |
537 | 2 | !numa_possible_nodes_ptr || |
538 | 2 | !numa_no_nodes_ptr) |
539 | 0 | return; |
540 | | |
541 | 2 | f = fopen(mask_size_file, "r"); |
542 | 2 | if (!f) { |
543 | | //numa_warn(W_cpumap, "Cannot parse %s", mask_size_file); |
544 | 0 | return; |
545 | 0 | } |
546 | | |
547 | 114 | while (getline(&buffer, &buflen, f) > 0) { |
548 | | /* mask starts after [last] tab */ |
549 | 112 | char *mask = strrchr(buffer,'\t') + 1; |
550 | | |
551 | 112 | if (strncmp(buffer,"Cpus_allowed:",13) == 0) |
552 | 2 | numproccpu = read_mask(mask, numa_all_cpus_ptr); |
553 | | |
554 | 112 | if (strncmp(buffer,"Mems_allowed:",13) == 0) { |
555 | 2 | numprocnode = read_mask(mask, numa_all_nodes_ptr); |
556 | 2 | } |
557 | 112 | } |
558 | 2 | fclose(f); |
559 | 2 | free(buffer); |
560 | | |
561 | 66 | for (i = 0; i <= hicpu; i++) |
562 | 64 | numa_bitmask_setbit(numa_possible_cpus_ptr, i); |
563 | 4 | for (i = 0; i <= maxconfigurednode; i++) |
564 | 2 | numa_bitmask_setbit(numa_possible_nodes_ptr, i); |
565 | | |
566 | | /* |
567 | | * Cpus_allowed in the kernel can be defined to all f's |
568 | | * i.e. it may be a superset of the actual available processors. |
569 | | * As such let's reduce numproccpu to the number of actual |
570 | | * available cpus. |
571 | | */ |
572 | 2 | if (numproccpu <= 0) { |
573 | 0 | for (i = 0; i <= hicpu; i++) |
574 | 0 | numa_bitmask_setbit(numa_all_cpus_ptr, i); |
575 | 0 | numproccpu = hicpu+1; |
576 | 0 | } |
577 | | |
578 | 2 | if (numproccpu > hicpu+1) { |
579 | 0 | numproccpu = hicpu+1; |
580 | 0 | for (i=hicpu+1; i<numa_all_cpus_ptr->size; i++) { |
581 | 0 | numa_bitmask_clearbit(numa_all_cpus_ptr, i); |
582 | 0 | } |
583 | 0 | } |
584 | | |
585 | 2 | if (numprocnode <= 0) { |
586 | 0 | for (i = 0; i <= maxconfigurednode; i++) |
587 | 0 | numa_bitmask_setbit(numa_all_nodes_ptr, i); |
588 | 0 | numprocnode = maxconfigurednode + 1; |
589 | 0 | } |
590 | | |
591 | 2 | return; |
592 | 2 | } |
593 | | |
594 | | /* |
595 | | * Find the highest cpu number possible (in other words the size |
596 | | * of a kernel cpumask_t (in bits) - 1) |
597 | | */ |
598 | | static void |
599 | | set_numa_max_cpu(void) |
600 | 2 | { |
601 | 2 | int len = 4096; |
602 | 2 | int n; |
603 | 2 | int olde = errno; |
604 | 2 | struct bitmask *buffer; |
605 | | |
606 | 2 | do { |
607 | 2 | buffer = numa_bitmask_alloc(len); |
608 | 2 | if (!buffer) |
609 | 0 | return; |
610 | 2 | n = numa_sched_getaffinity_v2_int(0, buffer); |
611 | | /* on success, returns size of kernel cpumask_t, in bytes */ |
612 | 2 | if (n < 0) { |
613 | 0 | if (errno == EINVAL) { |
614 | 0 | if (len >= 1024*1024) |
615 | 0 | break; |
616 | 0 | len *= 2; |
617 | 0 | numa_bitmask_free(buffer); |
618 | 0 | continue; |
619 | 0 | } else { |
620 | 0 | numa_warn(W_numcpus, "Unable to determine max cpu" |
621 | 0 | " (sched_getaffinity: %s); guessing...", |
622 | 0 | strerror(errno)); |
623 | 0 | n = sizeof(cpu_set_t); |
624 | 0 | break; |
625 | 0 | } |
626 | 0 | } |
627 | 2 | } while (n < 0); |
628 | 2 | numa_bitmask_free(buffer); |
629 | 2 | errno = olde; |
630 | 2 | cpumask_sz = n*8; |
631 | 2 | } |
632 | | |
633 | | /* |
634 | | * get the total (configured) number of cpus - both online and offline |
635 | | */ |
636 | | static void |
637 | | set_configured_cpus(void) |
638 | 2 | { |
639 | 2 | maxconfiguredcpu = sysconf(_SC_NPROCESSORS_CONF) - 1; |
640 | 2 | if (maxconfiguredcpu == -1) |
641 | 0 | numa_error("sysconf(NPROCESSORS_CONF) failed"); |
642 | 2 | } |
643 | | |
644 | | static void |
645 | | set_preferred_many(void) |
646 | 0 | { |
647 | 0 | int oldp; |
648 | 0 | struct bitmask *bmp, *tmp; |
649 | 0 | int old_errno; |
650 | |
|
651 | 0 | if (has_preferred_many >= 0) |
652 | 0 | return; |
653 | | |
654 | 0 | old_errno = errno; |
655 | |
|
656 | 0 | has_preferred_many = 0; |
657 | |
|
658 | 0 | bmp = numa_allocate_nodemask(); |
659 | 0 | tmp = numa_get_mems_allowed(); |
660 | 0 | if (!tmp || !bmp) |
661 | 0 | goto out; |
662 | | |
663 | 0 | if (get_mempolicy(&oldp, bmp->maskp, bmp->size, 0, 0) < 0) |
664 | 0 | goto out; |
665 | | |
666 | 0 | if (set_mempolicy(MPOL_PREFERRED_MANY, tmp->maskp, tmp->size) == 0) { |
667 | 0 | has_preferred_many = 1; |
668 | | /* reset the old memory policy ignoring error */ |
669 | 0 | (void)set_mempolicy(oldp, bmp->maskp, bmp->size+1); |
670 | 0 | } |
671 | |
|
672 | 0 | out: |
673 | 0 | numa_bitmask_free(tmp); |
674 | 0 | numa_bitmask_free(bmp); |
675 | 0 | errno = old_errno; |
676 | 0 | } |
677 | | |
678 | | /* |
679 | | * Initialize all the sizes. |
680 | | */ |
681 | | static void |
682 | | set_sizes(void) |
683 | 2 | { |
684 | 2 | sizes_set++; |
685 | 2 | set_nodemask_size(); /* size of kernel nodemask_t */ |
686 | 2 | set_configured_nodes(); /* configured nodes listed in /sys */ |
687 | 2 | set_numa_max_cpu(); /* size of kernel cpumask_t */ |
688 | 2 | set_configured_cpus(); /* cpus listed in /sys/devices/system/cpu */ |
689 | 2 | set_task_constraints(); /* cpus and nodes for current task */ |
690 | 2 | } |
691 | | |
692 | | int |
693 | | numa_num_configured_nodes(void) |
694 | 894 | { |
695 | | /* |
696 | | * NOTE: this function's behavior matches the documentation (ie: it |
697 | | * returns a count of nodes with memory) despite the poor function |
698 | | * naming. We also cannot use the similarly poorly named |
699 | | * numa_all_nodes_ptr as it only tracks nodes with memory from which |
700 | | * the calling process can allocate. Think sparse nodes, memory-less |
701 | | * nodes, cpusets... |
702 | | */ |
703 | 894 | int memnodecount=0, i; |
704 | | |
705 | 1.78k | for (i=0; i <= maxconfigurednode; i++) { |
706 | 894 | if (numa_bitmask_isbitset(numa_memnode_ptr, i)) |
707 | 894 | memnodecount++; |
708 | 894 | } |
709 | 894 | return memnodecount; |
710 | 894 | } |
711 | | |
712 | | int |
713 | | numa_num_configured_cpus(void) |
714 | 892 | { |
715 | | |
716 | 892 | return maxconfiguredcpu+1; |
717 | 892 | } |
718 | | |
719 | | int |
720 | | numa_num_possible_nodes(void) |
721 | 903 | { |
722 | 903 | return nodemask_sz; |
723 | 903 | } |
724 | | |
725 | | int |
726 | | numa_num_possible_cpus(void) |
727 | 897 | { |
728 | 897 | return cpumask_sz; |
729 | 897 | } |
730 | | |
731 | | int |
732 | | numa_num_task_nodes(void) |
733 | 0 | { |
734 | 0 | return numprocnode; |
735 | 0 | } |
736 | | |
737 | | /* |
738 | | * for backward compatibility |
739 | | */ |
740 | | int |
741 | | numa_num_thread_nodes(void) |
742 | 0 | { |
743 | 0 | return numa_num_task_nodes(); |
744 | 0 | } |
745 | | |
746 | | int |
747 | | numa_num_task_cpus(void) |
748 | 0 | { |
749 | 0 | return numproccpu; |
750 | 0 | } |
751 | | |
752 | | /* |
753 | | * for backward compatibility |
754 | | */ |
755 | | int |
756 | | numa_num_thread_cpus(void) |
757 | 0 | { |
758 | 0 | return numa_num_task_cpus(); |
759 | 0 | } |
760 | | |
761 | | /* |
762 | | * Return the number of the highest node in this running system, |
763 | | */ |
764 | | int |
765 | | numa_max_node(void) |
766 | 131 | { |
767 | 131 | return maxconfigurednode; |
768 | 131 | } |
769 | | |
770 | | make_internal_alias(numa_max_node); |
771 | | |
772 | | /* |
773 | | * Return the number of the highest possible node in a system, |
774 | | * which for v1 is the size of a numa.h nodemask_t(in bits)-1. |
775 | | * but for v2 is the size of a kernel nodemask_t(in bits)-1. |
776 | | */ |
777 | | SYMVER("numa_max_possible_node_v1", "numa_max_possible_node@libnuma_1.1") |
778 | | int |
779 | | numa_max_possible_node_v1(void) |
780 | 0 | { |
781 | 0 | return ((sizeof(nodemask_t)*8)-1); |
782 | 0 | } |
783 | | |
784 | | SYMVER("numa_max_possible_node_v2", "numa_max_possible_node@@libnuma_1.2") |
785 | | int |
786 | | numa_max_possible_node_v2(void) |
787 | 903 | { |
788 | 903 | return numa_num_possible_nodes()-1; |
789 | 903 | } |
790 | | |
791 | | make_internal_alias(numa_max_possible_node_v1); |
792 | | make_internal_alias(numa_max_possible_node_v2); |
793 | | |
794 | | /* |
795 | | * Allocate a bitmask for cpus, of a size large enough to |
796 | | * match the kernel's cpumask_t. |
797 | | */ |
798 | | struct bitmask * |
799 | | numa_allocate_cpumask() |
800 | 897 | { |
801 | 897 | int ncpus = numa_num_possible_cpus(); |
802 | | |
803 | 897 | return numa_bitmask_alloc(ncpus); |
804 | 897 | } |
805 | | |
806 | | /* |
807 | | * Allocate a bitmask the size of a libnuma nodemask_t |
808 | | */ |
809 | | static struct bitmask * |
810 | | allocate_nodemask_v1(void) |
811 | 0 | { |
812 | 0 | int nnodes = numa_max_possible_node_v1_int()+1; |
813 | |
|
814 | 0 | return numa_bitmask_alloc(nnodes); |
815 | 0 | } |
816 | | |
817 | | /* |
818 | | * Allocate a bitmask for nodes, of a size large enough to |
819 | | * match the kernel's nodemask_t. |
820 | | */ |
821 | | struct bitmask * |
822 | | numa_allocate_nodemask(void) |
823 | 902 | { |
824 | 902 | struct bitmask *bmp; |
825 | 902 | int nnodes = numa_max_possible_node_v2_int() + 1; |
826 | | |
827 | 902 | bmp = numa_bitmask_alloc(nnodes); |
828 | 902 | return bmp; |
829 | 902 | } |
830 | | |
831 | | /* (cache the result?) */ |
832 | | long long numa_node_size64(int node, long long *freep) |
833 | 0 | { |
834 | 0 | size_t len = 0; |
835 | 0 | char *line = NULL; |
836 | 0 | long long size = -1; |
837 | 0 | FILE *f; |
838 | 0 | char fn[64]; |
839 | 0 | int ok = 0; |
840 | 0 | int required = freep ? 2 : 1; |
841 | |
|
842 | 0 | if (freep) |
843 | 0 | *freep = 0; |
844 | 0 | sprintf(fn,"/sys/devices/system/node/node%d/meminfo", node); |
845 | 0 | f = fopen(fn, "r"); |
846 | 0 | if (!f) |
847 | 0 | return -1; |
848 | 0 | while (getdelim(&line, &len, '\n', f) > 0) { |
849 | 0 | char *end; |
850 | 0 | char *s = strcasestr(line, "kB"); |
851 | 0 | if (!s) |
852 | 0 | continue; |
853 | 0 | --s; |
854 | 0 | while (s > line && isspace(*s)) |
855 | 0 | --s; |
856 | 0 | while (s > line && isdigit(*s)) |
857 | 0 | --s; |
858 | 0 | if (strstr(line, "MemTotal")) { |
859 | 0 | size = strtoull(s,&end,0) << 10; |
860 | 0 | if (end == s) |
861 | 0 | size = -1; |
862 | 0 | else |
863 | 0 | ok++; |
864 | 0 | } |
865 | 0 | if (freep && strstr(line, "MemFree")) { |
866 | 0 | *freep = strtoull(s,&end,0) << 10; |
867 | 0 | if (end == s) |
868 | 0 | *freep = -1; |
869 | 0 | else |
870 | 0 | ok++; |
871 | 0 | } |
872 | 0 | } |
873 | 0 | fclose(f); |
874 | 0 | free(line); |
875 | 0 | if (ok != required) |
876 | 0 | numa_warn(W_badmeminfo, "Cannot parse sysfs meminfo (%d)", ok); |
877 | 0 | return size; |
878 | 0 | } |
879 | | |
880 | | make_internal_alias(numa_node_size64); |
881 | | |
882 | | long numa_node_size(int node, long *freep) |
883 | 0 | { |
884 | 0 | long long f2 = 0; |
885 | 0 | long sz = numa_node_size64_int(node, &f2); |
886 | 0 | if (freep) |
887 | 0 | *freep = f2; |
888 | 0 | return sz; |
889 | 0 | } |
890 | | |
891 | | int numa_available(void) |
892 | 0 | { |
893 | 0 | if (get_mempolicy(NULL, NULL, 0, 0, 0) < 0 && (errno == ENOSYS || errno == EPERM)) |
894 | 0 | return -1; |
895 | 0 | return 0; |
896 | 0 | } |
897 | | |
898 | | SYMVER("numa_interleave_memory_v1", "numa_interleave_memory@libnuma_1.1") |
899 | | void |
900 | | numa_interleave_memory_v1(void *mem, size_t size, const nodemask_t *mask) |
901 | 0 | { |
902 | 0 | struct bitmask bitmask; |
903 | |
|
904 | 0 | bitmask.size = sizeof(nodemask_t) * 8; |
905 | 0 | bitmask.maskp = (unsigned long *)mask; |
906 | 0 | dombind(mem, size, MPOL_INTERLEAVE, &bitmask); |
907 | 0 | } |
908 | | |
909 | | SYMVER("numa_interleave_memory_v2", "numa_interleave_memory@@libnuma_1.2") |
910 | | void |
911 | | numa_interleave_memory_v2(void *mem, size_t size, struct bitmask *bmp) |
912 | 0 | { |
913 | 0 | dombind(mem, size, MPOL_INTERLEAVE, bmp); |
914 | 0 | } |
915 | | |
916 | | void |
917 | | numa_weighted_interleave_memory(void *mem, size_t size, struct bitmask *bmp) |
918 | 0 | { |
919 | 0 | dombind(mem, size, MPOL_WEIGHTED_INTERLEAVE, bmp); |
920 | 0 | } |
921 | | |
922 | | void numa_tonode_memory(void *mem, size_t size, int node) |
923 | 0 | { |
924 | 0 | struct bitmask *nodes; |
925 | |
|
926 | 0 | nodes = numa_allocate_nodemask(); |
927 | 0 | if (!nodes) |
928 | 0 | return; |
929 | 0 | numa_bitmask_setbit(nodes, node); |
930 | 0 | dombind(mem, size, bind_policy, nodes); |
931 | 0 | numa_bitmask_free(nodes); |
932 | 0 | } |
933 | | |
934 | | SYMVER("numa_tonodemask_memory_v1", "numa_tonodemask_memory@libnuma_1.1") |
935 | | void |
936 | | numa_tonodemask_memory_v1(void *mem, size_t size, const nodemask_t *mask) |
937 | 0 | { |
938 | 0 | struct bitmask bitmask; |
939 | |
|
940 | 0 | bitmask.maskp = (unsigned long *)mask; |
941 | 0 | bitmask.size = sizeof(nodemask_t); |
942 | 0 | dombind(mem, size, bind_policy, &bitmask); |
943 | 0 | } |
944 | | |
945 | | SYMVER("numa_tonodemask_memory_v2", "numa_tonodemask_memory@@libnuma_1.2") |
946 | | void |
947 | | numa_tonodemask_memory_v2(void *mem, size_t size, struct bitmask *bmp) |
948 | 0 | { |
949 | 0 | dombind(mem, size, bind_policy, bmp); |
950 | 0 | } |
951 | | |
952 | | void numa_setlocal_memory(void *mem, size_t size) |
953 | 0 | { |
954 | 0 | dombind(mem, size, MPOL_LOCAL, NULL); |
955 | 0 | } |
956 | | |
957 | | void numa_police_memory(void *mem, size_t size) |
958 | 0 | { |
959 | 0 | int pagesize = numa_pagesize_int(); |
960 | 0 | unsigned long i; |
961 | 0 | char *p = mem; |
962 | 0 | for (i = 0; i < size; i += pagesize, p += pagesize) |
963 | 0 | __atomic_and_fetch(p, 0xff, __ATOMIC_RELAXED); |
964 | |
|
965 | 0 | } |
966 | | |
967 | | make_internal_alias(numa_police_memory); |
968 | | |
969 | | void *numa_alloc(size_t size) |
970 | 0 | { |
971 | 0 | char *mem; |
972 | 0 | mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, |
973 | 0 | 0, 0); |
974 | 0 | if (mem == (char *)-1) |
975 | 0 | return NULL; |
976 | 0 | numa_police_memory_int(mem, size); |
977 | 0 | return mem; |
978 | 0 | } |
979 | | |
980 | | void *numa_realloc(void *old_addr, size_t old_size, size_t new_size) |
981 | 0 | { |
982 | 0 | char *mem; |
983 | 0 | mem = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE); |
984 | 0 | if (mem == (char *)-1) |
985 | 0 | return NULL; |
986 | | /* |
987 | | * The memory policy of the allocated pages is preserved by mremap(), so |
988 | | * there is no need to (re)set it here. If the policy of the original |
989 | | * allocation is not set, the new pages will be allocated according to the |
990 | | * process' mempolicy. Trying to allocate explicitly the new pages on the |
991 | | * same node as the original ones would require changing the policy of the |
992 | | * newly allocated pages, which violates the numa_realloc() semantics. |
993 | | */ |
994 | 0 | return mem; |
995 | 0 | } |
996 | | |
997 | | SYMVER("numa_alloc_interleaved_subset_v1", "numa_alloc_interleaved_subset@libnuma_1.1") |
998 | | void *numa_alloc_interleaved_subset_v1(size_t size, const nodemask_t *mask) |
999 | 0 | { |
1000 | 0 | char *mem; |
1001 | 0 | struct bitmask bitmask; |
1002 | |
|
1003 | 0 | mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, |
1004 | 0 | 0, 0); |
1005 | 0 | if (mem == (char *)-1) |
1006 | 0 | return NULL; |
1007 | 0 | bitmask.maskp = (unsigned long *)mask; |
1008 | 0 | bitmask.size = sizeof(nodemask_t); |
1009 | 0 | mem = dombind_or_free(mem, size, MPOL_INTERLEAVE, &bitmask); |
1010 | 0 | return mem; |
1011 | 0 | } |
1012 | | |
1013 | | SYMVER("numa_alloc_interleaved_subset_v2", "numa_alloc_interleaved_subset@@libnuma_1.2") |
1014 | | void *numa_alloc_interleaved_subset_v2(size_t size, struct bitmask *bmp) |
1015 | 0 | { |
1016 | 0 | char *mem; |
1017 | |
|
1018 | 0 | mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, |
1019 | 0 | 0, 0); |
1020 | 0 | if (mem == (char *)-1) |
1021 | 0 | return NULL; |
1022 | 0 | mem = dombind_or_free(mem, size, MPOL_INTERLEAVE, bmp); |
1023 | 0 | return mem; |
1024 | 0 | } |
1025 | | |
1026 | | make_internal_alias(numa_alloc_interleaved_subset_v1); |
1027 | | make_internal_alias(numa_alloc_interleaved_subset_v2); |
1028 | | |
1029 | | void * |
1030 | | numa_alloc_interleaved(size_t size) |
1031 | 0 | { |
1032 | 0 | return numa_alloc_interleaved_subset_v2_int(size, numa_all_nodes_ptr); |
1033 | 0 | } |
1034 | | |
1035 | | void * |
1036 | | numa_alloc_weighted_interleaved_subset(size_t size, struct bitmask *bmp) |
1037 | 0 | { |
1038 | 0 | char *mem; |
1039 | |
|
1040 | 0 | mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, |
1041 | 0 | 0, 0); |
1042 | 0 | if (mem == (char *)-1) |
1043 | 0 | return NULL; |
1044 | 0 | mem = dombind_or_free(mem, size, MPOL_WEIGHTED_INTERLEAVE, bmp); |
1045 | 0 | return mem; |
1046 | 0 | } |
1047 | | |
1048 | | void * |
1049 | | numa_alloc_weighted_interleaved(size_t size) |
1050 | 0 | { |
1051 | 0 | return numa_alloc_weighted_interleaved_subset(size, numa_all_nodes_ptr); |
1052 | 0 | } |
1053 | | |
1054 | | /* |
1055 | | * given a user node mask, set memory policy to use those nodes |
1056 | | */ |
1057 | | SYMVER("numa_set_interleave_mask_v1", "numa_set_interleave_mask@libnuma_1.1") |
1058 | | void |
1059 | | numa_set_interleave_mask_v1(nodemask_t *mask) |
1060 | 0 | { |
1061 | 0 | struct bitmask *bmp; |
1062 | 0 | int nnodes = numa_max_possible_node_v1_int()+1; |
1063 | |
|
1064 | 0 | bmp = numa_bitmask_alloc(nnodes); |
1065 | 0 | copy_nodemask_to_bitmask(mask, bmp); |
1066 | 0 | if (numa_bitmask_equal(bmp, numa_no_nodes_ptr)) |
1067 | 0 | setpol(MPOL_DEFAULT, bmp); |
1068 | 0 | else |
1069 | 0 | setpol(MPOL_INTERLEAVE, bmp); |
1070 | 0 | numa_bitmask_free(bmp); |
1071 | 0 | } |
1072 | | |
1073 | | |
1074 | | SYMVER("numa_set_interleave_mask_v2", "numa_set_interleave_mask@@libnuma_1.2") |
1075 | | void |
1076 | | numa_set_interleave_mask_v2(struct bitmask *bmp) |
1077 | 0 | { |
1078 | 0 | if (numa_bitmask_equal(bmp, numa_no_nodes_ptr)) |
1079 | 0 | setpol(MPOL_DEFAULT, bmp); |
1080 | 0 | else |
1081 | 0 | setpol(MPOL_INTERLEAVE, bmp); |
1082 | 0 | } |
1083 | | |
1084 | | void |
1085 | | numa_set_weighted_interleave_mask(struct bitmask *bmp) |
1086 | 0 | { |
1087 | 0 | if (numa_bitmask_equal(bmp, numa_no_nodes_ptr)) |
1088 | 0 | setpol(MPOL_DEFAULT, bmp); |
1089 | 0 | else |
1090 | 0 | setpol(MPOL_WEIGHTED_INTERLEAVE, bmp); |
1091 | 0 | } |
1092 | | |
1093 | | SYMVER("numa_get_interleave_mask_v1", "numa_get_interleave_mask@libnuma_1.1") |
1094 | | nodemask_t |
1095 | | numa_get_interleave_mask_v1(void) |
1096 | 0 | { |
1097 | 0 | int oldpolicy = 0; |
1098 | 0 | struct bitmask *bmp; |
1099 | 0 | nodemask_t mask; |
1100 | |
|
1101 | 0 | bmp = allocate_nodemask_v1(); |
1102 | 0 | if (!bmp) |
1103 | 0 | return numa_no_nodes; |
1104 | 0 | getpol(&oldpolicy, bmp); |
1105 | 0 | if (oldpolicy == MPOL_INTERLEAVE) |
1106 | 0 | copy_bitmask_to_nodemask(bmp, &mask); |
1107 | 0 | else |
1108 | 0 | copy_bitmask_to_nodemask(numa_no_nodes_ptr, &mask); |
1109 | 0 | numa_bitmask_free(bmp); |
1110 | 0 | return mask; |
1111 | 0 | } |
1112 | | |
1113 | | SYMVER("numa_get_interleave_mask_v2", "numa_get_interleave_mask@@libnuma_1.2") |
1114 | | struct bitmask * |
1115 | | numa_get_interleave_mask_v2(void) |
1116 | 0 | { |
1117 | 0 | int oldpolicy = 0; |
1118 | 0 | struct bitmask *bmp; |
1119 | |
|
1120 | 0 | bmp = numa_allocate_nodemask(); |
1121 | 0 | if (!bmp) |
1122 | 0 | return NULL; |
1123 | 0 | getpol(&oldpolicy, bmp); |
1124 | 0 | if (oldpolicy != MPOL_INTERLEAVE) |
1125 | 0 | copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp); |
1126 | 0 | return bmp; |
1127 | 0 | } |
1128 | | |
1129 | | struct bitmask * |
1130 | | numa_get_weighted_interleave_mask(void) |
1131 | 0 | { |
1132 | 0 | int oldpolicy = 0; |
1133 | 0 | struct bitmask *bmp; |
1134 | |
|
1135 | 0 | bmp = numa_allocate_nodemask(); |
1136 | 0 | if (!bmp) |
1137 | 0 | return NULL; |
1138 | 0 | getpol(&oldpolicy, bmp); |
1139 | 0 | if (oldpolicy != MPOL_WEIGHTED_INTERLEAVE) |
1140 | 0 | copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp); |
1141 | 0 | return bmp; |
1142 | 0 | } |
1143 | | |
1144 | | /* (undocumented) */ |
1145 | | int numa_get_interleave_node(void) |
1146 | 0 | { |
1147 | 0 | int nd; |
1148 | 0 | if (get_mempolicy(&nd, NULL, 0, 0, MPOL_F_NODE) == 0) |
1149 | 0 | return nd; |
1150 | 0 | return 0; |
1151 | 0 | } |
1152 | | |
1153 | | void *numa_alloc_onnode(size_t size, int node) |
1154 | 0 | { |
1155 | 0 | char *mem; |
1156 | 0 | struct bitmask *bmp; |
1157 | |
|
1158 | 0 | bmp = numa_allocate_nodemask(); |
1159 | 0 | if (!bmp) |
1160 | 0 | return NULL; |
1161 | 0 | numa_bitmask_setbit(bmp, node); |
1162 | 0 | mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, |
1163 | 0 | 0, 0); |
1164 | 0 | if (mem == (char *)-1) |
1165 | 0 | mem = NULL; |
1166 | 0 | else |
1167 | 0 | mem = dombind_or_free(mem, size, bind_policy, bmp); |
1168 | |
|
1169 | 0 | numa_bitmask_free(bmp); |
1170 | 0 | return mem; |
1171 | 0 | } |
1172 | | |
1173 | | void *numa_alloc_local(size_t size) |
1174 | 0 | { |
1175 | 0 | char *mem; |
1176 | 0 | mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, |
1177 | 0 | 0, 0); |
1178 | 0 | if (mem == (char *)-1) |
1179 | 0 | mem = NULL; |
1180 | 0 | else |
1181 | 0 | mem = dombind_or_free(mem, size, MPOL_LOCAL, NULL); |
1182 | 0 | return mem; |
1183 | 0 | } |
1184 | | |
1185 | | void numa_set_bind_policy(int strict) |
1186 | 0 | { |
1187 | 0 | set_preferred_many(); |
1188 | 0 | if (strict) |
1189 | 0 | bind_policy = MPOL_BIND; |
1190 | 0 | else if (has_preferred_many) |
1191 | 0 | bind_policy = MPOL_PREFERRED_MANY; |
1192 | 0 | else |
1193 | 0 | bind_policy = MPOL_PREFERRED; |
1194 | 0 | } |
1195 | | |
1196 | | SYMVER("numa_set_membind_v1", "numa_set_membind@libnuma_1.1") |
1197 | | void |
1198 | | numa_set_membind_v1(const nodemask_t *mask) |
1199 | 0 | { |
1200 | 0 | struct bitmask bitmask; |
1201 | |
|
1202 | 0 | bitmask.maskp = (unsigned long *)mask; |
1203 | 0 | bitmask.size = sizeof(nodemask_t); |
1204 | 0 | setpol(MPOL_BIND, &bitmask); |
1205 | 0 | } |
1206 | | |
1207 | | SYMVER("numa_set_membind_v2", "numa_set_membind@@libnuma_1.2") |
1208 | | void |
1209 | | numa_set_membind_v2(struct bitmask *bmp) |
1210 | 0 | { |
1211 | 0 | setpol(MPOL_BIND, bmp); |
1212 | 0 | } |
1213 | | |
1214 | | make_internal_alias(numa_set_membind_v2); |
1215 | | |
1216 | | void |
1217 | | numa_set_membind_balancing(struct bitmask *bmp) |
1218 | 0 | { |
1219 | | /* MPOL_F_NUMA_BALANCING: ignore if unsupported */ |
1220 | 0 | if (set_mempolicy(MPOL_BIND | MPOL_F_NUMA_BALANCING, |
1221 | 0 | bmp->maskp, bmp->size + 1) < 0) { |
1222 | 0 | if (errno == EINVAL) { |
1223 | 0 | errno = 0; |
1224 | 0 | numa_set_membind_v2(bmp); |
1225 | 0 | } else |
1226 | 0 | numa_error("set_mempolicy"); |
1227 | 0 | } |
1228 | 0 | } |
1229 | | |
1230 | | /* |
1231 | | * copy a bitmask map body to a numa.h nodemask_t structure |
1232 | | */ |
1233 | | void |
1234 | | copy_bitmask_to_nodemask(struct bitmask *bmp, nodemask_t *nmp) |
1235 | 0 | { |
1236 | 0 | int max, i; |
1237 | |
|
1238 | 0 | memset(nmp, 0, sizeof(nodemask_t)); |
1239 | 0 | max = (sizeof(nodemask_t)*8); |
1240 | 0 | for (i=0; i<bmp->size; i++) { |
1241 | 0 | if (i >= max) |
1242 | 0 | break; |
1243 | 0 | if (numa_bitmask_isbitset(bmp, i)) |
1244 | 0 | nodemask_set_compat((nodemask_t *)nmp, i); |
1245 | 0 | } |
1246 | 0 | } |
1247 | | |
1248 | | /* |
1249 | | * copy a bitmask map body to another bitmask body |
1250 | | * fill a larger destination with zeroes |
1251 | | */ |
1252 | | void |
1253 | | copy_bitmask_to_bitmask(struct bitmask *bmpfrom, struct bitmask *bmpto) |
1254 | 136 | { |
1255 | 136 | int bytes; |
1256 | | |
1257 | 136 | if (bmpfrom->size >= bmpto->size) { |
1258 | 136 | memcpy(bmpto->maskp, bmpfrom->maskp, CPU_BYTES(bmpto->size)); |
1259 | 136 | } else if (bmpfrom->size < bmpto->size) { |
1260 | 0 | bytes = CPU_BYTES(bmpfrom->size); |
1261 | 0 | memcpy(bmpto->maskp, bmpfrom->maskp, bytes); |
1262 | 0 | memset(((char *)bmpto->maskp)+bytes, 0, |
1263 | 0 | CPU_BYTES(bmpto->size)-bytes); |
1264 | 0 | } |
1265 | 136 | } |
1266 | | |
1267 | | /* |
1268 | | * copy a numa.h nodemask_t structure to a bitmask map body |
1269 | | */ |
1270 | | void |
1271 | | copy_nodemask_to_bitmask(nodemask_t *nmp, struct bitmask *bmp) |
1272 | 0 | { |
1273 | 0 | int max, i; |
1274 | |
|
1275 | 0 | numa_bitmask_clearall(bmp); |
1276 | 0 | max = (sizeof(nodemask_t)*8); |
1277 | 0 | if (max > bmp->size) |
1278 | 0 | max = bmp->size; |
1279 | 0 | for (i=0; i<max; i++) { |
1280 | 0 | if (nodemask_isset_compat(nmp, i)) |
1281 | 0 | numa_bitmask_setbit(bmp, i); |
1282 | 0 | } |
1283 | 0 | } |
1284 | | |
1285 | | SYMVER("numa_get_membind_v1", "numa_get_membind@libnuma_1.1") |
1286 | | nodemask_t |
1287 | | numa_get_membind_v1(void) |
1288 | 0 | { |
1289 | 0 | int oldpolicy = 0; |
1290 | 0 | struct bitmask *bmp; |
1291 | 0 | nodemask_t nmp; |
1292 | |
|
1293 | 0 | bmp = allocate_nodemask_v1(); |
1294 | 0 | if (!bmp) |
1295 | 0 | return numa_no_nodes; |
1296 | 0 | getpol(&oldpolicy, bmp); |
1297 | 0 | if (oldpolicy == MPOL_BIND) { |
1298 | 0 | copy_bitmask_to_nodemask(bmp, &nmp); |
1299 | 0 | } else { |
1300 | | /* copy the body of the map to numa_all_nodes */ |
1301 | 0 | copy_bitmask_to_nodemask(bmp, &numa_all_nodes); |
1302 | 0 | nmp = numa_all_nodes; |
1303 | 0 | } |
1304 | 0 | numa_bitmask_free(bmp); |
1305 | 0 | return nmp; |
1306 | 0 | } |
1307 | | |
1308 | | SYMVER("numa_get_membind_v2", "numa_get_membind@@libnuma_1.2") |
1309 | | struct bitmask * |
1310 | | numa_get_membind_v2(void) |
1311 | 0 | { |
1312 | 0 | int oldpolicy = 0; |
1313 | 0 | struct bitmask *bmp = NULL; |
1314 | |
|
1315 | 0 | bmp = numa_allocate_nodemask(); |
1316 | 0 | if (!bmp) |
1317 | 0 | return NULL; |
1318 | 0 | getpol(&oldpolicy, bmp); |
1319 | 0 | if (oldpolicy != MPOL_BIND) |
1320 | 0 | copy_bitmask_to_bitmask(numa_all_nodes_ptr, bmp); |
1321 | 0 | return bmp; |
1322 | 0 | } |
1323 | | |
1324 | | //TODO: do we need a v1 nodemask_t version? |
1325 | | struct bitmask *numa_get_mems_allowed(void) |
1326 | 0 | { |
1327 | 0 | struct bitmask *bmp; |
1328 | | |
1329 | | /* |
1330 | | * can change, so query on each call. |
1331 | | */ |
1332 | 0 | bmp = numa_allocate_nodemask(); |
1333 | 0 | if (!bmp) |
1334 | 0 | return NULL; |
1335 | 0 | if (get_mempolicy(NULL, bmp->maskp, bmp->size, 0, |
1336 | 0 | MPOL_F_MEMS_ALLOWED) < 0) |
1337 | 0 | numa_error("get_mempolicy"); |
1338 | 0 | return bmp; |
1339 | 0 | } |
1340 | | make_internal_alias(numa_get_mems_allowed); |
1341 | | |
1342 | | void numa_free(void *mem, size_t size) |
1343 | 0 | { |
1344 | 0 | munmap(mem, size); |
1345 | 0 | } |
1346 | | |
1347 | | SYMVER("numa_parse_bitmap_v1", "numa_parse_bitmap@libnuma_1.1") |
1348 | | int |
1349 | | numa_parse_bitmap_v1(char *line, unsigned long *mask, int ncpus) |
1350 | 0 | { |
1351 | 0 | int i; |
1352 | 0 | char *p = strchr(line, '\n'); |
1353 | 0 | if (!p) |
1354 | 0 | return -1; |
1355 | | |
1356 | 0 | for (i = 0; p > line;i++) { |
1357 | 0 | char *oldp, *endp; |
1358 | 0 | oldp = p; |
1359 | 0 | if (*p == ',') |
1360 | 0 | --p; |
1361 | 0 | while (p > line && *p != ',') |
1362 | 0 | --p; |
1363 | | /* Eat two 32bit fields at a time to get longs */ |
1364 | 0 | if (p > line && sizeof(unsigned long) == 8) { |
1365 | 0 | oldp--; |
1366 | 0 | memmove(p, p+1, oldp-p+1); |
1367 | 0 | while (p > line && *p != ',') |
1368 | 0 | --p; |
1369 | 0 | } |
1370 | 0 | if (*p == ',') |
1371 | 0 | p++; |
1372 | 0 | if (i >= CPU_LONGS(ncpus)) |
1373 | 0 | return -1; |
1374 | 0 | mask[i] = strtoul(p, &endp, 16); |
1375 | 0 | if (endp != oldp) |
1376 | 0 | return -1; |
1377 | 0 | p--; |
1378 | 0 | } |
1379 | 0 | return 0; |
1380 | 0 | } |
1381 | | |
1382 | | SYMVER("numa_parse_bitmap_v2", "numa_parse_bitmap@@libnuma_1.2") |
1383 | | int |
1384 | | numa_parse_bitmap_v2(char *line, struct bitmask *mask) |
1385 | 1 | { |
1386 | 1 | int i, ncpus; |
1387 | 1 | char *p = strchr(line, '\n'); |
1388 | 1 | if (!p) |
1389 | 0 | return -1; |
1390 | 1 | ncpus = mask->size; |
1391 | | |
1392 | 2 | for (i = 0; p > line;i++) { |
1393 | 1 | char *oldp, *endp; |
1394 | 1 | oldp = p; |
1395 | 1 | if (*p == ',') |
1396 | 0 | --p; |
1397 | 9 | while (p > line && *p != ',') |
1398 | 8 | --p; |
1399 | | /* Eat two 32bit fields at a time to get longs */ |
1400 | 1 | if (p > line && sizeof(unsigned long) == 8) { |
1401 | 0 | oldp--; |
1402 | 0 | memmove(p, p+1, oldp-p+1); |
1403 | 0 | while (p > line && *p != ',') |
1404 | 0 | --p; |
1405 | 0 | } |
1406 | 1 | if (*p == ',') |
1407 | 0 | p++; |
1408 | 1 | if (i >= CPU_LONGS(ncpus)) |
1409 | 0 | return -1; |
1410 | 1 | mask->maskp[i] = strtoul(p, &endp, 16); |
1411 | 1 | if (endp != oldp) |
1412 | 0 | return -1; |
1413 | 1 | p--; |
1414 | 1 | } |
1415 | 1 | return 0; |
1416 | 1 | } |
1417 | | |
1418 | | static void init_node_cpu_mask_v2(void) |
1419 | 1 | { |
1420 | 1 | int nnodes = numa_max_possible_node_v2_int() + 1; |
1421 | 1 | struct bitmask **new_ncm, **null_ncm = NULL; |
1422 | 1 | new_ncm = calloc (nnodes, sizeof(struct bitmask *)); |
1423 | | /* Check for races with another thread */ |
1424 | 1 | if (new_ncm && !__atomic_compare_exchange_n(&node_cpu_mask_v2, &null_ncm, |
1425 | 1 | new_ncm, 0, |
1426 | 1 | __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) { |
1427 | 0 | free(new_ncm); |
1428 | 0 | } |
1429 | 1 | } |
1430 | | |
1431 | | static void cleanup_node_cpu_mask_v2(void) |
1432 | 0 | { |
1433 | 0 | if (node_cpu_mask_v2) { |
1434 | 0 | int i; |
1435 | 0 | int nnodes; |
1436 | 0 | nnodes = numa_max_possible_node_v2_int() + 1; |
1437 | 0 | for (i = 0; i < nnodes; i++) { |
1438 | 0 | FREE_AND_ZERO(node_cpu_mask_v2[i]); |
1439 | 0 | } |
1440 | 0 | free(node_cpu_mask_v2); |
1441 | 0 | node_cpu_mask_v2 = NULL; |
1442 | 0 | } |
1443 | 0 | } |
1444 | | |
1445 | | /* This would be better with some locking, but I don't want to make libnuma |
1446 | | dependent on pthreads right now. The races are relatively harmless. */ |
1447 | | SYMVER("numa_node_to_cpus_v1", "numa_node_to_cpus@libnuma_1.1") |
1448 | | int |
1449 | | numa_node_to_cpus_v1(int node, unsigned long *buffer, int bufferlen) |
1450 | 0 | { |
1451 | 0 | int err = 0; |
1452 | 0 | char fn[64]; |
1453 | 0 | FILE *f; |
1454 | 0 | char update; |
1455 | 0 | char *line = NULL; |
1456 | 0 | size_t len = 0; |
1457 | 0 | struct bitmask bitmask; |
1458 | 0 | int buflen_needed; |
1459 | 0 | unsigned long *mask; |
1460 | 0 | int ncpus = numa_num_possible_cpus(); |
1461 | 0 | int maxnode = numa_max_node_int(); |
1462 | |
|
1463 | 0 | buflen_needed = CPU_BYTES(ncpus); |
1464 | 0 | if ((unsigned)node > maxnode || bufferlen < buflen_needed) { |
1465 | 0 | errno = ERANGE; |
1466 | 0 | return -1; |
1467 | 0 | } |
1468 | 0 | if (bufferlen > buflen_needed) |
1469 | 0 | memset(buffer, 0, bufferlen); |
1470 | 0 | update = __atomic_fetch_and(&node_cpu_mask_v1_stale, 0, __ATOMIC_RELAXED); |
1471 | 0 | if (node_cpu_mask_v1[node] && !update) { |
1472 | 0 | memcpy(buffer, node_cpu_mask_v1[node], buflen_needed); |
1473 | 0 | return 0; |
1474 | 0 | } |
1475 | | |
1476 | 0 | mask = malloc(buflen_needed); |
1477 | 0 | if (!mask) |
1478 | 0 | mask = (unsigned long *)buffer; |
1479 | 0 | memset(mask, 0, buflen_needed); |
1480 | |
|
1481 | 0 | sprintf(fn, "/sys/devices/system/node/node%d/cpumap", node); |
1482 | 0 | f = fopen(fn, "r"); |
1483 | 0 | if (!f || getdelim(&line, &len, '\n', f) < 1) { |
1484 | 0 | if (numa_bitmask_isbitset(numa_nodes_ptr, node)) { |
1485 | 0 | numa_warn(W_nosysfs2, |
1486 | 0 | "/sys not mounted or invalid. Assuming one node: %s", |
1487 | 0 | strerror(errno)); |
1488 | 0 | numa_warn(W_nosysfs2, |
1489 | 0 | "(cannot open or correctly parse %s)", fn); |
1490 | 0 | } |
1491 | 0 | bitmask.maskp = (unsigned long *)mask; |
1492 | 0 | bitmask.size = buflen_needed * 8; |
1493 | 0 | numa_bitmask_setall(&bitmask); |
1494 | 0 | err = -1; |
1495 | 0 | } |
1496 | 0 | if (f) |
1497 | 0 | fclose(f); |
1498 | |
|
1499 | 0 | if (line && (numa_parse_bitmap_v1(line, mask, ncpus) < 0)) { |
1500 | 0 | numa_warn(W_cpumap, "Cannot parse cpumap. Assuming one node"); |
1501 | 0 | bitmask.maskp = (unsigned long *)mask; |
1502 | 0 | bitmask.size = buflen_needed * 8; |
1503 | 0 | numa_bitmask_setall(&bitmask); |
1504 | 0 | err = -1; |
1505 | 0 | } |
1506 | |
|
1507 | 0 | free(line); |
1508 | 0 | memmove(buffer, mask, buflen_needed); |
1509 | | |
1510 | | /* slightly racy, see above */ |
1511 | 0 | if (node_cpu_mask_v1[node]) { |
1512 | 0 | if (update) { |
1513 | | /* |
1514 | | * There may be readers on node_cpu_mask_v1[], hence it can not |
1515 | | * be freed. |
1516 | | */ |
1517 | 0 | memcpy(node_cpu_mask_v1[node], mask, buflen_needed); |
1518 | 0 | free(mask); |
1519 | 0 | mask = NULL; |
1520 | 0 | } else if (mask != buffer) |
1521 | 0 | free(mask); |
1522 | 0 | } else { |
1523 | 0 | node_cpu_mask_v1[node] = mask; |
1524 | 0 | } |
1525 | 0 | return err; |
1526 | 0 | } |
1527 | | |
1528 | | /* |
1529 | | * test whether a node has cpus |
1530 | | */ |
1531 | | /* This would be better with some locking, but I don't want to make libnuma |
1532 | | dependent on pthreads right now. The races are relatively harmless. */ |
1533 | | /* |
1534 | | * deliver a bitmask of cpus representing the cpus on a given node |
1535 | | */ |
1536 | | SYMVER("numa_node_to_cpus_v2", "numa_node_to_cpus@@libnuma_1.2") |
1537 | | int |
1538 | | numa_node_to_cpus_v2(int node, struct bitmask *buffer) |
1539 | 131 | { |
1540 | 131 | int err = 0; |
1541 | 131 | int nnodes = numa_max_node(); |
1542 | 131 | char fn[64], *line = NULL; |
1543 | 131 | FILE *f; |
1544 | 131 | char update; |
1545 | 131 | size_t len = 0; |
1546 | 131 | struct bitmask *mask; |
1547 | | |
1548 | 131 | if (!__atomic_load_n(&node_cpu_mask_v2, __ATOMIC_CONSUME)) |
1549 | 1 | init_node_cpu_mask_v2(); |
1550 | | |
1551 | 131 | if (node < 0 || node > nnodes) { |
1552 | 0 | errno = ERANGE; |
1553 | 0 | return -1; |
1554 | 0 | } |
1555 | 131 | numa_bitmask_clearall(buffer); |
1556 | | |
1557 | 131 | update = __atomic_fetch_and(&node_cpu_mask_v2_stale, 0, __ATOMIC_RELAXED); |
1558 | 131 | if (node_cpu_mask_v2[node] && !update) { |
1559 | | /* have already constructed a mask for this node */ |
1560 | 130 | if (buffer->size < node_cpu_mask_v2[node]->size) { |
1561 | 0 | errno = EINVAL; |
1562 | 0 | numa_error("map size mismatch"); |
1563 | 0 | return -1; |
1564 | 0 | } |
1565 | 130 | copy_bitmask_to_bitmask(node_cpu_mask_v2[node], buffer); |
1566 | 130 | return 0; |
1567 | 130 | } |
1568 | | |
1569 | | /* need a new mask for this node */ |
1570 | 1 | mask = numa_allocate_cpumask(); |
1571 | 1 | if (!mask) |
1572 | 0 | return -1; |
1573 | | |
1574 | | /* this is a kernel cpumask_t (see node_read_cpumap()) */ |
1575 | 1 | sprintf(fn, "/sys/devices/system/node/node%d/cpumap", node); |
1576 | 1 | f = fopen(fn, "r"); |
1577 | 1 | if (!f || getdelim(&line, &len, '\n', f) < 1) { |
1578 | 0 | if (numa_bitmask_isbitset(numa_nodes_ptr, node)) { |
1579 | 0 | numa_warn(W_nosysfs2, |
1580 | 0 | "/sys not mounted or invalid. Assuming one node: %s", |
1581 | 0 | strerror(errno)); |
1582 | 0 | numa_warn(W_nosysfs2, |
1583 | 0 | "(cannot open or correctly parse %s)", fn); |
1584 | 0 | } |
1585 | 0 | numa_bitmask_setall(mask); |
1586 | 0 | err = -1; |
1587 | 0 | } |
1588 | 1 | if (f) |
1589 | 1 | fclose(f); |
1590 | | |
1591 | 1 | if (line && (numa_parse_bitmap_v2(line, mask) < 0)) { |
1592 | 0 | numa_warn(W_cpumap, "Cannot parse cpumap. Assuming one node"); |
1593 | 0 | numa_bitmask_setall(mask); |
1594 | 0 | err = -1; |
1595 | 0 | } |
1596 | | |
1597 | 1 | free(line); |
1598 | 1 | copy_bitmask_to_bitmask(mask, buffer); |
1599 | | |
1600 | | /* slightly racy, see above */ |
1601 | | /* save the mask we created */ |
1602 | 1 | if (node_cpu_mask_v2[node]) { |
1603 | 0 | if (update) { |
1604 | 0 | copy_bitmask_to_bitmask(mask, node_cpu_mask_v2[node]); |
1605 | 0 | numa_bitmask_free(mask); |
1606 | 0 | mask = NULL; |
1607 | | /* how could this be? */ |
1608 | 0 | } else if (mask != buffer) |
1609 | 0 | numa_bitmask_free(mask); |
1610 | 1 | } else { |
1611 | | /* we don't want to cache faulty result */ |
1612 | 1 | if (!err) |
1613 | 1 | node_cpu_mask_v2[node] = mask; |
1614 | 0 | else |
1615 | 0 | numa_bitmask_free(mask); |
1616 | 1 | } |
1617 | 1 | return err; |
1618 | 1 | } |
1619 | | |
1620 | | make_internal_alias(numa_node_to_cpus_v1); |
1621 | | make_internal_alias(numa_node_to_cpus_v2); |
1622 | | |
1623 | | void numa_node_to_cpu_update(void) |
1624 | 0 | { |
1625 | 0 | __atomic_store_n(&node_cpu_mask_v1_stale, 1, __ATOMIC_RELAXED); |
1626 | 0 | __atomic_store_n(&node_cpu_mask_v2_stale, 1, __ATOMIC_RELAXED); |
1627 | 0 | } |
1628 | | |
1629 | | /* report the node of the specified cpu */ |
1630 | | int numa_node_of_cpu(int cpu) |
1631 | 0 | { |
1632 | 0 | struct bitmask *bmp; |
1633 | 0 | int ncpus, nnodes, node, ret; |
1634 | |
|
1635 | 0 | ncpus = numa_num_possible_cpus(); |
1636 | 0 | if (cpu > ncpus){ |
1637 | 0 | errno = EINVAL; |
1638 | 0 | return -1; |
1639 | 0 | } |
1640 | 0 | bmp = numa_bitmask_alloc(ncpus); |
1641 | 0 | nnodes = numa_max_node(); |
1642 | 0 | for (node = 0; node <= nnodes; node++){ |
1643 | 0 | if (numa_node_to_cpus_v2_int(node, bmp) < 0) { |
1644 | | /* It's possible for the node to not exist */ |
1645 | 0 | continue; |
1646 | 0 | } |
1647 | 0 | if (numa_bitmask_isbitset(bmp, cpu)){ |
1648 | 0 | ret = node; |
1649 | 0 | goto end; |
1650 | 0 | } |
1651 | 0 | } |
1652 | 0 | ret = -1; |
1653 | 0 | errno = EINVAL; |
1654 | 0 | end: |
1655 | 0 | numa_bitmask_free(bmp); |
1656 | 0 | return ret; |
1657 | 0 | } |
1658 | | |
1659 | | SYMVER("numa_run_on_node_mask_v1", "numa_run_on_node_mask@libnuma_1.1") |
1660 | | int |
1661 | | numa_run_on_node_mask_v1(const nodemask_t *mask) |
1662 | 0 | { |
1663 | 0 | int ncpus = numa_num_possible_cpus(); |
1664 | 0 | int i, k, err; |
1665 | 0 | unsigned long cpus[CPU_LONGS(ncpus)], nodecpus[CPU_LONGS(ncpus)]; |
1666 | 0 | memset(cpus, 0, CPU_BYTES(ncpus)); |
1667 | 0 | for (i = 0; i < NUMA_NUM_NODES; i++) { |
1668 | 0 | if (mask->n[i / BITS_PER_LONG] == 0) |
1669 | 0 | continue; |
1670 | 0 | if (nodemask_isset_compat(mask, i)) { |
1671 | 0 | if (numa_node_to_cpus_v1_int(i, nodecpus, CPU_BYTES(ncpus)) < 0) { |
1672 | 0 | numa_warn(W_noderunmask, |
1673 | 0 | "Cannot read node cpumask from sysfs"); |
1674 | 0 | continue; |
1675 | 0 | } |
1676 | 0 | for (k = 0; k < CPU_LONGS(ncpus); k++) |
1677 | 0 | cpus[k] |= nodecpus[k]; |
1678 | 0 | } |
1679 | 0 | } |
1680 | 0 | err = numa_sched_setaffinity_v1(0, CPU_BYTES(ncpus), cpus); |
1681 | | |
1682 | | /* The sched_setaffinity API is broken because it expects |
1683 | | the user to guess the kernel cpuset size. Do this in a |
1684 | | brute force way. */ |
1685 | 0 | if (err < 0 && errno == EINVAL) { |
1686 | 0 | int savederrno = errno; |
1687 | 0 | char *bigbuf; |
1688 | 0 | static int size = -1; |
1689 | 0 | if (size == -1) |
1690 | 0 | size = CPU_BYTES(ncpus) * 2; |
1691 | 0 | bigbuf = malloc(CPU_BUFFER_SIZE); |
1692 | 0 | if (!bigbuf) { |
1693 | 0 | errno = ENOMEM; |
1694 | 0 | return -1; |
1695 | 0 | } |
1696 | 0 | errno = savederrno; |
1697 | 0 | while (size <= CPU_BUFFER_SIZE) { |
1698 | 0 | memcpy(bigbuf, cpus, CPU_BYTES(ncpus)); |
1699 | 0 | memset(bigbuf + CPU_BYTES(ncpus), 0, |
1700 | 0 | CPU_BUFFER_SIZE - CPU_BYTES(ncpus)); |
1701 | 0 | err = numa_sched_setaffinity_v1_int(0, size, (unsigned long *)bigbuf); |
1702 | 0 | if (err == 0 || errno != EINVAL) |
1703 | 0 | break; |
1704 | 0 | size *= 2; |
1705 | 0 | } |
1706 | 0 | savederrno = errno; |
1707 | 0 | free(bigbuf); |
1708 | 0 | errno = savederrno; |
1709 | 0 | } |
1710 | 0 | return err; |
1711 | 0 | } |
1712 | | |
1713 | | /* |
1714 | | * Given a node mask (size of a kernel nodemask_t) (probably populated by |
1715 | | * a user argument list) set up a map of cpus (map "cpus") on those nodes. |
1716 | | * Then set affinity to those cpus. |
1717 | | */ |
1718 | | SYMVER("numa_run_on_node_mask_v2", "numa_run_on_node_mask@@libnuma_1.2") |
1719 | | int |
1720 | | numa_run_on_node_mask_v2(struct bitmask *bmp) |
1721 | 0 | { |
1722 | 0 | int ncpus, i, k, err; |
1723 | 0 | struct bitmask *cpus, *nodecpus; |
1724 | |
|
1725 | 0 | cpus = numa_allocate_cpumask(); |
1726 | 0 | ncpus = cpus->size; |
1727 | 0 | nodecpus = numa_allocate_cpumask(); |
1728 | 0 | if (!cpus || !nodecpus) |
1729 | 0 | return -1; |
1730 | | |
1731 | 0 | for (i = 0; i < bmp->size; i++) { |
1732 | 0 | if (bmp->maskp[i / BITS_PER_LONG] == 0) |
1733 | 0 | continue; |
1734 | 0 | if (numa_bitmask_isbitset(bmp, i)) { |
1735 | | /* |
1736 | | * numa_all_nodes_ptr is cpuset aware; use only |
1737 | | * these nodes |
1738 | | */ |
1739 | 0 | if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i)) { |
1740 | 0 | numa_warn(W_noderunmask, |
1741 | 0 | "node %d not allowed", i); |
1742 | 0 | continue; |
1743 | 0 | } |
1744 | 0 | if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) { |
1745 | 0 | numa_warn(W_noderunmask, |
1746 | 0 | "Cannot read node cpumask from sysfs"); |
1747 | 0 | continue; |
1748 | 0 | } |
1749 | 0 | for (k = 0; k < CPU_LONGS(ncpus); k++) |
1750 | 0 | cpus->maskp[k] |= nodecpus->maskp[k]; |
1751 | 0 | } |
1752 | 0 | } |
1753 | 0 | err = numa_sched_setaffinity_v2_int(0, cpus); |
1754 | |
|
1755 | 0 | numa_bitmask_free(cpus); |
1756 | 0 | numa_bitmask_free(nodecpus); |
1757 | | |
1758 | | /* used to have to consider that this could fail - it shouldn't now */ |
1759 | 0 | if (err < 0) { |
1760 | 0 | numa_error("numa_sched_setaffinity_v2_int() failed"); |
1761 | 0 | } |
1762 | |
|
1763 | 0 | return err; |
1764 | 0 | } |
1765 | | |
1766 | | make_internal_alias(numa_run_on_node_mask_v2); |
1767 | | |
1768 | | /* |
1769 | | * Given a node mask (size of a kernel nodemask_t) (probably populated by |
1770 | | * a user argument list) set up a map of cpus (map "cpus") on those nodes |
1771 | | * without any cpuset awareness. Then set affinity to those cpus. |
1772 | | */ |
1773 | | int |
1774 | | numa_run_on_node_mask_all(struct bitmask *bmp) |
1775 | 0 | { |
1776 | 0 | int ncpus, i, k, err; |
1777 | 0 | struct bitmask *cpus, *nodecpus; |
1778 | |
|
1779 | 0 | cpus = numa_allocate_cpumask(); |
1780 | 0 | ncpus = cpus->size; |
1781 | 0 | nodecpus = numa_allocate_cpumask(); |
1782 | 0 | if (!cpus || !nodecpus) |
1783 | 0 | return -1; |
1784 | | |
1785 | 0 | for (i = 0; i < bmp->size; i++) { |
1786 | 0 | if (bmp->maskp[i / BITS_PER_LONG] == 0) |
1787 | 0 | continue; |
1788 | 0 | if (numa_bitmask_isbitset(bmp, i)) { |
1789 | 0 | if (!numa_bitmask_isbitset(numa_possible_nodes_ptr, i)) { |
1790 | 0 | numa_warn(W_noderunmask, |
1791 | 0 | "node %d not allowed", i); |
1792 | 0 | continue; |
1793 | 0 | } |
1794 | 0 | if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) { |
1795 | 0 | numa_warn(W_noderunmask, |
1796 | 0 | "Cannot read node cpumask from sysfs"); |
1797 | 0 | continue; |
1798 | 0 | } |
1799 | 0 | for (k = 0; k < CPU_LONGS(ncpus); k++) |
1800 | 0 | cpus->maskp[k] |= nodecpus->maskp[k]; |
1801 | 0 | } |
1802 | 0 | } |
1803 | 0 | err = numa_sched_setaffinity_v2_int(0, cpus); |
1804 | |
|
1805 | 0 | numa_bitmask_free(cpus); |
1806 | 0 | numa_bitmask_free(nodecpus); |
1807 | | |
1808 | | /* With possible nodes freedom it can happen easily now */ |
1809 | 0 | if (err < 0) { |
1810 | 0 | numa_error("numa_sched_setaffinity_v2_int() failed"); |
1811 | 0 | } |
1812 | |
|
1813 | 0 | return err; |
1814 | 0 | } |
1815 | | |
1816 | | SYMVER("numa_get_run_node_mask_v1", "numa_get_run_node_mask@libnuma_1.1") |
1817 | | nodemask_t |
1818 | | numa_get_run_node_mask_v1(void) |
1819 | 0 | { |
1820 | 0 | int ncpus = numa_num_configured_cpus(); |
1821 | 0 | int i, k; |
1822 | 0 | int max = numa_max_node_int(); |
1823 | 0 | struct bitmask *bmp, *cpus, *nodecpus; |
1824 | 0 | nodemask_t nmp; |
1825 | |
|
1826 | 0 | cpus = numa_allocate_cpumask(); |
1827 | 0 | if (!cpus) |
1828 | 0 | return numa_no_nodes; |
1829 | 0 | if (numa_sched_getaffinity_v2_int(0, cpus) < 0){ |
1830 | 0 | nmp = numa_no_nodes; |
1831 | 0 | goto free_cpus; |
1832 | 0 | } |
1833 | | |
1834 | 0 | nodecpus = numa_allocate_cpumask(); |
1835 | 0 | if (!nodecpus) { |
1836 | 0 | nmp = numa_no_nodes; |
1837 | 0 | goto free_cpus; |
1838 | 0 | } |
1839 | | |
1840 | 0 | bmp = allocate_nodemask_v1(); /* the size of a nodemask_t */ |
1841 | 0 | if (!bmp) { |
1842 | 0 | nmp = numa_no_nodes; |
1843 | 0 | goto free_cpus2; |
1844 | 0 | } |
1845 | | |
1846 | 0 | for (i = 0; i <= max; i++) { |
1847 | 0 | if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) { |
1848 | | /* It's possible for the node to not exist */ |
1849 | 0 | continue; |
1850 | 0 | } |
1851 | 0 | for (k = 0; k < CPU_LONGS(ncpus); k++) { |
1852 | 0 | if (nodecpus->maskp[k] & cpus->maskp[k]) |
1853 | 0 | numa_bitmask_setbit(bmp, i); |
1854 | 0 | } |
1855 | 0 | } |
1856 | 0 | copy_bitmask_to_nodemask(bmp, &nmp); |
1857 | 0 | numa_bitmask_free(bmp); |
1858 | 0 | free_cpus2: |
1859 | 0 | numa_bitmask_free(nodecpus); |
1860 | 0 | free_cpus: |
1861 | 0 | numa_bitmask_free(cpus); |
1862 | 0 | return nmp; |
1863 | 0 | } |
1864 | | |
1865 | | SYMVER("numa_get_run_node_mask_v2", "numa_get_run_node_mask@@libnuma_1.2") |
1866 | | struct bitmask * |
1867 | | numa_get_run_node_mask_v2(void) |
1868 | 0 | { |
1869 | 0 | int i, k; |
1870 | 0 | int ncpus = numa_num_configured_cpus(); |
1871 | 0 | int max = numa_max_node_int(); |
1872 | 0 | struct bitmask *bmp, *cpus, *nodecpus; |
1873 | |
|
1874 | 0 | bmp = numa_allocate_cpumask(); |
1875 | 0 | cpus = numa_allocate_cpumask(); |
1876 | 0 | if (!bmp || !cpus) |
1877 | 0 | return NULL; |
1878 | 0 | if (numa_sched_getaffinity_v2_int(0, cpus) < 0){ |
1879 | 0 | copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp); |
1880 | 0 | goto free_cpus; |
1881 | 0 | } |
1882 | | |
1883 | 0 | nodecpus = numa_allocate_cpumask(); |
1884 | 0 | for (i = 0; i <= max; i++) { |
1885 | | /* |
1886 | | * numa_all_nodes_ptr is cpuset aware; show only |
1887 | | * these nodes |
1888 | | */ |
1889 | 0 | if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i)) { |
1890 | 0 | continue; |
1891 | 0 | } |
1892 | 0 | if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) { |
1893 | | /* It's possible for the node to not exist */ |
1894 | 0 | continue; |
1895 | 0 | } |
1896 | 0 | for (k = 0; k < CPU_LONGS(ncpus); k++) { |
1897 | 0 | if (nodecpus->maskp[k] & cpus->maskp[k]) |
1898 | 0 | numa_bitmask_setbit(bmp, i); |
1899 | 0 | } |
1900 | 0 | } |
1901 | 0 | numa_bitmask_free(nodecpus); |
1902 | 0 | free_cpus: |
1903 | 0 | numa_bitmask_free(cpus); |
1904 | 0 | return bmp; |
1905 | 0 | } |
1906 | | |
1907 | | int |
1908 | | numa_migrate_pages(int pid, struct bitmask *fromnodes, struct bitmask *tonodes) |
1909 | 0 | { |
1910 | 0 | int numa_num_nodes = numa_num_possible_nodes(); |
1911 | |
|
1912 | 0 | return migrate_pages(pid, numa_num_nodes + 1, fromnodes->maskp, |
1913 | 0 | tonodes->maskp); |
1914 | 0 | } |
1915 | | |
1916 | | int numa_move_pages(int pid, unsigned long count, |
1917 | | void **pages, const int *nodes, int *status, int flags) |
1918 | 0 | { |
1919 | 0 | return move_pages(pid, count, pages, nodes, status, flags); |
1920 | 0 | } |
1921 | | |
1922 | | int numa_run_on_node(int node) |
1923 | 0 | { |
1924 | 0 | int numa_num_nodes = numa_num_possible_nodes(); |
1925 | 0 | int ret = -1; |
1926 | 0 | struct bitmask *cpus; |
1927 | |
|
1928 | 0 | if (node < -1 || node >= numa_num_nodes){ |
1929 | 0 | errno = EINVAL; |
1930 | 0 | goto out; |
1931 | 0 | } |
1932 | | |
1933 | 0 | cpus = numa_allocate_cpumask(); |
1934 | 0 | if (!cpus) |
1935 | 0 | return -1; |
1936 | | |
1937 | 0 | if (node == -1) |
1938 | 0 | numa_bitmask_setall(cpus); |
1939 | 0 | else if (numa_node_to_cpus_v2_int(node, cpus) < 0){ |
1940 | 0 | numa_warn(W_noderunmask, "Cannot read node cpumask from sysfs"); |
1941 | 0 | goto free; |
1942 | 0 | } |
1943 | | |
1944 | 0 | ret = numa_sched_setaffinity_v2_int(0, cpus); |
1945 | 0 | free: |
1946 | 0 | numa_bitmask_free(cpus); |
1947 | 0 | out: |
1948 | 0 | return ret; |
1949 | 0 | } |
1950 | | |
1951 | | static struct bitmask *__numa_preferred(void) |
1952 | 0 | { |
1953 | 0 | int policy = 0; |
1954 | 0 | struct bitmask *bmp; |
1955 | |
|
1956 | 0 | bmp = numa_allocate_nodemask(); |
1957 | 0 | if (!bmp) |
1958 | 0 | return NULL; |
1959 | | /* could read the current CPU from /proc/self/status. Probably |
1960 | | not worth it. */ |
1961 | 0 | numa_bitmask_clearall(bmp); |
1962 | 0 | getpol(&policy, bmp); |
1963 | |
|
1964 | 0 | if (policy != MPOL_PREFERRED && |
1965 | 0 | policy != MPOL_PREFERRED_MANY && |
1966 | 0 | policy != MPOL_BIND) |
1967 | 0 | return bmp; |
1968 | | |
1969 | 0 | if (policy == MPOL_PREFERRED && numa_bitmask_weight(bmp) > 1) { |
1970 | 0 | errno = EINVAL; |
1971 | 0 | numa_error(__FILE__); |
1972 | 0 | } |
1973 | |
|
1974 | 0 | return bmp; |
1975 | 0 | } |
1976 | | |
1977 | | int numa_preferred_err(void) |
1978 | 0 | { |
1979 | 0 | int first_node = 0; |
1980 | 0 | struct bitmask *bmp; |
1981 | |
|
1982 | 0 | bmp = __numa_preferred(); |
1983 | 0 | first_node = numa_find_first(bmp); |
1984 | 0 | numa_bitmask_free(bmp); |
1985 | |
|
1986 | 0 | return first_node; |
1987 | 0 | } |
1988 | | |
1989 | | int numa_preferred(void) |
1990 | 0 | { |
1991 | 0 | int first_node = 0; |
1992 | |
|
1993 | 0 | first_node = numa_preferred_err(); |
1994 | 0 | first_node = first_node >= 0 ? first_node : 0; |
1995 | |
|
1996 | 0 | return first_node; |
1997 | 0 | } |
1998 | | |
1999 | | static void __numa_set_preferred(struct bitmask *bmp) |
2000 | 0 | { |
2001 | 0 | int nodes = numa_bitmask_weight(bmp); |
2002 | 0 | if (nodes > 1) { |
2003 | 0 | errno = EINVAL; |
2004 | 0 | numa_error(__FILE__); |
2005 | 0 | } |
2006 | |
|
2007 | 0 | setpol(nodes ? MPOL_PREFERRED : MPOL_LOCAL, bmp); |
2008 | 0 | } |
2009 | | |
2010 | | void numa_set_preferred(int node) |
2011 | 0 | { |
2012 | 0 | struct bitmask *bmp = numa_allocate_nodemask(); |
2013 | 0 | if (!bmp) |
2014 | 0 | return; |
2015 | 0 | numa_bitmask_setbit(bmp, node); |
2016 | 0 | __numa_set_preferred(bmp); |
2017 | 0 | numa_bitmask_free(bmp); |
2018 | 0 | } |
2019 | | |
2020 | | int numa_has_preferred_many(void) |
2021 | 0 | { |
2022 | 0 | set_preferred_many(); |
2023 | 0 | return has_preferred_many; |
2024 | 0 | } |
2025 | | |
2026 | | void numa_set_preferred_many(struct bitmask *bitmask) |
2027 | 0 | { |
2028 | 0 | int first_node = 0; |
2029 | |
|
2030 | 0 | set_preferred_many(); |
2031 | 0 | if (!has_preferred_many) { |
2032 | 0 | numa_warn(W_nodeparse, |
2033 | 0 | "Unable to handle MANY preferred nodes. Falling back to first node\n"); |
2034 | 0 | first_node = numa_find_first(bitmask); |
2035 | 0 | numa_set_preferred(first_node); |
2036 | 0 | return; |
2037 | 0 | } |
2038 | 0 | setpol(MPOL_PREFERRED_MANY, bitmask); |
2039 | 0 | } |
2040 | | |
2041 | | struct bitmask *numa_preferred_many() |
2042 | 0 | { |
2043 | 0 | return __numa_preferred(); |
2044 | 0 | } |
2045 | | |
2046 | | void numa_set_localalloc(void) |
2047 | 0 | { |
2048 | 0 | setpol(MPOL_LOCAL, numa_no_nodes_ptr); |
2049 | 0 | } |
2050 | | |
2051 | | SYMVER("numa_bind_v1", "numa_bind@libnuma_1.1") |
2052 | | void numa_bind_v1(const nodemask_t *nodemask) |
2053 | 0 | { |
2054 | 0 | struct bitmask bitmask; |
2055 | |
|
2056 | 0 | bitmask.maskp = (unsigned long *)nodemask; |
2057 | 0 | bitmask.size = sizeof(nodemask_t); |
2058 | 0 | numa_run_on_node_mask_v2_int(&bitmask); |
2059 | 0 | numa_set_membind_v2_int(&bitmask); |
2060 | 0 | } |
2061 | | |
2062 | | SYMVER("numa_bind_v2", "numa_bind@@libnuma_1.2") |
2063 | | void numa_bind_v2(struct bitmask *bmp) |
2064 | 0 | { |
2065 | 0 | numa_run_on_node_mask_v2_int(bmp); |
2066 | 0 | numa_set_membind_v2_int(bmp); |
2067 | 0 | } |
2068 | | |
2069 | | void numa_set_strict(int flag) |
2070 | 0 | { |
2071 | 0 | if (flag) |
2072 | 0 | mbind_flags |= MPOL_MF_STRICT; |
2073 | 0 | else |
2074 | 0 | mbind_flags &= ~MPOL_MF_STRICT; |
2075 | 0 | } |
2076 | | |
2077 | | /* |
2078 | | * Extract a node or processor number from the given string. |
2079 | | * Allow a relative node / processor specification within the allowed |
2080 | | * set if "relative" is nonzero |
2081 | | */ |
2082 | | static unsigned long get_nr(const char *s, char **end, struct bitmask *bmp, int relative) |
2083 | 10.9k | { |
2084 | 10.9k | long i, nr; |
2085 | | |
2086 | 10.9k | if (!relative) |
2087 | 6.53k | return strtoul(s, end, 0); |
2088 | | |
2089 | 4.39k | nr = strtoul(s, end, 0); |
2090 | 4.39k | if (s == *end) |
2091 | 5 | return nr; |
2092 | | /* Find the nth set bit */ |
2093 | 124k | for (i = 0; nr >= 0 && i <= bmp->size; i++) |
2094 | 120k | if (numa_bitmask_isbitset(bmp, i)) |
2095 | 6.12k | nr--; |
2096 | 4.39k | return i-1; |
2097 | 4.39k | } |
2098 | | |
2099 | | /* |
2100 | | * __numa_parse_nodestring() is called to create a node mask, given |
2101 | | * an ascii string such as 25 or 12-15 or 1,3,5-7 or +6-10. |
2102 | | * (the + indicates that the numbers are nodeset-relative) |
2103 | | * |
2104 | | * The nodes may be specified as absolute, or relative to the current nodeset. |
2105 | | * The list of available nodes is in a map pointed to by "allowed_nodes_ptr", |
2106 | | * which may represent all nodes or the nodes in the current nodeset. |
2107 | | * |
2108 | | * The caller must free the returned bitmask. |
2109 | | */ |
2110 | | static struct bitmask * |
2111 | | __numa_parse_nodestring(const char *s, struct bitmask *allowed_nodes_ptr) |
2112 | 892 | { |
2113 | 892 | int invert = 0, relative = 0; |
2114 | 892 | int conf_nodes = numa_num_configured_nodes(); |
2115 | 892 | char *end; |
2116 | 892 | struct bitmask *mask; |
2117 | | |
2118 | 892 | mask = numa_allocate_nodemask(); |
2119 | 892 | if (!mask) |
2120 | 0 | return NULL; |
2121 | | |
2122 | 892 | if (s[0] == 0){ |
2123 | 1 | copy_bitmask_to_bitmask(numa_no_nodes_ptr, mask); |
2124 | 1 | return mask; /* return freeable mask */ |
2125 | 1 | } |
2126 | 891 | if (*s == '!') { |
2127 | 3 | invert = 1; |
2128 | 3 | s++; |
2129 | 3 | } |
2130 | 891 | if (*s == '+') { |
2131 | 302 | relative++; |
2132 | 302 | s++; |
2133 | 302 | } |
2134 | 3.14k | do { |
2135 | 3.14k | unsigned long arg; |
2136 | 3.14k | int i; |
2137 | 3.14k | if (isalpha(*s)) { |
2138 | 140 | int n; |
2139 | 140 | if (!strcmp(s,"all")) { |
2140 | 2 | copy_bitmask_to_bitmask(allowed_nodes_ptr, |
2141 | 2 | mask); |
2142 | 2 | s+=4; |
2143 | 2 | break; |
2144 | 2 | } |
2145 | 138 | n = resolve_affinity(s, mask); |
2146 | 138 | if (n != NO_IO_AFFINITY) { |
2147 | 62 | if (n < 0) |
2148 | 62 | goto err; |
2149 | 0 | s += strlen(s) + 1; |
2150 | 0 | break; |
2151 | 62 | } |
2152 | 138 | } |
2153 | 3.07k | arg = get_nr(s, &end, allowed_nodes_ptr, relative); |
2154 | 3.07k | if (end == s) { |
2155 | 111 | numa_warn(W_nodeparse, "unparseable node description `%s'\n", s); |
2156 | 111 | goto err; |
2157 | 111 | } |
2158 | 2.96k | if (arg >= allowed_nodes_ptr->size || |
2159 | 2.67k | !numa_bitmask_isbitset(allowed_nodes_ptr, arg)) { |
2160 | 363 | numa_warn(W_nodeparse, "node argument %ld is out of range\n", arg); |
2161 | 363 | goto err; |
2162 | 363 | } |
2163 | 2.60k | i = arg; |
2164 | 2.60k | numa_bitmask_setbit(mask, i); |
2165 | 2.60k | s = end; |
2166 | 2.60k | if (*s == '-') { |
2167 | 1.31k | char *end2; |
2168 | 1.31k | unsigned long arg2; |
2169 | 1.31k | arg2 = get_nr(++s, &end2, allowed_nodes_ptr, relative); |
2170 | 1.31k | if (end2 == s) { |
2171 | 9 | numa_warn(W_nodeparse, "missing node argument %s\n", s); |
2172 | 9 | goto err; |
2173 | 9 | } |
2174 | 1.30k | if (arg2 >= allowed_nodes_ptr->size || |
2175 | 1.04k | !numa_bitmask_isbitset(allowed_nodes_ptr, arg2)) { |
2176 | 298 | numa_warn(W_nodeparse, "node argument %ld out of range\n", arg2); |
2177 | 298 | goto err; |
2178 | 298 | } |
2179 | 2.01k | while (arg <= arg2) { |
2180 | 1.00k | i = arg; |
2181 | 1.00k | if (numa_bitmask_isbitset(allowed_nodes_ptr,i)) |
2182 | 1.00k | numa_bitmask_setbit(mask, i); |
2183 | 1.00k | arg++; |
2184 | 1.00k | } |
2185 | 1.00k | s = end2; |
2186 | 1.00k | } |
2187 | 2.60k | } while (*s++ == ','); |
2188 | 48 | if (s[-1] != '\0') |
2189 | 13 | goto err; |
2190 | 35 | if (invert) { |
2191 | 2 | int i; |
2192 | 4 | for (i = 0; i < conf_nodes; i++) { |
2193 | 2 | if (numa_bitmask_isbitset(mask, i)) |
2194 | 2 | numa_bitmask_clearbit(mask, i); |
2195 | 0 | else |
2196 | 0 | numa_bitmask_setbit(mask, i); |
2197 | 2 | } |
2198 | 2 | } |
2199 | 35 | return mask; |
2200 | | |
2201 | 856 | err: |
2202 | 856 | numa_bitmask_free(mask); |
2203 | 856 | return NULL; |
2204 | 48 | } |
2205 | | |
2206 | | /* |
2207 | | * numa_parse_nodestring() is called to create a bitmask from nodes available |
2208 | | * for this task. |
2209 | | */ |
2210 | | |
2211 | | struct bitmask * numa_parse_nodestring(const char *s) |
2212 | 892 | { |
2213 | 892 | return __numa_parse_nodestring(s, numa_all_nodes_ptr); |
2214 | 892 | } |
2215 | | |
2216 | | /* |
2217 | | * numa_parse_nodestring_all() is called to create a bitmask from all nodes |
2218 | | * available. |
2219 | | */ |
2220 | | |
2221 | | struct bitmask * numa_parse_nodestring_all(const char *s) |
2222 | 0 | { |
2223 | 0 | return __numa_parse_nodestring(s, numa_possible_nodes_ptr); |
2224 | 0 | } |
2225 | | |
2226 | | /* |
2227 | | * __numa_parse_cpustring() is called to create a bitmask, given |
2228 | | * an ascii string such as 25 or 12-15 or 1,3,5-7 or +6-10. |
2229 | | * (the + indicates that the numbers are cpuset-relative) |
2230 | | * |
2231 | | * The cpus may be specified as absolute, or relative to the current cpuset. |
2232 | | * The list of available cpus for this task is in the map pointed to by |
2233 | | * "allowed_cpus_ptr", which may represent all cpus or the cpus in the |
2234 | | * current cpuset. |
2235 | | * |
2236 | | * The caller must free the returned bitmask. |
2237 | | */ |
2238 | | static struct bitmask * |
2239 | | __numa_parse_cpustring(const char *s, struct bitmask *allowed_cpus_ptr) |
2240 | 892 | { |
2241 | 892 | int invert = 0, relative=0; |
2242 | 892 | int conf_cpus = numa_num_configured_cpus(); |
2243 | 892 | char *end; |
2244 | 892 | struct bitmask *mask; |
2245 | 892 | int i; |
2246 | | |
2247 | 892 | mask = numa_allocate_cpumask(); |
2248 | 892 | if (!mask) |
2249 | 0 | return NULL; |
2250 | | |
2251 | 892 | if (s[0] == 0) |
2252 | 1 | return mask; |
2253 | 891 | if (*s == '!') { |
2254 | 3 | invert = 1; |
2255 | 3 | s++; |
2256 | 3 | } |
2257 | 891 | if (*s == '+') { |
2258 | 302 | relative++; |
2259 | 302 | s++; |
2260 | 302 | } |
2261 | 4.40k | do { |
2262 | 4.40k | unsigned long arg; |
2263 | | |
2264 | 4.40k | if (!strcmp(s,"all")) { |
2265 | 2 | copy_bitmask_to_bitmask(allowed_cpus_ptr, mask); |
2266 | 2 | s+=4; |
2267 | 2 | break; |
2268 | 2 | } |
2269 | 4.40k | arg = get_nr(s, &end, allowed_cpus_ptr, relative); |
2270 | 4.40k | if (end == s) { |
2271 | 184 | numa_warn(W_cpuparse, "unparseable cpu description `%s'\n", s); |
2272 | 184 | goto err; |
2273 | 184 | } |
2274 | 4.21k | if (arg >= conf_cpus) { |
2275 | 271 | numa_warn(W_cpuparse, "cpu argument %s is out of range\n", s); |
2276 | 271 | goto err; |
2277 | 271 | } |
2278 | 3.94k | i = arg; |
2279 | 3.94k | numa_bitmask_setbit(mask, i); |
2280 | 3.94k | s = end; |
2281 | 3.94k | if (*s == '-') { |
2282 | 2.14k | char *end2; |
2283 | 2.14k | unsigned long arg2; |
2284 | 2.14k | arg2 = get_nr(++s, &end2, allowed_cpus_ptr, relative); |
2285 | 2.14k | if (end2 == s) { |
2286 | 16 | numa_warn(W_cpuparse, "missing cpu argument %s\n", s); |
2287 | 16 | goto err; |
2288 | 16 | } |
2289 | 2.12k | if (arg2 >= conf_cpus) { |
2290 | 270 | numa_warn(W_cpuparse, "cpu argument %s out of range\n", s); |
2291 | 270 | goto err; |
2292 | 270 | } |
2293 | 4.08k | while (arg <= arg2) { |
2294 | 2.22k | i = arg; |
2295 | 2.22k | if (numa_bitmask_isbitset(allowed_cpus_ptr, i)) |
2296 | 2.22k | numa_bitmask_setbit(mask, i); |
2297 | 2.22k | arg++; |
2298 | 2.22k | } |
2299 | 1.85k | s = end2; |
2300 | 1.85k | } |
2301 | 3.94k | } while (*s++ == ','); |
2302 | 150 | if (s[-1] != '\0') |
2303 | 20 | goto err; |
2304 | 130 | if (invert) { |
2305 | 66 | for (i = 0; i < conf_cpus; i++) { |
2306 | 64 | if (numa_bitmask_isbitset(mask, i)) |
2307 | 33 | numa_bitmask_clearbit(mask, i); |
2308 | 31 | else |
2309 | 31 | numa_bitmask_setbit(mask, i); |
2310 | 64 | } |
2311 | 2 | } |
2312 | 130 | return mask; |
2313 | | |
2314 | 761 | err: |
2315 | 761 | numa_bitmask_free(mask); |
2316 | 761 | return NULL; |
2317 | 150 | } |
2318 | | |
2319 | | /* |
2320 | | * numa_parse_cpustring() is called to create a bitmask from cpus available |
2321 | | * for this task. |
2322 | | */ |
2323 | | |
2324 | | struct bitmask * numa_parse_cpustring(const char *s) |
2325 | 892 | { |
2326 | 892 | return __numa_parse_cpustring(s, numa_all_cpus_ptr); |
2327 | 892 | } |
2328 | | |
2329 | | /* |
2330 | | * numa_parse_cpustring_all() is called to create a bitmask from all cpus |
2331 | | * available. |
2332 | | */ |
2333 | | |
2334 | | struct bitmask * numa_parse_cpustring_all(const char *s) |
2335 | 0 | { |
2336 | 0 | return __numa_parse_cpustring(s, numa_possible_cpus_ptr); |
2337 | 0 | } |
2338 | | |
2339 | | int numa_has_home_node(void) |
2340 | 0 | { |
2341 | 0 | void *mem; |
2342 | 0 | static int has_home_node = -1; |
2343 | 0 | int page_size = numa_pagesize(); |
2344 | 0 | struct bitmask *tmp = numa_get_mems_allowed(); |
2345 | |
|
2346 | 0 | if (has_home_node >= 0) |
2347 | 0 | goto out; |
2348 | | |
2349 | 0 | has_home_node = 0; |
2350 | | /* Detect whether home_node is supported */ |
2351 | 0 | mem = mmap(0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); |
2352 | 0 | if (mem != MAP_FAILED) { |
2353 | 0 | dombind(mem, page_size, MPOL_BIND, tmp); |
2354 | 0 | if (set_mempolicy_home_node(mem, page_size, numa_find_first(tmp), 0) == 0) |
2355 | 0 | has_home_node = 1; |
2356 | 0 | munmap(mem, page_size); |
2357 | 0 | } |
2358 | |
|
2359 | 0 | out: |
2360 | 0 | numa_bitmask_free(tmp); |
2361 | 0 | return has_home_node; |
2362 | 0 | } |
2363 | | |
2364 | | int numa_set_mempolicy_home_node(void *start, unsigned long len, int home_node, int flags) |
2365 | 0 | { |
2366 | 0 | if (set_mempolicy_home_node(start, len, home_node, flags)) { |
2367 | 0 | numa_error("set_mempolicy_home_node"); |
2368 | 0 | return -1; |
2369 | 0 | } |
2370 | | |
2371 | 0 | return 0; |
2372 | 0 | } |
2373 | | |
2374 | | /* return 1 if the node is a cpuless node, 0 when it's not, and -1 on any error */ |
2375 | | int numa_is_cpuless_node(int node_id) |
2376 | 0 | { |
2377 | 0 | struct bitmask *cpumask; |
2378 | 0 | int ret = 0; |
2379 | |
|
2380 | 0 | if (node_id < 0 || node_id > numa_max_node()) { |
2381 | 0 | errno = ERANGE; |
2382 | 0 | return -1; |
2383 | 0 | } |
2384 | | |
2385 | 0 | cpumask = numa_allocate_cpumask(); |
2386 | 0 | if (!cpumask) |
2387 | 0 | return -1; |
2388 | | |
2389 | 0 | numa_node_to_cpus(node_id, cpumask); |
2390 | 0 | if (!numa_bitmask_weight(cpumask)) |
2391 | 0 | ret = 1; |
2392 | |
|
2393 | 0 | numa_bitmask_free(cpumask); |
2394 | 0 | return ret; |
2395 | 0 | } |