Coverage Report

Created: 2026-11-20 06:22

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/numactl/libnuma.c
Line
Count
Source
1
/* Simple NUMA library.
2
   Copyright (C) 2003,2004,2005,2008 Andi Kleen,SuSE Labs and
3
   Cliff Wickman,SGI.
4
5
   libnuma is free software; you can redistribute it and/or
6
   modify it under the terms of the GNU Lesser General Public
7
   License as published by the Free Software Foundation; version
8
   2.1.
9
10
   libnuma is distributed in the hope that it will be useful,
11
   but WITHOUT ANY WARRANTY; without even the implied warranty of
12
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13
   Lesser General Public License for more details.
14
15
   You should find a copy of v2.1 of the GNU Lesser General Public License
16
   somewhere on your Linux system; if not, write to the Free Software
17
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
19
   All calls are undefined when numa_available returns an error. */
20
#define _GNU_SOURCE
21
#include <stdlib.h>
22
#include <stdio.h>
23
#include <unistd.h>
24
#include <string.h>
25
#include <sched.h>
26
#include <dirent.h>
27
#include <errno.h>
28
#include <stdarg.h>
29
#include <ctype.h>
30
#include <assert.h>
31
32
#include <sys/mman.h>
33
#include <limits.h>
34
35
#include "config.h"
36
#include "numa.h"
37
#include "numaif.h"
38
#include "numaint.h"
39
#include "util.h"
40
#include "affinity.h"
41
42
#define WEAK __attribute__((weak))
43
44
0
#define CPU_BUFFER_SIZE 4096     /* This limits you to 32768 CPUs */
45
46
/* these are the old (version 1) masks */
47
nodemask_t numa_no_nodes;
48
nodemask_t numa_all_nodes;
49
/* these are now the default bitmask (pointers to) (version 2) */
50
struct bitmask *numa_no_nodes_ptr = NULL;
51
struct bitmask *numa_all_nodes_ptr = NULL;
52
struct bitmask *numa_possible_nodes_ptr = NULL;
53
struct bitmask *numa_all_cpus_ptr = NULL;
54
struct bitmask *numa_possible_cpus_ptr = NULL;
55
/* I would prefer to use symbol versioning to create v1 and v2 versions
56
   of numa_no_nodes and numa_all_nodes, but the loader does not correctly
57
   handle versioning of BSS versus small data items */
58
59
struct bitmask *numa_nodes_ptr = NULL;
60
static struct bitmask *numa_memnode_ptr = NULL;
61
static unsigned long *node_cpu_mask_v1[NUMA_NUM_NODES];
62
static char node_cpu_mask_v1_stale = 1;
63
static struct bitmask **node_cpu_mask_v2;
64
static char node_cpu_mask_v2_stale = 1;
65
66
WEAK void numa_error(char *where);
67
68
#ifndef TLS
69
#warning "not threadsafe"
70
#define __thread
71
#endif
72
73
static __thread int bind_policy = MPOL_BIND;
74
static __thread unsigned int mbind_flags = 0;
75
static int sizes_set=0;
76
static int maxconfigurednode = -1;
77
static int maxconfiguredcpu = -1;
78
static int numprocnode = -1;
79
static int numproccpu = -1;
80
static int nodemask_sz = 0;
81
static int cpumask_sz = 0;
82
83
static int has_preferred_many = -1;
84
85
int numa_exit_on_error = 0;
86
int numa_exit_on_warn = 0;
87
int numa_fail_alloc_on_error = 0;
88
static void set_sizes(void);
89
90
/*
91
 * There are two special functions, _init(void) and _fini(void), which
92
 * are called automatically by the dynamic loader whenever a library is loaded.
93
 *
94
 * The v1 library depends upon nodemask_t's of all nodes and no nodes.
95
 */
96
void __attribute__((constructor))
97
numa_init(void)
98
2
{
99
2
  int max,i;
100
101
2
  if (sizes_set)
102
0
    return;
103
104
2
  set_sizes();
105
  /* numa_all_nodes should represent existing nodes on this system */
106
2
        max = numa_num_configured_nodes();
107
4
        for (i = 0; i < max; i++)
108
2
                nodemask_set_compat((nodemask_t *)&numa_all_nodes, i);
109
2
  memset(&numa_no_nodes, 0, sizeof(numa_no_nodes));
110
111
  /* clear errno */
112
2
  errno = 0;
113
2
}
114
115
static void cleanup_node_cpu_mask_v2(void);
116
117
0
#define FREE_AND_ZERO(x) if (x) { \
118
0
    numa_bitmask_free(x); \
119
0
    x = NULL;   \
120
0
  }
121
122
void __attribute__((destructor))
123
numa_fini(void)
124
0
{
125
0
  FREE_AND_ZERO(numa_all_cpus_ptr);
126
0
  FREE_AND_ZERO(numa_possible_cpus_ptr);
127
0
  FREE_AND_ZERO(numa_all_nodes_ptr);
128
0
  FREE_AND_ZERO(numa_possible_nodes_ptr);
129
0
  FREE_AND_ZERO(numa_no_nodes_ptr);
130
0
  FREE_AND_ZERO(numa_memnode_ptr);
131
0
  FREE_AND_ZERO(numa_nodes_ptr);
132
0
  cleanup_node_cpu_mask_v2();
133
0
  numa_distance_cleanup();
134
0
}
135
136
static int numa_find_first(struct bitmask *mask)
137
0
{
138
0
  int i;
139
0
  for (i = 0; i < mask->size; i++)
140
0
    if (numa_bitmask_isbitset(mask, i))
141
0
      return i;
142
0
  return -1;
143
0
}
144
145
/*
146
 * The following bitmask declarations, bitmask_*() routines, and associated
147
 * _setbit() and _getbit() routines are:
148
 * Copyright (c) 2004_2007 Silicon Graphics, Inc. (SGI) All rights reserved.
149
 * SGI publishes it under the terms of the Library GNU General Public License,
150
 * v2, as published by the Free Software Foundation.
151
 */
152
static unsigned int
153
_getbit(const struct bitmask *bmp, unsigned int n)
154
130k
{
155
130k
  if (n < bmp->size)
156
130k
    return (bmp->maskp[n/bitsperlong] >> (n % bitsperlong)) & 1;
157
162
  else
158
162
    return 0;
159
130k
}
160
161
static void
162
_setbit(struct bitmask *bmp, unsigned int n, unsigned int v)
163
18.3k
{
164
18.3k
  if (n < bmp->size) {
165
18.3k
    if (v)
166
9.88k
      bmp->maskp[n/bitsperlong] |= 1UL << (n % bitsperlong);
167
8.41k
    else
168
8.41k
      bmp->maskp[n/bitsperlong] &= ~(1UL << (n % bitsperlong));
169
18.3k
  }
170
18.3k
}
171
172
int
173
numa_bitmask_isbitset(const struct bitmask *bmp, unsigned int i)
174
128k
{
175
128k
  return _getbit(bmp, i);
176
128k
}
177
178
struct bitmask *
179
numa_bitmask_setall(struct bitmask *bmp)
180
0
{
181
0
  unsigned int i;
182
0
  for (i = 0; i < bmp->size; i++)
183
0
    _setbit(bmp, i, 1);
184
0
  return bmp;
185
0
}
186
187
struct bitmask *
188
numa_bitmask_clearall(struct bitmask *bmp)
189
131
{
190
131
  unsigned int i;
191
8.51k
  for (i = 0; i < bmp->size; i++)
192
8.38k
    _setbit(bmp, i, 0);
193
131
  return bmp;
194
131
}
195
196
struct bitmask *
197
numa_bitmask_setbit(struct bitmask *bmp, unsigned int i)
198
9.88k
{
199
9.88k
  _setbit(bmp, i, 1);
200
9.88k
  return bmp;
201
9.88k
}
202
203
struct bitmask *
204
numa_bitmask_clearbit(struct bitmask *bmp, unsigned int i)
205
35
{
206
35
  _setbit(bmp, i, 0);
207
35
  return bmp;
208
35
}
209
210
unsigned int
211
numa_bitmask_nbytes(struct bitmask *bmp)
212
2
{
213
2
  return longsperbits(bmp->size) * sizeof(unsigned long);
214
2
}
215
216
/* where n is the number of bits in the map */
217
/* This function should not exit on failure, but right now we cannot really
218
   recover from this. */
219
struct bitmask *
220
numa_bitmask_alloc(unsigned int n)
221
1.80k
{
222
1.80k
  struct bitmask *bmp;
223
224
1.80k
  if (n < 1) {
225
0
    errno = EINVAL;
226
0
    numa_error("request to allocate mask for invalid number");
227
0
    return NULL;
228
0
  }
229
1.80k
  bmp = malloc(sizeof(*bmp));
230
1.80k
  if (bmp == 0)
231
0
    goto oom;
232
1.80k
  bmp->size = n;
233
1.80k
  bmp->maskp = calloc(longsperbits(n), sizeof(unsigned long));
234
1.80k
  if (bmp->maskp == 0) {
235
0
    free(bmp);
236
0
    goto oom;
237
0
  }
238
1.80k
  return bmp;
239
240
0
oom:
241
0
  numa_error("Out of memory allocating bitmask");
242
0
  exit(1);
243
1.80k
}
244
245
void
246
numa_bitmask_free(struct bitmask *bmp)
247
1.78k
{
248
1.78k
  if (bmp == 0)
249
0
    return;
250
1.78k
  free(bmp->maskp);
251
1.78k
  bmp->maskp = (unsigned long *)0xdeadcdef;  /* double free tripwire */
252
1.78k
  free(bmp);
253
1.78k
  return;
254
1.78k
}
255
256
/* True if two bitmasks are equal */
257
int
258
numa_bitmask_equal(const struct bitmask *bmp1, const struct bitmask *bmp2)
259
0
{
260
0
  unsigned int i;
261
0
  for (i = 0; i < bmp1->size || i < bmp2->size; i++)
262
0
    if (_getbit(bmp1, i) != _getbit(bmp2, i))
263
0
      return 0;
264
0
  return 1;
265
0
}
266
267
/* Hamming Weight: number of set bits */
268
unsigned int numa_bitmask_weight(const struct bitmask *bmp)
269
4
{
270
4
  unsigned int i;
271
4
  unsigned int w = 0;
272
2.18k
  for (i = 0; i < bmp->size; i++)
273
2.17k
    if (_getbit(bmp, i))
274
66
      w++;
275
4
  return w;
276
4
}
277
278
/* *****end of bitmask_  routines ************ */
279
280
/* Next two can be overwritten by the application for different error handling */
281
WEAK void numa_error(char *where)
282
0
{
283
0
  int olde = errno;
284
0
  perror(where);
285
0
  if (numa_exit_on_error)
286
0
    exit(1);
287
0
  errno = olde;
288
0
}
289
290
WEAK void numa_warn(int num, char *fmt, ...)
291
1.58k
{
292
1.58k
  static unsigned warned;
293
1.58k
  va_list ap;
294
1.58k
  int olde = errno;
295
296
  /* Give each warning only once */
297
1.58k
  if ((1<<num) & warned)
298
1.57k
    return;
299
9
  warned |= (1<<num);
300
301
9
  va_start(ap,fmt);
302
9
  fprintf(stderr, "libnuma: Warning: ");
303
9
  vfprintf(stderr, fmt, ap);
304
9
  fputc('\n', stderr);
305
9
  va_end(ap);
306
307
9
  if (numa_exit_on_warn)
308
0
    exit(1);
309
310
9
  errno = olde;
311
9
}
312
313
static void setpol(int policy, struct bitmask *bmp)
314
0
{
315
0
  if (set_mempolicy(policy, bmp->maskp, bmp->size) < 0)
316
0
    numa_error("set_mempolicy");
317
0
}
318
319
static void getpol(int *oldpolicy, struct bitmask *bmp)
320
0
{
321
0
  if (get_mempolicy(oldpolicy, bmp->maskp, bmp->size, 0, 0) < 0)
322
0
    numa_error("get_mempolicy");
323
0
}
324
325
static int dombind(void *mem, size_t size, int pol, struct bitmask *bmp)
326
0
{
327
0
  if (mbind(mem, size, pol, bmp ? bmp->maskp : NULL, bmp ? bmp->size : 0,
328
0
      mbind_flags) < 0) {
329
0
    numa_error("mbind");
330
0
    return -1;
331
0
  }
332
0
  return 0;
333
0
}
334
335
static void *dombind_or_free(void *mem, size_t size, int pol, struct bitmask *bmp)
336
0
{
337
0
  if (dombind(mem, size, pol, bmp) < 0 && numa_fail_alloc_on_error) {
338
0
    munmap(mem, size);
339
0
    return NULL;
340
0
  }
341
0
  return mem;
342
0
}
343
344
/* (undocumented) */
345
/* gives the wrong answer for hugetlbfs mappings. */
346
int numa_pagesize(void)
347
0
{
348
0
  static int pagesize;
349
0
  if (pagesize > 0)
350
0
    return pagesize;
351
0
  pagesize = getpagesize();
352
0
  return pagesize;
353
0
}
354
355
make_internal_alias(numa_pagesize);
356
357
/*
358
 * Find nodes (numa_nodes_ptr), nodes with memory (numa_memnode_ptr)
359
 * and the highest numbered existing node (maxconfigurednode).
360
 */
361
static void
362
set_configured_nodes(void)
363
2
{
364
2
  DIR *d;
365
2
  struct dirent *de;
366
367
2
  numa_memnode_ptr = numa_allocate_nodemask();
368
2
  numa_nodes_ptr = numa_allocate_nodemask();
369
2
  if (!numa_memnode_ptr || !numa_nodes_ptr)
370
0
    return;
371
372
2
  d = opendir("/sys/devices/system/node");
373
2
  if (!d) {
374
0
    maxconfigurednode = 0;
375
2
  } else {
376
24
    while ((de = readdir(d)) != NULL) {
377
22
      int nd;
378
22
      if (strncmp(de->d_name, "node", 4))
379
20
        continue;
380
2
      nd = strtoul(de->d_name+4, NULL, 0);
381
2
      numa_bitmask_setbit(numa_nodes_ptr, nd);
382
2
      numa_bitmask_setbit(numa_memnode_ptr, nd);
383
2
      if (maxconfigurednode < nd)
384
2
        maxconfigurednode = nd;
385
2
    }
386
2
    closedir(d);
387
2
  }
388
2
}
389
390
static inline int is_digit(char s)
391
574
{
392
574
  return (s >= '0' && s <= '9')
393
62
    || (s >= 'a' && s <= 'f')
394
62
    || (s >= 'A' && s <= 'F');
395
574
}
396
397
/* Is string 'pre' a prefix of string 's'? */
398
static int strprefix(const char *s, const char *pre)
399
112
{
400
112
  return strncmp(s, pre, strlen(pre)) == 0;
401
112
}
402
403
static const char *mask_size_file = "/proc/self/status";
404
static const char *nodemask_prefix = "Mems_allowed:\t";
405
/*
406
 * (do this the way Paul Jackson's libcpuset does it)
407
 * The nodemask values in /proc/self/status are in an
408
 * ascii format that uses 9 characters for each 32 bits of mask.
409
 * (this could also be used to find the cpumask size)
410
 */
411
static void
412
set_nodemask_size(void)
413
2
{
414
2
  FILE *fp;
415
2
  char *buf = NULL;
416
2
  char *tmp_buf = NULL;
417
2
  int digit_len = 0;
418
2
  size_t bufsize = 0;
419
420
2
  if ((fp = fopen(mask_size_file, "r")) == NULL)
421
0
    goto done;
422
423
114
  while (getline(&buf, &bufsize, fp) > 0) {
424
112
    if (strprefix(buf, nodemask_prefix)) {
425
2
      tmp_buf = buf;
426
2
      tmp_buf += strlen(nodemask_prefix);
427
576
      while (*tmp_buf != '\n' && *tmp_buf != '\0') {
428
574
        if (is_digit(*tmp_buf))
429
512
          digit_len++;
430
574
        tmp_buf++;
431
574
      }
432
2
      nodemask_sz = digit_len * 4;
433
2
    }
434
112
  }
435
2
  free(buf);
436
2
  fclose(fp);
437
2
done:
438
2
  if (nodemask_sz == 0) {/* fall back on error */
439
0
    int pol;
440
0
    unsigned long *mask = NULL, *origmask;
441
0
    nodemask_sz = 16;
442
0
    do {
443
0
      nodemask_sz <<= 1;
444
0
      origmask = mask;
445
0
      mask = realloc(mask, nodemask_sz / 8 + sizeof(unsigned long));
446
0
      if (!mask) {
447
0
        free(origmask);
448
0
        return;
449
0
      }
450
0
    } while (get_mempolicy(&pol, mask, nodemask_sz, 0, 0) < 0 && errno == EINVAL &&
451
0
        nodemask_sz < 4096*8);
452
0
    free(mask);
453
0
  }
454
2
}
455
456
/*
457
 * Read a mask consisting of a sequence of hexadecimal longs separated by
458
 * commas. Order them correctly and return the number of bits set.
459
 */
460
static int
461
read_mask(char *s, struct bitmask *bmp)
462
4
{
463
4
  char *end = s;
464
4
  int tmplen = (bmp->size + bitsperint - 1) / bitsperint;
465
4
  unsigned int tmp[tmplen];
466
4
  unsigned int *start = tmp;
467
4
  unsigned int i, n = 0, m = 0;
468
469
4
  if (!s)
470
0
    return 0; /* shouldn't happen */
471
472
4
  i = strtoul(s, &end, 16);
473
474
  /* Skip leading zeros */
475
66
  while (!i && *end++ == ',') {
476
62
    i = strtoul(end, &end, 16);
477
62
  }
478
479
4
  if (!i)
480
    /* End of string. No mask */
481
0
    return -1;
482
483
4
  if (n >= tmplen)
484
0
    return -1;
485
4
  start[n++] = i;
486
  /* Read sequence of ints */
487
4
  while (*end++ == ',') {
488
0
    i = strtoul(end, &end, 16);
489
490
    /* buffer overflow */
491
0
    if (n >= tmplen)
492
0
      return -1;
493
0
    start[n++] = i;
494
0
  }
495
496
  /*
497
   * Invert sequence of ints if necessary since the first int
498
   * is the highest and we put it first because we read it first.
499
   */
500
8
  while (n) {
501
4
    int w;
502
4
    unsigned long x = 0;
503
    /* read into long values in an endian-safe way */
504
8
    for (w = 0; n && w < bitsperlong; w += bitsperint)
505
4
      x |= ((unsigned long)start[n-- - 1] << w);
506
507
4
    bmp->maskp[m++] = x;
508
4
  }
509
  /*
510
   * Return the number of bits set
511
   */
512
4
  return numa_bitmask_weight(bmp);
513
4
}
514
515
/*
516
 * Read a processes constraints in terms of nodes and cpus from
517
 * /proc/self/status.
518
 */
519
static void
520
set_task_constraints(void)
521
2
{
522
2
  int hicpu = maxconfiguredcpu;
523
2
  int i;
524
2
  char *buffer = NULL;
525
2
  size_t buflen = 0;
526
2
  FILE *f;
527
528
2
  numa_all_cpus_ptr = numa_allocate_cpumask();
529
2
  numa_possible_cpus_ptr = numa_allocate_cpumask();
530
2
  numa_all_nodes_ptr = numa_allocate_nodemask();
531
2
  numa_possible_nodes_ptr = numa_allocate_nodemask();
532
2
  numa_no_nodes_ptr = numa_allocate_nodemask();
533
534
  // partial leak shouldn't happen because its transient
535
2
  if (!numa_all_cpus_ptr || !numa_possible_cpus_ptr ||
536
2
    !numa_all_nodes_ptr ||
537
2
    !numa_possible_nodes_ptr ||
538
2
    !numa_no_nodes_ptr)
539
0
    return;
540
541
2
  f = fopen(mask_size_file, "r");
542
2
  if (!f) {
543
    //numa_warn(W_cpumap, "Cannot parse %s", mask_size_file);
544
0
    return;
545
0
  }
546
547
114
  while (getline(&buffer, &buflen, f) > 0) {
548
    /* mask starts after [last] tab */
549
112
    char  *mask = strrchr(buffer,'\t') + 1;
550
551
112
    if (strncmp(buffer,"Cpus_allowed:",13) == 0)
552
2
      numproccpu = read_mask(mask, numa_all_cpus_ptr);
553
554
112
    if (strncmp(buffer,"Mems_allowed:",13) == 0) {
555
2
      numprocnode = read_mask(mask, numa_all_nodes_ptr);
556
2
    }
557
112
  }
558
2
  fclose(f);
559
2
  free(buffer);
560
561
66
  for (i = 0; i <= hicpu; i++)
562
64
    numa_bitmask_setbit(numa_possible_cpus_ptr, i);
563
4
  for (i = 0; i <= maxconfigurednode; i++)
564
2
    numa_bitmask_setbit(numa_possible_nodes_ptr, i);
565
566
  /*
567
   * Cpus_allowed in the kernel can be defined to all f's
568
   * i.e. it may be a superset of the actual available processors.
569
   * As such let's reduce numproccpu to the number of actual
570
   * available cpus.
571
   */
572
2
  if (numproccpu <= 0) {
573
0
    for (i = 0; i <= hicpu; i++)
574
0
      numa_bitmask_setbit(numa_all_cpus_ptr, i);
575
0
    numproccpu = hicpu+1;
576
0
  }
577
578
2
  if (numproccpu > hicpu+1) {
579
0
    numproccpu = hicpu+1;
580
0
    for (i=hicpu+1; i<numa_all_cpus_ptr->size; i++) {
581
0
      numa_bitmask_clearbit(numa_all_cpus_ptr, i);
582
0
    }
583
0
  }
584
585
2
  if (numprocnode <= 0) {
586
0
    for (i = 0; i <= maxconfigurednode; i++)
587
0
      numa_bitmask_setbit(numa_all_nodes_ptr, i);
588
0
    numprocnode = maxconfigurednode + 1;
589
0
  }
590
591
2
  return;
592
2
}
593
594
/*
595
 * Find the highest cpu number possible (in other words the size
596
 * of a kernel cpumask_t (in bits) - 1)
597
 */
598
static void
599
set_numa_max_cpu(void)
600
2
{
601
2
  int len = 4096;
602
2
  int n;
603
2
  int olde = errno;
604
2
  struct bitmask *buffer;
605
606
2
  do {
607
2
    buffer = numa_bitmask_alloc(len);
608
2
    if (!buffer)
609
0
      return;
610
2
    n = numa_sched_getaffinity_v2_int(0, buffer);
611
    /* on success, returns size of kernel cpumask_t, in bytes */
612
2
    if (n < 0) {
613
0
      if (errno == EINVAL) {
614
0
        if (len >= 1024*1024)
615
0
          break;
616
0
        len *= 2;
617
0
        numa_bitmask_free(buffer);
618
0
        continue;
619
0
      } else {
620
0
        numa_warn(W_numcpus, "Unable to determine max cpu"
621
0
            " (sched_getaffinity: %s); guessing...",
622
0
            strerror(errno));
623
0
        n = sizeof(cpu_set_t);
624
0
        break;
625
0
      }
626
0
    }
627
2
  } while (n < 0);
628
2
  numa_bitmask_free(buffer);
629
2
  errno = olde;
630
2
  cpumask_sz = n*8;
631
2
}
632
633
/*
634
 * get the total (configured) number of cpus - both online and offline
635
 */
636
static void
637
set_configured_cpus(void)
638
2
{
639
2
  maxconfiguredcpu = sysconf(_SC_NPROCESSORS_CONF) - 1;
640
2
  if (maxconfiguredcpu == -1)
641
0
    numa_error("sysconf(NPROCESSORS_CONF) failed");
642
2
}
643
644
static void
645
set_preferred_many(void)
646
0
{
647
0
  int oldp;
648
0
  struct bitmask *bmp, *tmp;
649
0
  int old_errno;
650
651
0
  if (has_preferred_many >= 0)
652
0
    return;
653
654
0
  old_errno = errno;
655
656
0
  has_preferred_many = 0;
657
658
0
  bmp = numa_allocate_nodemask();
659
0
  tmp = numa_get_mems_allowed();
660
0
  if (!tmp || !bmp)
661
0
    goto out;
662
663
0
  if (get_mempolicy(&oldp, bmp->maskp, bmp->size, 0, 0) < 0)
664
0
    goto out;
665
666
0
  if (set_mempolicy(MPOL_PREFERRED_MANY, tmp->maskp, tmp->size) == 0) {
667
0
    has_preferred_many = 1;
668
    /* reset the old memory policy ignoring error */
669
0
    (void)set_mempolicy(oldp, bmp->maskp, bmp->size+1);
670
0
  }
671
672
0
out:
673
0
  numa_bitmask_free(tmp);
674
0
  numa_bitmask_free(bmp);
675
0
  errno = old_errno;
676
0
}
677
678
/*
679
 * Initialize all the sizes.
680
 */
681
static void
682
set_sizes(void)
683
2
{
684
2
  sizes_set++;
685
2
  set_nodemask_size();  /* size of kernel nodemask_t */
686
2
  set_configured_nodes(); /* configured nodes listed in /sys */
687
2
  set_numa_max_cpu(); /* size of kernel cpumask_t */
688
2
  set_configured_cpus();  /* cpus listed in /sys/devices/system/cpu */
689
2
  set_task_constraints(); /* cpus and nodes for current task */
690
2
}
691
692
int
693
numa_num_configured_nodes(void)
694
894
{
695
  /*
696
  * NOTE: this function's behavior matches the documentation (ie: it
697
  * returns a count of nodes with memory) despite the poor function
698
  * naming.  We also cannot use the similarly poorly named
699
  * numa_all_nodes_ptr as it only tracks nodes with memory from which
700
  * the calling process can allocate.  Think sparse nodes, memory-less
701
  * nodes, cpusets...
702
  */
703
894
  int memnodecount=0, i;
704
705
1.78k
  for (i=0; i <= maxconfigurednode; i++) {
706
894
    if (numa_bitmask_isbitset(numa_memnode_ptr, i))
707
894
      memnodecount++;
708
894
  }
709
894
  return memnodecount;
710
894
}
711
712
int
713
numa_num_configured_cpus(void)
714
892
{
715
716
892
  return maxconfiguredcpu+1;
717
892
}
718
719
int
720
numa_num_possible_nodes(void)
721
903
{
722
903
  return nodemask_sz;
723
903
}
724
725
int
726
numa_num_possible_cpus(void)
727
897
{
728
897
  return cpumask_sz;
729
897
}
730
731
int
732
numa_num_task_nodes(void)
733
0
{
734
0
  return numprocnode;
735
0
}
736
737
/*
738
 * for backward compatibility
739
 */
740
int
741
numa_num_thread_nodes(void)
742
0
{
743
0
  return numa_num_task_nodes();
744
0
}
745
746
int
747
numa_num_task_cpus(void)
748
0
{
749
0
  return numproccpu;
750
0
}
751
752
/*
753
 * for backward compatibility
754
 */
755
int
756
numa_num_thread_cpus(void)
757
0
{
758
0
  return numa_num_task_cpus();
759
0
}
760
761
/*
762
 * Return the number of the highest node in this running system,
763
 */
764
int
765
numa_max_node(void)
766
131
{
767
131
  return maxconfigurednode;
768
131
}
769
770
make_internal_alias(numa_max_node);
771
772
/*
773
 * Return the number of the highest possible node in a system,
774
 * which for v1 is the size of a numa.h nodemask_t(in bits)-1.
775
 * but for v2 is the size of a kernel nodemask_t(in bits)-1.
776
 */
777
SYMVER("numa_max_possible_node_v1", "numa_max_possible_node@libnuma_1.1")
778
int
779
numa_max_possible_node_v1(void)
780
0
{
781
0
  return ((sizeof(nodemask_t)*8)-1);
782
0
}
783
784
SYMVER("numa_max_possible_node_v2", "numa_max_possible_node@@libnuma_1.2")
785
int
786
numa_max_possible_node_v2(void)
787
903
{
788
903
  return numa_num_possible_nodes()-1;
789
903
}
790
791
make_internal_alias(numa_max_possible_node_v1);
792
make_internal_alias(numa_max_possible_node_v2);
793
794
/*
795
 * Allocate a bitmask for cpus, of a size large enough to
796
 * match the kernel's cpumask_t.
797
 */
798
struct bitmask *
799
numa_allocate_cpumask()
800
897
{
801
897
  int ncpus = numa_num_possible_cpus();
802
803
897
  return numa_bitmask_alloc(ncpus);
804
897
}
805
806
/*
807
 * Allocate a bitmask the size of a libnuma nodemask_t
808
 */
809
static struct bitmask *
810
allocate_nodemask_v1(void)
811
0
{
812
0
  int nnodes = numa_max_possible_node_v1_int()+1;
813
814
0
  return numa_bitmask_alloc(nnodes);
815
0
}
816
817
/*
818
 * Allocate a bitmask for nodes, of a size large enough to
819
 * match the kernel's nodemask_t.
820
 */
821
struct bitmask *
822
numa_allocate_nodemask(void)
823
902
{
824
902
  struct bitmask *bmp;
825
902
  int nnodes = numa_max_possible_node_v2_int() + 1;
826
827
902
  bmp = numa_bitmask_alloc(nnodes);
828
902
  return bmp;
829
902
}
830
831
/* (cache the result?) */
832
long long numa_node_size64(int node, long long *freep)
833
0
{
834
0
  size_t len = 0;
835
0
  char *line = NULL;
836
0
  long long size = -1;
837
0
  FILE *f;
838
0
  char fn[64];
839
0
  int ok = 0;
840
0
  int required = freep ? 2 : 1;
841
842
0
  if (freep)
843
0
    *freep = 0;
844
0
  sprintf(fn,"/sys/devices/system/node/node%d/meminfo", node);
845
0
  f = fopen(fn, "r");
846
0
  if (!f)
847
0
    return -1;
848
0
  while (getdelim(&line, &len, '\n', f) > 0) {
849
0
    char *end;
850
0
    char *s = strcasestr(line, "kB");
851
0
    if (!s)
852
0
      continue;
853
0
    --s;
854
0
    while (s > line && isspace(*s))
855
0
      --s;
856
0
    while (s > line && isdigit(*s))
857
0
      --s;
858
0
    if (strstr(line, "MemTotal")) {
859
0
      size = strtoull(s,&end,0) << 10;
860
0
      if (end == s)
861
0
        size = -1;
862
0
      else
863
0
        ok++;
864
0
    }
865
0
    if (freep && strstr(line, "MemFree")) {
866
0
      *freep = strtoull(s,&end,0) << 10;
867
0
      if (end == s)
868
0
        *freep = -1;
869
0
      else
870
0
        ok++;
871
0
    }
872
0
  }
873
0
  fclose(f);
874
0
  free(line);
875
0
  if (ok != required)
876
0
    numa_warn(W_badmeminfo, "Cannot parse sysfs meminfo (%d)", ok);
877
0
  return size;
878
0
}
879
880
make_internal_alias(numa_node_size64);
881
882
long numa_node_size(int node, long *freep)
883
0
{
884
0
  long long f2 = 0;
885
0
  long sz = numa_node_size64_int(node, &f2);
886
0
  if (freep)
887
0
    *freep = f2;
888
0
  return sz;
889
0
}
890
891
int numa_available(void)
892
0
{
893
0
  if (get_mempolicy(NULL, NULL, 0, 0, 0) < 0 && (errno == ENOSYS || errno == EPERM))
894
0
    return -1;
895
0
  return 0;
896
0
}
897
898
SYMVER("numa_interleave_memory_v1", "numa_interleave_memory@libnuma_1.1")
899
void
900
numa_interleave_memory_v1(void *mem, size_t size, const nodemask_t *mask)
901
0
{
902
0
  struct bitmask bitmask;
903
904
0
  bitmask.size = sizeof(nodemask_t) * 8;
905
0
  bitmask.maskp = (unsigned long *)mask;
906
0
  dombind(mem, size, MPOL_INTERLEAVE, &bitmask);
907
0
}
908
909
SYMVER("numa_interleave_memory_v2", "numa_interleave_memory@@libnuma_1.2")
910
void
911
numa_interleave_memory_v2(void *mem, size_t size, struct bitmask *bmp)
912
0
{
913
0
  dombind(mem, size, MPOL_INTERLEAVE, bmp);
914
0
}
915
916
void
917
numa_weighted_interleave_memory(void *mem, size_t size, struct bitmask *bmp)
918
0
{
919
0
  dombind(mem, size, MPOL_WEIGHTED_INTERLEAVE, bmp);
920
0
}
921
922
void numa_tonode_memory(void *mem, size_t size, int node)
923
0
{
924
0
  struct bitmask *nodes;
925
926
0
  nodes = numa_allocate_nodemask();
927
0
  if (!nodes)
928
0
    return;
929
0
  numa_bitmask_setbit(nodes, node);
930
0
  dombind(mem, size, bind_policy, nodes);
931
0
  numa_bitmask_free(nodes);
932
0
}
933
934
SYMVER("numa_tonodemask_memory_v1", "numa_tonodemask_memory@libnuma_1.1")
935
void
936
numa_tonodemask_memory_v1(void *mem, size_t size, const nodemask_t *mask)
937
0
{
938
0
  struct bitmask bitmask;
939
940
0
  bitmask.maskp = (unsigned long *)mask;
941
0
  bitmask.size  = sizeof(nodemask_t);
942
0
  dombind(mem, size,  bind_policy, &bitmask);
943
0
}
944
945
SYMVER("numa_tonodemask_memory_v2", "numa_tonodemask_memory@@libnuma_1.2")
946
void
947
numa_tonodemask_memory_v2(void *mem, size_t size, struct bitmask *bmp)
948
0
{
949
0
  dombind(mem, size,  bind_policy, bmp);
950
0
}
951
952
void numa_setlocal_memory(void *mem, size_t size)
953
0
{
954
0
  dombind(mem, size, MPOL_LOCAL, NULL);
955
0
}
956
957
void numa_police_memory(void *mem, size_t size)
958
0
{
959
0
  int pagesize = numa_pagesize_int();
960
0
  unsigned long i;
961
0
  char *p = mem;
962
0
  for (i = 0; i < size; i += pagesize, p += pagesize)
963
0
    __atomic_and_fetch(p, 0xff, __ATOMIC_RELAXED);
964
965
0
}
966
967
make_internal_alias(numa_police_memory);
968
969
void *numa_alloc(size_t size)
970
0
{
971
0
  char *mem;
972
0
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
973
0
       0, 0);
974
0
  if (mem == (char *)-1)
975
0
    return NULL;
976
0
  numa_police_memory_int(mem, size);
977
0
  return mem;
978
0
}
979
980
void *numa_realloc(void *old_addr, size_t old_size, size_t new_size)
981
0
{
982
0
  char *mem;
983
0
  mem = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE);
984
0
  if (mem == (char *)-1)
985
0
    return NULL;
986
  /*
987
   *  The memory policy of the allocated pages is preserved by mremap(), so
988
   *  there is no need to (re)set it here. If the policy of the original
989
   *  allocation is not set, the new pages will be allocated according to the
990
   *  process' mempolicy. Trying to allocate explicitly the new pages on the
991
   *  same node as the original ones would require changing the policy of the
992
   *  newly allocated pages, which violates the numa_realloc() semantics.
993
   */
994
0
  return mem;
995
0
}
996
997
SYMVER("numa_alloc_interleaved_subset_v1", "numa_alloc_interleaved_subset@libnuma_1.1")
998
void *numa_alloc_interleaved_subset_v1(size_t size, const nodemask_t *mask)
999
0
{
1000
0
  char *mem;
1001
0
  struct bitmask bitmask;
1002
1003
0
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
1004
0
      0, 0);
1005
0
  if (mem == (char *)-1)
1006
0
    return NULL;
1007
0
  bitmask.maskp = (unsigned long *)mask;
1008
0
  bitmask.size  = sizeof(nodemask_t);
1009
0
  mem = dombind_or_free(mem, size, MPOL_INTERLEAVE, &bitmask);
1010
0
  return mem;
1011
0
}
1012
1013
SYMVER("numa_alloc_interleaved_subset_v2", "numa_alloc_interleaved_subset@@libnuma_1.2")
1014
void *numa_alloc_interleaved_subset_v2(size_t size, struct bitmask *bmp)
1015
0
{
1016
0
  char *mem;
1017
1018
0
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
1019
0
       0, 0);
1020
0
  if (mem == (char *)-1)
1021
0
    return NULL;
1022
0
  mem = dombind_or_free(mem, size, MPOL_INTERLEAVE, bmp);
1023
0
  return mem;
1024
0
}
1025
1026
make_internal_alias(numa_alloc_interleaved_subset_v1);
1027
make_internal_alias(numa_alloc_interleaved_subset_v2);
1028
1029
void *
1030
numa_alloc_interleaved(size_t size)
1031
0
{
1032
0
  return numa_alloc_interleaved_subset_v2_int(size, numa_all_nodes_ptr);
1033
0
}
1034
1035
void *
1036
numa_alloc_weighted_interleaved_subset(size_t size, struct bitmask *bmp)
1037
0
{
1038
0
  char *mem;
1039
1040
0
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
1041
0
       0, 0);
1042
0
  if (mem == (char *)-1)
1043
0
    return NULL;
1044
0
  mem = dombind_or_free(mem, size, MPOL_WEIGHTED_INTERLEAVE, bmp);
1045
0
  return mem;
1046
0
}
1047
1048
void *
1049
numa_alloc_weighted_interleaved(size_t size)
1050
0
{
1051
0
  return numa_alloc_weighted_interleaved_subset(size, numa_all_nodes_ptr);
1052
0
}
1053
1054
/*
1055
 * given a user node mask, set memory policy to use those nodes
1056
 */
1057
SYMVER("numa_set_interleave_mask_v1", "numa_set_interleave_mask@libnuma_1.1")
1058
void
1059
numa_set_interleave_mask_v1(nodemask_t *mask)
1060
0
{
1061
0
  struct bitmask *bmp;
1062
0
  int nnodes = numa_max_possible_node_v1_int()+1;
1063
1064
0
  bmp = numa_bitmask_alloc(nnodes);
1065
0
  copy_nodemask_to_bitmask(mask, bmp);
1066
0
  if (numa_bitmask_equal(bmp, numa_no_nodes_ptr))
1067
0
    setpol(MPOL_DEFAULT, bmp);
1068
0
  else
1069
0
    setpol(MPOL_INTERLEAVE, bmp);
1070
0
  numa_bitmask_free(bmp);
1071
0
}
1072
1073
1074
SYMVER("numa_set_interleave_mask_v2", "numa_set_interleave_mask@@libnuma_1.2")
1075
void
1076
numa_set_interleave_mask_v2(struct bitmask *bmp)
1077
0
{
1078
0
  if (numa_bitmask_equal(bmp, numa_no_nodes_ptr))
1079
0
    setpol(MPOL_DEFAULT, bmp);
1080
0
  else
1081
0
    setpol(MPOL_INTERLEAVE, bmp);
1082
0
}
1083
1084
void
1085
numa_set_weighted_interleave_mask(struct bitmask *bmp)
1086
0
{
1087
0
  if (numa_bitmask_equal(bmp, numa_no_nodes_ptr))
1088
0
    setpol(MPOL_DEFAULT, bmp);
1089
0
  else
1090
0
    setpol(MPOL_WEIGHTED_INTERLEAVE, bmp);
1091
0
}
1092
1093
SYMVER("numa_get_interleave_mask_v1", "numa_get_interleave_mask@libnuma_1.1")
1094
nodemask_t
1095
numa_get_interleave_mask_v1(void)
1096
0
{
1097
0
  int oldpolicy = 0;
1098
0
  struct bitmask *bmp;
1099
0
  nodemask_t mask;
1100
1101
0
  bmp = allocate_nodemask_v1();
1102
0
  if (!bmp)
1103
0
    return numa_no_nodes;
1104
0
  getpol(&oldpolicy, bmp);
1105
0
  if (oldpolicy == MPOL_INTERLEAVE)
1106
0
    copy_bitmask_to_nodemask(bmp, &mask);
1107
0
  else
1108
0
    copy_bitmask_to_nodemask(numa_no_nodes_ptr, &mask);
1109
0
  numa_bitmask_free(bmp);
1110
0
  return mask;
1111
0
}
1112
1113
SYMVER("numa_get_interleave_mask_v2", "numa_get_interleave_mask@@libnuma_1.2")
1114
struct bitmask *
1115
numa_get_interleave_mask_v2(void)
1116
0
{
1117
0
  int oldpolicy = 0;
1118
0
  struct bitmask *bmp;
1119
1120
0
  bmp = numa_allocate_nodemask();
1121
0
  if (!bmp)
1122
0
    return NULL;
1123
0
  getpol(&oldpolicy, bmp);
1124
0
  if (oldpolicy != MPOL_INTERLEAVE)
1125
0
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp);
1126
0
  return bmp;
1127
0
}
1128
1129
struct bitmask *
1130
numa_get_weighted_interleave_mask(void)
1131
0
{
1132
0
  int oldpolicy = 0;
1133
0
  struct bitmask *bmp;
1134
1135
0
  bmp = numa_allocate_nodemask();
1136
0
  if (!bmp)
1137
0
    return NULL;
1138
0
  getpol(&oldpolicy, bmp);
1139
0
  if (oldpolicy != MPOL_WEIGHTED_INTERLEAVE)
1140
0
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp);
1141
0
  return bmp;
1142
0
}
1143
1144
/* (undocumented) */
1145
int numa_get_interleave_node(void)
1146
0
{
1147
0
  int nd;
1148
0
  if (get_mempolicy(&nd, NULL, 0, 0, MPOL_F_NODE) == 0)
1149
0
    return nd;
1150
0
  return 0;
1151
0
}
1152
1153
void *numa_alloc_onnode(size_t size, int node)
1154
0
{
1155
0
  char *mem;
1156
0
  struct bitmask *bmp;
1157
1158
0
  bmp = numa_allocate_nodemask();
1159
0
  if (!bmp)
1160
0
    return NULL;
1161
0
  numa_bitmask_setbit(bmp, node);
1162
0
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
1163
0
       0, 0);
1164
0
  if (mem == (char *)-1)
1165
0
    mem = NULL;
1166
0
  else
1167
0
    mem = dombind_or_free(mem, size, bind_policy, bmp);
1168
1169
0
  numa_bitmask_free(bmp);
1170
0
  return mem;
1171
0
}
1172
1173
void *numa_alloc_local(size_t size)
1174
0
{
1175
0
  char *mem;
1176
0
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
1177
0
       0, 0);
1178
0
  if (mem == (char *)-1)
1179
0
    mem =  NULL;
1180
0
  else
1181
0
    mem = dombind_or_free(mem, size, MPOL_LOCAL, NULL);
1182
0
  return mem;
1183
0
}
1184
1185
void numa_set_bind_policy(int strict)
1186
0
{
1187
0
  set_preferred_many();
1188
0
  if (strict)
1189
0
    bind_policy = MPOL_BIND;
1190
0
  else if (has_preferred_many)
1191
0
    bind_policy = MPOL_PREFERRED_MANY;
1192
0
  else
1193
0
    bind_policy = MPOL_PREFERRED;
1194
0
}
1195
1196
SYMVER("numa_set_membind_v1", "numa_set_membind@libnuma_1.1")
1197
void
1198
numa_set_membind_v1(const nodemask_t *mask)
1199
0
{
1200
0
  struct bitmask bitmask;
1201
1202
0
  bitmask.maskp = (unsigned long *)mask;
1203
0
  bitmask.size  = sizeof(nodemask_t);
1204
0
  setpol(MPOL_BIND, &bitmask);
1205
0
}
1206
1207
SYMVER("numa_set_membind_v2", "numa_set_membind@@libnuma_1.2")
1208
void
1209
numa_set_membind_v2(struct bitmask *bmp)
1210
0
{
1211
0
  setpol(MPOL_BIND, bmp);
1212
0
}
1213
1214
make_internal_alias(numa_set_membind_v2);
1215
1216
void
1217
numa_set_membind_balancing(struct bitmask *bmp)
1218
0
{
1219
  /* MPOL_F_NUMA_BALANCING: ignore if unsupported */
1220
0
  if (set_mempolicy(MPOL_BIND | MPOL_F_NUMA_BALANCING,
1221
0
        bmp->maskp, bmp->size + 1) < 0) {
1222
0
    if (errno == EINVAL) {
1223
0
      errno = 0;
1224
0
      numa_set_membind_v2(bmp);
1225
0
    } else
1226
0
      numa_error("set_mempolicy");
1227
0
  }
1228
0
}
1229
1230
/*
1231
 * copy a bitmask map body to a numa.h nodemask_t structure
1232
 */
1233
void
1234
copy_bitmask_to_nodemask(struct bitmask *bmp, nodemask_t *nmp)
1235
0
{
1236
0
  int max, i;
1237
1238
0
  memset(nmp, 0, sizeof(nodemask_t));
1239
0
        max = (sizeof(nodemask_t)*8);
1240
0
  for (i=0; i<bmp->size; i++) {
1241
0
    if (i >= max)
1242
0
      break;
1243
0
    if (numa_bitmask_isbitset(bmp, i))
1244
0
      nodemask_set_compat((nodemask_t *)nmp, i);
1245
0
  }
1246
0
}
1247
1248
/*
1249
 * copy a bitmask map body to another bitmask body
1250
 * fill a larger destination with zeroes
1251
 */
1252
void
1253
copy_bitmask_to_bitmask(struct bitmask *bmpfrom, struct bitmask *bmpto)
1254
136
{
1255
136
  int bytes;
1256
1257
136
  if (bmpfrom->size >= bmpto->size) {
1258
136
    memcpy(bmpto->maskp, bmpfrom->maskp, CPU_BYTES(bmpto->size));
1259
136
  } else if (bmpfrom->size < bmpto->size) {
1260
0
    bytes = CPU_BYTES(bmpfrom->size);
1261
0
    memcpy(bmpto->maskp, bmpfrom->maskp, bytes);
1262
0
    memset(((char *)bmpto->maskp)+bytes, 0,
1263
0
          CPU_BYTES(bmpto->size)-bytes);
1264
0
  }
1265
136
}
1266
1267
/*
1268
 * copy a numa.h nodemask_t structure to a bitmask map body
1269
 */
1270
void
1271
copy_nodemask_to_bitmask(nodemask_t *nmp, struct bitmask *bmp)
1272
0
{
1273
0
  int max, i;
1274
1275
0
  numa_bitmask_clearall(bmp);
1276
0
        max = (sizeof(nodemask_t)*8);
1277
0
  if (max > bmp->size)
1278
0
    max = bmp->size;
1279
0
  for (i=0; i<max; i++) {
1280
0
    if (nodemask_isset_compat(nmp, i))
1281
0
      numa_bitmask_setbit(bmp, i);
1282
0
  }
1283
0
}
1284
1285
SYMVER("numa_get_membind_v1", "numa_get_membind@libnuma_1.1")
1286
nodemask_t
1287
numa_get_membind_v1(void)
1288
0
{
1289
0
  int oldpolicy = 0;
1290
0
  struct bitmask *bmp;
1291
0
  nodemask_t nmp;
1292
1293
0
  bmp = allocate_nodemask_v1();
1294
0
  if (!bmp)
1295
0
    return numa_no_nodes;
1296
0
  getpol(&oldpolicy, bmp);
1297
0
  if (oldpolicy == MPOL_BIND) {
1298
0
    copy_bitmask_to_nodemask(bmp, &nmp);
1299
0
  } else {
1300
    /* copy the body of the map to numa_all_nodes */
1301
0
    copy_bitmask_to_nodemask(bmp, &numa_all_nodes);
1302
0
    nmp = numa_all_nodes;
1303
0
  }
1304
0
  numa_bitmask_free(bmp);
1305
0
  return nmp;
1306
0
}
1307
1308
SYMVER("numa_get_membind_v2", "numa_get_membind@@libnuma_1.2")
1309
struct bitmask *
1310
numa_get_membind_v2(void)
1311
0
{
1312
0
  int oldpolicy = 0;
1313
0
  struct bitmask *bmp = NULL;
1314
1315
0
  bmp = numa_allocate_nodemask();
1316
0
  if (!bmp)
1317
0
    return NULL;
1318
0
  getpol(&oldpolicy, bmp);
1319
0
  if (oldpolicy != MPOL_BIND)
1320
0
    copy_bitmask_to_bitmask(numa_all_nodes_ptr, bmp);
1321
0
  return bmp;
1322
0
}
1323
1324
//TODO:  do we need a v1 nodemask_t version?
1325
struct bitmask *numa_get_mems_allowed(void)
1326
0
{
1327
0
  struct bitmask *bmp;
1328
1329
  /*
1330
   * can change, so query on each call.
1331
   */
1332
0
  bmp = numa_allocate_nodemask();
1333
0
  if (!bmp)
1334
0
    return NULL;
1335
0
  if (get_mempolicy(NULL, bmp->maskp, bmp->size, 0,
1336
0
        MPOL_F_MEMS_ALLOWED) < 0)
1337
0
    numa_error("get_mempolicy");
1338
0
  return bmp;
1339
0
}
1340
make_internal_alias(numa_get_mems_allowed);
1341
1342
void numa_free(void *mem, size_t size)
1343
0
{
1344
0
  munmap(mem, size);
1345
0
}
1346
1347
SYMVER("numa_parse_bitmap_v1", "numa_parse_bitmap@libnuma_1.1")
1348
int
1349
numa_parse_bitmap_v1(char *line, unsigned long *mask, int ncpus)
1350
0
{
1351
0
  int i;
1352
0
  char *p = strchr(line, '\n');
1353
0
  if (!p)
1354
0
    return -1;
1355
1356
0
  for (i = 0; p > line;i++) {
1357
0
    char *oldp, *endp;
1358
0
    oldp = p;
1359
0
    if (*p == ',')
1360
0
      --p;
1361
0
    while (p > line && *p != ',')
1362
0
      --p;
1363
    /* Eat two 32bit fields at a time to get longs */
1364
0
    if (p > line && sizeof(unsigned long) == 8) {
1365
0
      oldp--;
1366
0
      memmove(p, p+1, oldp-p+1);
1367
0
      while (p > line && *p != ',')
1368
0
        --p;
1369
0
    }
1370
0
    if (*p == ',')
1371
0
      p++;
1372
0
    if (i >= CPU_LONGS(ncpus))
1373
0
      return -1;
1374
0
    mask[i] = strtoul(p, &endp, 16);
1375
0
    if (endp != oldp)
1376
0
      return -1;
1377
0
    p--;
1378
0
  }
1379
0
  return 0;
1380
0
}
1381
1382
SYMVER("numa_parse_bitmap_v2", "numa_parse_bitmap@@libnuma_1.2")
1383
int
1384
numa_parse_bitmap_v2(char *line, struct bitmask *mask)
1385
1
{
1386
1
  int i, ncpus;
1387
1
  char *p = strchr(line, '\n');
1388
1
  if (!p)
1389
0
    return -1;
1390
1
  ncpus = mask->size;
1391
1392
2
  for (i = 0; p > line;i++) {
1393
1
    char *oldp, *endp;
1394
1
    oldp = p;
1395
1
    if (*p == ',')
1396
0
      --p;
1397
9
    while (p > line && *p != ',')
1398
8
      --p;
1399
    /* Eat two 32bit fields at a time to get longs */
1400
1
    if (p > line && sizeof(unsigned long) == 8) {
1401
0
      oldp--;
1402
0
      memmove(p, p+1, oldp-p+1);
1403
0
      while (p > line && *p != ',')
1404
0
        --p;
1405
0
    }
1406
1
    if (*p == ',')
1407
0
      p++;
1408
1
    if (i >= CPU_LONGS(ncpus))
1409
0
      return -1;
1410
1
    mask->maskp[i] = strtoul(p, &endp, 16);
1411
1
    if (endp != oldp)
1412
0
      return -1;
1413
1
    p--;
1414
1
  }
1415
1
  return 0;
1416
1
}
1417
1418
static void init_node_cpu_mask_v2(void)
1419
1
{
1420
1
  int nnodes = numa_max_possible_node_v2_int() + 1;
1421
1
  struct bitmask **new_ncm, **null_ncm = NULL;
1422
1
  new_ncm = calloc (nnodes, sizeof(struct bitmask *));
1423
  /* Check for races with another thread */
1424
1
  if (new_ncm && !__atomic_compare_exchange_n(&node_cpu_mask_v2, &null_ncm,
1425
1
      new_ncm, 0,
1426
1
      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) {
1427
0
    free(new_ncm);
1428
0
  }
1429
1
}
1430
1431
static void cleanup_node_cpu_mask_v2(void)
1432
0
{
1433
0
  if (node_cpu_mask_v2) {
1434
0
    int i;
1435
0
    int nnodes;
1436
0
    nnodes = numa_max_possible_node_v2_int() + 1;
1437
0
    for (i = 0; i < nnodes; i++) {
1438
0
      FREE_AND_ZERO(node_cpu_mask_v2[i]);
1439
0
    }
1440
0
    free(node_cpu_mask_v2);
1441
0
    node_cpu_mask_v2 = NULL;
1442
0
  }
1443
0
}
1444
1445
/* This would be better with some locking, but I don't want to make libnuma
1446
   dependent on pthreads right now. The races are relatively harmless. */
1447
SYMVER("numa_node_to_cpus_v1", "numa_node_to_cpus@libnuma_1.1")
1448
int
1449
numa_node_to_cpus_v1(int node, unsigned long *buffer, int bufferlen)
1450
0
{
1451
0
  int err = 0;
1452
0
  char fn[64];
1453
0
  FILE *f;
1454
0
  char update;
1455
0
  char *line = NULL;
1456
0
  size_t len = 0;
1457
0
  struct bitmask bitmask;
1458
0
  int buflen_needed;
1459
0
  unsigned long *mask;
1460
0
  int ncpus = numa_num_possible_cpus();
1461
0
  int maxnode = numa_max_node_int();
1462
1463
0
  buflen_needed = CPU_BYTES(ncpus);
1464
0
  if ((unsigned)node > maxnode || bufferlen < buflen_needed) {
1465
0
    errno = ERANGE;
1466
0
    return -1;
1467
0
  }
1468
0
  if (bufferlen > buflen_needed)
1469
0
    memset(buffer, 0, bufferlen);
1470
0
  update = __atomic_fetch_and(&node_cpu_mask_v1_stale, 0, __ATOMIC_RELAXED);
1471
0
  if (node_cpu_mask_v1[node] && !update) {
1472
0
    memcpy(buffer, node_cpu_mask_v1[node], buflen_needed);
1473
0
    return 0;
1474
0
  }
1475
1476
0
  mask = malloc(buflen_needed);
1477
0
  if (!mask)
1478
0
    mask = (unsigned long *)buffer;
1479
0
  memset(mask, 0, buflen_needed);
1480
1481
0
  sprintf(fn, "/sys/devices/system/node/node%d/cpumap", node);
1482
0
  f = fopen(fn, "r");
1483
0
  if (!f || getdelim(&line, &len, '\n', f) < 1) {
1484
0
    if (numa_bitmask_isbitset(numa_nodes_ptr, node)) {
1485
0
      numa_warn(W_nosysfs2,
1486
0
         "/sys not mounted or invalid. Assuming one node: %s",
1487
0
          strerror(errno));
1488
0
      numa_warn(W_nosysfs2,
1489
0
         "(cannot open or correctly parse %s)", fn);
1490
0
    }
1491
0
    bitmask.maskp = (unsigned long *)mask;
1492
0
    bitmask.size  = buflen_needed * 8;
1493
0
    numa_bitmask_setall(&bitmask);
1494
0
    err = -1;
1495
0
  }
1496
0
  if (f)
1497
0
    fclose(f);
1498
1499
0
  if (line && (numa_parse_bitmap_v1(line, mask, ncpus) < 0)) {
1500
0
    numa_warn(W_cpumap, "Cannot parse cpumap. Assuming one node");
1501
0
    bitmask.maskp = (unsigned long *)mask;
1502
0
    bitmask.size  = buflen_needed * 8;
1503
0
    numa_bitmask_setall(&bitmask);
1504
0
    err = -1;
1505
0
  }
1506
1507
0
  free(line);
1508
0
  memmove(buffer, mask, buflen_needed);
1509
1510
  /* slightly racy, see above */
1511
0
  if (node_cpu_mask_v1[node]) {
1512
0
    if (update) {
1513
      /*
1514
       * There may be readers on node_cpu_mask_v1[], hence it can not
1515
       * be freed.
1516
       */
1517
0
      memcpy(node_cpu_mask_v1[node], mask, buflen_needed);
1518
0
      free(mask);
1519
0
      mask = NULL;
1520
0
    } else if (mask != buffer)
1521
0
      free(mask);
1522
0
  } else {
1523
0
    node_cpu_mask_v1[node] = mask;
1524
0
  }
1525
0
  return err;
1526
0
}
1527
1528
/*
1529
 * test whether a node has cpus
1530
 */
1531
/* This would be better with some locking, but I don't want to make libnuma
1532
   dependent on pthreads right now. The races are relatively harmless. */
1533
/*
1534
 * deliver a bitmask of cpus representing the cpus on a given node
1535
 */
1536
SYMVER("numa_node_to_cpus_v2", "numa_node_to_cpus@@libnuma_1.2")
1537
int
1538
numa_node_to_cpus_v2(int node, struct bitmask *buffer)
1539
131
{
1540
131
  int err = 0;
1541
131
  int nnodes = numa_max_node();
1542
131
  char fn[64], *line = NULL;
1543
131
  FILE *f;
1544
131
  char update;
1545
131
  size_t len = 0;
1546
131
  struct bitmask *mask;
1547
1548
131
  if (!__atomic_load_n(&node_cpu_mask_v2, __ATOMIC_CONSUME))
1549
1
    init_node_cpu_mask_v2();
1550
1551
131
  if (node < 0 || node > nnodes) {
1552
0
    errno = ERANGE;
1553
0
    return -1;
1554
0
  }
1555
131
  numa_bitmask_clearall(buffer);
1556
1557
131
  update = __atomic_fetch_and(&node_cpu_mask_v2_stale, 0, __ATOMIC_RELAXED);
1558
131
  if (node_cpu_mask_v2[node] && !update) {
1559
    /* have already constructed a mask for this node */
1560
130
    if (buffer->size < node_cpu_mask_v2[node]->size) {
1561
0
      errno = EINVAL;
1562
0
      numa_error("map size mismatch");
1563
0
      return -1;
1564
0
    }
1565
130
    copy_bitmask_to_bitmask(node_cpu_mask_v2[node], buffer);
1566
130
    return 0;
1567
130
  }
1568
1569
  /* need a new mask for this node */
1570
1
  mask = numa_allocate_cpumask();
1571
1
  if (!mask)
1572
0
    return -1;
1573
1574
  /* this is a kernel cpumask_t (see node_read_cpumap()) */
1575
1
  sprintf(fn, "/sys/devices/system/node/node%d/cpumap", node);
1576
1
  f = fopen(fn, "r");
1577
1
  if (!f || getdelim(&line, &len, '\n', f) < 1) {
1578
0
    if (numa_bitmask_isbitset(numa_nodes_ptr, node)) {
1579
0
      numa_warn(W_nosysfs2,
1580
0
         "/sys not mounted or invalid. Assuming one node: %s",
1581
0
          strerror(errno));
1582
0
      numa_warn(W_nosysfs2,
1583
0
         "(cannot open or correctly parse %s)", fn);
1584
0
    }
1585
0
    numa_bitmask_setall(mask);
1586
0
    err = -1;
1587
0
  }
1588
1
  if (f)
1589
1
    fclose(f);
1590
1591
1
  if (line && (numa_parse_bitmap_v2(line, mask) < 0)) {
1592
0
    numa_warn(W_cpumap, "Cannot parse cpumap. Assuming one node");
1593
0
    numa_bitmask_setall(mask);
1594
0
    err = -1;
1595
0
  }
1596
1597
1
  free(line);
1598
1
  copy_bitmask_to_bitmask(mask, buffer);
1599
1600
  /* slightly racy, see above */
1601
  /* save the mask we created */
1602
1
  if (node_cpu_mask_v2[node]) {
1603
0
    if (update) {
1604
0
      copy_bitmask_to_bitmask(mask, node_cpu_mask_v2[node]);
1605
0
      numa_bitmask_free(mask);
1606
0
      mask = NULL;
1607
    /* how could this be? */
1608
0
    } else if (mask != buffer)
1609
0
      numa_bitmask_free(mask);
1610
1
  } else {
1611
    /* we don't want to cache faulty result */
1612
1
    if (!err)
1613
1
      node_cpu_mask_v2[node] = mask;
1614
0
    else
1615
0
      numa_bitmask_free(mask);
1616
1
  }
1617
1
  return err;
1618
1
}
1619
1620
make_internal_alias(numa_node_to_cpus_v1);
1621
make_internal_alias(numa_node_to_cpus_v2);
1622
1623
void numa_node_to_cpu_update(void)
1624
0
{
1625
0
  __atomic_store_n(&node_cpu_mask_v1_stale, 1, __ATOMIC_RELAXED);
1626
0
  __atomic_store_n(&node_cpu_mask_v2_stale, 1, __ATOMIC_RELAXED);
1627
0
}
1628
1629
/* report the node of the specified cpu */
1630
int numa_node_of_cpu(int cpu)
1631
0
{
1632
0
  struct bitmask *bmp;
1633
0
  int ncpus, nnodes, node, ret;
1634
1635
0
  ncpus = numa_num_possible_cpus();
1636
0
  if (cpu > ncpus){
1637
0
    errno = EINVAL;
1638
0
    return -1;
1639
0
  }
1640
0
  bmp = numa_bitmask_alloc(ncpus);
1641
0
  nnodes = numa_max_node();
1642
0
  for (node = 0; node <= nnodes; node++){
1643
0
    if (numa_node_to_cpus_v2_int(node, bmp) < 0) {
1644
      /* It's possible for the node to not exist */
1645
0
      continue;
1646
0
    }
1647
0
    if (numa_bitmask_isbitset(bmp, cpu)){
1648
0
      ret = node;
1649
0
      goto end;
1650
0
    }
1651
0
  }
1652
0
  ret = -1;
1653
0
  errno = EINVAL;
1654
0
end:
1655
0
  numa_bitmask_free(bmp);
1656
0
  return ret;
1657
0
}
1658
1659
SYMVER("numa_run_on_node_mask_v1", "numa_run_on_node_mask@libnuma_1.1")
1660
int
1661
numa_run_on_node_mask_v1(const nodemask_t *mask)
1662
0
{
1663
0
  int ncpus = numa_num_possible_cpus();
1664
0
  int i, k, err;
1665
0
  unsigned long cpus[CPU_LONGS(ncpus)], nodecpus[CPU_LONGS(ncpus)];
1666
0
  memset(cpus, 0, CPU_BYTES(ncpus));
1667
0
  for (i = 0; i < NUMA_NUM_NODES; i++) {
1668
0
    if (mask->n[i / BITS_PER_LONG] == 0)
1669
0
      continue;
1670
0
    if (nodemask_isset_compat(mask, i)) {
1671
0
      if (numa_node_to_cpus_v1_int(i, nodecpus, CPU_BYTES(ncpus)) < 0) {
1672
0
        numa_warn(W_noderunmask,
1673
0
            "Cannot read node cpumask from sysfs");
1674
0
        continue;
1675
0
      }
1676
0
      for (k = 0; k < CPU_LONGS(ncpus); k++)
1677
0
        cpus[k] |= nodecpus[k];
1678
0
    }
1679
0
  }
1680
0
  err = numa_sched_setaffinity_v1(0, CPU_BYTES(ncpus), cpus);
1681
1682
  /* The sched_setaffinity API is broken because it expects
1683
     the user to guess the kernel cpuset size. Do this in a
1684
     brute force way. */
1685
0
  if (err < 0 && errno == EINVAL) {
1686
0
    int savederrno = errno;
1687
0
    char *bigbuf;
1688
0
    static int size = -1;
1689
0
    if (size == -1)
1690
0
      size = CPU_BYTES(ncpus) * 2;
1691
0
    bigbuf = malloc(CPU_BUFFER_SIZE);
1692
0
    if (!bigbuf) {
1693
0
      errno = ENOMEM;
1694
0
      return -1;
1695
0
    }
1696
0
    errno = savederrno;
1697
0
    while (size <= CPU_BUFFER_SIZE) {
1698
0
      memcpy(bigbuf, cpus, CPU_BYTES(ncpus));
1699
0
      memset(bigbuf + CPU_BYTES(ncpus), 0,
1700
0
             CPU_BUFFER_SIZE - CPU_BYTES(ncpus));
1701
0
      err = numa_sched_setaffinity_v1_int(0, size, (unsigned long *)bigbuf);
1702
0
      if (err == 0 || errno != EINVAL)
1703
0
        break;
1704
0
      size *= 2;
1705
0
    }
1706
0
    savederrno = errno;
1707
0
    free(bigbuf);
1708
0
    errno = savederrno;
1709
0
  }
1710
0
  return err;
1711
0
}
1712
1713
/*
1714
 * Given a node mask (size of a kernel nodemask_t) (probably populated by
1715
 * a user argument list) set up a map of cpus (map "cpus") on those nodes.
1716
 * Then set affinity to those cpus.
1717
 */
1718
SYMVER("numa_run_on_node_mask_v2", "numa_run_on_node_mask@@libnuma_1.2")
1719
int
1720
numa_run_on_node_mask_v2(struct bitmask *bmp)
1721
0
{
1722
0
  int ncpus, i, k, err;
1723
0
  struct bitmask *cpus, *nodecpus;
1724
1725
0
  cpus = numa_allocate_cpumask();
1726
0
  ncpus = cpus->size;
1727
0
  nodecpus = numa_allocate_cpumask();
1728
0
  if (!cpus || !nodecpus)
1729
0
    return -1;
1730
1731
0
  for (i = 0; i < bmp->size; i++) {
1732
0
    if (bmp->maskp[i / BITS_PER_LONG] == 0)
1733
0
      continue;
1734
0
    if (numa_bitmask_isbitset(bmp, i)) {
1735
      /*
1736
       * numa_all_nodes_ptr is cpuset aware; use only
1737
       * these nodes
1738
       */
1739
0
      if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i)) {
1740
0
        numa_warn(W_noderunmask,
1741
0
          "node %d not allowed", i);
1742
0
        continue;
1743
0
      }
1744
0
      if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
1745
0
        numa_warn(W_noderunmask,
1746
0
          "Cannot read node cpumask from sysfs");
1747
0
        continue;
1748
0
      }
1749
0
      for (k = 0; k < CPU_LONGS(ncpus); k++)
1750
0
        cpus->maskp[k] |= nodecpus->maskp[k];
1751
0
    }
1752
0
  }
1753
0
  err = numa_sched_setaffinity_v2_int(0, cpus);
1754
1755
0
  numa_bitmask_free(cpus);
1756
0
  numa_bitmask_free(nodecpus);
1757
1758
  /* used to have to consider that this could fail - it shouldn't now */
1759
0
  if (err < 0) {
1760
0
    numa_error("numa_sched_setaffinity_v2_int() failed");
1761
0
  }
1762
1763
0
  return err;
1764
0
}
1765
1766
make_internal_alias(numa_run_on_node_mask_v2);
1767
1768
/*
1769
 * Given a node mask (size of a kernel nodemask_t) (probably populated by
1770
 * a user argument list) set up a map of cpus (map "cpus") on those nodes
1771
 * without any cpuset awareness. Then set affinity to those cpus.
1772
 */
1773
int
1774
numa_run_on_node_mask_all(struct bitmask *bmp)
1775
0
{
1776
0
  int ncpus, i, k, err;
1777
0
  struct bitmask *cpus, *nodecpus;
1778
1779
0
  cpus = numa_allocate_cpumask();
1780
0
  ncpus = cpus->size;
1781
0
  nodecpus = numa_allocate_cpumask();
1782
0
  if (!cpus || !nodecpus)
1783
0
    return -1;
1784
1785
0
  for (i = 0; i < bmp->size; i++) {
1786
0
    if (bmp->maskp[i / BITS_PER_LONG] == 0)
1787
0
      continue;
1788
0
    if (numa_bitmask_isbitset(bmp, i)) {
1789
0
      if (!numa_bitmask_isbitset(numa_possible_nodes_ptr, i)) {
1790
0
        numa_warn(W_noderunmask,
1791
0
          "node %d not allowed", i);
1792
0
        continue;
1793
0
      }
1794
0
      if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
1795
0
        numa_warn(W_noderunmask,
1796
0
          "Cannot read node cpumask from sysfs");
1797
0
        continue;
1798
0
      }
1799
0
      for (k = 0; k < CPU_LONGS(ncpus); k++)
1800
0
        cpus->maskp[k] |= nodecpus->maskp[k];
1801
0
    }
1802
0
  }
1803
0
  err = numa_sched_setaffinity_v2_int(0, cpus);
1804
1805
0
  numa_bitmask_free(cpus);
1806
0
  numa_bitmask_free(nodecpus);
1807
1808
  /* With possible nodes freedom it can happen easily now */
1809
0
  if (err < 0) {
1810
0
    numa_error("numa_sched_setaffinity_v2_int() failed");
1811
0
  }
1812
1813
0
  return err;
1814
0
}
1815
1816
SYMVER("numa_get_run_node_mask_v1", "numa_get_run_node_mask@libnuma_1.1")
1817
nodemask_t
1818
numa_get_run_node_mask_v1(void)
1819
0
{
1820
0
  int ncpus = numa_num_configured_cpus();
1821
0
  int i, k;
1822
0
  int max = numa_max_node_int();
1823
0
  struct bitmask *bmp, *cpus, *nodecpus;
1824
0
  nodemask_t nmp;
1825
1826
0
  cpus = numa_allocate_cpumask();
1827
0
  if (!cpus)
1828
0
    return numa_no_nodes;
1829
0
  if (numa_sched_getaffinity_v2_int(0, cpus) < 0){
1830
0
    nmp = numa_no_nodes;
1831
0
    goto free_cpus;
1832
0
  }
1833
1834
0
  nodecpus = numa_allocate_cpumask();
1835
0
  if (!nodecpus) {
1836
0
    nmp = numa_no_nodes;
1837
0
    goto free_cpus;
1838
0
  }
1839
1840
0
  bmp = allocate_nodemask_v1(); /* the size of a nodemask_t */
1841
0
  if (!bmp) {
1842
0
    nmp = numa_no_nodes;
1843
0
    goto free_cpus2;
1844
0
  }
1845
1846
0
  for (i = 0; i <= max; i++) {
1847
0
    if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
1848
      /* It's possible for the node to not exist */
1849
0
      continue;
1850
0
    }
1851
0
    for (k = 0; k < CPU_LONGS(ncpus); k++) {
1852
0
      if (nodecpus->maskp[k] & cpus->maskp[k])
1853
0
        numa_bitmask_setbit(bmp, i);
1854
0
    }
1855
0
  }
1856
0
  copy_bitmask_to_nodemask(bmp, &nmp);
1857
0
  numa_bitmask_free(bmp);
1858
0
free_cpus2:
1859
0
  numa_bitmask_free(nodecpus);
1860
0
free_cpus:
1861
0
  numa_bitmask_free(cpus);
1862
0
  return nmp;
1863
0
}
1864
1865
SYMVER("numa_get_run_node_mask_v2", "numa_get_run_node_mask@@libnuma_1.2")
1866
struct bitmask *
1867
numa_get_run_node_mask_v2(void)
1868
0
{
1869
0
  int i, k;
1870
0
  int ncpus = numa_num_configured_cpus();
1871
0
  int max = numa_max_node_int();
1872
0
  struct bitmask *bmp, *cpus, *nodecpus;
1873
1874
0
  bmp = numa_allocate_cpumask();
1875
0
  cpus = numa_allocate_cpumask();
1876
0
  if (!bmp || !cpus)
1877
0
    return NULL;
1878
0
  if (numa_sched_getaffinity_v2_int(0, cpus) < 0){
1879
0
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp);
1880
0
    goto free_cpus;
1881
0
  }
1882
1883
0
  nodecpus = numa_allocate_cpumask();
1884
0
  for (i = 0; i <= max; i++) {
1885
    /*
1886
     * numa_all_nodes_ptr is cpuset aware; show only
1887
     * these nodes
1888
     */
1889
0
    if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i)) {
1890
0
      continue;
1891
0
    }
1892
0
    if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
1893
      /* It's possible for the node to not exist */
1894
0
      continue;
1895
0
    }
1896
0
    for (k = 0; k < CPU_LONGS(ncpus); k++) {
1897
0
      if (nodecpus->maskp[k] & cpus->maskp[k])
1898
0
        numa_bitmask_setbit(bmp, i);
1899
0
    }
1900
0
  }
1901
0
  numa_bitmask_free(nodecpus);
1902
0
free_cpus:
1903
0
  numa_bitmask_free(cpus);
1904
0
  return bmp;
1905
0
}
1906
1907
int
1908
numa_migrate_pages(int pid, struct bitmask *fromnodes, struct bitmask *tonodes)
1909
0
{
1910
0
  int numa_num_nodes = numa_num_possible_nodes();
1911
1912
0
  return migrate_pages(pid, numa_num_nodes + 1, fromnodes->maskp,
1913
0
              tonodes->maskp);
1914
0
}
1915
1916
int numa_move_pages(int pid, unsigned long count,
1917
  void **pages, const int *nodes, int *status, int flags)
1918
0
{
1919
0
  return move_pages(pid, count, pages, nodes, status, flags);
1920
0
}
1921
1922
int numa_run_on_node(int node)
1923
0
{
1924
0
  int numa_num_nodes = numa_num_possible_nodes();
1925
0
  int ret = -1;
1926
0
  struct bitmask *cpus;
1927
1928
0
  if (node < -1 || node >= numa_num_nodes){
1929
0
    errno = EINVAL;
1930
0
    goto out;
1931
0
  }
1932
1933
0
  cpus = numa_allocate_cpumask();
1934
0
  if (!cpus)
1935
0
    return -1;
1936
1937
0
  if (node == -1)
1938
0
    numa_bitmask_setall(cpus);
1939
0
  else if (numa_node_to_cpus_v2_int(node, cpus) < 0){
1940
0
    numa_warn(W_noderunmask, "Cannot read node cpumask from sysfs");
1941
0
    goto free;
1942
0
  }
1943
1944
0
  ret = numa_sched_setaffinity_v2_int(0, cpus);
1945
0
free:
1946
0
  numa_bitmask_free(cpus);
1947
0
out:
1948
0
  return ret;
1949
0
}
1950
1951
static struct bitmask *__numa_preferred(void)
1952
0
{
1953
0
  int policy = 0;
1954
0
  struct bitmask *bmp;
1955
1956
0
  bmp = numa_allocate_nodemask();
1957
0
  if (!bmp)
1958
0
    return NULL;
1959
  /* could read the current CPU from /proc/self/status. Probably
1960
     not worth it. */
1961
0
  numa_bitmask_clearall(bmp);
1962
0
  getpol(&policy, bmp);
1963
1964
0
  if (policy != MPOL_PREFERRED &&
1965
0
      policy != MPOL_PREFERRED_MANY &&
1966
0
      policy != MPOL_BIND)
1967
0
    return bmp;
1968
1969
0
  if (policy == MPOL_PREFERRED && numa_bitmask_weight(bmp) > 1) {
1970
0
    errno = EINVAL;
1971
0
    numa_error(__FILE__);
1972
0
  }
1973
1974
0
  return bmp;
1975
0
}
1976
1977
int numa_preferred_err(void)
1978
0
{
1979
0
  int first_node = 0;
1980
0
  struct bitmask *bmp;
1981
1982
0
  bmp = __numa_preferred();
1983
0
  first_node = numa_find_first(bmp);
1984
0
  numa_bitmask_free(bmp);
1985
1986
0
  return first_node;
1987
0
}
1988
1989
int numa_preferred(void)
1990
0
{
1991
0
  int first_node = 0;
1992
1993
0
  first_node = numa_preferred_err();
1994
0
  first_node = first_node >= 0 ? first_node : 0;
1995
1996
0
  return first_node;
1997
0
}
1998
1999
static void __numa_set_preferred(struct bitmask *bmp)
2000
0
{
2001
0
  int nodes = numa_bitmask_weight(bmp);
2002
0
  if (nodes > 1) {
2003
0
    errno = EINVAL;
2004
0
    numa_error(__FILE__);
2005
0
  }
2006
2007
0
  setpol(nodes ? MPOL_PREFERRED : MPOL_LOCAL, bmp);
2008
0
}
2009
2010
void numa_set_preferred(int node)
2011
0
{
2012
0
  struct bitmask *bmp = numa_allocate_nodemask();
2013
0
  if (!bmp)
2014
0
    return;
2015
0
  numa_bitmask_setbit(bmp, node);
2016
0
  __numa_set_preferred(bmp);
2017
0
  numa_bitmask_free(bmp);
2018
0
}
2019
2020
int numa_has_preferred_many(void)
2021
0
{
2022
0
  set_preferred_many();
2023
0
  return has_preferred_many;
2024
0
}
2025
2026
void numa_set_preferred_many(struct bitmask *bitmask)
2027
0
{
2028
0
  int first_node = 0;
2029
2030
0
  set_preferred_many();
2031
0
  if (!has_preferred_many) {
2032
0
    numa_warn(W_nodeparse,
2033
0
      "Unable to handle MANY preferred nodes. Falling back to first node\n");
2034
0
    first_node = numa_find_first(bitmask);
2035
0
    numa_set_preferred(first_node);
2036
0
    return;
2037
0
  }
2038
0
  setpol(MPOL_PREFERRED_MANY, bitmask);
2039
0
}
2040
2041
struct bitmask *numa_preferred_many()
2042
0
{
2043
0
  return __numa_preferred();
2044
0
}
2045
2046
void numa_set_localalloc(void)
2047
0
{
2048
0
  setpol(MPOL_LOCAL, numa_no_nodes_ptr);
2049
0
}
2050
2051
SYMVER("numa_bind_v1", "numa_bind@libnuma_1.1")
2052
void numa_bind_v1(const nodemask_t *nodemask)
2053
0
{
2054
0
  struct bitmask bitmask;
2055
2056
0
  bitmask.maskp = (unsigned long *)nodemask;
2057
0
  bitmask.size  = sizeof(nodemask_t);
2058
0
  numa_run_on_node_mask_v2_int(&bitmask);
2059
0
  numa_set_membind_v2_int(&bitmask);
2060
0
}
2061
2062
SYMVER("numa_bind_v2", "numa_bind@@libnuma_1.2")
2063
void numa_bind_v2(struct bitmask *bmp)
2064
0
{
2065
0
  numa_run_on_node_mask_v2_int(bmp);
2066
0
  numa_set_membind_v2_int(bmp);
2067
0
}
2068
2069
void numa_set_strict(int flag)
2070
0
{
2071
0
  if (flag)
2072
0
    mbind_flags |= MPOL_MF_STRICT;
2073
0
  else
2074
0
    mbind_flags &= ~MPOL_MF_STRICT;
2075
0
}
2076
2077
/*
2078
 * Extract a node or processor number from the given string.
2079
 * Allow a relative node / processor specification within the allowed
2080
 * set if "relative" is nonzero
2081
 */
2082
static unsigned long get_nr(const char *s, char **end, struct bitmask *bmp, int relative)
2083
10.9k
{
2084
10.9k
  long i, nr;
2085
2086
10.9k
  if (!relative)
2087
6.53k
    return strtoul(s, end, 0);
2088
2089
4.39k
  nr = strtoul(s, end, 0);
2090
4.39k
  if (s == *end)
2091
5
    return nr;
2092
  /* Find the nth set bit */
2093
124k
  for (i = 0; nr >= 0 && i <= bmp->size; i++)
2094
120k
    if (numa_bitmask_isbitset(bmp, i))
2095
6.12k
      nr--;
2096
4.39k
  return i-1;
2097
4.39k
}
2098
2099
/*
2100
 * __numa_parse_nodestring() is called to create a node mask, given
2101
 * an ascii string such as 25 or 12-15 or 1,3,5-7 or +6-10.
2102
 * (the + indicates that the numbers are nodeset-relative)
2103
 *
2104
 * The nodes may be specified as absolute, or relative to the current nodeset.
2105
 * The list of available nodes is in a map pointed to by "allowed_nodes_ptr",
2106
 * which may represent all nodes or the nodes in the current nodeset.
2107
 *
2108
 * The caller must free the returned bitmask.
2109
 */
2110
static struct bitmask *
2111
__numa_parse_nodestring(const char *s, struct bitmask *allowed_nodes_ptr)
2112
892
{
2113
892
  int invert = 0, relative = 0;
2114
892
  int conf_nodes = numa_num_configured_nodes();
2115
892
  char *end;
2116
892
  struct bitmask *mask;
2117
2118
892
  mask = numa_allocate_nodemask();
2119
892
  if (!mask)
2120
0
    return NULL;
2121
2122
892
  if (s[0] == 0){
2123
1
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, mask);
2124
1
    return mask; /* return freeable mask */
2125
1
  }
2126
891
  if (*s == '!') {
2127
3
    invert = 1;
2128
3
    s++;
2129
3
  }
2130
891
  if (*s == '+') {
2131
302
    relative++;
2132
302
    s++;
2133
302
  }
2134
3.14k
  do {
2135
3.14k
    unsigned long arg;
2136
3.14k
    int i;
2137
3.14k
    if (isalpha(*s)) {
2138
140
      int n;
2139
140
      if (!strcmp(s,"all")) {
2140
2
        copy_bitmask_to_bitmask(allowed_nodes_ptr,
2141
2
              mask);
2142
2
        s+=4;
2143
2
        break;
2144
2
      }
2145
138
      n = resolve_affinity(s, mask);
2146
138
      if (n != NO_IO_AFFINITY) {
2147
62
        if (n < 0)
2148
62
          goto err;
2149
0
        s += strlen(s) + 1;
2150
0
        break;
2151
62
      }
2152
138
    }
2153
3.07k
    arg = get_nr(s, &end, allowed_nodes_ptr, relative);
2154
3.07k
    if (end == s) {
2155
111
      numa_warn(W_nodeparse, "unparseable node description `%s'\n", s);
2156
111
      goto err;
2157
111
    }
2158
2.96k
    if (arg >= allowed_nodes_ptr->size ||
2159
2.67k
        !numa_bitmask_isbitset(allowed_nodes_ptr, arg)) {
2160
363
      numa_warn(W_nodeparse, "node argument %ld is out of range\n", arg);
2161
363
      goto err;
2162
363
    }
2163
2.60k
    i = arg;
2164
2.60k
    numa_bitmask_setbit(mask, i);
2165
2.60k
    s = end;
2166
2.60k
    if (*s == '-') {
2167
1.31k
      char *end2;
2168
1.31k
      unsigned long arg2;
2169
1.31k
      arg2 = get_nr(++s, &end2, allowed_nodes_ptr, relative);
2170
1.31k
      if (end2 == s) {
2171
9
        numa_warn(W_nodeparse, "missing node argument %s\n", s);
2172
9
        goto err;
2173
9
      }
2174
1.30k
      if (arg2 >= allowed_nodes_ptr->size ||
2175
1.04k
          !numa_bitmask_isbitset(allowed_nodes_ptr, arg2)) {
2176
298
        numa_warn(W_nodeparse, "node argument %ld out of range\n", arg2);
2177
298
        goto err;
2178
298
      }
2179
2.01k
      while (arg <= arg2) {
2180
1.00k
        i = arg;
2181
1.00k
        if (numa_bitmask_isbitset(allowed_nodes_ptr,i))
2182
1.00k
          numa_bitmask_setbit(mask, i);
2183
1.00k
        arg++;
2184
1.00k
      }
2185
1.00k
      s = end2;
2186
1.00k
    }
2187
2.60k
  } while (*s++ == ',');
2188
48
  if (s[-1] != '\0')
2189
13
    goto err;
2190
35
  if (invert) {
2191
2
    int i;
2192
4
    for (i = 0; i < conf_nodes; i++) {
2193
2
      if (numa_bitmask_isbitset(mask, i))
2194
2
        numa_bitmask_clearbit(mask, i);
2195
0
      else
2196
0
        numa_bitmask_setbit(mask, i);
2197
2
    }
2198
2
  }
2199
35
  return mask;
2200
2201
856
err:
2202
856
  numa_bitmask_free(mask);
2203
856
  return NULL;
2204
48
}
2205
2206
/*
2207
 * numa_parse_nodestring() is called to create a bitmask from nodes available
2208
 * for this task.
2209
 */
2210
2211
struct bitmask * numa_parse_nodestring(const char *s)
2212
892
{
2213
892
  return __numa_parse_nodestring(s, numa_all_nodes_ptr);
2214
892
}
2215
2216
/*
2217
 * numa_parse_nodestring_all() is called to create a bitmask from all nodes
2218
 * available.
2219
 */
2220
2221
struct bitmask * numa_parse_nodestring_all(const char *s)
2222
0
{
2223
0
  return __numa_parse_nodestring(s, numa_possible_nodes_ptr);
2224
0
}
2225
2226
/*
2227
 * __numa_parse_cpustring() is called to create a bitmask, given
2228
 * an ascii string such as 25 or 12-15 or 1,3,5-7 or +6-10.
2229
 * (the + indicates that the numbers are cpuset-relative)
2230
 *
2231
 * The cpus may be specified as absolute, or relative to the current cpuset.
2232
 * The list of available cpus for this task is in the map pointed to by
2233
 * "allowed_cpus_ptr", which may represent all cpus or the cpus in the
2234
 * current cpuset.
2235
 *
2236
 * The caller must free the returned bitmask.
2237
 */
2238
static struct bitmask *
2239
__numa_parse_cpustring(const char *s, struct bitmask *allowed_cpus_ptr)
2240
892
{
2241
892
  int invert = 0, relative=0;
2242
892
  int conf_cpus = numa_num_configured_cpus();
2243
892
  char *end;
2244
892
  struct bitmask *mask;
2245
892
  int i;
2246
2247
892
  mask = numa_allocate_cpumask();
2248
892
  if (!mask)
2249
0
    return NULL;
2250
2251
892
  if (s[0] == 0)
2252
1
    return mask;
2253
891
  if (*s == '!') {
2254
3
    invert = 1;
2255
3
    s++;
2256
3
  }
2257
891
  if (*s == '+') {
2258
302
    relative++;
2259
302
    s++;
2260
302
  }
2261
4.40k
  do {
2262
4.40k
    unsigned long arg;
2263
2264
4.40k
    if (!strcmp(s,"all")) {
2265
2
      copy_bitmask_to_bitmask(allowed_cpus_ptr, mask);
2266
2
      s+=4;
2267
2
      break;
2268
2
    }
2269
4.40k
    arg = get_nr(s, &end, allowed_cpus_ptr, relative);
2270
4.40k
    if (end == s) {
2271
184
      numa_warn(W_cpuparse, "unparseable cpu description `%s'\n", s);
2272
184
      goto err;
2273
184
    }
2274
4.21k
    if (arg >= conf_cpus) {
2275
271
      numa_warn(W_cpuparse, "cpu argument %s is out of range\n", s);
2276
271
      goto err;
2277
271
    }
2278
3.94k
    i = arg;
2279
3.94k
    numa_bitmask_setbit(mask, i);
2280
3.94k
    s = end;
2281
3.94k
    if (*s == '-') {
2282
2.14k
      char *end2;
2283
2.14k
      unsigned long arg2;
2284
2.14k
      arg2 = get_nr(++s, &end2, allowed_cpus_ptr, relative);
2285
2.14k
      if (end2 == s) {
2286
16
        numa_warn(W_cpuparse, "missing cpu argument %s\n", s);
2287
16
        goto err;
2288
16
      }
2289
2.12k
      if (arg2 >= conf_cpus) {
2290
270
        numa_warn(W_cpuparse, "cpu argument %s out of range\n", s);
2291
270
        goto err;
2292
270
      }
2293
4.08k
      while (arg <= arg2) {
2294
2.22k
        i = arg;
2295
2.22k
        if (numa_bitmask_isbitset(allowed_cpus_ptr, i))
2296
2.22k
          numa_bitmask_setbit(mask, i);
2297
2.22k
        arg++;
2298
2.22k
      }
2299
1.85k
      s = end2;
2300
1.85k
    }
2301
3.94k
  } while (*s++ == ',');
2302
150
  if (s[-1] != '\0')
2303
20
    goto err;
2304
130
  if (invert) {
2305
66
    for (i = 0; i < conf_cpus; i++) {
2306
64
      if (numa_bitmask_isbitset(mask, i))
2307
33
        numa_bitmask_clearbit(mask, i);
2308
31
      else
2309
31
        numa_bitmask_setbit(mask, i);
2310
64
    }
2311
2
  }
2312
130
  return mask;
2313
2314
761
err:
2315
761
  numa_bitmask_free(mask);
2316
761
  return NULL;
2317
150
}
2318
2319
/*
2320
 * numa_parse_cpustring() is called to create a bitmask from cpus available
2321
 * for this task.
2322
 */
2323
2324
struct bitmask * numa_parse_cpustring(const char *s)
2325
892
{
2326
892
  return __numa_parse_cpustring(s, numa_all_cpus_ptr);
2327
892
}
2328
2329
/*
2330
 * numa_parse_cpustring_all() is called to create a bitmask from all cpus
2331
 * available.
2332
 */
2333
2334
struct bitmask * numa_parse_cpustring_all(const char *s)
2335
0
{
2336
0
  return __numa_parse_cpustring(s, numa_possible_cpus_ptr);
2337
0
}
2338
2339
int numa_has_home_node(void)
2340
0
{
2341
0
  void *mem;
2342
0
  static int has_home_node = -1;
2343
0
  int page_size = numa_pagesize();
2344
0
  struct bitmask *tmp = numa_get_mems_allowed();
2345
2346
0
  if (has_home_node >= 0)
2347
0
    goto out;
2348
2349
0
  has_home_node = 0;
2350
  /* Detect whether home_node is supported */
2351
0
  mem = mmap(0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
2352
0
  if (mem != MAP_FAILED) {
2353
0
    dombind(mem, page_size, MPOL_BIND, tmp);
2354
0
    if (set_mempolicy_home_node(mem, page_size, numa_find_first(tmp), 0) == 0)
2355
0
      has_home_node = 1;
2356
0
    munmap(mem, page_size);
2357
0
  }
2358
2359
0
out:
2360
0
  numa_bitmask_free(tmp);
2361
0
  return has_home_node;
2362
0
}
2363
2364
int numa_set_mempolicy_home_node(void *start, unsigned long len, int home_node, int flags)
2365
0
{
2366
0
  if (set_mempolicy_home_node(start, len, home_node, flags)) {
2367
0
    numa_error("set_mempolicy_home_node");
2368
0
    return -1;
2369
0
  }
2370
2371
0
  return 0;
2372
0
}
2373
2374
 /* return 1 if the node is a cpuless node, 0 when it's not, and -1 on any error */
2375
int numa_is_cpuless_node(int node_id)
2376
0
{
2377
0
  struct bitmask *cpumask;
2378
0
  int ret = 0;
2379
2380
0
  if (node_id < 0 || node_id > numa_max_node()) {
2381
0
    errno = ERANGE;
2382
0
    return -1;
2383
0
  }
2384
2385
0
  cpumask = numa_allocate_cpumask();
2386
0
  if (!cpumask)
2387
0
    return -1;
2388
2389
0
  numa_node_to_cpus(node_id, cpumask);
2390
0
  if (!numa_bitmask_weight(cpumask))
2391
0
    ret = 1;
2392
2393
0
  numa_bitmask_free(cpumask);
2394
0
  return ret;
2395
0
}