/src/numactl/libnuma.c

Source (jump to first uncovered line)
/* Simple NUMA library.
   Copyright (C) 2003,2004,2005,2008 Andi Kleen,SuSE Labs and
   Cliff Wickman,SGI.

   libnuma is free software; you can redistribute it and/or
   modify it under the terms of the GNU Lesser General Public
   License as published by the Free Software Foundation; version
   2.1.

   libnuma is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   Lesser General Public License for more details.

   You should find a copy of v2.1 of the GNU Lesser General Public License
   somewhere on your Linux system; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA

   All calls are undefined when numa_available returns an error. */
#define _GNU_SOURCE 1
#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <sched.h>
#include <dirent.h>
#include <errno.h>
#include <stdarg.h>
#include <ctype.h>
#include <assert.h>

#include <sys/mman.h>
#include <limits.h>

#include "config.h"
#include "numa.h"
#include "numaif.h"
#include "numaint.h"
#include "util.h"
#include "affinity.h"

#define WEAK __attribute__((weak))

#define CPU_BUFFER_SIZE 4096     /* This limits you to 32768 CPUs */

/* these are the old (version 1) masks */
nodemask_t numa_no_nodes;
nodemask_t numa_all_nodes;
/* these are now the default bitmask (pointers to) (version 2) */
struct bitmask *numa_no_nodes_ptr = NULL;
struct bitmask *numa_all_nodes_ptr = NULL;
struct bitmask *numa_possible_nodes_ptr = NULL;
struct bitmask *numa_all_cpus_ptr = NULL;
struct bitmask *numa_possible_cpus_ptr = NULL;
/* I would prefer to use symbol versioning to create v1 and v2 versions
   of numa_no_nodes and numa_all_nodes, but the loader does not correctly
   handle versioning of BSS versus small data items */

struct bitmask *numa_nodes_ptr = NULL;
static struct bitmask *numa_memnode_ptr = NULL;
static unsigned long *node_cpu_mask_v1[NUMA_NUM_NODES];
static char node_cpu_mask_v1_stale = 1;
static struct bitmask **node_cpu_mask_v2;
static char node_cpu_mask_v2_stale = 1;

WEAK void numa_error(char *where);

#ifndef TLS
#warning "not threadsafe"
#define __thread
#endif

static __thread int bind_policy = MPOL_BIND;
static __thread unsigned int mbind_flags = 0;
static int sizes_set=0;
static int maxconfigurednode = -1;
static int maxconfiguredcpu = -1;
static int numprocnode = -1;
static int numproccpu = -1;
static int nodemask_sz = 0;
static int cpumask_sz = 0;

static int has_preferred_many = -1;

int numa_exit_on_error = 0;
int numa_exit_on_warn = 0;
int numa_fail_alloc_on_error = 0;
static void set_sizes(void);

/*
 * There are two special functions, _init(void) and _fini(void), which
 * are called automatically by the dynamic loader whenever a library is loaded.
 *
 * The v1 library depends upon nodemask_t's of all nodes and no nodes.
 */
void __attribute__((constructor))
numa_init(void)
{
  int max,i;

  if (sizes_set)
    return;

  set_sizes();
  /* numa_all_nodes should represent existing nodes on this system */
        max = numa_num_configured_nodes();
        for (i = 0; i < max; i++)
                nodemask_set_compat((nodemask_t *)&numa_all_nodes, i);
  memset(&numa_no_nodes, 0, sizeof(numa_no_nodes));

  /* clear errno */
  errno = 0;
}

static void cleanup_node_cpu_mask_v2(void);

#define FREE_AND_ZERO(x) if (x) { \
    numa_bitmask_free(x); \
    x = NULL;   \
  }

void __attribute__((destructor))
numa_fini(void)
{
  FREE_AND_ZERO(numa_all_cpus_ptr);
  FREE_AND_ZERO(numa_possible_cpus_ptr);
  FREE_AND_ZERO(numa_all_nodes_ptr);
  FREE_AND_ZERO(numa_possible_nodes_ptr);
  FREE_AND_ZERO(numa_no_nodes_ptr);
  FREE_AND_ZERO(numa_memnode_ptr);
  FREE_AND_ZERO(numa_nodes_ptr);
  cleanup_node_cpu_mask_v2();
}

static int numa_find_first(struct bitmask *mask)
{
  int i;
  for (i = 0; i < mask->size; i++)
    if (numa_bitmask_isbitset(mask, i))
      return i;
  return -1;
}

/*
 * The following bitmask declarations, bitmask_*() routines, and associated
 * _setbit() and _getbit() routines are:
 * Copyright (c) 2004_2007 Silicon Graphics, Inc. (SGI) All rights reserved.
 * SGI publishes it under the terms of the Library GNU General Public License,
 * v2, as published by the Free Software Foundation.
 */
static unsigned int
_getbit(const struct bitmask *bmp, unsigned int n)
{
  if (n < bmp->size)
    return (bmp->maskp[n/bitsperlong] >> (n % bitsperlong)) & 1;
  else
    return 0;
}

static void
_setbit(struct bitmask *bmp, unsigned int n, unsigned int v)
{
  if (n < bmp->size) {
    if (v)
      bmp->maskp[n/bitsperlong] |= 1UL << (n % bitsperlong);
    else
      bmp->maskp[n/bitsperlong] &= ~(1UL << (n % bitsperlong));
  }
}

int
numa_bitmask_isbitset(const struct bitmask *bmp, unsigned int i)
{
  return _getbit(bmp, i);
}

struct bitmask *
numa_bitmask_setall(struct bitmask *bmp)
{
  unsigned int i;
  for (i = 0; i < bmp->size; i++)
    _setbit(bmp, i, 1);
  return bmp;
}

struct bitmask *
numa_bitmask_clearall(struct bitmask *bmp)
{
  unsigned int i;
  for (i = 0; i < bmp->size; i++)
    _setbit(bmp, i, 0);
  return bmp;
}

struct bitmask *
numa_bitmask_setbit(struct bitmask *bmp, unsigned int i)
{
  _setbit(bmp, i, 1);
  return bmp;
}

struct bitmask *
numa_bitmask_clearbit(struct bitmask *bmp, unsigned int i)
{
  _setbit(bmp, i, 0);
  return bmp;
}

unsigned int
numa_bitmask_nbytes(struct bitmask *bmp)
{
  return longsperbits(bmp->size) * sizeof(unsigned long);
}

/* where n is the number of bits in the map */
/* This function should not exit on failure, but right now we cannot really
   recover from this. */
struct bitmask *
numa_bitmask_alloc(unsigned int n)
{
  struct bitmask *bmp;

  if (n < 1) {
    errno = EINVAL;
    numa_error("request to allocate mask for invalid number");
    return NULL;
  }
  bmp = malloc(sizeof(*bmp));
  if (bmp == 0)
    goto oom;
  bmp->size = n;
  bmp->maskp = calloc(longsperbits(n), sizeof(unsigned long));
  if (bmp->maskp == 0) {
    free(bmp);
    goto oom;
  }
  return bmp;

oom:
  numa_error("Out of memory allocating bitmask");
  exit(1);
}

void
numa_bitmask_free(struct bitmask *bmp)
{
  if (bmp == 0)
    return;
  free(bmp->maskp);
  bmp->maskp = (unsigned long *)0xdeadcdef;  /* double free tripwire */
  free(bmp);
  return;
}

/* True if two bitmasks are equal */
int
numa_bitmask_equal(const struct bitmask *bmp1, const struct bitmask *bmp2)
{
  unsigned int i;
  for (i = 0; i < bmp1->size || i < bmp2->size; i++)
    if (_getbit(bmp1, i) != _getbit(bmp2, i))
      return 0;
  return 1;
}

/* Hamming Weight: number of set bits */
unsigned int numa_bitmask_weight(const struct bitmask *bmp)
{
  unsigned int i;
  unsigned int w = 0;
  for (i = 0; i < bmp->size; i++)
    if (_getbit(bmp, i))
      w++;
  return w;
}

/* *****end of bitmask_  routines ************ */

/* Next two can be overwritten by the application for different error handling */
WEAK void numa_error(char *where)
{
  int olde = errno;
  perror(where);
  if (numa_exit_on_error)
    exit(1);
  errno = olde;
}

WEAK void numa_warn(int num, char *fmt, ...)
{
  static unsigned warned;
  va_list ap;
  int olde = errno;

  /* Give each warning only once */
  if ((1<<num) & warned)
    return;
  warned |= (1<<num);

  va_start(ap,fmt);
  fprintf(stderr, "libnuma: Warning: ");
  vfprintf(stderr, fmt, ap);
  fputc('\n', stderr);
  va_end(ap);

  if (numa_exit_on_warn)
    exit(1);

  errno = olde;
}

static void setpol(int policy, struct bitmask *bmp)
{
  if (set_mempolicy(policy, bmp->maskp, bmp->size + 1) < 0)
    numa_error("set_mempolicy");
}

static void getpol(int *oldpolicy, struct bitmask *bmp)
{
  if (get_mempolicy(oldpolicy, bmp->maskp, bmp->size + 1, 0, 0) < 0)
    numa_error("get_mempolicy");
}

static int dombind(void *mem, size_t size, int pol, struct bitmask *bmp)
{
  if (mbind(mem, size, pol, bmp ? bmp->maskp : NULL, bmp ? bmp->size + 1 : 0,
      mbind_flags) < 0) {
    numa_error("mbind");
    return -1;
  }
  return 0;
}

static void *dombind_or_free(void *mem, size_t size, int pol, struct bitmask *bmp)
{
  if (dombind(mem, size, pol, bmp) < 0 && numa_fail_alloc_on_error) {
    munmap(mem, size);
    return NULL;
  }
  return mem;
}

/* (undocumented) */
/* gives the wrong answer for hugetlbfs mappings. */
int numa_pagesize(void)
{
  static int pagesize;
  if (pagesize > 0)
    return pagesize;
  pagesize = getpagesize();
  return pagesize;
}

make_internal_alias(numa_pagesize);

/*
 * Find nodes (numa_nodes_ptr), nodes with memory (numa_memnode_ptr)
 * and the highest numbered existing node (maxconfigurednode).
 */
static void
set_configured_nodes(void)
{
  DIR *d;
  struct dirent *de;

  numa_memnode_ptr = numa_allocate_nodemask();
  numa_nodes_ptr = numa_allocate_nodemask();
  if (!numa_memnode_ptr || !numa_nodes_ptr)
    return;

  d = opendir("/sys/devices/system/node");
  if (!d) {
    maxconfigurednode = 0;
  } else {
    while ((de = readdir(d)) != NULL) {
      int nd;
      if (strncmp(de->d_name, "node", 4))
        continue;
      nd = strtoul(de->d_name+4, NULL, 0);
      numa_bitmask_setbit(numa_nodes_ptr, nd);
      numa_bitmask_setbit(numa_memnode_ptr, nd);
      if (maxconfigurednode < nd)
        maxconfigurednode = nd;
    }
    closedir(d);
  }
}

static inline int is_digit(char s)
{
  return (s >= '0' && s <= '9')
    || (s >= 'a' && s <= 'f')
    || (s >= 'A' && s <= 'F');
}

/* Is string 'pre' a prefix of string 's'? */
static int strprefix(const char *s, const char *pre)
{
  return strncmp(s, pre, strlen(pre)) == 0;
}

static const char *mask_size_file = "/proc/self/status";
static const char *nodemask_prefix = "Mems_allowed:\t";
/*
 * (do this the way Paul Jackson's libcpuset does it)
 * The nodemask values in /proc/self/status are in an
 * ascii format that uses 9 characters for each 32 bits of mask.
 * (this could also be used to find the cpumask size)
 */
static void
set_nodemask_size(void)
{
  FILE *fp;
  char *buf = NULL;
  char *tmp_buf = NULL;
  int digit_len = 0;
  size_t bufsize = 0;

  if ((fp = fopen(mask_size_file, "r")) == NULL)
    goto done;

  while (getline(&buf, &bufsize, fp) > 0) {
    if (strprefix(buf, nodemask_prefix)) {
      tmp_buf = buf;
      tmp_buf += strlen(nodemask_prefix);
      while (*tmp_buf != '\n' && *tmp_buf != '\0') {
        if (is_digit(*tmp_buf))
          digit_len++;
        tmp_buf++;
      }
      nodemask_sz = digit_len * 4;
    }
  }
  free(buf);
  fclose(fp);
done:
  if (nodemask_sz == 0) {/* fall back on error */
    int pol;
    unsigned long *mask = NULL, *origmask;
    nodemask_sz = 16;
    do {
      nodemask_sz <<= 1;
      origmask = mask;
      mask = realloc(mask, nodemask_sz / 8 + sizeof(unsigned long));
      if (!mask) {
        free(origmask);
        return;
      }
    } while (get_mempolicy(&pol, mask, nodemask_sz + 1, 0, 0) < 0 && errno == EINVAL &&
        nodemask_sz < 4096*8);
    free(mask);
  }
}

/*
 * Read a mask consisting of a sequence of hexadecimal longs separated by
 * commas. Order them correctly and return the number of bits set.
 */
static int
read_mask(char *s, struct bitmask *bmp)
{
  char *end = s;
  int tmplen = (bmp->size + bitsperint - 1) / bitsperint;
  unsigned int tmp[tmplen];
  unsigned int *start = tmp;
  unsigned int i, n = 0, m = 0;

  if (!s)
    return 0; /* shouldn't happen */

  i = strtoul(s, &end, 16);

  /* Skip leading zeros */
  while (!i && *end++ == ',') {
    i = strtoul(end, &end, 16);
  }

  if (!i)
    /* End of string. No mask */
    return -1;

  start[n++] = i;
  /* Read sequence of ints */
  while (*end++ == ',') {
    i = strtoul(end, &end, 16);
    start[n++] = i;

    /* buffer overflow */
    if (n > tmplen)
      return -1;
  }

  /*
   * Invert sequence of ints if necessary since the first int
   * is the highest and we put it first because we read it first.
   */
  while (n) {
    int w;
    unsigned long x = 0;
    /* read into long values in an endian-safe way */
    for (w = 0; n && w < bitsperlong; w += bitsperint)
      x |= ((unsigned long)start[n-- - 1] << w);

    bmp->maskp[m++] = x;
  }
  /*
   * Return the number of bits set
   */
  return numa_bitmask_weight(bmp);
}

/*
 * Read a processes constraints in terms of nodes and cpus from
 * /proc/self/status.
 */
static void
set_task_constraints(void)
{
  int hicpu = maxconfiguredcpu;
  int i;
  char *buffer = NULL;
  size_t buflen = 0;
  FILE *f;

  numa_all_cpus_ptr = numa_allocate_cpumask();
  numa_possible_cpus_ptr = numa_allocate_cpumask();
  numa_all_nodes_ptr = numa_allocate_nodemask();
  numa_possible_nodes_ptr = numa_allocate_cpumask();
  numa_no_nodes_ptr = numa_allocate_nodemask();

  // partial leak shouldn't happen because its transient
  if (!numa_all_cpus_ptr || !numa_possible_cpus_ptr ||
    !numa_all_nodes_ptr ||
    !numa_possible_nodes_ptr ||
    !numa_no_nodes_ptr)
    return;

  f = fopen(mask_size_file, "r");
  if (!f) {
    //numa_warn(W_cpumap, "Cannot parse %s", mask_size_file);
    return;
  }

  while (getline(&buffer, &buflen, f) > 0) {
    /* mask starts after [last] tab */
    char  *mask = strrchr(buffer,'\t') + 1;

    if (strncmp(buffer,"Cpus_allowed:",13) == 0)
      numproccpu = read_mask(mask, numa_all_cpus_ptr);

    if (strncmp(buffer,"Mems_allowed:",13) == 0) {
      numprocnode = read_mask(mask, numa_all_nodes_ptr);
    }
  }
  fclose(f);
  free(buffer);

  for (i = 0; i <= hicpu; i++)
    numa_bitmask_setbit(numa_possible_cpus_ptr, i);
  for (i = 0; i <= maxconfigurednode; i++)
    numa_bitmask_setbit(numa_possible_nodes_ptr, i);

  /*
   * Cpus_allowed in the kernel can be defined to all f's
   * i.e. it may be a superset of the actual available processors.
   * As such let's reduce numproccpu to the number of actual
   * available cpus.
   */
  if (numproccpu <= 0) {
    for (i = 0; i <= hicpu; i++)
      numa_bitmask_setbit(numa_all_cpus_ptr, i);
    numproccpu = hicpu+1;
  }

  if (numproccpu > hicpu+1) {
    numproccpu = hicpu+1;
    for (i=hicpu+1; i<numa_all_cpus_ptr->size; i++) {
      numa_bitmask_clearbit(numa_all_cpus_ptr, i);
    }
  }

  if (numprocnode <= 0) {
    for (i = 0; i <= maxconfigurednode; i++)
      numa_bitmask_setbit(numa_all_nodes_ptr, i);
    numprocnode = maxconfigurednode + 1;
  }

  return;
}

/*
 * Find the highest cpu number possible (in other words the size
 * of a kernel cpumask_t (in bits) - 1)
 */
static void
set_numa_max_cpu(void)
{
  int len = 4096;
  int n;
  int olde = errno;
  struct bitmask *buffer;

  do {
    buffer = numa_bitmask_alloc(len);
    if (!buffer)
      return;
    n = numa_sched_getaffinity_v2_int(0, buffer);
    /* on success, returns size of kernel cpumask_t, in bytes */
    if (n < 0) {
      if (errno == EINVAL) {
        if (len >= 1024*1024)
          break;
        len *= 2;
        numa_bitmask_free(buffer);
        continue;
      } else {
        numa_warn(W_numcpus, "Unable to determine max cpu"
            " (sched_getaffinity: %s); guessing...",
            strerror(errno));
        n = sizeof(cpu_set_t);
        break;
      }
    }
  } while (n < 0);
  numa_bitmask_free(buffer);
  errno = olde;
  cpumask_sz = n*8;
}

/*
 * get the total (configured) number of cpus - both online and offline
 */
static void
set_configured_cpus(void)
{
  maxconfiguredcpu = sysconf(_SC_NPROCESSORS_CONF) - 1;
  if (maxconfiguredcpu == -1)
    numa_error("sysconf(NPROCESSORS_CONF) failed");
}

static void
set_preferred_many(void)
{
  int oldp;
  struct bitmask *bmp, *tmp;
  int old_errno;

  if (has_preferred_many >= 0)
    return;

  old_errno = errno;

  has_preferred_many = 0;

  bmp = numa_allocate_nodemask();
  tmp = numa_get_mems_allowed();
  if (!tmp || !bmp)
    goto out;

  if (get_mempolicy(&oldp, bmp->maskp, bmp->size + 1, 0, 0) < 0)
    goto out;

  if (set_mempolicy(MPOL_PREFERRED_MANY, tmp->maskp, tmp->size) == 0) {
    has_preferred_many = 1;
    /* reset the old memory policy ignoring error */
    (void)set_mempolicy(oldp, bmp->maskp, bmp->size+1);
  }

out:
  numa_bitmask_free(tmp);
  numa_bitmask_free(bmp);
  errno = old_errno;
}

/*
 * Initialize all the sizes.
 */
static void
set_sizes(void)
{
  sizes_set++;
  set_nodemask_size();  /* size of kernel nodemask_t */
  set_configured_nodes(); /* configured nodes listed in /sys */
  set_numa_max_cpu(); /* size of kernel cpumask_t */
  set_configured_cpus();  /* cpus listed in /sys/devices/system/cpu */
  set_task_constraints(); /* cpus and nodes for current task */
}

int
numa_num_configured_nodes(void)
{
  /*
  * NOTE: this function's behavior matches the documentation (ie: it
  * returns a count of nodes with memory) despite the poor function
  * naming.  We also cannot use the similarly poorly named
  * numa_all_nodes_ptr as it only tracks nodes with memory from which
  * the calling process can allocate.  Think sparse nodes, memory-less
  * nodes, cpusets...
  */
  int memnodecount=0, i;

  for (i=0; i <= maxconfigurednode; i++) {
    if (numa_bitmask_isbitset(numa_memnode_ptr, i))
      memnodecount++;
  }
  return memnodecount;
}

int
numa_num_configured_cpus(void)
{

  return maxconfiguredcpu+1;
}

int
numa_num_possible_nodes(void)
{
  return nodemask_sz;
}

int
numa_num_possible_cpus(void)
{
  return cpumask_sz;
}

int
numa_num_task_nodes(void)
{
  return numprocnode;
}

/*
 * for backward compatibility
 */
int
numa_num_thread_nodes(void)
{
  return numa_num_task_nodes();
}

int
numa_num_task_cpus(void)
{
  return numproccpu;
}

/*
 * for backward compatibility
 */
int
numa_num_thread_cpus(void)
{
  return numa_num_task_cpus();
}

/*
 * Return the number of the highest node in this running system,
 */
int
numa_max_node(void)
{
  return maxconfigurednode;
}

make_internal_alias(numa_max_node);

/*
 * Return the number of the highest possible node in a system,
 * which for v1 is the size of a numa.h nodemask_t(in bits)-1.
 * but for v2 is the size of a kernel nodemask_t(in bits)-1.
 */
SYMVER("numa_max_possible_node_v1", "numa_max_possible_node@libnuma_1.1")
int
numa_max_possible_node_v1(void)
{
  return ((sizeof(nodemask_t)*8)-1);
}

SYMVER("numa_max_possible_node_v2", "numa_max_possible_node@@libnuma_1.2")
int
numa_max_possible_node_v2(void)
{
  return numa_num_possible_nodes()-1;
}

make_internal_alias(numa_max_possible_node_v1);
make_internal_alias(numa_max_possible_node_v2);

/*
 * Allocate a bitmask for cpus, of a size large enough to
 * match the kernel's cpumask_t.
 */
struct bitmask *
numa_allocate_cpumask()
{
  int ncpus = numa_num_possible_cpus();

  return numa_bitmask_alloc(ncpus);
}

/*
 * Allocate a bitmask the size of a libnuma nodemask_t
 */
static struct bitmask *
allocate_nodemask_v1(void)
{
  int nnodes = numa_max_possible_node_v1_int()+1;

  return numa_bitmask_alloc(nnodes);
}

/*
 * Allocate a bitmask for nodes, of a size large enough to
 * match the kernel's nodemask_t.
 */
struct bitmask *
numa_allocate_nodemask(void)
{
  struct bitmask *bmp;
  int nnodes = numa_max_possible_node_v2_int() + 1;

  bmp = numa_bitmask_alloc(nnodes);
  return bmp;
}

/* (cache the result?) */
long long numa_node_size64(int node, long long *freep)
{
  size_t len = 0;
  char *line = NULL;
  long long size = -1;
  FILE *f;
  char fn[64];
  int ok = 0;
  int required = freep ? 2 : 1;

  if (freep)
    *freep = 0;
  sprintf(fn,"/sys/devices/system/node/node%d/meminfo", node);
  f = fopen(fn, "r");
  if (!f)
    return -1;
  while (getdelim(&line, &len, '\n', f) > 0) {
    char *end;
    char *s = strcasestr(line, "kB");
    if (!s)
      continue;
    --s;
    while (s > line && isspace(*s))
      --s;
    while (s > line && isdigit(*s))
      --s;
    if (strstr(line, "MemTotal")) {
      size = strtoull(s,&end,0) << 10;
      if (end == s)
        size = -1;
      else
        ok++;
    }
    if (freep && strstr(line, "MemFree")) {
      *freep = strtoull(s,&end,0) << 10;
      if (end == s)
        *freep = -1;
      else
        ok++;
    }
  }
  fclose(f);
  free(line);
  if (ok != required)
    numa_warn(W_badmeminfo, "Cannot parse sysfs meminfo (%d)", ok);
  return size;
}

make_internal_alias(numa_node_size64);

long numa_node_size(int node, long *freep)
{
  long long f2 = 0;
  long sz = numa_node_size64_int(node, &f2);
  if (freep)
    *freep = f2;
  return sz;
}

int numa_available(void)
{
  if (get_mempolicy(NULL, NULL, 0, 0, 0) < 0 && (errno == ENOSYS || errno == EPERM))
    return -1;
  return 0;
}

SYMVER("numa_interleave_memory_v1", "numa_interleave_memory@libnuma_1.1")
void
numa_interleave_memory_v1(void *mem, size_t size, const nodemask_t *mask)
{
  struct bitmask bitmask;

  bitmask.size = sizeof(nodemask_t) * 8;
  bitmask.maskp = (unsigned long *)mask;
  dombind(mem, size, MPOL_INTERLEAVE, &bitmask);
}

SYMVER("numa_interleave_memory_v2", "numa_interleave_memory@@libnuma_1.2")
void
numa_interleave_memory_v2(void *mem, size_t size, struct bitmask *bmp)
{
  dombind(mem, size, MPOL_INTERLEAVE, bmp);
}

void
numa_weighted_interleave_memory(void *mem, size_t size, struct bitmask *bmp)
{
  dombind(mem, size, MPOL_WEIGHTED_INTERLEAVE, bmp);
}

void numa_tonode_memory(void *mem, size_t size, int node)
{
  struct bitmask *nodes;

  nodes = numa_allocate_nodemask();
  if (!nodes)
    return;
  numa_bitmask_setbit(nodes, node);
  dombind(mem, size, bind_policy, nodes);
  numa_bitmask_free(nodes);
}

SYMVER("numa_tonodemask_memory_v1", "numa_tonodemask_memory@libnuma_1.1")
void
numa_tonodemask_memory_v1(void *mem, size_t size, const nodemask_t *mask)
{
  struct bitmask bitmask;

  bitmask.maskp = (unsigned long *)mask;
  bitmask.size  = sizeof(nodemask_t);
  dombind(mem, size,  bind_policy, &bitmask);
}

SYMVER("numa_tonodemask_memory_v2", "numa_tonodemask_memory@@libnuma_1.2")
void
numa_tonodemask_memory_v2(void *mem, size_t size, struct bitmask *bmp)
{
  dombind(mem, size,  bind_policy, bmp);
}

void numa_setlocal_memory(void *mem, size_t size)
{
  dombind(mem, size, MPOL_LOCAL, NULL);
}

void numa_police_memory(void *mem, size_t size)
{
  int pagesize = numa_pagesize_int();
  unsigned long i;
  char *p = mem;
  for (i = 0; i < size; i += pagesize, p += pagesize)
    __atomic_and_fetch(p, 0xff, __ATOMIC_RELAXED);

}

make_internal_alias(numa_police_memory);

void *numa_alloc(size_t size)
{
  char *mem;
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
       0, 0);
  if (mem == (char *)-1)
    return NULL;
  numa_police_memory_int(mem, size);
  return mem;
}

void *numa_realloc(void *old_addr, size_t old_size, size_t new_size)
{
  char *mem;
  mem = mremap(old_addr, old_size, new_size, MREMAP_MAYMOVE);
  if (mem == (char *)-1)
    return NULL;
  /*
   *  The memory policy of the allocated pages is preserved by mremap(), so
   *  there is no need to (re)set it here. If the policy of the original
   *  allocation is not set, the new pages will be allocated according to the
   *  process' mempolicy. Trying to allocate explicitly the new pages on the
   *  same node as the original ones would require changing the policy of the
   *  newly allocated pages, which violates the numa_realloc() semantics.
   */
  return mem;
}

SYMVER("numa_alloc_interleaved_subset_v1", "numa_alloc_interleaved_subset@libnuma_1.1")
void *numa_alloc_interleaved_subset_v1(size_t size, const nodemask_t *mask)
{
  char *mem;
  struct bitmask bitmask;

  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
      0, 0);
  if (mem == (char *)-1)
    return NULL;
  bitmask.maskp = (unsigned long *)mask;
  bitmask.size  = sizeof(nodemask_t);
  mem = dombind_or_free(mem, size, MPOL_INTERLEAVE, &bitmask);
  return mem;
}

SYMVER("numa_alloc_interleaved_subset_v2", "numa_alloc_interleaved_subset@@libnuma_1.2")
void *numa_alloc_interleaved_subset_v2(size_t size, struct bitmask *bmp)
{
  char *mem;

  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
       0, 0);
  if (mem == (char *)-1)
    return NULL;
  mem = dombind_or_free(mem, size, MPOL_INTERLEAVE, bmp);
  return mem;
}

make_internal_alias(numa_alloc_interleaved_subset_v1);
make_internal_alias(numa_alloc_interleaved_subset_v2);

void *
numa_alloc_interleaved(size_t size)
{
  return numa_alloc_interleaved_subset_v2_int(size, numa_all_nodes_ptr);
}

void *
numa_alloc_weighted_interleaved_subset(size_t size, struct bitmask *bmp)
{
  char *mem;

  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
       0, 0);
  if (mem == (char *)-1)
    return NULL;
  mem = dombind_or_free(mem, size, MPOL_WEIGHTED_INTERLEAVE, bmp);
  return mem;
}

void *
numa_alloc_weighted_interleaved(size_t size)
{
  return numa_alloc_weighted_interleaved_subset(size, numa_all_nodes_ptr);
}

/*
 * given a user node mask, set memory policy to use those nodes
 */
SYMVER("numa_set_interleave_mask_v1", "numa_set_interleave_mask@libnuma_1.1")
void
numa_set_interleave_mask_v1(nodemask_t *mask)
{
  struct bitmask *bmp;
  int nnodes = numa_max_possible_node_v1_int()+1;

  bmp = numa_bitmask_alloc(nnodes);
  copy_nodemask_to_bitmask(mask, bmp);
  if (numa_bitmask_equal(bmp, numa_no_nodes_ptr))
    setpol(MPOL_DEFAULT, bmp);
  else
    setpol(MPOL_INTERLEAVE, bmp);
  numa_bitmask_free(bmp);
}


SYMVER("numa_set_interleave_mask_v2", "numa_set_interleave_mask@@libnuma_1.2")
void
numa_set_interleave_mask_v2(struct bitmask *bmp)
{
  if (numa_bitmask_equal(bmp, numa_no_nodes_ptr))
    setpol(MPOL_DEFAULT, bmp);
  else
    setpol(MPOL_INTERLEAVE, bmp);
}

void
numa_set_weighted_interleave_mask(struct bitmask *bmp)
{
  if (numa_bitmask_equal(bmp, numa_no_nodes_ptr))
    setpol(MPOL_DEFAULT, bmp);
  else
    setpol(MPOL_WEIGHTED_INTERLEAVE, bmp);
}

SYMVER("numa_get_interleave_mask_v1", "numa_get_interleave_mask@libnuma_1.1")
nodemask_t
numa_get_interleave_mask_v1(void)
{
  int oldpolicy = 0;
  struct bitmask *bmp;
  nodemask_t mask;

  bmp = allocate_nodemask_v1();
  if (!bmp)
    return numa_no_nodes;
  getpol(&oldpolicy, bmp);
  if (oldpolicy == MPOL_INTERLEAVE)
    copy_bitmask_to_nodemask(bmp, &mask);
  else
    copy_bitmask_to_nodemask(numa_no_nodes_ptr, &mask);
  numa_bitmask_free(bmp);
  return mask;
}

SYMVER("numa_get_interleave_mask_v2", "numa_get_interleave_mask@@libnuma_1.2")
struct bitmask *
numa_get_interleave_mask_v2(void)
{
  int oldpolicy = 0;
  struct bitmask *bmp;

  bmp = numa_allocate_nodemask();
  if (!bmp)
    return NULL;
  getpol(&oldpolicy, bmp);
  if (oldpolicy != MPOL_INTERLEAVE)
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp);
  return bmp;
}

struct bitmask *
numa_get_weighted_interleave_mask(void)
{
  int oldpolicy = 0;
  struct bitmask *bmp;

  bmp = numa_allocate_nodemask();
  if (!bmp)
    return NULL;
  getpol(&oldpolicy, bmp);
  if (oldpolicy != MPOL_WEIGHTED_INTERLEAVE)
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp);
  return bmp;
}

/* (undocumented) */
int numa_get_interleave_node(void)
{
  int nd;
  if (get_mempolicy(&nd, NULL, 0, 0, MPOL_F_NODE) == 0)
    return nd;
  return 0;
}

void *numa_alloc_onnode(size_t size, int node)
{
  char *mem;
  struct bitmask *bmp;

  bmp = numa_allocate_nodemask();
  if (!bmp)
    return NULL;
  numa_bitmask_setbit(bmp, node);
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
       0, 0);
  if (mem == (char *)-1)
    mem = NULL;
  else
    mem = dombind_or_free(mem, size, bind_policy, bmp);

  numa_bitmask_free(bmp);
  return mem;
}

void *numa_alloc_local(size_t size)
{
  char *mem;
  mem = mmap(0, size, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
       0, 0);
  if (mem == (char *)-1)
    mem =  NULL;
  else
    mem = dombind_or_free(mem, size, MPOL_LOCAL, NULL);
  return mem;
}

void numa_set_bind_policy(int strict)
{
  set_preferred_many();
  if (strict)
    bind_policy = MPOL_BIND;
  else if (has_preferred_many)
    bind_policy = MPOL_PREFERRED_MANY;
  else
    bind_policy = MPOL_PREFERRED;
}

SYMVER("numa_set_membind_v1", "numa_set_membind@libnuma_1.1")
void
numa_set_membind_v1(const nodemask_t *mask)
{
  struct bitmask bitmask;

  bitmask.maskp = (unsigned long *)mask;
  bitmask.size  = sizeof(nodemask_t);
  setpol(MPOL_BIND, &bitmask);
}

SYMVER("numa_set_membind_v2", "numa_set_membind@@libnuma_1.2")
void
numa_set_membind_v2(struct bitmask *bmp)
{
  setpol(MPOL_BIND, bmp);
}

make_internal_alias(numa_set_membind_v2);

void
numa_set_membind_balancing(struct bitmask *bmp)
{
  /* MPOL_F_NUMA_BALANCING: ignore if unsupported */
  if (set_mempolicy(MPOL_BIND | MPOL_F_NUMA_BALANCING,
        bmp->maskp, bmp->size + 1) < 0) {
    if (errno == EINVAL) {
      errno = 0;
      numa_set_membind_v2(bmp);
    } else
      numa_error("set_mempolicy");
  }
}

/*
 * copy a bitmask map body to a numa.h nodemask_t structure
 */
void
copy_bitmask_to_nodemask(struct bitmask *bmp, nodemask_t *nmp)
{
  int max, i;

  memset(nmp, 0, sizeof(nodemask_t));
        max = (sizeof(nodemask_t)*8);
  for (i=0; i<bmp->size; i++) {
    if (i >= max)
      break;
    if (numa_bitmask_isbitset(bmp, i))
      nodemask_set_compat((nodemask_t *)nmp, i);
  }
}

/*
 * copy a bitmask map body to another bitmask body
 * fill a larger destination with zeroes
 */
void
copy_bitmask_to_bitmask(struct bitmask *bmpfrom, struct bitmask *bmpto)
{
  int bytes;

  if (bmpfrom->size >= bmpto->size) {
    memcpy(bmpto->maskp, bmpfrom->maskp, CPU_BYTES(bmpto->size));
  } else if (bmpfrom->size < bmpto->size) {
    bytes = CPU_BYTES(bmpfrom->size);
    memcpy(bmpto->maskp, bmpfrom->maskp, bytes);
    memset(((char *)bmpto->maskp)+bytes, 0,
          CPU_BYTES(bmpto->size)-bytes);
  }
}

/*
 * copy a numa.h nodemask_t structure to a bitmask map body
 */
void
copy_nodemask_to_bitmask(nodemask_t *nmp, struct bitmask *bmp)
{
  int max, i;

  numa_bitmask_clearall(bmp);
        max = (sizeof(nodemask_t)*8);
  if (max > bmp->size)
    max = bmp->size;
  for (i=0; i<max; i++) {
    if (nodemask_isset_compat(nmp, i))
      numa_bitmask_setbit(bmp, i);
  }
}

SYMVER("numa_get_membind_v1", "numa_get_membind@libnuma_1.1")
nodemask_t
numa_get_membind_v1(void)
{
  int oldpolicy = 0;
  struct bitmask *bmp;
  nodemask_t nmp;

  bmp = allocate_nodemask_v1();
  if (!bmp)
    return numa_no_nodes;
  getpol(&oldpolicy, bmp);
  if (oldpolicy == MPOL_BIND) {
    copy_bitmask_to_nodemask(bmp, &nmp);
  } else {
    /* copy the body of the map to numa_all_nodes */
    copy_bitmask_to_nodemask(bmp, &numa_all_nodes);
    nmp = numa_all_nodes;
  }
  numa_bitmask_free(bmp);
  return nmp;
}

SYMVER("numa_get_membind_v2", "numa_get_membind@@libnuma_1.2")
struct bitmask *
numa_get_membind_v2(void)
{
  int oldpolicy = 0;
  struct bitmask *bmp = NULL;

  bmp = numa_allocate_nodemask();
  if (!bmp)
    return NULL;
  getpol(&oldpolicy, bmp);
  if (oldpolicy != MPOL_BIND)
    copy_bitmask_to_bitmask(numa_all_nodes_ptr, bmp);
  return bmp;
}

//TODO:  do we need a v1 nodemask_t version?
struct bitmask *numa_get_mems_allowed(void)
{
  struct bitmask *bmp;

  /*
   * can change, so query on each call.
   */
  bmp = numa_allocate_nodemask();
  if (!bmp)
    return NULL;
  if (get_mempolicy(NULL, bmp->maskp, bmp->size + 1, 0,
        MPOL_F_MEMS_ALLOWED) < 0)
    numa_error("get_mempolicy");
  return bmp;
}
make_internal_alias(numa_get_mems_allowed);

void numa_free(void *mem, size_t size)
{
  munmap(mem, size);
}

SYMVER("numa_parse_bitmap_v1", "numa_parse_bitmap@libnuma_1.1")
int
numa_parse_bitmap_v1(char *line, unsigned long *mask, int ncpus)
{
  int i;
  char *p = strchr(line, '\n');
  if (!p)
    return -1;

  for (i = 0; p > line;i++) {
    char *oldp, *endp;
    oldp = p;
    if (*p == ',')
      --p;
    while (p > line && *p != ',')
      --p;
    /* Eat two 32bit fields at a time to get longs */
    if (p > line && sizeof(unsigned long) == 8) {
      oldp--;
      memmove(p, p+1, oldp-p+1);
      while (p > line && *p != ',')
        --p;
    }
    if (*p == ',')
      p++;
    if (i >= CPU_LONGS(ncpus))
      return -1;
    mask[i] = strtoul(p, &endp, 16);
    if (endp != oldp)
      return -1;
    p--;
  }
  return 0;
}

SYMVER("numa_parse_bitmap_v2", "numa_parse_bitmap@@libnuma_1.2")
int
numa_parse_bitmap_v2(char *line, struct bitmask *mask)
{
  int i, ncpus;
  char *p = strchr(line, '\n');
  if (!p)
    return -1;
  ncpus = mask->size;

  for (i = 0; p > line;i++) {
    char *oldp, *endp;
    oldp = p;
    if (*p == ',')
      --p;
    while (p > line && *p != ',')
      --p;
    /* Eat two 32bit fields at a time to get longs */
    if (p > line && sizeof(unsigned long) == 8) {
      oldp--;
      memmove(p, p+1, oldp-p+1);
      while (p > line && *p != ',')
        --p;
    }
    if (*p == ',')
      p++;
    if (i >= CPU_LONGS(ncpus))
      return -1;
    mask->maskp[i] = strtoul(p, &endp, 16);
    if (endp != oldp)
      return -1;
    p--;
  }
  return 0;
}

static void init_node_cpu_mask_v2(void)
{
  int nnodes = numa_max_possible_node_v2_int() + 1;
  struct bitmask **new_ncm, **null_ncm = NULL;
  new_ncm = calloc (nnodes, sizeof(struct bitmask *));
  /* Check for races with another thread */
  if (new_ncm && !__atomic_compare_exchange_n(&node_cpu_mask_v2, &null_ncm,
      new_ncm, 1,
      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE)) {
    free(new_ncm);
  }
}

static void cleanup_node_cpu_mask_v2(void)
{
  if (node_cpu_mask_v2) {
    int i;
    int nnodes;
    nnodes = numa_max_possible_node_v2_int() + 1;
    for (i = 0; i < nnodes; i++) {
      FREE_AND_ZERO(node_cpu_mask_v2[i]);
    }
    free(node_cpu_mask_v2);
    node_cpu_mask_v2 = NULL;
  }
}

/* This would be better with some locking, but I don't want to make libnuma
   dependent on pthreads right now. The races are relatively harmless. */
SYMVER("numa_node_to_cpus_v1", "numa_node_to_cpus@libnuma_1.1")
int
numa_node_to_cpus_v1(int node, unsigned long *buffer, int bufferlen)
{
  int err = 0;
  char fn[64];
  FILE *f;
  char update;
  char *line = NULL;
  size_t len = 0;
  struct bitmask bitmask;
  int buflen_needed;
  unsigned long *mask;
  int ncpus = numa_num_possible_cpus();
  int maxnode = numa_max_node_int();

  buflen_needed = CPU_BYTES(ncpus);
  if ((unsigned)node > maxnode || bufferlen < buflen_needed) {
    errno = ERANGE;
    return -1;
  }
  if (bufferlen > buflen_needed)
    memset(buffer, 0, bufferlen);
  update = __atomic_fetch_and(&node_cpu_mask_v1_stale, 0, __ATOMIC_RELAXED);
  if (node_cpu_mask_v1[node] && !update) {
    memcpy(buffer, node_cpu_mask_v1[node], buflen_needed);
    return 0;
  }

  mask = malloc(buflen_needed);
  if (!mask)
    mask = (unsigned long *)buffer;
  memset(mask, 0, buflen_needed);

  sprintf(fn, "/sys/devices/system/node/node%d/cpumap", node);
  f = fopen(fn, "r");
  if (!f || getdelim(&line, &len, '\n', f) < 1) {
    if (numa_bitmask_isbitset(numa_nodes_ptr, node)) {
      numa_warn(W_nosysfs2,
         "/sys not mounted or invalid. Assuming one node: %s",
          strerror(errno));
      numa_warn(W_nosysfs2,
         "(cannot open or correctly parse %s)", fn);
    }
    bitmask.maskp = (unsigned long *)mask;
    bitmask.size  = buflen_needed * 8;
    numa_bitmask_setall(&bitmask);
    err = -1;
  }
  if (f)
    fclose(f);

  if (line && (numa_parse_bitmap_v1(line, mask, ncpus) < 0)) {
    numa_warn(W_cpumap, "Cannot parse cpumap. Assuming one node");
    bitmask.maskp = (unsigned long *)mask;
    bitmask.size  = buflen_needed * 8;
    numa_bitmask_setall(&bitmask);
    err = -1;
  }

  free(line);
  memmove(buffer, mask, buflen_needed);

  /* slightly racy, see above */
  if (node_cpu_mask_v1[node]) {
    if (update) {
      /*
       * There may be readers on node_cpu_mask_v1[], hence it can not
       * be freed.
       */
      memcpy(node_cpu_mask_v1[node], mask, buflen_needed);
      free(mask);
      mask = NULL;
    } else if (mask != buffer)
      free(mask);
  } else {
    node_cpu_mask_v1[node] = mask;
  }
  return err;
}

/*
 * test whether a node has cpus
 */
/* This would be better with some locking, but I don't want to make libnuma
   dependent on pthreads right now. The races are relatively harmless. */
/*
 * deliver a bitmask of cpus representing the cpus on a given node
 */
SYMVER("numa_node_to_cpus_v2", "numa_node_to_cpus@@libnuma_1.2")
int
numa_node_to_cpus_v2(int node, struct bitmask *buffer)
{
  int err = 0;
  int nnodes = numa_max_node();
  char fn[64], *line = NULL;
  FILE *f;
  char update;
  size_t len = 0;
  struct bitmask *mask;

  if (!__atomic_load_n(&node_cpu_mask_v2, __ATOMIC_CONSUME))
    init_node_cpu_mask_v2();

  if (node > nnodes) {
    errno = ERANGE;
    return -1;
  }
  numa_bitmask_clearall(buffer);

  update = __atomic_fetch_and(&node_cpu_mask_v2_stale, 0, __ATOMIC_RELAXED);
  if (node_cpu_mask_v2[node] && !update) {
    /* have already constructed a mask for this node */
    if (buffer->size < node_cpu_mask_v2[node]->size) {
      errno = EINVAL;
      numa_error("map size mismatch");
      return -1;
    }
    copy_bitmask_to_bitmask(node_cpu_mask_v2[node], buffer);
    return 0;
  }

  /* need a new mask for this node */
  mask = numa_allocate_cpumask();
  if (!mask)
    return -1;

  /* this is a kernel cpumask_t (see node_read_cpumap()) */
  sprintf(fn, "/sys/devices/system/node/node%d/cpumap", node);
  f = fopen(fn, "r");
  if (!f || getdelim(&line, &len, '\n', f) < 1) {
    if (numa_bitmask_isbitset(numa_nodes_ptr, node)) {
      numa_warn(W_nosysfs2,
         "/sys not mounted or invalid. Assuming one node: %s",
          strerror(errno));
      numa_warn(W_nosysfs2,
         "(cannot open or correctly parse %s)", fn);
    }
    numa_bitmask_setall(mask);
    err = -1;
  }
  if (f)
    fclose(f);

  if (line && (numa_parse_bitmap_v2(line, mask) < 0)) {
    numa_warn(W_cpumap, "Cannot parse cpumap. Assuming one node");
    numa_bitmask_setall(mask);
    err = -1;
  }

  free(line);
  copy_bitmask_to_bitmask(mask, buffer);

  /* slightly racy, see above */
  /* save the mask we created */
  if (node_cpu_mask_v2[node]) {
    if (update) {
      copy_bitmask_to_bitmask(mask, node_cpu_mask_v2[node]);
      numa_bitmask_free(mask);
      mask = NULL;
    /* how could this be? */
    } else if (mask != buffer)
      numa_bitmask_free(mask);
  } else {
    /* we don't want to cache faulty result */
    if (!err)
      node_cpu_mask_v2[node] = mask;
    else
      numa_bitmask_free(mask);
  }
  return err;
}

make_internal_alias(numa_node_to_cpus_v1);
make_internal_alias(numa_node_to_cpus_v2);

void numa_node_to_cpu_update(void)
{
  __atomic_store_n(&node_cpu_mask_v1_stale, 1, __ATOMIC_RELAXED);
  __atomic_store_n(&node_cpu_mask_v2_stale, 1, __ATOMIC_RELAXED);
}

/* report the node of the specified cpu */
int numa_node_of_cpu(int cpu)
{
  struct bitmask *bmp;
  int ncpus, nnodes, node, ret;

  ncpus = numa_num_possible_cpus();
  if (cpu > ncpus){
    errno = EINVAL;
    return -1;
  }
  bmp = numa_bitmask_alloc(ncpus);
  nnodes = numa_max_node();
  for (node = 0; node <= nnodes; node++){
    if (numa_node_to_cpus_v2_int(node, bmp) < 0) {
      /* It's possible for the node to not exist */
      continue;
    }
    if (numa_bitmask_isbitset(bmp, cpu)){
      ret = node;
      goto end;
    }
  }
  ret = -1;
  errno = EINVAL;
end:
  numa_bitmask_free(bmp);
  return ret;
}

SYMVER("numa_run_on_node_mask_v1", "numa_run_on_node_mask@libnuma_1.1")
int
numa_run_on_node_mask_v1(const nodemask_t *mask)
{
  int ncpus = numa_num_possible_cpus();
  int i, k, err;
  unsigned long cpus[CPU_LONGS(ncpus)], nodecpus[CPU_LONGS(ncpus)];
  memset(cpus, 0, CPU_BYTES(ncpus));
  for (i = 0; i < NUMA_NUM_NODES; i++) {
    if (mask->n[i / BITS_PER_LONG] == 0)
      continue;
    if (nodemask_isset_compat(mask, i)) {
      if (numa_node_to_cpus_v1_int(i, nodecpus, CPU_BYTES(ncpus)) < 0) {
        numa_warn(W_noderunmask,
            "Cannot read node cpumask from sysfs");
        continue;
      }
      for (k = 0; k < CPU_LONGS(ncpus); k++)
        cpus[k] |= nodecpus[k];
    }
  }
  err = numa_sched_setaffinity_v1(0, CPU_BYTES(ncpus), cpus);

  /* The sched_setaffinity API is broken because it expects
     the user to guess the kernel cpuset size. Do this in a
     brute force way. */
  if (err < 0 && errno == EINVAL) {
    int savederrno = errno;
    char *bigbuf;
    static int size = -1;
    if (size == -1)
      size = CPU_BYTES(ncpus) * 2;
    bigbuf = malloc(CPU_BUFFER_SIZE);
    if (!bigbuf) {
      errno = ENOMEM;
      return -1;
    }
    errno = savederrno;
    while (size <= CPU_BUFFER_SIZE) {
      memcpy(bigbuf, cpus, CPU_BYTES(ncpus));
      memset(bigbuf + CPU_BYTES(ncpus), 0,
             CPU_BUFFER_SIZE - CPU_BYTES(ncpus));
      err = numa_sched_setaffinity_v1_int(0, size, (unsigned long *)bigbuf);
      if (err == 0 || errno != EINVAL)
        break;
      size *= 2;
    }
    savederrno = errno;
    free(bigbuf);
    errno = savederrno;
  }
  return err;
}

/*
 * Given a node mask (size of a kernel nodemask_t) (probably populated by
 * a user argument list) set up a map of cpus (map "cpus") on those nodes.
 * Then set affinity to those cpus.
 */
SYMVER("numa_run_on_node_mask_v2", "numa_run_on_node_mask@@libnuma_1.2")
int
numa_run_on_node_mask_v2(struct bitmask *bmp)
{
  int ncpus, i, k, err;
  struct bitmask *cpus, *nodecpus;

  cpus = numa_allocate_cpumask();
  ncpus = cpus->size;
  nodecpus = numa_allocate_cpumask();
  if (!cpus || !nodecpus)
    return -1;

  for (i = 0; i < bmp->size; i++) {
    if (bmp->maskp[i / BITS_PER_LONG] == 0)
      continue;
    if (numa_bitmask_isbitset(bmp, i)) {
      /*
       * numa_all_nodes_ptr is cpuset aware; use only
       * these nodes
       */
      if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i)) {
        numa_warn(W_noderunmask,
          "node %d not allowed", i);
        continue;
      }
      if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
        numa_warn(W_noderunmask,
          "Cannot read node cpumask from sysfs");
        continue;
      }
      for (k = 0; k < CPU_LONGS(ncpus); k++)
        cpus->maskp[k] |= nodecpus->maskp[k];
    }
  }
  err = numa_sched_setaffinity_v2_int(0, cpus);

  numa_bitmask_free(cpus);
  numa_bitmask_free(nodecpus);

  /* used to have to consider that this could fail - it shouldn't now */
  if (err < 0) {
    numa_error("numa_sched_setaffinity_v2_int() failed");
  }

  return err;
}

make_internal_alias(numa_run_on_node_mask_v2);

/*
 * Given a node mask (size of a kernel nodemask_t) (probably populated by
 * a user argument list) set up a map of cpus (map "cpus") on those nodes
 * without any cpuset awareness. Then set affinity to those cpus.
 */
int
numa_run_on_node_mask_all(struct bitmask *bmp)
{
  int ncpus, i, k, err;
  struct bitmask *cpus, *nodecpus;

  cpus = numa_allocate_cpumask();
  ncpus = cpus->size;
  nodecpus = numa_allocate_cpumask();
  if (!cpus || !nodecpus)
    return -1;

  for (i = 0; i < bmp->size; i++) {
    if (bmp->maskp[i / BITS_PER_LONG] == 0)
      continue;
    if (numa_bitmask_isbitset(bmp, i)) {
      if (!numa_bitmask_isbitset(numa_possible_nodes_ptr, i)) {
        numa_warn(W_noderunmask,
          "node %d not allowed", i);
        continue;
      }
      if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
        numa_warn(W_noderunmask,
          "Cannot read node cpumask from sysfs");
        continue;
      }
      for (k = 0; k < CPU_LONGS(ncpus); k++)
        cpus->maskp[k] |= nodecpus->maskp[k];
    }
  }
  err = numa_sched_setaffinity_v2_int(0, cpus);

  numa_bitmask_free(cpus);
  numa_bitmask_free(nodecpus);

  /* With possible nodes freedom it can happen easily now */
  if (err < 0) {
    numa_error("numa_sched_setaffinity_v2_int() failed");
  }

  return err;
}

SYMVER("numa_get_run_node_mask_v1", "numa_get_run_node_mask@libnuma_1.1")
nodemask_t
numa_get_run_node_mask_v1(void)
{
  int ncpus = numa_num_configured_cpus();
  int i, k;
  int max = numa_max_node_int();
  struct bitmask *bmp, *cpus, *nodecpus;
  nodemask_t nmp;

  cpus = numa_allocate_cpumask();
  if (!cpus)
    return numa_no_nodes;
  if (numa_sched_getaffinity_v2_int(0, cpus) < 0){
    nmp = numa_no_nodes;
    goto free_cpus;
  }

  nodecpus = numa_allocate_cpumask();
  if (!nodecpus) {
    nmp = numa_no_nodes;
    goto free_cpus;
  }

  bmp = allocate_nodemask_v1(); /* the size of a nodemask_t */
  if (!bmp) {
    nmp = numa_no_nodes;
    goto free_cpus2;
  }

  for (i = 0; i <= max; i++) {
    if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
      /* It's possible for the node to not exist */
      continue;
    }
    for (k = 0; k < CPU_LONGS(ncpus); k++) {
      if (nodecpus->maskp[k] & cpus->maskp[k])
        numa_bitmask_setbit(bmp, i);
    }
  }
  copy_bitmask_to_nodemask(bmp, &nmp);
  numa_bitmask_free(bmp);
free_cpus2:
  numa_bitmask_free(nodecpus);
free_cpus:
  numa_bitmask_free(cpus);
  return nmp;
}

SYMVER("numa_get_run_node_mask_v2", "numa_get_run_node_mask@@libnuma_1.2")
struct bitmask *
numa_get_run_node_mask_v2(void)
{
  int i, k;
  int ncpus = numa_num_configured_cpus();
  int max = numa_max_node_int();
  struct bitmask *bmp, *cpus, *nodecpus;

  bmp = numa_allocate_cpumask();
  cpus = numa_allocate_cpumask();
  if (!bmp || !cpus)
    return NULL;
  if (numa_sched_getaffinity_v2_int(0, cpus) < 0){
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, bmp);
    goto free_cpus;
  }

  nodecpus = numa_allocate_cpumask();
  for (i = 0; i <= max; i++) {
    /*
     * numa_all_nodes_ptr is cpuset aware; show only
     * these nodes
     */
    if (!numa_bitmask_isbitset(numa_all_nodes_ptr, i)) {
      continue;
    }
    if (numa_node_to_cpus_v2_int(i, nodecpus) < 0) {
      /* It's possible for the node to not exist */
      continue;
    }
    for (k = 0; k < CPU_LONGS(ncpus); k++) {
      if (nodecpus->maskp[k] & cpus->maskp[k])
        numa_bitmask_setbit(bmp, i);
    }
  }
  numa_bitmask_free(nodecpus);
free_cpus:
  numa_bitmask_free(cpus);
  return bmp;
}

int
numa_migrate_pages(int pid, struct bitmask *fromnodes, struct bitmask *tonodes)
{
  int numa_num_nodes = numa_num_possible_nodes();

  return migrate_pages(pid, numa_num_nodes + 1, fromnodes->maskp,
              tonodes->maskp);
}

int numa_move_pages(int pid, unsigned long count,
  void **pages, const int *nodes, int *status, int flags)
{
  return move_pages(pid, count, pages, nodes, status, flags);
}

int numa_run_on_node(int node)
{
  int numa_num_nodes = numa_num_possible_nodes();
  int ret = -1;
  struct bitmask *cpus;

  if (node >= numa_num_nodes){
    errno = EINVAL;
    goto out;
  }

  cpus = numa_allocate_cpumask();
  if (!cpus)
    return -1;

  if (node == -1)
    numa_bitmask_setall(cpus);
  else if (numa_node_to_cpus_v2_int(node, cpus) < 0){
    numa_warn(W_noderunmask, "Cannot read node cpumask from sysfs");
    goto free;
  }

  ret = numa_sched_setaffinity_v2_int(0, cpus);
free:
  numa_bitmask_free(cpus);
out:
  return ret;
}

static struct bitmask *__numa_preferred(void)
{
  int policy = 0;
  struct bitmask *bmp;

  bmp = numa_allocate_nodemask();
  if (!bmp)
    return NULL;
  /* could read the current CPU from /proc/self/status. Probably
     not worth it. */
  numa_bitmask_clearall(bmp);
  getpol(&policy, bmp);

  if (policy != MPOL_PREFERRED &&
      policy != MPOL_PREFERRED_MANY &&
      policy != MPOL_BIND)
    return bmp;

  if (policy == MPOL_PREFERRED && numa_bitmask_weight(bmp) > 1) {
    errno = EINVAL;
    numa_error(__FILE__);
  }

  return bmp;
}

int numa_preferred_err(void)
{
  int first_node = 0;
  struct bitmask *bmp;

  bmp = __numa_preferred();
  first_node = numa_find_first(bmp);
  numa_bitmask_free(bmp);
  
  return first_node;
}

int numa_preferred(void)
{
  int first_node = 0;

  first_node = numa_preferred_err();
  first_node = first_node >= 0 ? first_node : 0;

  return first_node;
}

static void __numa_set_preferred(struct bitmask *bmp)
{
  int nodes = numa_bitmask_weight(bmp);
  if (nodes > 1) {
    errno = EINVAL;
    numa_error(__FILE__);
  }

  setpol(nodes ? MPOL_PREFERRED : MPOL_LOCAL, bmp);
}

void numa_set_preferred(int node)
{
  struct bitmask *bmp = numa_allocate_nodemask();
  if (!bmp)
    return;
  numa_bitmask_setbit(bmp, node);
  __numa_set_preferred(bmp);
  numa_bitmask_free(bmp);
}

int numa_has_preferred_many(void)
{
  set_preferred_many();
  return has_preferred_many;
}

void numa_set_preferred_many(struct bitmask *bitmask)
{
  int first_node = 0;

  set_preferred_many();
  if (!has_preferred_many) {
    numa_warn(W_nodeparse,
      "Unable to handle MANY preferred nodes. Falling back to first node\n");
    first_node = numa_find_first(bitmask);
    numa_set_preferred(first_node);
    return;
  }
  setpol(MPOL_PREFERRED_MANY, bitmask);
}

struct bitmask *numa_preferred_many()
{
  return __numa_preferred();
}

void numa_set_localalloc(void)
{
  setpol(MPOL_LOCAL, numa_no_nodes_ptr);
}

SYMVER("numa_bind_v1", "numa_bind@libnuma_1.1")
void numa_bind_v1(const nodemask_t *nodemask)
{
  struct bitmask bitmask;

  bitmask.maskp = (unsigned long *)nodemask;
  bitmask.size  = sizeof(nodemask_t);
  numa_run_on_node_mask_v2_int(&bitmask);
  numa_set_membind_v2_int(&bitmask);
}

SYMVER("numa_bind_v2", "numa_bind@@libnuma_1.2")
void numa_bind_v2(struct bitmask *bmp)
{
  numa_run_on_node_mask_v2_int(bmp);
  numa_set_membind_v2_int(bmp);
}

void numa_set_strict(int flag)
{
  if (flag)
    mbind_flags |= MPOL_MF_STRICT;
  else
    mbind_flags &= ~MPOL_MF_STRICT;
}

/*
 * Extract a node or processor number from the given string.
 * Allow a relative node / processor specification within the allowed
 * set if "relative" is nonzero
 */
static unsigned long get_nr(const char *s, char **end, struct bitmask *bmp, int relative)
{
  long i, nr;

  if (!relative)
    return strtoul(s, end, 0);

  nr = strtoul(s, end, 0);
  if (s == *end)
    return nr;
  /* Find the nth set bit */
  for (i = 0; nr >= 0 && i <= bmp->size; i++)
    if (numa_bitmask_isbitset(bmp, i))
      nr--;
  return i-1;
}

/*
 * __numa_parse_nodestring() is called to create a node mask, given
 * an ascii string such as 25 or 12-15 or 1,3,5-7 or +6-10.
 * (the + indicates that the numbers are nodeset-relative)
 *
 * The nodes may be specified as absolute, or relative to the current nodeset.
 * The list of available nodes is in a map pointed to by "allowed_nodes_ptr",
 * which may represent all nodes or the nodes in the current nodeset.
 *
 * The caller must free the returned bitmask.
 */
static struct bitmask *
__numa_parse_nodestring(const char *s, struct bitmask *allowed_nodes_ptr)
{
  int invert = 0, relative = 0;
  int conf_nodes = numa_num_configured_nodes();
  char *end;
  struct bitmask *mask;

  mask = numa_allocate_nodemask();
  if (!mask)
    return NULL;

  if (s[0] == 0){
    copy_bitmask_to_bitmask(numa_no_nodes_ptr, mask);
    return mask; /* return freeable mask */
  }
  if (*s == '!') {
    invert = 1;
    s++;
  }
  if (*s == '+') {
    relative++;
    s++;
  }
  do {
    unsigned long arg;
    int i;
    if (isalpha(*s)) {
      int n;
      if (!strcmp(s,"all")) {
        copy_bitmask_to_bitmask(allowed_nodes_ptr,
              mask);
        s+=4;
        break;
      }
      n = resolve_affinity(s, mask);
      if (n != NO_IO_AFFINITY) {
        if (n < 0)
          goto err;
        s += strlen(s) + 1;
        break;
      }
    }
    arg = get_nr(s, &end, allowed_nodes_ptr, relative);
    if (end == s) {
      numa_warn(W_nodeparse, "unparseable node description `%s'\n", s);
      goto err;
    }
    if (!numa_bitmask_isbitset(allowed_nodes_ptr, arg)) {
      numa_warn(W_nodeparse, "node argument %ld is out of range\n", arg);
      goto err;
    }
    i = arg;
    numa_bitmask_setbit(mask, i);
    s = end;
    if (*s == '-') {
      char *end2;
      unsigned long arg2;
      arg2 = get_nr(++s, &end2, allowed_nodes_ptr, relative);
      if (end2 == s) {
        numa_warn(W_nodeparse, "missing node argument %s\n", s);
        goto err;
      }
      if (!numa_bitmask_isbitset(allowed_nodes_ptr, arg2)) {
        numa_warn(W_nodeparse, "node argument %ld out of range\n", arg2);
        goto err;
      }
      while (arg <= arg2) {
        i = arg;
        if (numa_bitmask_isbitset(allowed_nodes_ptr,i))
          numa_bitmask_setbit(mask, i);
        arg++;
      }
      s = end2;
    }
  } while (*s++ == ',');
  if (s[-1] != '\0')
    goto err;
  if (invert) {
    int i;
    for (i = 0; i < conf_nodes; i++) {
      if (numa_bitmask_isbitset(mask, i))
        numa_bitmask_clearbit(mask, i);
      else
        numa_bitmask_setbit(mask, i);
    }
  }
  return mask;

err:
  numa_bitmask_free(mask);
  return NULL;
}

/*
 * numa_parse_nodestring() is called to create a bitmask from nodes available
 * for this task.
 */

struct bitmask * numa_parse_nodestring(const char *s)
{
  return __numa_parse_nodestring(s, numa_all_nodes_ptr);
}

/*
 * numa_parse_nodestring_all() is called to create a bitmask from all nodes
 * available.
 */

struct bitmask * numa_parse_nodestring_all(const char *s)
{
  return __numa_parse_nodestring(s, numa_possible_nodes_ptr);
}

/*
 * __numa_parse_cpustring() is called to create a bitmask, given
 * an ascii string such as 25 or 12-15 or 1,3,5-7 or +6-10.
 * (the + indicates that the numbers are cpuset-relative)
 *
 * The cpus may be specified as absolute, or relative to the current cpuset.
 * The list of available cpus for this task is in the map pointed to by
 * "allowed_cpus_ptr", which may represent all cpus or the cpus in the
 * current cpuset.
 *
 * The caller must free the returned bitmask.
 */
static struct bitmask *
__numa_parse_cpustring(const char *s, struct bitmask *allowed_cpus_ptr)
{
  int invert = 0, relative=0;
  int conf_cpus = numa_num_configured_cpus();
  char *end;
  struct bitmask *mask;
  int i;

  mask = numa_allocate_cpumask();
  if (!mask)
    return NULL;

  if (s[0] == 0)
    return mask;
  if (*s == '!') {
    invert = 1;
    s++;
  }
  if (*s == '+') {
    relative++;
    s++;
  }
  do {
    unsigned long arg;

    if (!strcmp(s,"all")) {
      copy_bitmask_to_bitmask(allowed_cpus_ptr, mask);
      s+=4;
      break;
    }
    arg = get_nr(s, &end, allowed_cpus_ptr, relative);
    if (end == s) {
      numa_warn(W_cpuparse, "unparseable cpu description `%s'\n", s);
      goto err;
    }
    if (!numa_bitmask_isbitset(allowed_cpus_ptr, arg)) {
      numa_warn(W_cpuparse, "cpu argument %s is out of range\n", s);
      goto err;
    }
    i = arg;
    numa_bitmask_setbit(mask, i);
    s = end;
    if (*s == '-') {
      char *end2;
      unsigned long arg2;
      arg2 = get_nr(++s, &end2, allowed_cpus_ptr, relative);
      if (end2 == s) {
        numa_warn(W_cpuparse, "missing cpu argument %s\n", s);
        goto err;
      }
      if (!numa_bitmask_isbitset(allowed_cpus_ptr, arg2)) {
        numa_warn(W_cpuparse, "cpu argument %s out of range\n", s);
        goto err;
      }
      while (arg <= arg2) {
        i = arg;
        if (numa_bitmask_isbitset(allowed_cpus_ptr, i))
          numa_bitmask_setbit(mask, i);
        arg++;
      }
      s = end2;
    }
  } while (*s++ == ',');
  if (s[-1] != '\0')
    goto err;
  if (invert) {
    for (i = 0; i < conf_cpus; i++) {
      if (numa_bitmask_isbitset(mask, i))
        numa_bitmask_clearbit(mask, i);
      else
        numa_bitmask_setbit(mask, i);
    }
  }
  return mask;

err:
  numa_bitmask_free(mask);
  return NULL;
}

/*
 * numa_parse_cpustring() is called to create a bitmask from cpus available
 * for this task.
 */

struct bitmask * numa_parse_cpustring(const char *s)
{
  return __numa_parse_cpustring(s, numa_all_cpus_ptr);
}

/*
 * numa_parse_cpustring_all() is called to create a bitmask from all cpus
 * available.
 */

struct bitmask * numa_parse_cpustring_all(const char *s)
{
  return __numa_parse_cpustring(s, numa_possible_cpus_ptr);
}

int numa_has_home_node(void)
{
  void *mem;
  static int has_home_node = -1;
  int page_size = numa_pagesize();
  struct bitmask *tmp = numa_get_mems_allowed();

  if (has_home_node >= 0)
    goto out;

  has_home_node = 0;
  /* Detect whether home_node is supported */
  mem = mmap(0, page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  if (mem != MAP_FAILED) {
    dombind(mem, page_size, MPOL_BIND, tmp);
    if (set_mempolicy_home_node(mem, page_size, numa_find_first(tmp), 0) == 0)
      has_home_node = 1;
    munmap(mem, page_size);
  }

out:
  numa_bitmask_free(tmp);
  return has_home_node;
}

int numa_set_mempolicy_home_node(void *start, unsigned long len, int home_node, int flags)
{
  if (set_mempolicy_home_node(start, len, home_node, flags)) {
    numa_error("set_mempolicy_home_node");
    return -1;
  }

  return 0;
}

Coverage Report

Created: 2025-07-18 06:29