/src/haproxy/src/pool.c

Source
/*
 * Memory management functions.
 *
 * Copyright 2000-2007 Willy Tarreau <w@1wt.eu>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 *
 */

#include <errno.h>

#include <import/plock.h>

#include <haproxy/activity.h>
#include <haproxy/api.h>
#include <haproxy/applet-t.h>
#include <haproxy/cfgparse.h>
#include <haproxy/channel.h>
#include <haproxy/cli.h>
#include <haproxy/clock.h>
#include <haproxy/errors.h>
#include <haproxy/global.h>
#include <haproxy/list.h>
#include <haproxy/pool.h>
#include <haproxy/pool-os.h>
#include <haproxy/sc_strm.h>
#include <haproxy/stconn.h>
#include <haproxy/sym.h>
#include <haproxy/thread.h>
#include <haproxy/tools.h>


/* These ones are initialized per-thread on startup by init_pools() */
THREAD_LOCAL size_t pool_cache_bytes = 0;                /* total cache size */
THREAD_LOCAL size_t pool_cache_count = 0;                /* #cache objects   */

struct list pools __read_mostly = LIST_HEAD_INIT(pools);
int mem_poison_byte __read_mostly = 'P';
int pool_trim_in_progress = 0;
uint pool_debugging __read_mostly =               /* set of POOL_DBG_* flags */
#if defined(DEBUG_FAIL_ALLOC) && (DEBUG_FAIL_ALLOC > 0)
  POOL_DBG_FAIL_ALLOC |
#endif
#if defined(DEBUG_DONT_SHARE_POOLS) && (DEBUG_DONT_SHARE_POOLS > 0)
  POOL_DBG_DONT_MERGE |
#endif
#if defined(DEBUG_POOL_INTEGRITY) && (DEBUG_POOL_INTEGRITY > 0)
  POOL_DBG_COLD_FIRST |
  POOL_DBG_INTEGRITY  |
#endif
#if defined(CONFIG_HAP_NO_GLOBAL_POOLS)
  POOL_DBG_NO_GLOBAL  |
#endif
#if defined(DEBUG_NO_POOLS) && (DEBUG_NO_POOLS > 0)
  POOL_DBG_NO_CACHE   |
#endif
#if defined(DEBUG_POOL_TRACING) && (DEBUG_POOL_TRACING > 0)
  POOL_DBG_CALLER     |
#endif
#if defined(DEBUG_MEMORY_POOLS) && (DEBUG_MEMORY_POOLS > 0)
  POOL_DBG_TAG        |
#endif
#if defined(DEBUG_UAF) && (DEBUG_UAF > 0)
  POOL_DBG_NO_CACHE   |
  POOL_DBG_UAF        |
#endif
  0;

static const struct {
  uint flg;
  const char *set;
  const char *clr;
  const char *hlp;
} dbg_options[] = {
  /* flg,                 set,          clr,            hlp */
  { POOL_DBG_FAIL_ALLOC, "fail",       "no-fail",      "randomly fail allocations" },
  { POOL_DBG_DONT_MERGE, "no-merge",   "merge",        "disable merging of similar pools" },
  { POOL_DBG_COLD_FIRST, "cold-first", "hot-first",    "pick cold objects first" },
  { POOL_DBG_INTEGRITY,  "integrity",  "no-integrity", "enable cache integrity checks" },
  { POOL_DBG_NO_GLOBAL,  "no-global",  "global",       "disable global shared cache" },
  { POOL_DBG_NO_CACHE,   "no-cache",   "cache",        "disable thread-local cache" },
  { POOL_DBG_CALLER,     "caller",     "no-caller",    "save caller information in cache" },
  { POOL_DBG_TAG,        "tag",        "no-tag",       "add tag at end of allocated objects" },
  { POOL_DBG_POISON,     "poison",     "no-poison",    "poison newly allocated objects" },
  { POOL_DBG_UAF,        "uaf",        "no-uaf",       "enable use-after-free checks (slow)" },
  { POOL_DBG_BACKUP,     "backup",     "no-backup",    "compare object contents on re-alloc" },
  { 0 /* end */ }
};

/* describes a snapshot of a pool line about to be dumped by "show pools" */
struct pool_dump_info {
  const struct pool_head *entry;
  ulong alloc_items;
  ulong alloc_bytes;
  ulong used_items;
  ulong cached_items;
  ulong need_avg;
  ulong failed_items;
};

/* context used by "show pools" */
struct show_pools_ctx {
  char *prefix;  /* if non-null, match this prefix name for the pool */
  int how;     /* bits 0..3: 0=no sort, 1=by name, 2=by item size, 3=by total alloc */
  int maxcnt;  /* 0=no limit, other=max number of output entries */
  int pool_idx; /* Index of the pool to restart the dump  (-1 means the header must be dumped) */
  unsigned int nbpools; /* total number of pools */
  unsigned long long allocated; /* total number of bytes allocated*/
  unsigned long long used; /* total number of bytes used */
  unsigned long long cached_bytes; /* total number of bytes cached */
  struct pool_dump_info *pool_info; /* array containing info about all pools (contains nbpools entries) */
};

static int mem_fail_rate __read_mostly = 0;
static int using_default_allocator __read_mostly = 1; // linked-in allocator or LD_PRELOADed one ?
static int disable_trim __read_mostly = 0;
static int(*my_mallctl)(const char *, void *, size_t *, void *, size_t) = NULL;
static int(*_malloc_trim)(size_t) = NULL;

/* returns the pool hash bucket an object should use based on its pointer.
 * Objects will needed consistent bucket assignment so that they may be
 * allocated on one thread and released on another one. Thus only the
 * pointer is usable.
 */
static forceinline unsigned int pool_pbucket(const void *ptr)
{
  return ptr_hash(ptr, CONFIG_HAP_POOL_BUCKETS_BITS);
}

/* returns the pool hash bucket to use for the current thread. This should only
 * be used when no pointer is available (e.g. count alloc failures).
 */
static forceinline unsigned int pool_tbucket(void)
{
  return tid % CONFIG_HAP_POOL_BUCKETS;
}

/* ask the allocator to trim memory pools.
 * This must run under thread isolation so that competing threads trying to
 * allocate or release memory do not prevent the allocator from completing
 * its job. We just have to be careful as callers might already be isolated
 * themselves.
 */
void trim_all_pools(void)
{
  int isolated = thread_isolated();

  if (!isolated)
    thread_isolate();

  malloc_trim(0);

  if (!isolated)
    thread_release();
}

/* check if we're using the same allocator as the one that provides
 * malloc_trim() and mallinfo(). The principle is that on glibc, both
 * malloc_trim() and mallinfo() are provided, and using mallinfo() we
 * can check if malloc() is performed through glibc or any other one
 * the executable was linked against (e.g. jemalloc). Prior to this we
 * have to check whether we're running on jemalloc by verifying if the
 * mallctl() function is provided. Its pointer will be used later.
 */
static void detect_allocator(void)
{
#if defined(__ELF__)
  extern int mallctl(const char *, void *, size_t *, void *, size_t) __attribute__((weak));

  my_mallctl = mallctl;
#endif
  if (!my_mallctl) {
    /* trick: we won't enter here if mallctl() is known at link
     * time. This allows to detect if the symbol was changed since
     * the program was linked, indicating it's not running on the
     * expected allocator (due to an LD_PRELOAD) and that we must
     * be extra cautious and avoid some optimizations that are
     * known to break such as malloc_trim().
     */
    my_mallctl = get_sym_curr_addr("mallctl");
    using_default_allocator = (my_mallctl == NULL);
  }

  if (!my_mallctl) {
#if defined(HA_HAVE_MALLOC_TRIM)
#ifdef HA_HAVE_MALLINFO2
    struct mallinfo2 mi1, mi2;
#else
    struct mallinfo mi1, mi2;
#endif
    void *ptr;

#ifdef HA_HAVE_MALLINFO2
    mi1 = mallinfo2();
#else
    mi1 = mallinfo();
#endif
    ptr = DISGUISE(malloc(1));
#ifdef HA_HAVE_MALLINFO2
    mi2 = mallinfo2();
#else
    mi2 = mallinfo();
#endif
    free(DISGUISE(ptr));

    using_default_allocator = !!memcmp(&mi1, &mi2, sizeof(mi1));
#elif defined(HA_HAVE_MALLOC_ZONE)
    using_default_allocator = (malloc_default_zone() != NULL);
#endif
  }

  /* detect presence of malloc_trim() */
  _malloc_trim = get_sym_next_addr("malloc_trim");
}

/* replace the libc's malloc_trim() so that we can also intercept the calls
 * from child libraries when the allocator is not the default one.
 */
int malloc_trim(size_t pad)
{
  int ret = 0;

  if (disable_trim)
    return ret;

  HA_ATOMIC_INC(&pool_trim_in_progress);

  if (my_mallctl) {
    /* here we're on jemalloc and malloc_trim() is called either
     * by haproxy or another dependency (the worst case that
     * normally crashes). Instead of just failing, we can actually
     * emulate it so let's do it now.
     */
    unsigned int i, narenas = 0;
    size_t len = sizeof(narenas);

    if (my_mallctl("arenas.narenas", &narenas, &len, NULL, 0) == 0) {
      for (i = 0; i < narenas; i ++) {
        char mib[32] = {0};
        snprintf(mib, sizeof(mib), "arena.%u.purge", i);
        (void)my_mallctl(mib, NULL, NULL, NULL, 0);
        ret = 1; // success
      }
    }
  }
  else if (!using_default_allocator) {
    /* special allocators that can be LD_PRELOADed end here */
    ret = 0; // did nothing
  }
  else if (_malloc_trim) {
    /* we're typically on glibc and not overridden */
    ret = _malloc_trim(pad);
  }
#if defined(HA_HAVE_MALLOC_ZONE)
  else {
    /* we're on MacOS, there's an equivalent mechanism */
    vm_address_t *zones;
    unsigned int i, nzones;

    if (malloc_get_all_zones(0, NULL, &zones, &nzones) == KERN_SUCCESS) {
      for (i = 0; i < nzones; i ++) {
        malloc_zone_t *zone = (malloc_zone_t *)zones[i];

        /* we cannot purge anonymous zones */
        if (zone->zone_name) {
          malloc_zone_pressure_relief(zone, 0);
          ret = 1; // success
        }
      }
    }
  }
#endif
  HA_ATOMIC_DEC(&pool_trim_in_progress);

  /* here we have ret=0 if nothing was release, or 1 if some were */
  return ret;
}

static int mem_should_fail(const struct pool_head *pool)
{
  int ret = 0;

  if (mem_fail_rate > 0 && !(global.mode & MODE_STARTING)) {
    if (mem_fail_rate > statistical_prng_range(100))
      ret = 1;
    else
      ret = 0;
  }
  return ret;
}

/* Try to find an existing shared pool with the same characteristics and
 * returns it, otherwise creates this one. NULL is returned if no memory
 * is available for a new creation. Two flags are supported :
 *   - MEM_F_SHARED to indicate that the pool may be shared with other users
 *   - MEM_F_EXACT to indicate that the size must not be rounded up
 * The name must be a stable pointer during all the program's life time.
 * The file and line are passed to store the registration location in the
 * registration struct. Use create_pool() instead which does it for free.
 * The alignment will be stored as-is in the registration.
 */
struct pool_head *create_pool_with_loc(const char *name, ullong size,
               unsigned int align, unsigned int flags,
               const char *file, unsigned int line)
{
  struct pool_registration *reg;
  struct pool_head *pool;

  reg = calloc(1, sizeof(*reg));
  if (!reg)
    return NULL;

  reg->name = name;
  reg->file = file;
  reg->line = line;
  reg->size = size;
  reg->flags = flags;
  reg->align = align;

  pool = create_pool_from_reg(name, reg);
  if (!pool)
    free(reg);
  return pool;
}

/* create a pool from a pool registration. All configuration is taken from
 * there. The alignment will automatically be raised to sizeof(void*) or the
 * next power of two so that it's always possible to lazily pass alignof() or
 * sizeof(). Alignments are always respected when merging pools.
 */
struct pool_head *create_pool_from_reg(const char *name, struct pool_registration *reg)
{
  unsigned int extra_mark, extra_caller, extra;
  unsigned int flags = reg->flags;
  ullong reg_size = reg->size; // copy of the originally requested size
  ullong size = reg_size;
  unsigned int alignment = reg->align;
  struct pool_head *pool = NULL;
  struct pool_head *entry;
  struct list *start;
  unsigned int align;
  unsigned int best_diff;
  int thr __maybe_unused;

  /* alignment of zero means type alignment */
  if (!alignment)
    alignment = reg->type_align;

  /* extend alignment if needed */
  if (alignment < sizeof(void*))
    alignment = sizeof(void*);
  else if (alignment & (alignment - 1)) {
    /* not power of two! round up to next power of two by filling
     * all LSB in O(log(log(N))) then increment the result.
     */
    int shift = 1;
    do {
      alignment |= alignment >> shift;
      shift *= 2;
    } while (alignment & (alignment + 1));
    alignment++;
  }

  if (reg->type_align && alignment < reg->type_align) {
    ha_alert("BUG in the code: at %s:%u, requested creation of pool '%s' aligned to %u "
       "while type requires alignment of %u! Please report to developers. Aborting.\n",
       reg->file, reg->line, name, alignment, reg->type_align);
    return NULL;
  }

  if (invalid_char(name)) {
    ha_alert("Pool '%s' declared at %s:%u contains invalid chars in its name and "
       "cannot be registered. Please report to developers. Aborting.\n",
       name, reg->file, reg->line);
    return NULL;
  }

  extra_mark = (pool_debugging & POOL_DBG_TAG) ? POOL_EXTRA_MARK : 0;
  extra_caller = (pool_debugging & POOL_DBG_CALLER) ? POOL_EXTRA_CALLER : 0;
  extra = extra_mark + extra_caller;

  if (size > 0xFFFFFFFFULL || (size + extra) > 0xFFFFFFFFULL || (uint)(size + extra) < (uint)reg_size)
    goto ovf;

  if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
    /* we'll store two lists there, we need the room for this. Let's
     * make sure it's always OK even when including the extra word
     * that is stored after the pci struct.
     */
    if (size + extra - extra_caller < sizeof(struct pool_cache_item))
      size = sizeof(struct pool_cache_item) + extra_caller - extra;
  }

  /* Now we know our size is set to the strict minimum possible. It may
   * be OK for elements allocated with an exact size (e.g. buffers), but
   * we're going to round the size up 16 bytes to merge almost identical
   * pools together. We only round up however when we add the debugging
   * tag since it's used to detect overflows. Otherwise we only round up
   * to the size of a word to preserve alignment.
   */
  if (!(flags & MEM_F_EXACT)) {
    align = (pool_debugging & POOL_DBG_TAG) ? sizeof(void *) : 16;
    size  = ((size + align - 1) & -(ullong)align);
  }

  if (pool_debugging & POOL_DBG_BACKUP) {
    /* keep a full backup of the pool at release time. We need
     * a word-aligned size, so it's fine to do it now.
     */
    extra += size;
  }

  if (size > 0xFFFFFFFFULL || (size + extra) > 0xFFFFFFFFULL || (uint)(size + extra) < (uint)reg_size)
    goto ovf;

  /* TODO: thread: we do not lock pool list for now because all pools are
   * created during HAProxy startup (so before threads creation) */
  start = &pools;
  best_diff = ~0U;

  list_for_each_entry(entry, &pools, list) {
    if (entry->size == size ||
        (!(flags & MEM_F_EXACT) && !pool_allocated(entry) &&
         /* size within 1% of avg size */
         (((ullong)entry->sum_size * 100ULL < (ullong)size * entry->users * 101ULL &&
           (ullong)entry->sum_size * 101ULL > (ullong)size * entry->users * 100ULL) ||
          /* or +/- 16 compared to the current avg size */
          (entry->sum_size - 16 * entry->users < size * entry->users &&
           entry->sum_size + 16 * entry->users > size * entry->users)))) {

      /* either we can share this place and we take it, or
       * we look for a shareable one or for the next position
       * before which we will insert a new one.
       */
      if ((flags & entry->flags & MEM_F_SHARED) &&
          (!(pool_debugging & POOL_DBG_DONT_MERGE) ||
           strcmp(name, entry->name) == 0)) {
        /* we can share this one */
        uint diff = (abs((int)(size * entry->users - entry->size)) + entry->users / 2) / entry->users;

        /* the principle here is:
         *   - if the best pool is smaller and the current
         *     candidate larger, we prefer the larger one
         *     so as not to grow an existing pool;
         *   - otherwise we go for the smallest distance
         *     from the existing one.
         */
        if (!pool || entry->size == size ||
            (pool->size != size &&
             ((pool->size < size && entry->size >= size) ||
              (diff == best_diff && entry->size >= size) ||
              (diff < best_diff)))) {
          best_diff = diff;
          pool = entry;
        }
      }
    }
    else if (entry->size > size) {
      /* insert before this one */
      start = &entry->list;
    }
  }

  if (!pool) {
    void *pool_addr;

    pool_addr = calloc(1, sizeof(*pool) + __alignof__(*pool));
    if (!pool_addr)
      goto fail;

    /* always provide an aligned pool */
    pool = (struct pool_head*)((((size_t)pool_addr) + __alignof__(*pool)) & -(size_t)__alignof__(*pool));
    pool->base_addr = pool_addr; // keep it, it's the address to free later

    if (name)
      strlcpy2(pool->name, name, sizeof(pool->name));
    pool->alloc_sz = size + extra;
    pool->size = size;
    pool->align = alignment;
    pool->flags = flags;
    LIST_APPEND(start, &pool->list);
    LIST_INIT(&pool->regs);

    if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
      /* update per-thread pool cache if necessary */
      for (thr = 0; thr < MAX_THREADS; thr++) {
        LIST_INIT(&pool->cache[thr].list);
        pool->cache[thr].tid = thr;
        pool->cache[thr].pool = pool;
      }
    }
  }
  else {
    /* we found the best one */
    if (size > pool->size) {
      pool->size = size;
      pool->alloc_sz = size + extra;
    }
    if (alignment > pool->align)
      pool->align = alignment;
    DPRINTF(stderr, "Sharing %s with %s\n", name, pool->name);
  }

  LIST_APPEND(&pool->regs, &reg->list);
  pool->users++;
  pool->sum_size += size;

 fail:
  return pool;
 ovf:
  ha_alert("Failed to create pool '%s' of size '%llu': overflow detected due to too large "
           "a configured size and/or configured pool options. Aborting.\n",
           name, reg_size);
  return NULL;
}

/* Tries to allocate an object for the pool <pool> using the system's allocator
 * and directly returns it. The pool's allocated counter is checked but NOT
 * updated, this is left to the caller, and but no other checks are performed.
 */
void *pool_get_from_os_noinc(struct pool_head *pool)
{
  if (!pool->limit || pool_allocated(pool) < pool->limit) {
    void *ptr;

    if ((pool_debugging & POOL_DBG_UAF) || (pool->flags & MEM_F_UAF))
      ptr = pool_alloc_area_uaf(pool->alloc_sz, pool->align);
    else
      ptr = pool_alloc_area(pool->alloc_sz, pool->align);
    if (ptr)
      return ptr;
    _HA_ATOMIC_INC(&pool->buckets[pool_tbucket()].failed);
  }
  activity[tid].pool_fail++;
  return NULL;

}

/* Releases a pool item back to the operating system but DOES NOT update
 * the allocation counter, it's left to the caller to do it. It may be
 * done before or after, it doesn't matter, the function does not use it.
 */
void pool_put_to_os_nodec(struct pool_head *pool, void *ptr)
{
  if ((pool_debugging & POOL_DBG_UAF) || (pool->flags & MEM_F_UAF))
    pool_free_area_uaf(ptr, pool->alloc_sz);
  else
    pool_free_area(ptr, pool->alloc_sz);
}

/* Tries to allocate an object for the pool <pool> using the system's allocator
 * and directly returns it. The pool's counters are updated but the object is
 * never cached, so this is usable with and without local or shared caches.
 */
void *pool_alloc_nocache(struct pool_head *pool, const void *caller)
{
  void *ptr = NULL;
  uint bucket;
  uint used;

  ptr = pool_get_from_os_noinc(pool);
  if (!ptr)
    return NULL;

  bucket = pool_pbucket(ptr);

  _HA_ATOMIC_INC(&pool->buckets[bucket].allocated);
  used = _HA_ATOMIC_FETCH_ADD(&pool->buckets[bucket].used, 1);
  swrate_add_scaled_opportunistic(&pool->buckets[bucket].needed_avg, POOL_AVG_SAMPLES, used, POOL_AVG_SAMPLES/4);

  /* keep track of where the element was allocated from */
  POOL_DEBUG_SET_MARK(pool, ptr);
  POOL_DEBUG_TRACE_CALLER(pool, (struct pool_cache_item *)ptr, caller);
  return ptr;
}

/* Release a pool item back to the OS and keeps the pool's counters up to date.
 * This is always defined even when pools are not enabled (their usage stats
 * are maintained).
 */
void pool_free_nocache(struct pool_head *pool, void *ptr)
{
  uint bucket = pool_pbucket(ptr);
  uint used;

  used = _HA_ATOMIC_SUB_FETCH(&pool->buckets[bucket].used, 1);
  _HA_ATOMIC_DEC(&pool->buckets[bucket].allocated);
  swrate_add_opportunistic(&pool->buckets[bucket].needed_avg, POOL_AVG_SAMPLES, used);

  pool_put_to_os_nodec(pool, ptr);
}

/* Copies the contents of <item> to the reserved area after it to have a backup.
 * The item part is left untouched.
 */
void pool_copy_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size)
{
  ulong *ptr = (ulong *)item;
  ulong extra;

  if (size <= sizeof(*item))
    return;

  extra = !!(pool_debugging & (POOL_DBG_TAG|POOL_DBG_CALLER));
  memcpy(&ptr[size/sizeof(*ptr) + extra], ptr, size);
}

/* Updates <pch>'s fill_pattern and fills the free area after <item> with it,
 * up to <size> bytes. The item part is left untouched.
 */
void pool_fill_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size)
{
  ulong *ptr = (ulong *)item;
  uint ofs;
  ulong u;

  if (size <= sizeof(*item))
    return;

  /* Upgrade the fill_pattern to change about half of the bits
   * (to be sure to catch static flag corruption), and apply it.
   */
  u = pch->fill_pattern += ~0UL / 3; // 0x55...55
  ofs = sizeof(*item) / sizeof(*ptr);
  while (ofs < size / sizeof(*ptr))
    ptr[ofs++] = u;
}

/* check for a pool_cache_item integrity after extracting it from the cache. It
 * must have been previously initialized using either pool_fill_pattern() or
 * pool_copy_pattern(). If any corruption is detected, the function provokes an
 * immediate crash.
 */
void pool_check_pattern(struct pool_cache_head *pch, struct pool_head *pool, struct pool_cache_item *item, const void *caller)
{
  const ulong *ptr = (const ulong *)item;
  uint size = pool->size;
  uint ofs;
  ulong u;

  if (size <= sizeof(*item))
    return;

  if (pool_debugging & POOL_DBG_INTEGRITY) {
    /* let's check that all words past *item are equal */
    ofs = sizeof(*item) / sizeof(*ptr);
    u = ptr[ofs++];
    while (ofs < size / sizeof(*ptr)) {
      if (unlikely(ptr[ofs] != u)) {
        pool_inspect_item("cache corruption detected", pool, item, caller, ofs * sizeof(*ptr));
        ABORT_NOW();
      }
      ofs++;
    }
  } else {
    /* the pattern was backed up */
    ofs = sizeof(*item) / sizeof(*ptr);
    u = !!(pool_debugging & (POOL_DBG_TAG|POOL_DBG_CALLER));
    while (ofs < size / sizeof(*ptr)) {
      if (unlikely(ptr[ofs] != ptr[size/sizeof(*ptr) + u + ofs])) {
        pool_inspect_item("cache corruption detected", pool, item, caller, ofs * sizeof(*ptr));
        ABORT_NOW();
      }
      ofs++;
    }
  }
}

/* removes up to <count> items from the end of the local pool cache <ph> for
 * pool <pool>. The shared pool is refilled with these objects in the limit
 * of the number of acceptable objects, and the rest will be released to the
 * OS. It is not a problem is <count> is larger than the number of objects in
 * the local cache. The counters are automatically updated. Must not be used
 * with pools disabled.
 */
static void pool_evict_last_items(struct pool_head *pool, struct pool_cache_head *ph, uint count)
{
  struct pool_cache_item *item;
  struct pool_item *pi, *head = NULL;
  void *caller = __builtin_return_address(0);
  uint released = 0;
  uint cluster = 0;
  uint to_free_max;
  uint bucket;
  uint used;

  BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

  /* Note: this will be zero when global pools are disabled */
  to_free_max = pool_releasable(pool);

  while (released < count && !LIST_ISEMPTY(&ph->list)) {
    item = LIST_PREV(&ph->list, typeof(item), by_pool);
    BUG_ON(&item->by_pool == &ph->list);
    if (unlikely(pool_debugging & (POOL_DBG_INTEGRITY|POOL_DBG_BACKUP)))
      pool_check_pattern(ph, pool, item, caller);
    LIST_DELETE(&item->by_pool);
    LIST_DELETE(&item->by_lru);

    bucket = pool_pbucket(item);
    used = _HA_ATOMIC_SUB_FETCH(&pool->buckets[bucket].used, 1);
    swrate_add_opportunistic(&pool->buckets[bucket].needed_avg, POOL_AVG_SAMPLES, used);

    if (to_free_max > released || cluster) {
      /* will never match when global pools are disabled */
      pi = (struct pool_item *)item;
      pi->next = NULL;
      pi->down = head;
      head = pi;
      cluster++;
      if (cluster >= CONFIG_HAP_POOL_CLUSTER_SIZE) {
        /* enough to make a cluster */
        pool_put_to_shared_cache(pool, head);
        cluster = 0;
        head = NULL;
      }
    } else {
      /* does pool_free_nocache() with a known bucket */
      _HA_ATOMIC_DEC(&pool->buckets[bucket].allocated);
      pool_put_to_os_nodec(pool, item);
    }

    released++;
  }

  /* incomplete cluster left */
  if (cluster)
    pool_put_to_shared_cache(pool, head);

  ph->count -= released;
  pool_cache_count -= released;
  pool_cache_bytes -= released * pool->size;
}

/* Evicts some of the oldest objects from one local cache, until its number of
 * objects is no more than 16+1/8 of the total number of locally cached objects
 * or the total size of the local cache is no more than 75% of its maximum (i.e.
 * we don't want a single cache to use all the cache for itself). For this, the
 * list is scanned in reverse. If <full> is non-null, all objects are evicted.
 * Must not be used when pools are disabled.
 */
void pool_evict_from_local_cache(struct pool_head *pool, int full)
{
  struct pool_cache_head *ph = &pool->cache[tid];

  BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

  while ((ph->count && full) ||
         (ph->count >= CONFIG_HAP_POOL_CLUSTER_SIZE &&
          ph->count >= 16 + pool_cache_count / 8 &&
          pool_cache_bytes > global.tune.pool_cache_size * 3 / 4)) {
    pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE);
  }
}

/* Evicts some of the oldest objects from the local cache, pushing them to the
 * global pool. Must not be used when pools are disabled.
 */
void pool_evict_from_local_caches()
{
  struct pool_cache_item *item;
  struct pool_cache_head *ph;
  struct pool_head *pool;

  BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

  do {
    item = LIST_PREV(&th_ctx->pool_lru_head, struct pool_cache_item *, by_lru);
    BUG_ON(&item->by_lru == &th_ctx->pool_lru_head);
    /* note: by definition we remove oldest objects so they also are the
     * oldest in their own pools, thus their next is the pool's head.
     */
    ph = LIST_NEXT(&item->by_pool, struct pool_cache_head *, list);
    BUG_ON(ph->tid != tid);

    pool = container_of(ph - tid, struct pool_head, cache);
    BUG_ON(pool != ph->pool);

    pool_evict_last_items(pool, ph, CONFIG_HAP_POOL_CLUSTER_SIZE);
  } while (pool_cache_bytes > global.tune.pool_cache_size * 7 / 8);
}

/* Frees an object to the local cache, possibly pushing oldest objects to the
 * shared cache, which itself may decide to release some of them to the OS.
 * While it is unspecified what the object becomes past this point, it is
 * guaranteed to be released from the users' perspective. A caller address may
 * be passed and stored into the area when DEBUG_POOL_TRACING is set. Must not
 * be used with pools disabled.
 */
void pool_put_to_cache(struct pool_head *pool, void *ptr, const void *caller)
{
  struct pool_cache_item *item = (struct pool_cache_item *)ptr;
  struct pool_cache_head *ph = &pool->cache[tid];

  BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

  LIST_INSERT(&ph->list, &item->by_pool);
  LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru);
  POOL_DEBUG_TRACE_CALLER(pool, item, caller);
  ph->count++;
  if (unlikely(pool_debugging & POOL_DBG_BACKUP))
    pool_copy_pattern(ph, item, pool->size);

  if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
    pool_fill_pattern(ph, item, pool->size);

  pool_cache_count++;
  pool_cache_bytes += pool->size;

  if (unlikely(pool_cache_bytes > global.tune.pool_cache_size * 3 / 4)) {
    uint64_t mem_wait_start = 0;

    if (unlikely((th_ctx->flags & (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)) ==
                 (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)))
      mem_wait_start = now_mono_time();

    if (ph->count >= 16 + pool_cache_count / 8 + CONFIG_HAP_POOL_CLUSTER_SIZE)
      pool_evict_from_local_cache(pool, 0);
    if (pool_cache_bytes > global.tune.pool_cache_size)
      pool_evict_from_local_caches();

    if (unlikely(mem_wait_start))
      th_ctx->mem_wait_total += now_mono_time() - mem_wait_start;
  }
}

/* Tries to refill the local cache <pch> from the shared one for pool <pool>.
 * This is only used when pools are in use and shared pools are enabled. No
 * malloc() is attempted, and poisonning is never performed. The purpose is to
 * get the fastest possible refilling so that the caller can easily check if
 * the cache has enough objects for its use. Must not be used when pools are
 * disabled.
 */
void pool_refill_local_from_shared(struct pool_head *pool, struct pool_cache_head *pch)
{
  struct pool_cache_item *item;
  struct pool_item *ret, *down;
  uint bucket;
  uint count;

  BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

  /* we'll need to reference the first element to figure the next one. We
   * must temporarily lock it so that nobody allocates then releases it,
   * or the dereference could fail. In order to limit the locking,
   * threads start from a bucket that depends on their ID.
   */

  bucket = pool_tbucket();
  ret = _HA_ATOMIC_LOAD(&pool->buckets[bucket].free_list);
  count = 0;
  do {
    /* look for an apparently non-busy entry. If we hit a busy pool
     * we retry with another random bucket. And if we encounter a
     * NULL, we retry once with another random bucket. This is in
     * order to prevent object accumulation in other buckets.
     */
    while (unlikely(ret == POOL_BUSY || (ret == NULL && count++ < 1))) {
      bucket = statistical_prng() % CONFIG_HAP_POOL_BUCKETS;
      ret = _HA_ATOMIC_LOAD(&pool->buckets[bucket].free_list);
    }
    if (ret == NULL)
      return;
  } while (unlikely((ret = _HA_ATOMIC_XCHG(&pool->buckets[bucket].free_list, POOL_BUSY)) == POOL_BUSY));

  if (unlikely(ret == NULL)) {
    HA_ATOMIC_STORE(&pool->buckets[bucket].free_list, NULL);
    return;
  }

  /* this releases the lock */
  HA_ATOMIC_STORE(&pool->buckets[bucket].free_list, ret->next);

  /* now store the retrieved object(s) into the local cache. Note that
   * they don't all have the same hash and that it doesn't necessarily
   * match the one from the pool.
   */
  count = 0;
  for (; ret; ret = down) {
    down = ret->down;
    item = (struct pool_cache_item *)ret;
    POOL_DEBUG_TRACE_CALLER(pool, item, NULL);
    LIST_INSERT(&pch->list, &item->by_pool);
    LIST_INSERT(&th_ctx->pool_lru_head, &item->by_lru);
    _HA_ATOMIC_INC(&pool->buckets[pool_pbucket(item)].used);
    count++;
    if (unlikely(pool_debugging & POOL_DBG_INTEGRITY))
      pool_fill_pattern(pch, item, pool->size);

  }
  pch->count += count;
  pool_cache_count += count;
  pool_cache_bytes += count * pool->size;
}

/* Adds pool item cluster <item> to the shared cache, which contains <count>
 * elements. The caller is advised to first check using pool_releasable() if
 * it's wise to add this series of objects there. Both the pool and the item's
 * head must be valid.
 */
void pool_put_to_shared_cache(struct pool_head *pool, struct pool_item *item)
{
  struct pool_item *free_list;
  uint bucket = pool_pbucket(item);

  /* we prefer to put the item into the entry that corresponds to its own
   * hash so that on return it remains in the right place, but that's not
   * mandatory.
   */
  free_list = _HA_ATOMIC_LOAD(&pool->buckets[bucket].free_list);
  do {
    /* look for an apparently non-busy entry */
    while (unlikely(free_list == POOL_BUSY)) {
      bucket = (bucket + 1) % CONFIG_HAP_POOL_BUCKETS;
      free_list = _HA_ATOMIC_LOAD(&pool->buckets[bucket].free_list);
    }
    _HA_ATOMIC_STORE(&item->next, free_list);
    __ha_barrier_atomic_store();
  } while (!_HA_ATOMIC_CAS(&pool->buckets[bucket].free_list, &free_list, item));
  __ha_barrier_atomic_store();
}

/*
 * This function frees whatever can be freed in pool <pool>.
 */
void pool_flush(struct pool_head *pool)
{
  struct pool_item *next, *temp, *down;
  uint bucket;

  if (!pool || (pool_debugging & (POOL_DBG_NO_CACHE|POOL_DBG_NO_GLOBAL)))
    return;

  /* The loop below atomically detaches the head of the free list and
   * replaces it with a NULL. Then the list can be released.
   */
  for (bucket = 0; bucket < CONFIG_HAP_POOL_BUCKETS; bucket++) {
    next = pool->buckets[bucket].free_list;
    while (1) {
      while (unlikely(next == POOL_BUSY))
        next = (void*)pl_wait_new_long((ulong*)&pool->buckets[bucket].free_list, (ulong)next);

      if (next == NULL)
        break;

      next = _HA_ATOMIC_XCHG(&pool->buckets[bucket].free_list, POOL_BUSY);
      if (next != POOL_BUSY) {
        HA_ATOMIC_STORE(&pool->buckets[bucket].free_list, NULL);
        break;
      }
    }

    while (next) {
      temp = next;
      next = temp->next;
      for (; temp; temp = down) {
        down = temp->down;
        _HA_ATOMIC_DEC(&pool->buckets[pool_pbucket(temp)].allocated);
        pool_put_to_os_nodec(pool, temp);
      }
    }
  }
  /* here, we should have pool->allocated == pool->used */
}

/*
 * This function frees whatever can be freed in all pools, but respecting
 * the minimum thresholds imposed by owners. It makes sure to be alone to
 * run by using thread_isolate(). <pool_ctx> is unused.
 */
void pool_gc(struct pool_head *pool_ctx)
{
  struct pool_head *entry;
  uint64_t mem_wait_start = 0;
  int isolated = thread_isolated();

  if (unlikely((th_ctx->flags & (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)) ==
               (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)))
    mem_wait_start = now_mono_time();

  if (!isolated)
    thread_isolate();

  list_for_each_entry(entry, &pools, list) {
    struct pool_item *temp, *down;
    uint allocated = pool_allocated(entry);
    uint used = pool_used(entry);
    int bucket = 0;

    while ((int)(allocated - used) > (int)entry->minavail) {
      /* ok let's find next entry to evict */
      while (!entry->buckets[bucket].free_list && bucket < CONFIG_HAP_POOL_BUCKETS)
        bucket++;

      if (bucket >= CONFIG_HAP_POOL_BUCKETS)
        break;

      temp = entry->buckets[bucket].free_list;
      entry->buckets[bucket].free_list = temp->next;
      for (; temp; temp = down) {
        down = temp->down;
        allocated--;
        _HA_ATOMIC_DEC(&entry->buckets[pool_pbucket(temp)].allocated);
        pool_put_to_os_nodec(entry, temp);
      }
    }
  }

  trim_all_pools();

  if (!isolated)
    thread_release();

  if (mem_wait_start)
    th_ctx->mem_wait_total += now_mono_time() - mem_wait_start;
}

/*
 * Returns a pointer to type <type> taken from the pool <pool_type> or
 * dynamically allocated. In the first case, <pool_type> is updated to point to
 * the next element in the list. <flags> is a binary-OR of POOL_F_* flags.
 * Prefer using pool_alloc() which does the right thing without flags.
 */
void *__pool_alloc(struct pool_head *pool, unsigned int flags)
{
  void *p = NULL;
  void *caller = __builtin_return_address(0);

  if (unlikely(pool_debugging & POOL_DBG_FAIL_ALLOC))
    if (!(flags & POOL_F_NO_FAIL) && mem_should_fail(pool))
      return NULL;

  if (likely(!(pool_debugging & POOL_DBG_NO_CACHE)) && !p)
    p = pool_get_from_cache(pool, caller);

  if (unlikely(!p)) {
    /* count allocation time only for cache misses */
    uint64_t mem_wait_start = 0;

    if (unlikely((th_ctx->flags & (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)) ==
                       (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)))
      mem_wait_start = now_mono_time();

    p = pool_alloc_nocache(pool, caller);

    if (unlikely(mem_wait_start))
      th_ctx->mem_wait_total += now_mono_time() - mem_wait_start;
  }

  if (likely(p)) {
#ifdef USE_MEMORY_PROFILING
    if (unlikely(profiling & HA_PROF_MEMORY)) {
      extern struct memprof_stats memprof_stats[MEMPROF_HASH_BUCKETS + 1];
      struct memprof_stats *bin;

      bin = memprof_get_bin(__builtin_return_address(0), MEMPROF_METH_P_ALLOC);
      if (unlikely(th_ctx->lock_level & 0x7F))
        _HA_ATOMIC_ADD(&bin->locked_calls, 1);
      _HA_ATOMIC_ADD(&bin->alloc_calls, 1);
      _HA_ATOMIC_ADD(&bin->alloc_tot, pool->size);
      _HA_ATOMIC_STORE(&bin->info, pool);
      /* replace the caller with the allocated bin: this way
       * we'll the pool_free() call will be able to update our
       * entry. We only do it for non-colliding entries though,
       * since these ones store the true caller location.
       */
      if (bin >= &memprof_stats[0] && bin < &memprof_stats[MEMPROF_HASH_BUCKETS])
        POOL_DEBUG_TRACE_CALLER(pool, (struct pool_cache_item *)p, bin);
    }
#endif
    if (unlikely(flags & POOL_F_MUST_ZERO))
      memset(p, 0, pool->size);
    else if (unlikely(!(flags & POOL_F_NO_POISON) && (pool_debugging & POOL_DBG_POISON)))
      memset(p, mem_poison_byte, pool->size);
  }
  return p;
}

/*
 * Puts a memory area back to the corresponding pool. <ptr> be valid. Using
 * pool_free() is preferred.
 */
void __pool_free(struct pool_head *pool, void *ptr)
{
  const void *caller = __builtin_return_address(0);

  /* we'll get late corruption if we refill to the wrong pool or double-free */
  POOL_DEBUG_CHECK_MARK(pool, ptr, caller);
  POOL_DEBUG_RESET_MARK(pool, ptr);

#ifdef USE_MEMORY_PROFILING
  if (unlikely(profiling & HA_PROF_MEMORY) && ptr) {
    extern struct memprof_stats memprof_stats[MEMPROF_HASH_BUCKETS + 1];
    struct memprof_stats *bin;

    bin = memprof_get_bin(__builtin_return_address(0), MEMPROF_METH_P_FREE);
    if (unlikely(th_ctx->lock_level & 0x7F))
      _HA_ATOMIC_ADD(&bin->locked_calls, 1);
    _HA_ATOMIC_ADD(&bin->free_calls, 1);
    _HA_ATOMIC_ADD(&bin->free_tot, pool->size);
    _HA_ATOMIC_STORE(&bin->info, pool);

    /* check if the caller is an allocator, and if so, let's update
     * its free() count.
     */
    bin = *(struct memprof_stats**)(((char *)ptr) + pool->alloc_sz - sizeof(void*));
    if (bin >= &memprof_stats[0] && bin < &memprof_stats[MEMPROF_HASH_BUCKETS]) {
      _HA_ATOMIC_ADD(&bin->free_calls, 1);
      _HA_ATOMIC_ADD(&bin->free_tot, pool->size);
    }
  }
#endif

  if (unlikely((pool_debugging & POOL_DBG_NO_CACHE) ||
               (pool->flags & MEM_F_UAF) ||
         global.tune.pool_cache_size < pool->size)) {
    uint64_t mem_wait_start = 0;

    if (unlikely((th_ctx->flags & (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)) ==
                       (TH_FL_TASK_PROFILING|TH_FL_TASK_PROFILING_M)))
      mem_wait_start = now_mono_time();

    pool_free_nocache(pool, ptr);

    if (unlikely(mem_wait_start))
      th_ctx->mem_wait_total += now_mono_time() - mem_wait_start;
  }
  else {
    /* this one will count its own time itself */
    pool_put_to_cache(pool, ptr, caller);
  }
}

/*
 * This function destroys a pool by freeing it completely, unless it's still
 * in use. This should be called only under extreme circumstances. It always
 * returns NULL if the resulting pool is empty, easing the clearing of the old
 * pointer, otherwise it returns the pool.
 * .
 */
void *pool_destroy(struct pool_head *pool)
{
  if (pool) {
    if (!(pool_debugging & POOL_DBG_NO_CACHE))
      pool_evict_from_local_cache(pool, 1);

    pool_flush(pool);
    if (pool_used(pool))
      return pool;
    pool->users--;
    if (!pool->users) {
      /* remove all registrations at once */
      struct pool_registration *reg, *back;

      list_for_each_entry_safe(reg, back, &pool->regs, list) {
        LIST_DELETE(&reg->list);
        if (!(reg->flags & MEM_F_STATREG))
          free(reg);
      }

      LIST_DELETE(&pool->list);

      /* note that if used == 0, the cache is empty */
      free(pool->base_addr);
    }

    /* make sure this pool is no longer referenced in memory profiling */
    memprof_remove_stale_info(pool);
  }
  return NULL;
}

/* This destroys all pools on exit. It is *not* thread safe. */
void pool_destroy_all()
{
  struct pool_head *entry, *back;

  list_for_each_entry_safe(entry, back, &pools, list) {
    /* there's only one occurrence of each pool in the list,
     * and we're existing instead of looping on the whole
     * list just to decrement users, force it to 1 here.
     */
    entry->users = 1;
    pool_destroy(entry);
  }
}

/* carefully inspects an item upon fatal error and emit diagnostics.
 * If ofs < 0, no hint is provided regarding the content location. However if
 * ofs >= 0, then we also try to inspect around that place where corruption
 * was detected.
 */
void pool_inspect_item(const char *msg, struct pool_head *pool, const void *item, const void *caller, ssize_t ofs)
{
  const struct pool_head *the_pool = NULL;

  chunk_printf(&trash,
         "FATAL: pool inconsistency detected in thread %d: %s.\n"
         "  caller: %p (",
         tid + 1, msg, caller);

  resolve_sym_name(&trash, NULL, caller);

  chunk_appendf(&trash,
          ")\n"
          "  item: %p\n"
          "  pool: %p ('%s', size %u, real %u, users %u)\n",
          item, pool, pool->name, pool->size, pool->alloc_sz, pool->users);

  if (ofs >= 0) {
    chunk_printf(&trash, "Contents around first corrupted address relative to pool item:.\n");
    dump_area_with_syms(&trash, item, item + ofs, NULL, NULL, NULL);
  }

  if (pool_debugging & POOL_DBG_TAG) {
    const void **pool_mark;
    struct pool_head *ph;
    const void *tag;

    pool_mark = (const void **)(((char *)item) + pool->size);
    tag =  may_access(pool_mark) ? *pool_mark : NULL;
    if (tag == pool) {
      chunk_appendf(&trash, "  tag: @%p = %p (%s)\n", pool_mark, tag, pool->name);
      the_pool = pool;
    }
    else {
      if (!may_access(pool_mark))
        chunk_appendf(&trash, "Tag not accessible. ");
      else
        chunk_appendf(&trash, "Tag does not match (%p). ", tag);

      list_for_each_entry(ph, &pools, list) {
        pool_mark = (const void **)(((char *)item) + ph->size);
        if (!may_access(pool_mark))
          continue;
        tag =  *pool_mark;

        if (tag == ph) {
          if (!the_pool)
            chunk_appendf(&trash, "Possible origin pool(s):\n");

          chunk_appendf(&trash, "  tag: @%p = %p (%s, size %u, real %u, users %u)\n",
                  pool_mark, tag, ph->name, ph->size, ph->alloc_sz, ph->users);
          if (!the_pool || the_pool->size < ph->size)
            the_pool = ph;
        }
      }

      if (!the_pool) {
        chunk_appendf(&trash,
                "Tag does not match any other pool.\n");

        pool_mark = (const void **)(((char *)item) + pool->size);
        if (resolve_sym_name(&trash, "Resolving the tag as a pool_free() location: ", *pool_mark))
          chunk_appendf(&trash, "\n");
        else
          chunk_appendf(&trash, " (no match).\n");

        dump_area_with_syms(&trash, item, pool_mark, pool, "pool", pool->name);
      }
    }
  }

  if (pool_debugging & POOL_DBG_CALLER) {
    struct buffer *trash2 = get_trash_chunk();
    const struct pool_head *ph;
    const void **pool_mark;
    const void *tag, *rec_tag;

    ph = the_pool ? the_pool : pool;
    pool_mark = (const void **)(((char *)item) + ph->alloc_sz - sizeof(void*));
    rec_tag =  may_access(pool_mark) ? *pool_mark : NULL;

    if (rec_tag && resolve_sym_name(trash2, NULL, rec_tag))
      chunk_appendf(&trash,
              "Recorded caller if pool '%s':\n  @%p (+%04u) = %p (%s)\n",
              ph->name, pool_mark, (uint)(ph->alloc_sz - sizeof(void*)),
              rec_tag, trash2->area);

    if (!the_pool) {
      /* the pool couldn't be formally verified */
      chunk_appendf(&trash, "Other possible callers:\n");
      list_for_each_entry(ph, &pools, list) {
        if (ph == pool)
          continue;
        pool_mark = (const void **)(((char *)item) + ph->alloc_sz - sizeof(void*));
        if (!may_access(pool_mark))
          continue;
        tag = *pool_mark;
        if (tag == rec_tag)
          continue;

        /* see if we can resolve something */
        chunk_printf(trash2, "@%p (+%04u) = %p (", pool_mark, (uint)(ph->alloc_sz - sizeof(void*)), tag);
        if (resolve_sym_name(trash2, NULL, tag)) {
          chunk_appendf(trash2, ")");
          chunk_appendf(&trash,
                  "  %s [as pool %s, size %u, real %u, users %u]\n",
                  trash2->area, ph->name, ph->size, ph->alloc_sz, ph->users);
        }
      }
    }
  }

  chunk_appendf(&trash, "\n");
  DISGUISE(write(2, trash.area, trash.data));
}

/* used by qsort in "show pools" to sort by name */
static int cmp_dump_pools_name(const void *a, const void *b)
{
  const struct pool_dump_info *l = (const struct pool_dump_info *)a;
  const struct pool_dump_info *r = (const struct pool_dump_info *)b;

  return strcmp(l->entry->name, r->entry->name);
}

/* used by qsort in "show pools" to sort by item size */
static int cmp_dump_pools_size(const void *a, const void *b)
{
  const struct pool_dump_info *l = (const struct pool_dump_info *)a;
  const struct pool_dump_info *r = (const struct pool_dump_info *)b;

  if (l->entry->size > r->entry->size)
    return -1;
  else if (l->entry->size < r->entry->size)
    return 1;
  else
    return 0;
}

/* used by qsort in "show pools" to sort by usage */
static int cmp_dump_pools_usage(const void *a, const void *b)
{
  const struct pool_dump_info *l = (const struct pool_dump_info *)a;
  const struct pool_dump_info *r = (const struct pool_dump_info *)b;

  if (l->alloc_bytes > r->alloc_bytes)
    return -1;
  else if (l->alloc_bytes < r->alloc_bytes)
    return 1;
  else
    return 0;
}

/* This function fills the show_pools_ctx context by looping on pools. It will
 * count the number of pools and allocate the array to store info about each
 * pool. It returns 0 on success and -1 on error.
 *
 * It may sort pools by a criterion if bits 0..3 of <ctx->how> are non-zero, and
 * limit the number of output lines if <ctx->maxcnt> is non-zero. It may limit
 * only to pools whose names start with <ctx->prefix> if not null.
 */
static int get_show_pools_info(struct show_pools_ctx *ctx)
{
  struct pool_head *entry;
  int by_what = (ctx->how & 0xF); // bits 0..3 = sorting criterion
  int n = 0;

  ctx->nbpools = 0;
  list_for_each_entry(entry, &pools, list)
    ctx->nbpools++;

  ctx->pool_info = calloc(ctx->nbpools, sizeof(*ctx->pool_info));
  if (ctx->pool_info == NULL)
    return -1;

  list_for_each_entry(entry, &pools, list) {
    uint cached = 0;
    uint alloc_items;

    alloc_items = pool_allocated(entry);
    /* do not dump unused entries when sorting by usage */
    if (by_what == 3 && !alloc_items)
      continue;

    /* verify the pool name if a prefix is requested */
    if (ctx->prefix && strncmp(entry->name, ctx->prefix, strlen(ctx->prefix)) != 0)
      continue;

    if (!(pool_debugging & POOL_DBG_NO_CACHE)) {
      int i;

      for (i = 0; i < global.nbthread; i++)
        cached += entry->cache[i].count;
    }
    ctx->pool_info[n].entry = entry;
    ctx->pool_info[n].alloc_items = alloc_items;
    ctx->pool_info[n].alloc_bytes = (ulong)entry->size * alloc_items;
    ctx->pool_info[n].used_items = pool_used(entry);
    ctx->pool_info[n].cached_items = cached;
    ctx->pool_info[n].need_avg = swrate_avg(pool_needed_avg(entry), POOL_AVG_SAMPLES);
    ctx->pool_info[n].failed_items = pool_failed(entry);

    ctx->cached_bytes += ctx->pool_info[n].cached_items * (ulong)ctx->pool_info[n].entry->size;
    ctx->allocated    += ctx->pool_info[n].alloc_items  * (ulong)ctx->pool_info[n].entry->size;
    ctx->used         += ctx->pool_info[n].used_items   * (ulong)ctx->pool_info[n].entry->size;
    n++;
  }

  /* adjust the number of entries to be dumped */
  ctx->nbpools = n;

  if (by_what == 1)  /* sort by name */
    qsort(ctx->pool_info, ctx->nbpools, sizeof(*ctx->pool_info), cmp_dump_pools_name);
  else if (by_what == 2)  /* sort by item size */
    qsort(ctx->pool_info, ctx->nbpools, sizeof(*ctx->pool_info), cmp_dump_pools_size);
  else if (by_what == 3)  /* sort by total usage */
    qsort(ctx->pool_info, ctx->nbpools, sizeof(*ctx->pool_info), cmp_dump_pools_usage);

  if (!ctx->maxcnt || ctx->maxcnt > ctx->nbpools)
    ctx->maxcnt = ctx->nbpools;
  ctx->pool_idx = -1;
  return 0;
}

/* This function dumps information about pools found in the applet context.
 */
int dump_pools_info(struct appctx *appctx)
{
  struct show_pools_ctx *ctx = appctx->svcctx;
  int i;

  if (ctx->pool_idx == -1) {
    chunk_printf(&trash, "Dumping pools usage");
    if (ctx->nbpools > ctx->maxcnt)
      chunk_appendf(&trash, " (limited to the first %u entries)", ctx->maxcnt);
    chunk_appendf(&trash, ". Use SIGQUIT to flush them.\n");

    if (applet_putchk(appctx, &trash) == -1)
      return 0;

    ctx->pool_idx = 0;
  }

  for (i = ctx->pool_idx; i < ctx->nbpools && i < ctx->maxcnt; i++) {
    chunk_printf(&trash, "  - Pool %s (%u bytes/%u) : %lu allocated (%lu bytes), %lu used"
           " (~%lu by thread caches)"
           ", needed_avg %lu, %lu failures, %u users, @%p%s\n",
           ctx->pool_info[i].entry->name, ctx->pool_info[i].entry->size, ctx->pool_info[i].entry->align,
           ctx->pool_info[i].alloc_items, ctx->pool_info[i].alloc_bytes,
           ctx->pool_info[i].used_items, ctx->pool_info[i].cached_items,
           ctx->pool_info[i].need_avg, ctx->pool_info[i].failed_items,
           ctx->pool_info[i].entry->users, ctx->pool_info[i].entry,
           (ctx->pool_info[i].entry->flags & MEM_F_SHARED) ? " [SHARED]" : "");

    if (ctx->how & 0x10) { // print details
      struct pool_registration *reg;

      list_for_each_entry(reg, &ctx->pool_info[i].entry->regs, list) {
        chunk_appendf(&trash, "      >  %-12s: size=%llu flags=%#x align=%u", reg->name, reg->size, reg->flags, reg->align);
        if (reg->file && reg->line)
          chunk_appendf(&trash, " [%s:%u]", reg->file, reg->line);
        chunk_appendf(&trash, "\n");
      }
    }

    if (applet_putchk(appctx, &trash) == -1) {
      ctx->pool_idx = i;
      return 0;
    }
  }

  chunk_printf(&trash, "Total: %d pools, %llu bytes allocated, %llu used"
         " (~%llu by thread caches)"
         ".\n",
         ctx->nbpools, ctx->allocated, ctx->used, ctx->cached_bytes
    );
  if (applet_putchk(appctx, &trash) == -1)
    return 0;

  return 1;
}

/* This function returns the total number of failed pool allocations */
int pool_total_failures()
{
  struct pool_head *entry;
  int failed = 0;

  list_for_each_entry(entry, &pools, list)
    failed += pool_failed(entry);
  return failed;
}

/* This function returns the total amount of memory allocated in pools (in bytes) */
unsigned long long pool_total_allocated()
{
  struct pool_head *entry;
  unsigned long long allocated = 0;

  list_for_each_entry(entry, &pools, list)
    allocated += pool_allocated(entry) * (ullong)entry->size;
  return allocated;
}

/* This function returns the total amount of memory used in pools (in bytes) */
unsigned long long pool_total_used()
{
  struct pool_head *entry;
  unsigned long long used = 0;

  list_for_each_entry(entry, &pools, list)
    used += pool_used(entry) * (ullong)entry->size;
  return used;
}

/* This function parses a string made of a set of debugging features as
 * specified after -dM on the command line, and will set pool_debugging
 * accordingly. On success it returns a strictly positive value. It may zero
 * with the first warning in <err>, -1 with a help message in <err>, or -2 with
 * the first error in <err> return the first error in <err>. <err> is undefined
 * on success, and will be non-null and locally allocated on help/error/warning.
 * The caller must free it. Warnings are used to report features that were not
 * enabled at build time, and errors are used to report unknown features.
 */
int pool_parse_debugging(const char *str, char **err)
{
  struct ist args;
  char *end;
  uint new_dbg;
  int v;


  /* if it's empty or starts with a number, it's the mem poisonning byte */
  v = strtol(str, &end, 0);
  if (!*end || *end == ',') {
    mem_poison_byte = *str ? v : 'P';
    if (mem_poison_byte >= 0)
      pool_debugging |=  POOL_DBG_POISON;
    else
      pool_debugging &= ~POOL_DBG_POISON;
    str = end;
  }

  new_dbg = pool_debugging;

  for (args = ist(str); istlen(args); args = istadv(istfind(args, ','), 1)) {
    struct ist feat = iststop(args, ',');

    if (!istlen(feat))
      continue;

    if (isteq(feat, ist("help"))) {
      ha_free(err);
      memprintf(err,
          "-dM alone enables memory poisonning with byte 0x50 on allocation. A numeric\n"
          "value may be appended immediately after -dM to use another value (0 supported).\n"
          "Then an optional list of comma-delimited keywords may be appended to set or\n"
          "clear some debugging options ('*' marks the current setting):\n\n"
          "    set               clear            description\n"
          "  -----------------+-----------------+-----------------------------------------\n");

      for (v = 0; dbg_options[v].flg; v++) {
        memprintf(err, "%s  %c %-15s|%c %-15s| %s\n",
            *err,
            (pool_debugging & dbg_options[v].flg) ? '*' : ' ',
            dbg_options[v].set,
            (pool_debugging & dbg_options[v].flg) ? ' ' : '*',
            dbg_options[v].clr,
            dbg_options[v].hlp);
      }

      memprintf(err,
                "%s  -----------------+-----------------+-----------------------------------------\n"
          "Examples:\n"
          "  Disable merging and enable poisonning with byte 'P': -dM0x50,no-merge\n"
          "  Randomly fail allocations: -dMfail\n"
          "  Detect out-of-bound corruptions: -dMno-merge,tag\n"
          "  Detect post-free cache corruptions: -dMno-merge,cold-first,integrity,caller\n"
          "  Detect all cache corruptions: -dMno-merge,cold-first,integrity,tag,caller\n"
          "  Detect UAF (disables cache, very slow): -dMuaf\n"
          "  Detect post-cache UAF: -dMuaf,cache,no-merge,cold-first,integrity,tag,caller\n"
          "  Detect post-free cache corruptions: -dMno-merge,cold-first,integrity,caller\n",
                *err);
      return -1;
    }

    for (v = 0; dbg_options[v].flg; v++) {
      if (isteq(feat, ist(dbg_options[v].set))) {
        new_dbg |= dbg_options[v].flg;
        /* UAF implicitly disables caching, but it's
         * still possible to forcefully re-enable it.
         */
        if (dbg_options[v].flg == POOL_DBG_UAF)
          new_dbg |= POOL_DBG_NO_CACHE;
        /* fail should preset the tune.fail-alloc ratio to 1%  */
        if (dbg_options[v].flg == POOL_DBG_FAIL_ALLOC)
          mem_fail_rate = 1;
        break;
      }
      else if (isteq(feat, ist(dbg_options[v].clr))) {
        new_dbg &= ~dbg_options[v].flg;
        /* no-fail should reset the tune.fail-alloc ratio */
        if (dbg_options[v].flg == POOL_DBG_FAIL_ALLOC)
          mem_fail_rate = 0;
        break;
      }
    }

    if (!dbg_options[v].flg) {
      memprintf(err, "unknown pool debugging feature <%.*s>", (int)istlen(feat), istptr(feat));
      return -2;
    }
  }

  pool_debugging = new_dbg;
  return 1;
}

/* parse a "show pools" command. It returns 1 on failure, 0 if it starts to dump. */
static int cli_parse_show_pools(char **args, char *payload, struct appctx *appctx, void *private)
{
  struct show_pools_ctx *ctx = applet_reserve_svcctx(appctx, sizeof(*ctx));
  int arg;

  for (arg = 2; *args[arg]; arg++) {
    if (strcmp(args[arg], "byname") == 0) {
      ctx->how = (ctx->how & ~0xF) | 1; // sort output by name
    }
    else if (strcmp(args[arg], "bysize") == 0) {
      ctx->how = (ctx->how & ~0xF) | 2; // sort output by item size
    }
    else if (strcmp(args[arg], "byusage") == 0) {
      ctx->how = (ctx->how & ~0xF) | 3; // sort output by total allocated size
    }
    else if (strcmp(args[arg], "detailed") == 0) {
      ctx->how |= 0x10;                 // print detailed registrations
    }
    else if (strcmp(args[arg], "match") == 0 && *args[arg+1]) {
      ctx->prefix = strdup(args[arg+1]); // only pools starting with this
      if (!ctx->prefix)
        return cli_err(appctx, "Out of memory.\n");
      arg++;
    }
    else if (isdigit((unsigned char)*args[arg])) {
      ctx->maxcnt = atoi(args[arg]); // number of entries to dump
    }
    else
      return cli_err(appctx, "Expects either 'byname', 'bysize', 'byusage', 'match <pfx>', 'detailed', or a max number of output lines.\n");
  }

  if (get_show_pools_info(ctx) == -1)
    return cli_err(appctx, "Failed to get info about pools.\n");
  return 0;
}

/* release the "show pools" context */
static void cli_release_show_pools(struct appctx *appctx)
{
  struct show_pools_ctx *ctx = appctx->svcctx;

  ha_free(&ctx->pool_info);
  ha_free(&ctx->prefix);
}

/* This function dumps memory usage information onto the stream connector's
 * read buffer. It returns 0 as long as it does not complete, non-zero upon
 * completion.
 */
static int cli_io_handler_dump_pools(struct appctx *appctx)
{
  return dump_pools_info(appctx);
}

/* callback used to create early pool <name> of size <size> and store the
 * resulting pointer into <ptr>. If the allocation fails, it quits with after
 * emitting an error message.
 */
void create_pool_callback(struct pool_head **ptr, char *name, struct pool_registration *reg)
{
  *ptr = create_pool_from_reg(name, reg);
  if (!*ptr) {
    ha_alert("Failed to allocate pool '%s' of size %llu : %s. Aborting.\n",
       name, reg->size, strerror(errno));
    exit(1);
  }
}

/* Initializes all per-thread arrays on startup */
static void init_pools()
{
  int thr;

  for (thr = 0; thr < MAX_THREADS; thr++) {
    LIST_INIT(&ha_thread_ctx[thr].pool_lru_head);
  }

  detect_allocator();
}

INITCALL0(STG_PREPARE, init_pools);

/* Report in build options if trim is supported */
static void pools_register_build_options(void)
{
  if (!using_default_allocator) {
    char *ptr = NULL;
    memprintf(&ptr, "Running with a replaced memory allocator (e.g. via LD_PRELOAD).");
    hap_register_build_opts(ptr, 1);
    mark_tainted(TAINTED_REPLACED_MEM_ALLOCATOR);
  }
}
INITCALL0(STG_REGISTER, pools_register_build_options);

/* register cli keywords */
static struct cli_kw_list cli_kws = {{ },{
  { { "show", "pools",  NULL }, "show pools [by*] [match <pfx>] [nb]     : report information about the memory pools usage", cli_parse_show_pools, cli_io_handler_dump_pools, cli_release_show_pools },
  {{},}
}};

INITCALL1(STG_REGISTER, cli_register_kw, &cli_kws);


/* config parser for global "tune.fail-alloc" */
static int mem_parse_global_fail_alloc(char **args, int section_type, struct proxy *curpx,
                                       const struct proxy *defpx, const char *file, int line,
                                       char **err)
{
  if (too_many_args(1, args, err, NULL))
    return -1;
  mem_fail_rate = atoi(args[1]);
  if (mem_fail_rate < 0 || mem_fail_rate > 100) {
      memprintf(err, "'%s' expects a numeric value between 0 and 100.", args[0]);
      return -1;
  }
  return 0;
}

/* config parser for global "tune.memory.hot-size" */
static int mem_parse_global_hot_size(char **args, int section_type, struct proxy *curpx,
                                       const struct proxy *defpx, const char *file, int line,
                                       char **err)
{
  long size;

  if (too_many_args(1, args, err, NULL))
    return -1;

  size = atol(args[1]);
  if (size <= 0) {
      memprintf(err, "'%s' expects a strictly positive value.", args[0]);
      return -1;
  }

  global.tune.pool_cache_size = size;
  return 0;
}

/* config parser for global "no-memory-trimming" */
static int mem_parse_global_no_mem_trim(char **args, int section_type, struct proxy *curpx,
                                       const struct proxy *defpx, const char *file, int line,
                                       char **err)
{
  if (too_many_args(0, args, err, NULL))
    return -1;
  disable_trim = 1;
  return 0;
}

/* register global config keywords */
static struct cfg_kw_list mem_cfg_kws = {ILH, {
  { CFG_GLOBAL, "tune.fail-alloc", mem_parse_global_fail_alloc },
  { CFG_GLOBAL, "tune.memory.hot-size", mem_parse_global_hot_size },
  { CFG_GLOBAL, "no-memory-trimming", mem_parse_global_no_mem_trim },
  { 0, NULL, NULL }
}};

INITCALL1(STG_REGISTER, cfg_register_keywords, &mem_cfg_kws);

/*
 * Local variables:
 *  c-indent-level: 8
 *  c-basic-offset: 8
 * End:
 */

Coverage Report

Created: 2026-10-16 06:09