/src/haproxy/include/haproxy/pool.h

Source
/*
 * include/haproxy/pool.h
 * Memory management definitions..
 *
 * Copyright (C) 2000-2020 Willy Tarreau - w@1wt.eu
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation, version 2.1
 * exclusively.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 */

#ifndef _HAPROXY_POOL_H
#define _HAPROXY_POOL_H

#include <string.h>

#include <haproxy/api.h>
#include <haproxy/freq_ctr.h>
#include <haproxy/list.h>
#include <haproxy/pool-t.h>
#include <haproxy/thread.h>

/* This creates a pool_reg registers a call to create_pool_callback(ptr) with it.
 * Do not use this one, use REGISTER_POOL() instead.
 */
#define __REGISTER_POOL(_line, _ptr, _name, _size, _type_align, _align)    \
  static struct pool_registration __pool_reg_##_line = { \
    .name = _name,               \
    .file = __FILE__,            \
    .line = __LINE__,            \
    .size = _size,               \
    .flags = MEM_F_STATREG,            \
    .type_align = _type_align,           \
    .align = _align,             \
  };                   \
  INITCALL3(STG_POOL, create_pool_callback, (_ptr), (_name), &__pool_reg_##_line);

/* intermediary level for line number resolution, do not use this one, use
 * REGISTER_POOL() instead.
 */
#define _REGISTER_POOL(line, ptr, name, size, align, type_align)  \
  __REGISTER_POOL(line, ptr, name, size, align, type_align)

/* This registers a call to create_pool_callback(ptr) with these args */
#define REGISTER_POOL(ptr, name, size)  \
  _REGISTER_POOL(__LINE__, ptr, name, size, 0, 0)

/* This macro declares a pool head <ptr> and registers its creation */
#define DECLARE_POOL(ptr, name, size)   \
  struct pool_head *(ptr) __read_mostly = NULL; \
  _REGISTER_POOL(__LINE__, &ptr, name, size, 0, 0)

/* This macro declares a static pool head <ptr> and registers its creation */
#define DECLARE_STATIC_POOL(ptr, name, size) \
  static struct pool_head *(ptr) __read_mostly; \
  _REGISTER_POOL(__LINE__, &ptr, name, size, 0, 0)

/*** below are the aligned pool macros, taking one extra arg for alignment ***/

/* This registers a call to create_pool_callback(ptr) with these args */
#define REGISTER_ALIGNED_POOL(ptr, name, size, align) \
  _REGISTER_POOL(__LINE__, ptr, name, size, 0, align)

/* This macro declares an aligned pool head <ptr> and registers its creation */
#define DECLARE_ALIGNED_POOL(ptr, name, size, align)        \
  struct pool_head *(ptr) __read_mostly = NULL; \
  _REGISTER_POOL(__LINE__, &ptr, name, size, 0, align)

/* This macro declares a static aligned pool head <ptr> and registers its creation */
#define DECLARE_STATIC_ALIGNED_POOL(ptr, name, size, align) \
  static struct pool_head *(ptr) __read_mostly; \
  _REGISTER_POOL(__LINE__, &ptr, name, size, 0, align)

/*** below are the typed pool macros, taking a type and an extra size ***/

/* This is only used by REGISTER_TYPED_POOL below */
#define _REGISTER_TYPED_POOL(ptr, name, type, extra, align, ...)    \
  _REGISTER_POOL(__LINE__, ptr, name, sizeof(type) + extra, __alignof__(type), align)

/* This registers a call to create_pool_callback(ptr) with these args.
 * It supports two optional args:
 *  - extra: the extra size to be allocated at the end of the type. Def: 0.
 *  - align: the desired alignment on the type. Def: 0 = same as type.
 */
#define REGISTER_TYPED_POOL(ptr, name, type, args...) \
  _REGISTER_TYPED_POOL(ptr, name, type, ##args, 0, 0)

/* This macro declares an aligned pool head <ptr> and registers its creation */
#define DECLARE_TYPED_POOL(ptr, name, type, args...)        \
  struct pool_head *(ptr) __read_mostly = NULL; \
  _REGISTER_TYPED_POOL(&ptr, name, type, ##args, 0, 0)

/* This macro declares a static aligned pool head <ptr> and registers its creation */
#define DECLARE_STATIC_TYPED_POOL(ptr, name, type, args...)   \
  static struct pool_head *(ptr) __read_mostly; \
  _REGISTER_TYPED_POOL(&ptr, name, type, ##args, 0, 0)

/* By default, free objects are linked by a pointer stored at the beginning of
 * the memory area. When DEBUG_MEMORY_POOLS is set, the allocated area is
 * inflated by the size of a pointer so that the link is placed at the end
 * of the objects. Hence free objects in pools remain intact. In addition,
 * this location is used to keep a pointer to the pool the object was
 * allocated from, and verify it's freed into the appropriate one.
 */
# define POOL_EXTRA_MARK (sizeof(void *))
# define POOL_DEBUG_SET_MARK(pool, item)        \
  do {               \
    typeof(pool) __p = (pool);        \
    typeof(item) __i = (item);        \
    if (likely(!(pool_debugging & POOL_DBG_TAG)))   \
      break;           \
    *(typeof(pool)*)(((char *)__i) + __p->size) = __p;  \
  } while (0)

# define POOL_DEBUG_RESET_MARK(pool, item)        \
  do {               \
    typeof(pool) __p = (pool);        \
    typeof(item) __i = (item);        \
    if (likely(!(pool_debugging & POOL_DBG_TAG)))   \
      break;           \
    *(typeof(pool)*)(((char *)__i) + __p->size) = __builtin_return_address(0); \
  } while (0)

# define POOL_DEBUG_CHECK_MARK(pool, item, caller)        \
  do {               \
    typeof(pool) __p = (pool);        \
    typeof(item) __i = (item);        \
    if (likely(!(pool_debugging & POOL_DBG_TAG)))   \
      break;           \
    if (*(typeof(pool)*)(((char *)__i) + __p->size) != __p) { \
      pool_inspect_item("tag mismatch on free()", __p, __i, caller, -1); \
      ABORT_NOW();         \
    }             \
  } while (0)

/* It's possible to trace callers of pool_free() by placing their pointer
 * after the end of the area and the optional mark above, which means the
 * end of the allocated array.
 */
# define POOL_EXTRA_CALLER (sizeof(void *))
# define POOL_DEBUG_TRACE_CALLER(pool, item, caller)      \
  do {               \
    typeof(pool) __p = (pool);        \
    typeof(item) __i = (item);        \
    typeof(caller) __c = (caller);        \
    if (likely(!(pool_debugging & POOL_DBG_CALLER)))  \
      break;           \
    *(typeof(caller)*)(((char *)__i) + __p->alloc_sz - sizeof(void*)) = __c; \
  } while (0)

/* poison each newly allocated area with this byte if >= 0 */
extern int mem_poison_byte;

/* trim() in progress */
extern int pool_trim_in_progress;

/* set of POOL_DBG_* flags */
extern uint pool_debugging;

/* pools are listed here */
extern struct list pools;

int malloc_trim(size_t pad);
void trim_all_pools(void);

void *pool_get_from_os_noinc(struct pool_head *pool);
void pool_put_to_os_nodec(struct pool_head *pool, void *ptr);
void *pool_alloc_nocache(struct pool_head *pool, const void *caller);
void pool_free_nocache(struct pool_head *pool, void *ptr);
int pool_parse_debugging(const char *str, char **err);
int pool_total_failures(void);
unsigned long long pool_total_allocated(void);
unsigned long long pool_total_used(void);
void pool_flush(struct pool_head *pool);
void pool_gc(struct pool_head *pool_ctx);
struct pool_head *create_pool_with_loc(const char *name, unsigned int size, unsigned int align,
                                       unsigned int flags, const char *file, unsigned int line);
struct pool_head *create_pool_from_reg(const char *name, struct pool_registration *reg);
void create_pool_callback(struct pool_head **ptr, char *name, struct pool_registration *reg);
void *pool_destroy(struct pool_head *pool);
void pool_destroy_all(void);
void *__pool_alloc(struct pool_head *pool, unsigned int flags);
void __pool_free(struct pool_head *pool, void *ptr);
void pool_inspect_item(const char *msg, struct pool_head *pool, const void *item, const void *caller, ssize_t ofs);

#define create_pool(name, size, flags) \
  create_pool_with_loc(name, size, 0, flags, __FILE__, __LINE__)

#define create_aligned_pool(name, size, align, flags)     \
  create_pool_with_loc(name, size, align, flags, __FILE__, __LINE__)


/****************** Thread-local cache management ******************/

extern THREAD_LOCAL size_t pool_cache_bytes;   /* total cache size */
extern THREAD_LOCAL size_t pool_cache_count;   /* #cache objects   */

void pool_evict_from_local_cache(struct pool_head *pool, int full);
void pool_evict_from_local_caches(void);
void pool_put_to_cache(struct pool_head *pool, void *ptr, const void *caller);
void pool_fill_pattern(struct pool_cache_head *pch, struct pool_cache_item *item, uint size);
void pool_check_pattern(struct pool_cache_head *pch, struct pool_head *pool, struct pool_cache_item *item, const void *caller);
void pool_refill_local_from_shared(struct pool_head *pool, struct pool_cache_head *pch);
void pool_put_to_shared_cache(struct pool_head *pool, struct pool_item *item);

/* returns the total number of allocated entries for a pool across all buckets */
static inline uint pool_allocated(const struct pool_head *pool)
{
  int bucket;
  uint ret;

  for (bucket = ret = 0; bucket < CONFIG_HAP_POOL_BUCKETS; bucket++)
    ret += HA_ATOMIC_LOAD(&pool->buckets[bucket].allocated);
  return ret;
}

/* returns the total number of used entries for a pool across all buckets */
static inline uint pool_used(const struct pool_head *pool)
{
  int bucket;
  uint ret;

  for (bucket = ret = 0; bucket < CONFIG_HAP_POOL_BUCKETS; bucket++)
    ret += HA_ATOMIC_LOAD(&pool->buckets[bucket].used);
  return ret;
}

/* returns the raw total number needed entries across all buckets. It must
 * be passed to swrate_avg() to get something usable.
 */
static inline uint pool_needed_avg(const struct pool_head *pool)
{
  int bucket;
  uint ret;

  for (bucket = ret = 0; bucket < CONFIG_HAP_POOL_BUCKETS; bucket++)
    ret += HA_ATOMIC_LOAD(&pool->buckets[bucket].needed_avg);
  return ret;
}

/* returns the total number of failed allocations for a pool across all buckets */
static inline uint pool_failed(const struct pool_head *pool)
{
  int bucket;
  uint ret;

  for (bucket = ret = 0; bucket < CONFIG_HAP_POOL_BUCKETS; bucket++)
    ret += HA_ATOMIC_LOAD(&pool->buckets[bucket].failed);
  return ret;
}

/* Returns the max number of entries that may be brought back to the pool
 * before it's considered as full. Note that it is only usable for releasing
 * objects, hence the function assumes that no more than ->used entries will
 * be released in the worst case, and that this value is always lower than or
 * equal to ->allocated. It's important to understand that under thread
 * contention these values may not always be accurate but the principle is that
 * any deviation remains contained. When global pools are disabled, this
 * function always returns zero so that the caller knows it must free the
 * object via other ways.
 */
static inline uint pool_releasable(const struct pool_head *pool)
{
  uint alloc, used;
  uint needed_raw;

  if (unlikely(pool_debugging & (POOL_DBG_NO_CACHE|POOL_DBG_NO_GLOBAL)))
    return 0;

  alloc = pool_allocated(pool);
  used  = pool_used(pool);
  if (used > alloc)
    alloc = used;

  needed_raw = pool_needed_avg(pool);
  if (alloc < swrate_avg(needed_raw + needed_raw / 4, POOL_AVG_SAMPLES))
    return used; // less than needed is allocated, can release everything

  if ((uint)(alloc - used) < pool->minavail)
    return pool->minavail - (alloc - used); // less than minimum available

  /* there are enough objects in this pool */
  return 0;
}

/* These are generic cache-aware wrappers that allocate/free from/to the local
 * cache first, then from the second level if it exists.
 */

/* Tries to retrieve an object from the local pool cache corresponding to pool
 * <pool>. If none is available, tries to allocate from the shared cache if any
 * and returns NULL if nothing is available. Must not be used when pools are
 * disabled.
 */
static inline void *pool_get_from_cache(struct pool_head *pool, const void *caller)
{
  struct pool_cache_item *item;
  struct pool_cache_head *ph;

  BUG_ON(pool_debugging & POOL_DBG_NO_CACHE);

  ph = &pool->cache[tid];
  if (unlikely(LIST_ISEMPTY(&ph->list))) {
    if (!(pool_debugging & POOL_DBG_NO_GLOBAL))
      pool_refill_local_from_shared(pool, ph);
    if (LIST_ISEMPTY(&ph->list))
      return NULL;
  }

  /* allocate hottest objects first */
  item = LIST_NEXT(&ph->list, typeof(item), by_pool);

  if (unlikely(pool_debugging & (POOL_DBG_COLD_FIRST|POOL_DBG_INTEGRITY))) {
    /* allocate oldest objects first so as to keep them as long as possible
     * in the cache before being reused and maximizing the chance to detect
     * an overwrite.
     */
    if (pool_debugging & POOL_DBG_COLD_FIRST)
      item = LIST_PREV(&ph->list, typeof(item), by_pool);

    if (pool_debugging & POOL_DBG_INTEGRITY)
      pool_check_pattern(ph, pool, item, caller);
  }

  BUG_ON(&item->by_pool == &ph->list);
  LIST_DELETE(&item->by_pool);
  LIST_DELETE(&item->by_lru);

  /* keep track of where the element was allocated from */
  POOL_DEBUG_SET_MARK(pool, item);
  POOL_DEBUG_TRACE_CALLER(pool, item, caller);

  ph->count--;
  pool_cache_bytes -= pool->size;
  pool_cache_count--;

  return item;
}


/****************** Common high-level code ******************/

#if !defined(DEBUG_MEM_STATS)

/*
 * Returns a pointer to an object from pool <pool> allocated using
 * flags <flag> from the POOL_F_* set.
 */
#define pool_alloc_flag(pool, flag)  __pool_alloc((pool), (flag))

/*
 * Returns a pointer to type <type> taken from the pool <pool_type> or
 * dynamically allocated. Memory poisonning is performed if enabled.
 */
#define pool_alloc(pool) __pool_alloc((pool), 0)

/*
 * Returns a pointer to type <type> taken from the pool <pool_type> or
 * dynamically allocated. The area is zeroed.
 */
#define pool_zalloc(pool) __pool_alloc((pool), POOL_F_MUST_ZERO)

/*
 * Puts a memory area back to the corresponding pool. Just like with the libc's
 * free(), <ptr> may be NULL.
 */
#define pool_free(pool, ptr)        \
  do {           \
    typeof(ptr) __ptr = (ptr);    \
    if (likely((__ptr) != NULL))    \
      __pool_free(pool, __ptr); \
  } while (0)


#else /* DEBUG_MEM_STATS is set below */

#define pool_free(pool, ptr)  ({          \
  struct pool_head *__pool = (pool);        \
  typeof(ptr) __ptr = (ptr);          \
  static struct mem_stats _ __attribute__((used,__section__("mem_stats"),__aligned__(sizeof(void*)))) = { \
    .caller = {           \
      .file = __FILE__, .line = __LINE__,   \
      .what = MEM_STATS_TYPE_P_FREE,      \
      .func = __func__,       \
    },              \
  };                \
  _.extra = __pool;           \
  HA_WEAK(__start_mem_stats);         \
  HA_WEAK(__stop_mem_stats);          \
  if (__ptr)  {             \
    _HA_ATOMIC_INC(&_.calls);       \
    _HA_ATOMIC_ADD(&_.size, __pool->size);      \
    __pool_free(__pool, __ptr);       \
  }               \
})

#define pool_alloc_flag(pool, flag)  ({         \
  struct pool_head *__pool = (pool);        \
  uint __flag = (flag);           \
  size_t __x = __pool->size;          \
  static struct mem_stats _ __attribute__((used,__section__("mem_stats"),__aligned__(sizeof(void*)))) = { \
    .caller = {           \
      .file = __FILE__, .line = __LINE__,   \
      .what = MEM_STATS_TYPE_P_ALLOC,     \
      .func = __func__,       \
    },              \
  };                \
  _.extra = __pool;           \
  HA_WEAK(__start_mem_stats);         \
  HA_WEAK(__stop_mem_stats);          \
  _HA_ATOMIC_INC(&_.calls);         \
  _HA_ATOMIC_ADD(&_.size, __x);         \
  __pool_alloc(__pool, __flag);         \
})

#define pool_alloc(pool) pool_alloc_flag(pool, 0)

#define pool_zalloc(pool) pool_alloc_flag(pool, POOL_F_MUST_ZERO)

#endif /* DEBUG_MEM_STATS */

#endif /* _HAPROXY_POOL_H */

/*
 * Local variables:
 *  c-indent-level: 8
 *  c-basic-offset: 8
 * End:
 */

Coverage Report

Created: 2026-01-03 06:29

Line	Count	Source
1		/*
2		* include/haproxy/pool.h
3		* Memory management definitions..
4		*
5		* Copyright (C) 2000-2020 Willy Tarreau - w@1wt.eu
6		*
7		* This library is free software; you can redistribute it and/or
8		* modify it under the terms of the GNU Lesser General Public
9		* License as published by the Free Software Foundation, version 2.1
10		* exclusively.
11		*
12		* This library is distributed in the hope that it will be useful,
13		* but WITHOUT ANY WARRANTY; without even the implied warranty of
14		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15		* Lesser General Public License for more details.
16		*
17		* You should have received a copy of the GNU Lesser General Public
18		* License along with this library; if not, write to the Free Software
19		* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20		*/
21
22		#ifndef _HAPROXY_POOL_H
23		#define _HAPROXY_POOL_H
24
25		#include <string.h>
26
27		#include <haproxy/api.h>
28		#include <haproxy/freq_ctr.h>
29		#include <haproxy/list.h>
30		#include <haproxy/pool-t.h>
31		#include <haproxy/thread.h>
32
33		/* This creates a pool_reg registers a call to create_pool_callback(ptr) with it.
34		* Do not use this one, use REGISTER_POOL() instead.
35		*/
36		#define __REGISTER_POOL(_line, _ptr, _name, _size, _type_align, _align) \
37		static struct pool_registration __pool_reg_##_line = { \
38		.name = _name, \
39		.file = __FILE__, \