/src/haproxy/include/import/eb64tree.h

Source
/*
 * Elastic Binary Trees - macros and structures for operations on 64bit nodes.
 * Version 6.0.6
 * (C) 2002-2011 - Willy Tarreau <w@1wt.eu>
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation, version 2.1
 * exclusively.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#ifndef _EB64TREE_H
#define _EB64TREE_H

#include "ebtree.h"


/* Return the structure of type <type> whose member <member> points to <ptr> */
#define eb64_entry(ptr, type, member) container_of(ptr, type, member)

/*
 * Exported functions and macros.
 * Many of them are always inlined because they are extremely small, and
 * are generally called at most once or twice in a program.
 */

/* Return leftmost node in the tree, or NULL if none */
static inline struct eb64_node *eb64_first(struct eb_root *root)
{
  return eb64_entry(eb_first(root), struct eb64_node, node);
}

/* Return rightmost node in the tree, or NULL if none */
static inline struct eb64_node *eb64_last(struct eb_root *root)
{
  return eb64_entry(eb_last(root), struct eb64_node, node);
}

/* Return next node in the tree, or NULL if none */
static inline struct eb64_node *eb64_next(struct eb64_node *eb64)
{
  return eb64_entry(eb_next(&eb64->node), struct eb64_node, node);
}

/* Return previous node in the tree, or NULL if none */
static inline struct eb64_node *eb64_prev(struct eb64_node *eb64)
{
  return eb64_entry(eb_prev(&eb64->node), struct eb64_node, node);
}

/* Return next leaf node within a duplicate sub-tree, or NULL if none. */
static inline struct eb64_node *eb64_next_dup(struct eb64_node *eb64)
{
  return eb64_entry(eb_next_dup(&eb64->node), struct eb64_node, node);
}

/* Return previous leaf node within a duplicate sub-tree, or NULL if none. */
static inline struct eb64_node *eb64_prev_dup(struct eb64_node *eb64)
{
  return eb64_entry(eb_prev_dup(&eb64->node), struct eb64_node, node);
}

/* Return next node in the tree, skipping duplicates, or NULL if none */
static inline struct eb64_node *eb64_next_unique(struct eb64_node *eb64)
{
  return eb64_entry(eb_next_unique(&eb64->node), struct eb64_node, node);
}

/* Return previous node in the tree, skipping duplicates, or NULL if none */
static inline struct eb64_node *eb64_prev_unique(struct eb64_node *eb64)
{
  return eb64_entry(eb_prev_unique(&eb64->node), struct eb64_node, node);
}

/* Delete node from the tree if it was linked in. Mark the node unused. Note
 * that this function relies on a non-inlined generic function: eb_delete.
 */
static inline void eb64_delete(struct eb64_node *eb64)
{
  eb_delete(&eb64->node);
}

/*
 * The following functions are not inlined by default. They are declared
 * in eb64tree.c, which simply relies on their inline version.
 */
struct eb64_node *eb64_lookup(struct eb_root *root, u64 x);
struct eb64_node *eb64i_lookup(struct eb_root *root, s64 x);
struct eb64_node *eb64_lookup_le(struct eb_root *root, u64 x);
struct eb64_node *eb64_lookup_ge(struct eb_root *root, u64 x);
struct eb64_node *eb64_insert(struct eb_root *root, struct eb64_node *new);
struct eb64_node *eb64i_insert(struct eb_root *root, struct eb64_node *new);

/*
 * The following functions are less likely to be used directly, because their
 * code is larger. The non-inlined version is preferred.
 */

/* Delete node from the tree if it was linked in. Mark the node unused. */
static forceinline void __eb64_delete(struct eb64_node *eb64)
{
  __eb_delete(&eb64->node);
}

/*
 * Find the first occurrence of a key in the tree <root>. If none can be
 * found, return NULL.
 */
static forceinline struct eb64_node *__eb64_lookup(struct eb_root *root, u64 x)
{
  struct eb64_node *node;
  eb_troot_t *troot;
  u64 y, z;

  troot = root->b[EB_LEFT];
  if (unlikely(troot == NULL))
    return NULL;

  while (1) {
    if (unlikely(eb_gettag(troot) == EB_LEAF)) {
      node = container_of(eb_untag(troot, EB_LEAF),
              struct eb64_node, node.branches);
      if (node->key == x)
        return node;
      else
        return NULL;
    }
    node = container_of(eb_untag(troot, EB_NODE),
            struct eb64_node, node.branches);

    eb_prefetch(node->node.branches.b[0], 0);
    eb_prefetch(node->node.branches.b[1], 0);

    y = node->key ^ x;
    z = 1ULL << (node->node.bit & 63);
    troot = (x & z) ? node->node.branches.b[1] : node->node.branches.b[0];

    if (!y) {
      /* Either we found the node which holds the key, or
       * we have a dup tree. In the later case, we have to
       * walk it down left to get the first entry.
       */
      if (node->node.bit < 0) {
        troot = node->node.branches.b[EB_LEFT];
        while (eb_gettag(troot) != EB_LEAF)
          troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT];
        node = container_of(eb_untag(troot, EB_LEAF),
                struct eb64_node, node.branches);
      }
      return node;
    }

    if (y & -(z << 1))
      return NULL; /* no more common bits */
  }
}

/*
 * Find the first occurrence of a signed key in the tree <root>. If none can
 * be found, return NULL.
 */
static forceinline struct eb64_node *__eb64i_lookup(struct eb_root *root, s64 x)
{
  struct eb64_node *node;
  eb_troot_t *troot;
  u64 key = x ^ (1ULL << 63);
  u64 y, z;

  troot = root->b[EB_LEFT];
  if (unlikely(troot == NULL))
    return NULL;

  while (1) {
    if (unlikely(eb_gettag(troot) == EB_LEAF)) {
      node = container_of(eb_untag(troot, EB_LEAF),
              struct eb64_node, node.branches);
      if (node->key == (u64)x)
        return node;
      else
        return NULL;
    }
    node = container_of(eb_untag(troot, EB_NODE),
            struct eb64_node, node.branches);

    eb_prefetch(node->node.branches.b[0], 0);
    eb_prefetch(node->node.branches.b[1], 0);

    y = node->key ^ x;
    z = 1ULL << (node->node.bit & 63);
    troot = (key & z) ? node->node.branches.b[1] : node->node.branches.b[0];

    if (!y) {
      /* Either we found the node which holds the key, or
       * we have a dup tree. In the later case, we have to
       * walk it down left to get the first entry.
       */
      if (node->node.bit < 0) {
        troot = node->node.branches.b[EB_LEFT];
        while (eb_gettag(troot) != EB_LEAF)
          troot = (eb_untag(troot, EB_NODE))->b[EB_LEFT];
        node = container_of(eb_untag(troot, EB_LEAF),
                struct eb64_node, node.branches);
      }
      return node;
    }

    if (y & -(z << 1))
      return NULL; /* no more common bits */
  }
}

/* Insert eb64_node <new> into subtree starting at node root <root>.
 * Only new->key needs be set with the key. The eb64_node is returned.
 * If root->b[EB_RGHT]==1, the tree may only contain unique keys.
 */
static forceinline struct eb64_node *
__eb64_insert(struct eb_root *root, struct eb64_node *new) {
  struct eb64_node *old;
  unsigned int side;
  eb_troot_t *troot;
  u64 newkey; /* caching the key saves approximately one cycle */
  eb_troot_t *root_right;
  int old_node_bit;

  side = EB_LEFT;
  troot = root->b[EB_LEFT];
  root_right = root->b[EB_RGHT];
  if (unlikely(troot == NULL)) {
    /* Tree is empty, insert the leaf part below the left branch */
    root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF);
    new->node.leaf_p = eb_dotag(root, EB_LEFT);
    new->node.node_p = NULL; /* node part unused */
    return new;
  }

  /* The tree descent is fairly easy :
   *  - first, check if we have reached a leaf node
   *  - second, check if we have gone too far
   *  - third, reiterate
   * Everywhere, we use <new> for the node node we are inserting, <root>
   * for the node we attach it to, and <old> for the node we are
   * displacing below <new>. <troot> will always point to the future node
   * (tagged with its type). <side> carries the side the node <new> is
   * attached to below its parent, which is also where previous node
   * was attached. <newkey> carries the key being inserted.
   */
  newkey = new->key;

  while (1) {
    if (unlikely(eb_gettag(troot) == EB_LEAF)) {
      eb_troot_t *new_left, *new_rght;
      eb_troot_t *new_leaf, *old_leaf;

      old = container_of(eb_untag(troot, EB_LEAF),
              struct eb64_node, node.branches);

      new_left = eb_dotag(&new->node.branches, EB_LEFT);
      new_rght = eb_dotag(&new->node.branches, EB_RGHT);
      new_leaf = eb_dotag(&new->node.branches, EB_LEAF);
      old_leaf = eb_dotag(&old->node.branches, EB_LEAF);

      new->node.node_p = old->node.leaf_p;

      /* Right here, we have 3 possibilities :
         - the tree does not contain the key, and we have
           new->key < old->key. We insert new above old, on
           the left ;

         - the tree does not contain the key, and we have
           new->key > old->key. We insert new above old, on
           the right ;

         - the tree does contain the key, which implies it
           is alone. We add the new key next to it as a
           first duplicate.

         The last two cases can easily be partially merged.
      */
       
      if (new->key < old->key) {
        new->node.leaf_p = new_left;
        old->node.leaf_p = new_rght;
        new->node.branches.b[EB_LEFT] = new_leaf;
        new->node.branches.b[EB_RGHT] = old_leaf;
      } else {
        /* we may refuse to duplicate this key if the tree is
         * tagged as containing only unique keys.
         */
        if ((new->key == old->key) && eb_gettag(root_right))
          return old;

        /* new->key >= old->key, new goes the right */
        old->node.leaf_p = new_left;
        new->node.leaf_p = new_rght;
        new->node.branches.b[EB_LEFT] = old_leaf;
        new->node.branches.b[EB_RGHT] = new_leaf;

        if (new->key == old->key) {
          new->node.bit = -1;
          root->b[side] = eb_dotag(&new->node.branches, EB_NODE);
          return new;
        }
      }
      break;
    }

    /* OK we're walking down this link */
    old = container_of(eb_untag(troot, EB_NODE),
            struct eb64_node, node.branches);

    eb_prefetch(old->node.branches.b[0], 0);
    eb_prefetch(old->node.branches.b[1], 0);

    old_node_bit = old->node.bit;

    /* Stop going down when we don't have common bits anymore. We
     * also stop in front of a duplicates tree because it means we
     * have to insert above.
     */

    if ((old_node_bit < 0) || /* we're above a duplicate tree, stop here */
        (((new->key ^ old->key) >> old_node_bit) >= EB_NODE_BRANCHES)) {
      /* The tree did not contain the key, so we insert <new> before the node
       * <old>, and set ->bit to designate the lowest bit position in <new>
       * which applies to ->branches.b[].
       */
      eb_troot_t *new_left, *new_rght;
      eb_troot_t *new_leaf, *old_node;

      new_left = eb_dotag(&new->node.branches, EB_LEFT);
      new_rght = eb_dotag(&new->node.branches, EB_RGHT);
      new_leaf = eb_dotag(&new->node.branches, EB_LEAF);
      old_node = eb_dotag(&old->node.branches, EB_NODE);

      new->node.node_p = old->node.node_p;

      if (new->key < old->key) {
        new->node.leaf_p = new_left;
        old->node.node_p = new_rght;
        new->node.branches.b[EB_LEFT] = new_leaf;
        new->node.branches.b[EB_RGHT] = old_node;
      }
      else if (new->key > old->key) {
        old->node.node_p = new_left;
        new->node.leaf_p = new_rght;
        new->node.branches.b[EB_LEFT] = old_node;
        new->node.branches.b[EB_RGHT] = new_leaf;
      }
      else {
        struct eb_node *ret;
        ret = eb_insert_dup(&old->node, &new->node);
        return container_of(ret, struct eb64_node, node);
      }
      break;
    }

    /* walk down */

    if (sizeof(long) >= 8) {
      side = newkey >> old_node_bit;
    } else {
      /* note: provides the best code on low-register count archs
       * such as i386.
       */
      side = newkey;
      side >>= old_node_bit;
      if (old_node_bit >= 32) {
        side = newkey >> 32;
        side >>= old_node_bit & 0x1F;
      }
    }
    side &= EB_NODE_BRANCH_MASK;
    troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
    root = &old->node.branches;
  }

  /* Ok, now we are inserting <new> between <root> and <old>. <old>'s
   * parent is already set to <new>, and the <root>'s branch is still in
   * <side>. Update the root's leaf till we have it. Note that we can also
   * find the side by checking the side of new->node.node_p.
   */

  /* We need the common higher bits between new->key and old->key.
   * What differences are there between new->key and the node here ?
   * NOTE that bit(new) is always < bit(root) because highest
   * bit of new->key and old->key are identical here (otherwise they
   * would sit on different branches).
   */
  // note that if EB_NODE_BITS > 1, we should check that it's still >= 0
  new->node.bit = fls64(new->key ^ old->key) - EB_NODE_BITS;
  root->b[side] = eb_dotag(&new->node.branches, EB_NODE);

  return new;
}

/* Insert eb64_node <new> into subtree starting at node root <root>, using
 * signed keys. Only new->key needs be set with the key. The eb64_node
 * is returned. If root->b[EB_RGHT]==1, the tree may only contain unique keys.
 */
static forceinline struct eb64_node *
__eb64i_insert(struct eb_root *root, struct eb64_node *new) {
  struct eb64_node *old;
  unsigned int side;
  eb_troot_t *troot;
  u64 newkey; /* caching the key saves approximately one cycle */
  eb_troot_t *root_right;
  int old_node_bit;

  side = EB_LEFT;
  troot = root->b[EB_LEFT];
  root_right = root->b[EB_RGHT];
  if (unlikely(troot == NULL)) {
    /* Tree is empty, insert the leaf part below the left branch */
    root->b[EB_LEFT] = eb_dotag(&new->node.branches, EB_LEAF);
    new->node.leaf_p = eb_dotag(root, EB_LEFT);
    new->node.node_p = NULL; /* node part unused */
    return new;
  }

  /* The tree descent is fairly easy :
   *  - first, check if we have reached a leaf node
   *  - second, check if we have gone too far
   *  - third, reiterate
   * Everywhere, we use <new> for the node node we are inserting, <root>
   * for the node we attach it to, and <old> for the node we are
   * displacing below <new>. <troot> will always point to the future node
   * (tagged with its type). <side> carries the side the node <new> is
   * attached to below its parent, which is also where previous node
   * was attached. <newkey> carries a high bit shift of the key being
   * inserted in order to have negative keys stored before positive
   * ones.
   */
  newkey = new->key ^ (1ULL << 63);

  while (1) {
    if (unlikely(eb_gettag(troot) == EB_LEAF)) {
      eb_troot_t *new_left, *new_rght;
      eb_troot_t *new_leaf, *old_leaf;

      old = container_of(eb_untag(troot, EB_LEAF),
              struct eb64_node, node.branches);

      new_left = eb_dotag(&new->node.branches, EB_LEFT);
      new_rght = eb_dotag(&new->node.branches, EB_RGHT);
      new_leaf = eb_dotag(&new->node.branches, EB_LEAF);
      old_leaf = eb_dotag(&old->node.branches, EB_LEAF);

      new->node.node_p = old->node.leaf_p;

      /* Right here, we have 3 possibilities :
         - the tree does not contain the key, and we have
           new->key < old->key. We insert new above old, on
           the left ;

         - the tree does not contain the key, and we have
           new->key > old->key. We insert new above old, on
           the right ;

         - the tree does contain the key, which implies it
           is alone. We add the new key next to it as a
           first duplicate.

         The last two cases can easily be partially merged.
      */
       
      if ((s64)new->key < (s64)old->key) {
        new->node.leaf_p = new_left;
        old->node.leaf_p = new_rght;
        new->node.branches.b[EB_LEFT] = new_leaf;
        new->node.branches.b[EB_RGHT] = old_leaf;
      } else {
        /* we may refuse to duplicate this key if the tree is
         * tagged as containing only unique keys.
         */
        if ((new->key == old->key) && eb_gettag(root_right))
          return old;

        /* new->key >= old->key, new goes the right */
        old->node.leaf_p = new_left;
        new->node.leaf_p = new_rght;
        new->node.branches.b[EB_LEFT] = old_leaf;
        new->node.branches.b[EB_RGHT] = new_leaf;

        if (new->key == old->key) {
          new->node.bit = -1;
          root->b[side] = eb_dotag(&new->node.branches, EB_NODE);
          return new;
        }
      }
      break;
    }

    /* OK we're walking down this link */
    old = container_of(eb_untag(troot, EB_NODE),
            struct eb64_node, node.branches);

    eb_prefetch(old->node.branches.b[0], 0);
    eb_prefetch(old->node.branches.b[1], 0);

    old_node_bit = old->node.bit;

    /* Stop going down when we don't have common bits anymore. We
     * also stop in front of a duplicates tree because it means we
     * have to insert above.
     */

    if ((old_node_bit < 0) || /* we're above a duplicate tree, stop here */
        (((new->key ^ old->key) >> old_node_bit) >= EB_NODE_BRANCHES)) {
      /* The tree did not contain the key, so we insert <new> before the node
       * <old>, and set ->bit to designate the lowest bit position in <new>
       * which applies to ->branches.b[].
       */
      eb_troot_t *new_left, *new_rght;
      eb_troot_t *new_leaf, *old_node;

      new_left = eb_dotag(&new->node.branches, EB_LEFT);
      new_rght = eb_dotag(&new->node.branches, EB_RGHT);
      new_leaf = eb_dotag(&new->node.branches, EB_LEAF);
      old_node = eb_dotag(&old->node.branches, EB_NODE);

      new->node.node_p = old->node.node_p;

      if ((s64)new->key < (s64)old->key) {
        new->node.leaf_p = new_left;
        old->node.node_p = new_rght;
        new->node.branches.b[EB_LEFT] = new_leaf;
        new->node.branches.b[EB_RGHT] = old_node;
      }
      else if ((s64)new->key > (s64)old->key) {
        old->node.node_p = new_left;
        new->node.leaf_p = new_rght;
        new->node.branches.b[EB_LEFT] = old_node;
        new->node.branches.b[EB_RGHT] = new_leaf;
      }
      else {
        struct eb_node *ret;
        ret = eb_insert_dup(&old->node, &new->node);
        return container_of(ret, struct eb64_node, node);
      }
      break;
    }

    /* walk down */

    if (sizeof(long) >= 8) {
      side = newkey >> old_node_bit;
    } else {
      /* note: provides the best code on low-register count archs
       * such as i386.
       */
      side = newkey;
      side >>= old_node_bit;
      if (old_node_bit >= 32) {
        side = newkey >> 32;
        side >>= old_node_bit & 0x1F;
      }
    }
    side &= EB_NODE_BRANCH_MASK;
    troot = side ? old->node.branches.b[1] : old->node.branches.b[0];
    root = &old->node.branches;
  }

  /* Ok, now we are inserting <new> between <root> and <old>. <old>'s
   * parent is already set to <new>, and the <root>'s branch is still in
   * <side>. Update the root's leaf till we have it. Note that we can also
   * find the side by checking the side of new->node.node_p.
   */

  /* We need the common higher bits between new->key and old->key.
   * What differences are there between new->key and the node here ?
   * NOTE that bit(new) is always < bit(root) because highest
   * bit of new->key and old->key are identical here (otherwise they
   * would sit on different branches).
   */
  // note that if EB_NODE_BITS > 1, we should check that it's still >= 0
  new->node.bit = fls64(new->key ^ old->key) - EB_NODE_BITS;
  root->b[side] = eb_dotag(&new->node.branches, EB_NODE);

  return new;
}

#endif /* _EB64_TREE_H */

Coverage Report

Created: 2026-01-09 06:23

Line	Count	Source
1		/*
2		* Elastic Binary Trees - macros and structures for operations on 64bit nodes.
3		* Version 6.0.6
4		* (C) 2002-2011 - Willy Tarreau <w@1wt.eu>
5		*
6		* This library is free software; you can redistribute it and/or
7		* modify it under the terms of the GNU Lesser General Public
8		* License as published by the Free Software Foundation, version 2.1
9		* exclusively.
10		*
11		* This library is distributed in the hope that it will be useful,
12		* but WITHOUT ANY WARRANTY; without even the implied warranty of
13		* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14		* Lesser General Public License for more details.
15		*
16		* You should have received a copy of the GNU Lesser General Public
17		* License along with this library; if not, write to the Free Software
18		* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19		*/
20
21		#ifndef _EB64TREE_H
22		#define _EB64TREE_H
23
24		#include "ebtree.h"
25
26
27		/* Return the structure of type <type> whose member <member> points to <ptr> */
28	0	#define eb64_entry(ptr, type, member) container_of(ptr, type, member)
29
30		/*
31		* Exported functions and macros.
32		* Many of them are always inlined because they are extremely small, and
33		* are generally called at most once or twice in a program.
34		*/
35
36		/* Return leftmost node in the tree, or NULL if none */
37		static inline struct eb64_node eb64_first(struct eb_root root)
38	0	{
39	0	return eb64_entry(eb_first(root), struct eb64_node, node);