// Copyright 2018 The Abseil Authors.

//

// Licensed under the Apache License, Version 2.0 (the "License");

// you may not use this file except in compliance with the License.

// You may obtain a copy of the License at

//

//      https://www.apache.org/licenses/LICENSE-2.0

//

// Unless required by applicable law or agreed to in writing, software

// distributed under the License is distributed on an "AS IS" BASIS,

// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

// See the License for the specific language governing permissions and

// limitations under the License.

#include "absl/container/internal/raw_hash_set.h"

#include <atomic>

#include <cassert>

#include <cstddef>

#include <cstdint>

#include <cstring>

#include "absl/base/attributes.h"

#include "absl/base/config.h"

#include "absl/base/dynamic_annotations.h"

#include "absl/base/internal/endian.h"

#include "absl/base/optimization.h"

#include "absl/container/internal/container_memory.h"

#include "absl/container/internal/hashtablez_sampler.h"

#include "absl/hash/hash.h"

namespace absl {

ABSL_NAMESPACE_BEGIN

namespace container_internal {

// Represents a control byte corresponding to a full slot with arbitrary hash.

constexpr ctrl_t ZeroCtrlT() { return static_cast<ctrl_t>(0); }

// We have space for `growth_info` before a single block of control bytes. A

// single block of empty control bytes for tables without any slots allocated.

// This enables removing a branch in the hot path of find(). In order to ensure

// that the control bytes are aligned to 16, we have 16 bytes before the control

// bytes even though growth_info only needs 8.

alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[32] = {

    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),

    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),

    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),

    ZeroCtrlT(),       ZeroCtrlT(),    ZeroCtrlT(),    ZeroCtrlT(),

    ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,

    ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,

    ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty,

    ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty};

// We need one full byte followed by a sentinel byte for iterator::operator++ to

// work. We have a full group after kSentinel to be safe (in case operator++ is

// changed to read a full group).

ABSL_CONST_INIT ABSL_DLL const ctrl_t kSooControl[17] = {

    ZeroCtrlT(),    ctrl_t::kSentinel, ZeroCtrlT(),    ctrl_t::kEmpty,

    ctrl_t::kEmpty, ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty,

    ctrl_t::kEmpty, ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty,

    ctrl_t::kEmpty, ctrl_t::kEmpty,    ctrl_t::kEmpty, ctrl_t::kEmpty,

    ctrl_t::kEmpty};

static_assert(NumControlBytes(SooCapacity()) <= 17,

              "kSooControl capacity too small");

#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL

constexpr size_t Group::kWidth;

#endif

namespace {

// Returns "random" seed.

inline size_t RandomSeed() {

#ifdef ABSL_HAVE_THREAD_LOCAL

  static thread_local size_t counter = 0;

  // On Linux kernels >= 5.4 the MSAN runtime has a false-positive when

  // accessing thread local storage data from loaded libraries

  // (https://github.com/google/sanitizers/issues/1265), for this reason counter

  // needs to be annotated as initialized.

  ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(&counter, sizeof(size_t));

  size_t value = ++counter;

#else   // ABSL_HAVE_THREAD_LOCAL

  static std::atomic<size_t> counter(0);

  size_t value = counter.fetch_add(1, std::memory_order_relaxed);

#endif  // ABSL_HAVE_THREAD_LOCAL

  return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter));

}

bool ShouldRehashForBugDetection(const ctrl_t* ctrl, size_t capacity) {

  // Note: we can't use the abseil-random library because abseil-random

  // depends on swisstable. We want to return true with probability

  // `min(1, RehashProbabilityConstant() / capacity())`. In order to do this,

  // we probe based on a random hash and see if the offset is less than

  // RehashProbabilityConstant().

  return probe(ctrl, capacity, absl::HashOf(RandomSeed())).offset() <

         RehashProbabilityConstant();

}

}  // namespace

GenerationType* EmptyGeneration() {

  if (SwisstableGenerationsEnabled()) {

    constexpr size_t kNumEmptyGenerations = 1024;

    static constexpr GenerationType kEmptyGenerations[kNumEmptyGenerations]{};

    return const_cast<GenerationType*>(

        &kEmptyGenerations[RandomSeed() % kNumEmptyGenerations]);

  }

  return nullptr;

}

bool CommonFieldsGenerationInfoEnabled::

    should_rehash_for_bug_detection_on_insert(const ctrl_t* ctrl,

                                              size_t capacity) const {

  if (reserved_growth_ == kReservedGrowthJustRanOut) return true;

  if (reserved_growth_ > 0) return false;

  return ShouldRehashForBugDetection(ctrl, capacity);

}

bool CommonFieldsGenerationInfoEnabled::should_rehash_for_bug_detection_on_move(

    const ctrl_t* ctrl, size_t capacity) const {

  return ShouldRehashForBugDetection(ctrl, capacity);

}

bool ShouldInsertBackwardsForDebug(size_t capacity, size_t hash,

                                   const ctrl_t* ctrl) {

  // To avoid problems with weak hashes and single bit tests, we use % 13.

  // TODO(kfm,sbenza): revisit after we do unconditional mixing

  return !is_small(capacity) && (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6;

}

size_t PrepareInsertAfterSoo(size_t hash, size_t slot_size,

                             CommonFields& common) {

  assert(common.capacity() == NextCapacity(SooCapacity()));

  // After resize from capacity 1 to 3, we always have exactly the slot with

  // index 1 occupied, so we need to insert either at index 0 or index 2.

  assert(HashSetResizeHelper::SooSlotIndex() == 1);

  PrepareInsertCommon(common);

  const size_t offset = H1(hash, common.control()) & 2;

  common.growth_info().OverwriteEmptyAsFull();

  SetCtrlInSingleGroupTable(common, offset, H2(hash), slot_size);

  common.infoz().RecordInsert(hash, /*distance_from_desired=*/0);

  return offset;

}

void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) {

  assert(ctrl[capacity] == ctrl_t::kSentinel);

  assert(IsValidCapacity(capacity));

  for (ctrl_t* pos = ctrl; pos < ctrl + capacity; pos += Group::kWidth) {

    Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos);

  }

  // Copy the cloned ctrl bytes.

  std::memcpy(ctrl + capacity + 1, ctrl, NumClonedBytes());

  ctrl[capacity] = ctrl_t::kSentinel;

}

// Extern template instantiation for inline function.

template FindInfo find_first_non_full(const CommonFields&, size_t);

FindInfo find_first_non_full_outofline(const CommonFields& common,

                                       size_t hash) {

  return find_first_non_full(common, hash);

}

namespace {

// Returns the address of the slot just after slot assuming each slot has the

// specified size.

static inline void* NextSlot(void* slot, size_t slot_size) {

  return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) + slot_size);

}

// Returns the address of the slot just before slot assuming each slot has the

// specified size.

static inline void* PrevSlot(void* slot, size_t slot_size) {

  return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(slot) - slot_size);

}

// Finds guaranteed to exists empty slot from the given position.

// NOTE: this function is almost never triggered inside of the

// DropDeletesWithoutResize, so we keep it simple.

// The table is rather sparse, so empty slot will be found very quickly.

size_t FindEmptySlot(size_t start, size_t end, const ctrl_t* ctrl) {

  for (size_t i = start; i < end; ++i) {

    if (IsEmpty(ctrl[i])) {

      return i;

    }

  }

  assert(false && "no empty slot");

  return ~size_t{};

}

void DropDeletesWithoutResize(CommonFields& common,

                              const PolicyFunctions& policy) {

  void* set = &common;

  void* slot_array = common.slot_array();

  const size_t capacity = common.capacity();

  assert(IsValidCapacity(capacity));

  assert(!is_small(capacity));

  // Algorithm:

  // - mark all DELETED slots as EMPTY

  // - mark all FULL slots as DELETED

  // - for each slot marked as DELETED

  //     hash = Hash(element)

  //     target = find_first_non_full(hash)

  //     if target is in the same group

  //       mark slot as FULL

  //     else if target is EMPTY

  //       transfer element to target

  //       mark slot as EMPTY

  //       mark target as FULL

  //     else if target is DELETED

  //       swap current element with target element

  //       mark target as FULL

  //       repeat procedure for current slot with moved from element (target)

  ctrl_t* ctrl = common.control();

  ConvertDeletedToEmptyAndFullToDeleted(ctrl, capacity);

  const void* hash_fn = policy.hash_fn(common);

  auto hasher = policy.hash_slot;

  auto transfer = policy.transfer;

  const size_t slot_size = policy.slot_size;

  size_t total_probe_length = 0;

  void* slot_ptr = SlotAddress(slot_array, 0, slot_size);

  // The index of an empty slot that can be used as temporary memory for

  // the swap operation.

  constexpr size_t kUnknownId = ~size_t{};

  size_t tmp_space_id = kUnknownId;

  for (size_t i = 0; i != capacity;

       ++i, slot_ptr = NextSlot(slot_ptr, slot_size)) {

    assert(slot_ptr == SlotAddress(slot_array, i, slot_size));

    if (IsEmpty(ctrl[i])) {

      tmp_space_id = i;

      continue;

    }

    if (!IsDeleted(ctrl[i])) continue;

    const size_t hash = (*hasher)(hash_fn, slot_ptr);

    const FindInfo target = find_first_non_full(common, hash);

    const size_t new_i = target.offset;

    total_probe_length += target.probe_length;

    // Verify if the old and new i fall within the same group wrt the hash.

    // If they do, we don't need to move the object as it falls already in the

    // best probe we can.

    const size_t probe_offset = probe(common, hash).offset();

    const auto probe_index = [probe_offset, capacity](size_t pos) {

      return ((pos - probe_offset) & capacity) / Group::kWidth;

    };

    // Element doesn't move.

    if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) {

      SetCtrl(common, i, H2(hash), slot_size);

      continue;

    }

    void* new_slot_ptr = SlotAddress(slot_array, new_i, slot_size);

    if (IsEmpty(ctrl[new_i])) {

      // Transfer element to the empty spot.

      // SetCtrl poisons/unpoisons the slots so we have to call it at the

      // right time.

      SetCtrl(common, new_i, H2(hash), slot_size);

      (*transfer)(set, new_slot_ptr, slot_ptr);

      SetCtrl(common, i, ctrl_t::kEmpty, slot_size);

      // Initialize or change empty space id.

      tmp_space_id = i;

    } else {

      assert(IsDeleted(ctrl[new_i]));

      SetCtrl(common, new_i, H2(hash), slot_size);

      // Until we are done rehashing, DELETED marks previously FULL slots.

      if (tmp_space_id == kUnknownId) {

        tmp_space_id = FindEmptySlot(i + 1, capacity, ctrl);

      }

      void* tmp_space = SlotAddress(slot_array, tmp_space_id, slot_size);

      SanitizerUnpoisonMemoryRegion(tmp_space, slot_size);

      // Swap i and new_i elements.

      (*transfer)(set, tmp_space, new_slot_ptr);

      (*transfer)(set, new_slot_ptr, slot_ptr);

      (*transfer)(set, slot_ptr, tmp_space);

      SanitizerPoisonMemoryRegion(tmp_space, slot_size);

      // repeat the processing of the ith slot

      --i;

      slot_ptr = PrevSlot(slot_ptr, slot_size);

    }

  }

  ResetGrowthLeft(common);

  common.infoz().RecordRehash(total_probe_length);

}

static bool WasNeverFull(CommonFields& c, size_t index) {

  if (is_single_group(c.capacity())) {

    return true;

  }

  const size_t index_before = (index - Group::kWidth) & c.capacity();

  const auto empty_after = Group(c.control() + index).MaskEmpty();

  const auto empty_before = Group(c.control() + index_before).MaskEmpty();

  // We count how many consecutive non empties we have to the right and to the

  // left of `it`. If the sum is >= kWidth then there is at least one probe

  // window that might have seen a full group.

  return empty_before && empty_after &&

         static_cast<size_t>(empty_after.TrailingZeros()) +

                 empty_before.LeadingZeros() <

             Group::kWidth;

}

}  // namespace

void EraseMetaOnly(CommonFields& c, size_t index, size_t slot_size) {

  assert(IsFull(c.control()[index]) && "erasing a dangling iterator");

  c.decrement_size();

  c.infoz().RecordErase();

  if (WasNeverFull(c, index)) {

    SetCtrl(c, index, ctrl_t::kEmpty, slot_size);

    c.growth_info().OverwriteFullAsEmpty();

    return;

  }

  c.growth_info().OverwriteFullAsDeleted();

  SetCtrl(c, index, ctrl_t::kDeleted, slot_size);

}

void ClearBackingArray(CommonFields& c, const PolicyFunctions& policy,

                       bool reuse, bool soo_enabled) {

  c.set_size(0);

  if (reuse) {

    assert(!soo_enabled || c.capacity() > SooCapacity());

    ResetCtrl(c, policy.slot_size);

    ResetGrowthLeft(c);

    c.infoz().RecordStorageChanged(0, c.capacity());

  } else {

    // We need to record infoz before calling dealloc, which will unregister

    // infoz.

    c.infoz().RecordClearedReservation();

    c.infoz().RecordStorageChanged(0, soo_enabled ? SooCapacity() : 0);

    (*policy.dealloc)(c, policy);

    c = soo_enabled ? CommonFields{soo_tag_t{}} : CommonFields{non_soo_tag_t{}};

  }

}

void HashSetResizeHelper::GrowIntoSingleGroupShuffleControlBytes(

    ctrl_t* __restrict new_ctrl, size_t new_capacity) const {

  assert(is_single_group(new_capacity));

  constexpr size_t kHalfWidth = Group::kWidth / 2;

  constexpr size_t kQuarterWidth = Group::kWidth / 4;

  assert(old_capacity_ < kHalfWidth);

  static_assert(sizeof(uint64_t) >= kHalfWidth,

                "Group size is too large. The ctrl bytes for half a group must "

                "fit into a uint64_t for this implementation.");

  static_assert(sizeof(uint64_t) <= Group::kWidth,

                "Group size is too small. The ctrl bytes for a group must "

                "cover a uint64_t for this implementation.");

  const size_t half_old_capacity = old_capacity_ / 2;

  // NOTE: operations are done with compile time known size = kHalfWidth.

  // Compiler optimizes that into single ASM operation.

  // Load the bytes from half_old_capacity + 1. This contains the last half of

  // old_ctrl bytes, followed by the sentinel byte, and then the first half of

  // the cloned bytes. This effectively shuffles the control bytes.

  uint64_t copied_bytes = 0;

  copied_bytes =

      absl::little_endian::Load64(old_ctrl() + half_old_capacity + 1);

  // We change the sentinel byte to kEmpty before storing to both the start of

  // the new_ctrl, and past the end of the new_ctrl later for the new cloned

  // bytes. Note that this is faster than setting the sentinel byte to kEmpty

  // after the copy directly in new_ctrl because we are limited on store

  // bandwidth.

  constexpr uint64_t kEmptyXorSentinel =

      static_cast<uint8_t>(ctrl_t::kEmpty) ^

      static_cast<uint8_t>(ctrl_t::kSentinel);

  const uint64_t mask_convert_old_sentinel_to_empty =

      kEmptyXorSentinel << (half_old_capacity * 8);

  copied_bytes ^= mask_convert_old_sentinel_to_empty;

  // Copy second half of bytes to the beginning. This correctly sets the bytes

  // [0, old_capacity]. We potentially copy more bytes in order to have compile

  // time known size. Mirrored bytes from the old_ctrl() will also be copied. In

  // case of old_capacity_ == 3, we will copy 1st element twice.

  // Examples:

  // (old capacity = 1)

  // old_ctrl = 0S0EEEEEEE...

  // new_ctrl = E0EEEEEE??...

  //

  // (old capacity = 3)

  // old_ctrl = 012S012EEEEE...

  // new_ctrl = 12E012EE????...

  //

  // (old capacity = 7)

  // old_ctrl = 0123456S0123456EE...

  // new_ctrl = 456E0123?????????...

  absl::little_endian::Store64(new_ctrl, copied_bytes);

  // Set the space [old_capacity + 1, new_capacity] to empty as these bytes will

  // not be written again. This is safe because

  // NumControlBytes = new_capacity + kWidth and new_capacity >=

  // old_capacity+1.

  // Examples:

  // (old_capacity = 3, new_capacity = 15)

  // new_ctrl  = 12E012EE?????????????...??

  // *new_ctrl = 12E0EEEEEEEEEEEEEEEE?...??

  // position        /          S

  //

  // (old_capacity = 7, new_capacity = 15)

  // new_ctrl  = 456E0123?????????????????...??

  // *new_ctrl = 456E0123EEEEEEEEEEEEEEEE?...??

  // position            /      S

  std::memset(new_ctrl + old_capacity_ + 1, static_cast<int8_t>(ctrl_t::kEmpty),

              Group::kWidth);

  // Set the last kHalfWidth bytes to empty, to ensure the bytes all the way to

  // the end are initialized.

  // Examples:

  // new_ctrl  = 12E0EEEEEEEEEEEEEEEE?...???????

  // *new_ctrl = 12E0EEEEEEEEEEEEEEEE???EEEEEEEE

  // position                   S       /

  //

  // new_ctrl  = 456E0123EEEEEEEEEEEEEEEE???????

  // *new_ctrl = 456E0123EEEEEEEEEEEEEEEEEEEEEEE

  // position                   S       /

  std::memset(new_ctrl + NumControlBytes(new_capacity) - kHalfWidth,

              static_cast<int8_t>(ctrl_t::kEmpty), kHalfWidth);

  // Copy the first bytes to the end (starting at new_capacity +1) to set the

  // cloned bytes. Note that we use the already copied bytes from old_ctrl here

  // rather than copying from new_ctrl to avoid a Read-after-Write hazard, since

  // new_ctrl was just written to. The first old_capacity-1 bytes are set

  // correctly. Then there may be up to old_capacity bytes that need to be

  // overwritten, and any remaining bytes will be correctly set to empty. This

  // sets [new_capacity + 1, new_capacity +1 + old_capacity] correctly.

  // Examples:

  // new_ctrl  = 12E0EEEEEEEEEEEEEEEE?...???????

  // *new_ctrl = 12E0EEEEEEEEEEEE12E012EEEEEEEEE

  // position                   S/

  //

  // new_ctrl  = 456E0123EEEEEEEE?...???EEEEEEEE

  // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE

  // position                   S/

  absl::little_endian::Store64(new_ctrl + new_capacity + 1, copied_bytes);

  // Set The remaining bytes at the end past the cloned bytes to empty. The

  // incorrectly set bytes are [new_capacity + old_capacity + 2,

  // min(new_capacity + 1 + kHalfWidth, new_capacity + old_capacity + 2 +

  // half_old_capacity)]. Taking the difference, we need to set min(kHalfWidth -

  // (old_capacity + 1), half_old_capacity)]. Since old_capacity < kHalfWidth,

  // half_old_capacity < kQuarterWidth, so we set kQuarterWidth beginning at

  // new_capacity + old_capacity + 2 to kEmpty.

  // Examples:

  // new_ctrl  = 12E0EEEEEEEEEEEE12E012EEEEEEEEE

  // *new_ctrl = 12E0EEEEEEEEEEEE12E0EEEEEEEEEEE

  // position                   S    /

  //

  // new_ctrl  = 456E0123EEEEEEEE456E0123EEEEEEE

  // *new_ctrl = 456E0123EEEEEEEE456E0123EEEEEEE (no change)

  // position                   S        /

  std::memset(new_ctrl + new_capacity + old_capacity_ + 2,

              static_cast<int8_t>(ctrl_t::kEmpty), kQuarterWidth);

  // Finally, we set the new sentinel byte.

  new_ctrl[new_capacity] = ctrl_t::kSentinel;

}

void HashSetResizeHelper::InitControlBytesAfterSoo(ctrl_t* new_ctrl, ctrl_t h2,

                                                   size_t new_capacity) {

  assert(is_single_group(new_capacity));

  std::memset(new_ctrl, static_cast<int8_t>(ctrl_t::kEmpty),

              NumControlBytes(new_capacity));

  assert(HashSetResizeHelper::SooSlotIndex() == 1);

  // This allows us to avoid branching on had_soo_slot_.

  assert(had_soo_slot_ || h2 == ctrl_t::kEmpty);

  new_ctrl[1] = new_ctrl[new_capacity + 2] = h2;

  new_ctrl[new_capacity] = ctrl_t::kSentinel;

}

void HashSetResizeHelper::GrowIntoSingleGroupShuffleTransferableSlots(

    void* new_slots, size_t slot_size) const {

  assert(old_capacity_ > 0);

  const size_t half_old_capacity = old_capacity_ / 2;

  SanitizerUnpoisonMemoryRegion(old_slots(), slot_size * old_capacity_);

  std::memcpy(new_slots,

              SlotAddress(old_slots(), half_old_capacity + 1, slot_size),

              slot_size * half_old_capacity);

  std::memcpy(SlotAddress(new_slots, half_old_capacity + 1, slot_size),

              old_slots(), slot_size * (half_old_capacity + 1));

}

void HashSetResizeHelper::GrowSizeIntoSingleGroupTransferable(

    CommonFields& c, size_t slot_size) {

  assert(old_capacity_ < Group::kWidth / 2);

  assert(is_single_group(c.capacity()));

  assert(IsGrowingIntoSingleGroupApplicable(old_capacity_, c.capacity()));

  GrowIntoSingleGroupShuffleControlBytes(c.control(), c.capacity());

  GrowIntoSingleGroupShuffleTransferableSlots(c.slot_array(), slot_size);

  // We poison since GrowIntoSingleGroupShuffleTransferableSlots

  // may leave empty slots unpoisoned.

  PoisonSingleGroupEmptySlots(c, slot_size);

}

void HashSetResizeHelper::TransferSlotAfterSoo(CommonFields& c,

                                               size_t slot_size) {

  assert(was_soo_);

  assert(had_soo_slot_);

  assert(is_single_group(c.capacity()));

  std::memcpy(SlotAddress(c.slot_array(), SooSlotIndex(), slot_size),

              old_soo_data(), slot_size);

  PoisonSingleGroupEmptySlots(c, slot_size);

}

namespace {

// Called whenever the table needs to vacate empty slots either by removing

// tombstones via rehash or growth.

ABSL_ATTRIBUTE_NOINLINE

FindInfo FindInsertPositionWithGrowthOrRehash(CommonFields& common, size_t hash,

                                              const PolicyFunctions& policy) {

  const size_t cap = common.capacity();

  if (cap > Group::kWidth &&

      // Do these calculations in 64-bit to avoid overflow.

      common.size() * uint64_t{32} <= cap * uint64_t{25}) {

    // Squash DELETED without growing if there is enough capacity.

    //

    // Rehash in place if the current size is <= 25/32 of capacity.

    // Rationale for such a high factor: 1) DropDeletesWithoutResize() is

    // faster than resize, and 2) it takes quite a bit of work to add

    // tombstones.  In the worst case, seems to take approximately 4

    // insert/erase pairs to create a single tombstone and so if we are

    // rehashing because of tombstones, we can afford to rehash-in-place as

    // long as we are reclaiming at least 1/8 the capacity without doing more

    // than 2X the work.  (Where "work" is defined to be size() for rehashing

    // or rehashing in place, and 1 for an insert or erase.)  But rehashing in

    // place is faster per operation than inserting or even doubling the size

    // of the table, so we actually afford to reclaim even less space from a

    // resize-in-place.  The decision is to rehash in place if we can reclaim

    // at about 1/8th of the usable capacity (specifically 3/28 of the

    // capacity) which means that the total cost of rehashing will be a small

    // fraction of the total work.

    //

    // Here is output of an experiment using the BM_CacheInSteadyState

    // benchmark running the old case (where we rehash-in-place only if we can

    // reclaim at least 7/16*capacity) vs. this code (which rehashes in place

    // if we can recover 3/32*capacity).

    //

    // Note that although in the worst-case number of rehashes jumped up from

    // 15 to 190, but the number of operations per second is almost the same.

    //

    // Abridged output of running BM_CacheInSteadyState benchmark from

    // raw_hash_set_benchmark.   N is the number of insert/erase operations.

    //

    //      | OLD (recover >= 7/16        | NEW (recover >= 3/32)

    // size |    N/s LoadFactor NRehashes |    N/s LoadFactor NRehashes

    //  448 | 145284       0.44        18 | 140118       0.44        19

    //  493 | 152546       0.24        11 | 151417       0.48        28

    //  538 | 151439       0.26        11 | 151152       0.53        38

    //  583 | 151765       0.28        11 | 150572       0.57        50

    //  628 | 150241       0.31        11 | 150853       0.61        66

    //  672 | 149602       0.33        12 | 150110       0.66        90

    //  717 | 149998       0.35        12 | 149531       0.70       129

    //  762 | 149836       0.37        13 | 148559       0.74       190

    //  807 | 149736       0.39        14 | 151107       0.39        14

    //  852 | 150204       0.42        15 | 151019       0.42        15

    DropDeletesWithoutResize(common, policy);

  } else {

    // Otherwise grow the container.

    policy.resize(common, NextCapacity(cap), HashtablezInfoHandle{});

  }

  // This function is typically called with tables containing deleted slots.

  // The table will be big and `FindFirstNonFullAfterResize` will always

  // fallback to `find_first_non_full`. So using `find_first_non_full` directly.

  return find_first_non_full(common, hash);

}

}  // namespace

const void* GetHashRefForEmptyHasher(const CommonFields& common) {

  // Empty base optimization typically make the empty base class address to be

  // the same as the first address of the derived class object.

  // But we generally assume that for empty hasher we can return any valid

  // pointer.

  return &common;

}

size_t PrepareInsertNonSoo(CommonFields& common, size_t hash, FindInfo target,

                           const PolicyFunctions& policy) {

  // When there are no deleted slots in the table

  // and growth_left is positive, we can insert at the first

  // empty slot in the probe sequence (target).

  const bool use_target_hint =

      // Optimization is disabled when generations are enabled.

      // We have to rehash even sparse tables randomly in such mode.

      !SwisstableGenerationsEnabled() &&

      common.growth_info().HasNoDeletedAndGrowthLeft();

  if (ABSL_PREDICT_FALSE(!use_target_hint)) {

    // Notes about optimized mode when generations are disabled:

    // We do not enter this branch if table has no deleted slots

    // and growth_left is positive.

    // We enter this branch in the following cases listed in decreasing

    // frequency:

    // 1. Table without deleted slots (>95% cases) that needs to be resized.

    // 2. Table with deleted slots that has space for the inserting element.

    // 3. Table with deleted slots that needs to be rehashed or resized.

    if (ABSL_PREDICT_TRUE(common.growth_info().HasNoGrowthLeftAndNoDeleted())) {

      const size_t old_capacity = common.capacity();

      policy.resize(common, NextCapacity(old_capacity), HashtablezInfoHandle{});

      target = HashSetResizeHelper::FindFirstNonFullAfterResize(

          common, old_capacity, hash);

    } else {

      // Note: the table may have no deleted slots here when generations

      // are enabled.

      const bool rehash_for_bug_detection =

          common.should_rehash_for_bug_detection_on_insert();

      if (rehash_for_bug_detection) {

        // Move to a different heap allocation in order to detect bugs.

        const size_t cap = common.capacity();

        policy.resize(common,

                      common.growth_left() > 0 ? cap : NextCapacity(cap),

                      HashtablezInfoHandle{});

      }

      if (ABSL_PREDICT_TRUE(common.growth_left() > 0)) {

        target = find_first_non_full(common, hash);

      } else {

        target = FindInsertPositionWithGrowthOrRehash(common, hash, policy);

      }

    }

  }

  PrepareInsertCommon(common);

  common.growth_info().OverwriteControlAsFull(common.control()[target.offset]);

  SetCtrl(common, target.offset, H2(hash), policy.slot_size);

  common.infoz().RecordInsert(hash, target.probe_length);

  return target.offset;

}

}  // namespace container_internal

ABSL_NAMESPACE_END

}  // namespace absl