: alloc_(alloc), tag_producer_(std::make_unique<TagProducerImpl>()),

      stats_matcher_(std::make_unique<StatsMatcherImpl>()),

      histogram_settings_(std::make_unique<HistogramSettingsImpl>()),

      null_counter_(alloc.symbolTable()), null_gauge_(alloc.symbolTable()),

      null_histogram_(alloc.symbolTable()), null_text_readout_(alloc.symbolTable()),

      well_known_tags_(alloc.symbolTable().makeSet("well_known_tags")) {

  for (const auto& desc : Config::TagNames::get().descriptorVec()) {

    well_known_tags_->rememberBuiltin(desc.name_);

  }

  StatNameManagedStorage empty("", alloc.symbolTable());

  auto new_scope = std::make_shared<ScopeImpl>(*this, StatName(empty.statName()), false);

  addScope(new_scope);

  default_scope_ = new_scope;

}

ThreadLocalStoreImpl::~ThreadLocalStoreImpl() {

  ASSERT(shutting_down_ || !threading_ever_initialized_);

  default_scope_.reset();

  ASSERT(scopes_.empty());

  ASSERT(scopes_to_cleanup_.empty());

  ASSERT(central_cache_entries_to_cleanup_.empty());

  ASSERT(histograms_to_cleanup_.empty());

}

void ThreadLocalStoreImpl::setHistogramSettings(HistogramSettingsConstPtr&& histogram_settings) {

  iterateScopes([this](const ScopeImplSharedPtr& scope)

                    ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) -> bool {

                      assertLocked(*scope);

                      ASSERT(scope->centralCacheLockHeld()->histograms_.empty());

                      return true;

                    });

  histogram_settings_ = std::move(histogram_settings);

}

void ThreadLocalStoreImpl::setStatsMatcher(StatsMatcherPtr&& stats_matcher) {

  stats_matcher_ = std::move(stats_matcher);

  if (stats_matcher_->acceptsAll()) {

    return;

  }

  Thread::LockGuard lock(lock_);

  const uint32_t first_histogram_index = deleted_histograms_.size();

  iterateScopesLockHeld([this](const ScopeImplSharedPtr& scope) ABSL_EXCLUSIVE_LOCKS_REQUIRED(

                            lock_) -> bool {

    assertLocked(*scope);

    if (scope->scope_matcher_ != nullptr) {

      return true;

    }

    const CentralCacheEntrySharedPtr& central_cache = scope->centralCacheLockHeld();

    removeRejectedStats<CounterSharedPtr>(central_cache->counters_,

                                          [this](const CounterSharedPtr& counter) mutable {

                                            alloc_.markCounterForDeletion(counter);

                                          });

    removeRejectedStats<GaugeSharedPtr>(

        central_cache->gauges_,

        [this](const GaugeSharedPtr& gauge) mutable { alloc_.markGaugeForDeletion(gauge); });

    removeRejectedStats(central_cache->histograms_, deleted_histograms_);

    removeRejectedStats<TextReadoutSharedPtr>(

        central_cache->text_readouts_, [this](const TextReadoutSharedPtr& text_readout) mutable {

          alloc_.markTextReadoutForDeletion(text_readout);

        });

    return true;

  });

  {

    Thread::LockGuard hist_lock(hist_mutex_);

    for (uint32_t i = first_histogram_index; i < deleted_histograms_.size(); ++i) {

      uint32_t erased = histogram_set_.erase(deleted_histograms_[i].get());

      ASSERT(erased == 1);

      sinked_histograms_.erase(deleted_histograms_[i].get());

    }

  }

}

void ThreadLocalStoreImpl::removeRejectedStats(StatMapClass& map, StatListClass& list) {

  StatNameVec remove_list;

  for (auto& stat : map) {

    if (rejects(stat.first)) {

      remove_list.push_back(stat.first);

    }

  }

  for (StatName stat_name : remove_list) {

    auto iter = map.find(stat_name);

    ASSERT(iter != map.end());

    list.push_back(iter->second); // Save SharedPtr to the list to avoid invalidating refs to stat.

    map.erase(iter);

  }

}

    StatNameHashMap<StatSharedPtr>& map, std::function<void(const StatSharedPtr&)> f_deletion) {

  StatNameVec remove_list;

  for (auto& stat : map) {

    if (rejects(stat.first)) {

      remove_list.push_back(stat.first);

    }

  }

  for (StatName stat_name : remove_list) {

    auto iter = map.find(stat_name);

    ASSERT(iter != map.end());

    f_deletion(iter->second);

    map.erase(iter);

  }

}

std::vector<CounterSharedPtr> ThreadLocalStoreImpl::counters() const {

  std::vector<CounterSharedPtr> ret;

  forEachCounter([&ret](std::size_t size) { ret.reserve(size); },

                 [&ret](Counter& counter) { ret.emplace_back(CounterSharedPtr(&counter)); });

  return ret;

}

                                                            StatsMatcherSharedPtr matcher) {

  StatNameManagedStorage stat_name_storage(Utility::sanitizeStatsName(name), symbolTable());

  return scopeFromStatName(stat_name_storage.statName(), evictable, limits, std::move(matcher));

}

                                                   StatsMatcherSharedPtr matcher) {

  SymbolTable::StoragePtr joined = symbolTable().join({prefix_.statName(), name});

  StatsMatcherSharedPtr child_matcher = matcher ? std::move(matcher) : scope_matcher_;

  auto new_scope = std::make_shared<ScopeImpl>(parent_, StatName(joined.get()), evictable, limits,

                                               std::move(child_matcher));

  parent_.addScope(new_scope);

  return new_scope;

}

void ThreadLocalStoreImpl::addScope(std::shared_ptr<ScopeImpl>& new_scope) {

  Thread::LockGuard lock(lock_);

  scopes_[new_scope.get()] = std::weak_ptr<ScopeImpl>(new_scope);

}

std::vector<GaugeSharedPtr> ThreadLocalStoreImpl::gauges() const {

  std::vector<GaugeSharedPtr> ret;

  forEachGauge([&ret](std::size_t size) { ret.reserve(size); },

               [&ret](Gauge& gauge) { ret.emplace_back(GaugeSharedPtr(&gauge)); });

  return ret;

}

std::vector<TextReadoutSharedPtr> ThreadLocalStoreImpl::textReadouts() const {

  std::vector<TextReadoutSharedPtr> ret;

  forEachTextReadout(

      [&ret](std::size_t size) { ret.reserve(size); },

      [&ret](TextReadout& text_readout) { ret.emplace_back(TextReadoutSharedPtr(&text_readout)); });

  return ret;

}

std::vector<ParentHistogramSharedPtr> ThreadLocalStoreImpl::histograms() const {

  std::vector<ParentHistogramSharedPtr> ret;

  forEachHistogram([&ret](std::size_t size) mutable { ret.reserve(size); },

                   [&ret](ParentHistogram& histogram) mutable {

                     ret.emplace_back(ParentHistogramSharedPtr(&histogram));

                   });

  return ret;

}

                                               ThreadLocal::Instance& tls) {

  threading_ever_initialized_ = true;

  main_thread_dispatcher_ = &main_thread_dispatcher;

  tls_cache_ = ThreadLocal::TypedSlot<TlsCache>::makeUnique(tls);

  tls_cache_->set(

      [](Event::Dispatcher&) -> std::shared_ptr<TlsCache> { return std::make_shared<TlsCache>(); });

  tls_ = tls;

}

void ThreadLocalStoreImpl::shutdownThreading() {

  shutting_down_ = true;

  ASSERT(!tls_.has_value() || tls_->isShutdown());

  {

    Thread::LockGuard lock(lock_);

    scopes_to_cleanup_.clear();

    central_cache_entries_to_cleanup_.clear();

  }

  Thread::LockGuard lock(hist_mutex_);

  for (ParentHistogramImpl* histogram : histogram_set_) {

    histogram->setShuttingDown(true);

  }

  histogram_set_.clear();

  sinked_histograms_.clear();

}

void ThreadLocalStoreImpl::mergeHistograms(PostMergeCb merge_complete_cb) {

  if (!shutting_down_) {

    ASSERT(!merge_in_progress_);

    merge_in_progress_ = true;

    tls_cache_->runOnAllThreads(

        [](OptRef<TlsCache> tls_cache) {

          for (const auto& id_hist : tls_cache->tls_histogram_cache_) {

            const TlsHistogramSharedPtr& tls_hist = id_hist.second;

            tls_hist->beginMerge();

          }

        },

        [this, merge_complete_cb]() -> void { mergeInternal(merge_complete_cb); });

  } else {

    merge_complete_cb();

  }

}

void ThreadLocalStoreImpl::mergeInternal(PostMergeCb merge_complete_cb) {

  if (!shutting_down_) {

    forEachHistogram(nullptr, [](ParentHistogram& histogram) { histogram.merge(); });

    merge_complete_cb();

    merge_in_progress_ = false;

  }

}

ThreadLocalStoreImpl::CentralCacheEntry::~CentralCacheEntry() {

  ASSERT(symbol_table_.toString(StatNameManagedStorage("Hello.world", symbol_table_).statName()) ==

         "Hello.world");

  rejected_stats_.free(symbol_table_); // NOLINT(clang-analyzer-unix.Malloc)

}

void ThreadLocalStoreImpl::releaseScopeCrossThread(ScopeImpl* scope) {

  Thread::ReleasableLockGuard lock(lock_);

  ASSERT(scopes_.count(scope) == 1);

  scopes_.erase(scope);

  if (!shutting_down_ && main_thread_dispatcher_) {

    bool need_post = scopes_to_cleanup_.empty();

    scopes_to_cleanup_.push_back(scope->scope_id_);

    assertLocked(*scope);

    central_cache_entries_to_cleanup_.push_back(scope->centralCacheLockHeld());

    lock.release();

    if (need_post) {

      main_thread_dispatcher_->post([this]() {

        sync_.syncPoint(MainDispatcherCleanupSync);

        clearScopesFromCaches();

      });

    }

  }

}

void ThreadLocalStoreImpl::releaseHistogramCrossThread(uint64_t histogram_id) {

  if (!shutting_down_ && main_thread_dispatcher_) {

    bool need_post = false;

    {

      Thread::LockGuard lock(hist_mutex_);

      need_post = histograms_to_cleanup_.empty();

      histograms_to_cleanup_.push_back(histogram_id);

    }

    if (need_post) {

      main_thread_dispatcher_->post([this]() { clearHistogramsFromCaches(); });

    }

  }

}

ThreadLocalStoreImpl::TlsCache::insertScope(uint64_t scope_id) {

  return scope_cache_[scope_id];

}

void ThreadLocalStoreImpl::TlsCache::eraseScopes(const std::vector<uint64_t>& scope_ids) {

  for (uint64_t scope_id : scope_ids) {

    scope_cache_.erase(scope_id);

  }

}

void ThreadLocalStoreImpl::TlsCache::eraseHistograms(const std::vector<uint64_t>& histograms) {

  for (uint64_t histogram_id : histograms) {

    tls_histogram_cache_.erase(histogram_id);

  }

}

void ThreadLocalStoreImpl::clearScopesFromCaches() {

  if (!shutting_down_) {

    auto scope_ids = std::make_shared<std::vector<uint64_t>>();

    auto central_caches = std::make_shared<std::vector<CentralCacheEntrySharedPtr>>();

    {

      Thread::LockGuard lock(lock_);

      *scope_ids = std::move(scopes_to_cleanup_);

      scopes_to_cleanup_.clear();

      *central_caches = std::move(central_cache_entries_to_cleanup_);

      central_cache_entries_to_cleanup_.clear();

    }

    tls_cache_->runOnAllThreads(

        [scope_ids](OptRef<TlsCache> tls_cache) { tls_cache->eraseScopes(*scope_ids); },

        [central_caches]() { /* Holds onto central_caches until all tls caches are clear */ });

  }

}

void ThreadLocalStoreImpl::clearHistogramsFromCaches() {

  if (!shutting_down_) {

    auto histograms = std::make_shared<std::vector<uint64_t>>();

    {

      Thread::LockGuard lock(hist_mutex_);

      histograms->swap(histograms_to_cleanup_);

    }

    tls_cache_->runOnAllThreads(

        [histograms](OptRef<TlsCache> tls_cache) { tls_cache->eraseHistograms(*histograms); });

  }

}

    : scope_id_(parent.next_scope_id_++), parent_(parent), evictable_(evictable), limits_(limits),

      scope_matcher_(std::move(scope_matcher)), prefix_(prefix, parent.alloc_.symbolTable()),

      central_cache_(new CentralCacheEntry(parent.alloc_.symbolTable())) {

  parent_.ensureOverflowStats(limits_);

}

ThreadLocalStoreImpl::ScopeImpl::~ScopeImpl() {

  parent_.sync_.syncPoint(DeleteScopeSync);

  parent_.releaseScopeCrossThread(this);

  prefix_.free(symbolTable());

}

      : pool_(tls.symbolTable()), stat_name_tags_(stat_name_tags.value_or(StatNameTagVector())) {

    if (!stat_name_tags) {

      TagVector tags;

      tag_extracted_name_ =

          pool_.add(tls.tagProducer().produceTags(tls.symbolTable().toString(name), tags));

      StatName empty;

      for (const auto& tag : tags) {

        StatName tag_name = tls.wellKnownTags().getBuiltin(tag.name_, empty);

        if (tag_name.empty()) {

          tag_name = pool_.add(tag.name_);

        }

        stat_name_tags_.emplace_back(tag_name, pool_.add(tag.value_));

      }

    } else {

      tag_extracted_name_ = name;

    }

  }

  const StatNameTagVector& statNameTags() const { return stat_name_tags_; }

  StatName tagExtractedName() const { return tag_extracted_name_; }

                                                     const StatsMatcher& matcher) {

  if (matcher.acceptsAll()) {

    return false;

  }

  auto iter = central_rejected_stats.find(name);

  const StatNameStorage* rejected_name = nullptr;

  if (iter != central_rejected_stats.end()) {

    rejected_name = &(*iter);

  } else {

    if (matcher.slowRejects(fast_reject_result, name)) {

      auto insertion = central_rejected_stats.insert(StatNameStorage(name, symbolTable()));

      const StatNameStorage& rejected_name_ref = *(insertion.first);

      rejected_name = &rejected_name_ref;

    }

  }

  if (rejected_name != nullptr) {

    if (tls_rejected_stats != nullptr) {

      tls_rejected_stats->insert(rejected_name->statName());

    }

    return true;

  }

  return false;

}

    StatNameHashSet* tls_rejected_stats, StatType& null_stat) {

  if (tls_rejected_stats != nullptr &&

      tls_rejected_stats->find(full_stat_name) != tls_rejected_stats->end()) {

    return null_stat;

  }

  if (tls_cache) {

    auto pos = tls_cache->find(full_stat_name);

    if (pos != tls_cache->end()) {

      return pos->second;

    }

  }

  Thread::LockGuard lock(parent_.lock_);

  auto iter = central_cache_map.find(full_stat_name);

  RefcountPtr<StatType>* central_ref = nullptr;

  if (iter != central_cache_map.end()) {

    central_ref = &(iter->second);

  } else if (parent_.checkAndRememberRejection(full_stat_name, fast_reject_result,

                                               central_rejected_stats, tls_rejected_stats,

                                               effectiveMatcher())) {

    return null_stat;

  } else {

    if constexpr (std::is_same_v<StatType, Counter>) {

      if (limits_.max_counters != 0 && central_cache_map.size() >= limits_.max_counters) {

        parent_.counters_overflow_->inc();

        return null_stat;

      }

    } else if constexpr (std::is_same_v<StatType, Gauge>) {

      if (limits_.max_gauges != 0 && central_cache_map.size() >= limits_.max_gauges) {

        parent_.gauges_overflow_->inc();

        return null_stat;

      }

    } else {

      static_assert(std::is_same_v<StatType, TextReadout>, "Unexpected StatType");

    }

    StatNameTagHelper tag_helper(parent_, name_no_tags, stat_name_tags);

    RefcountPtr<StatType> stat = make_stat(

        parent_.alloc_, full_stat_name, tag_helper.tagExtractedName(), tag_helper.statNameTags());

    ASSERT(stat != nullptr);

    central_ref = &central_cache_map[stat->statName()];

    *central_ref = stat;

  }

  StatType& ret = **central_ref;

  if (tls_cache) {

    tls_cache->insert(std::make_pair(ret.statName(), std::reference_wrapper<StatType>(ret)));

  }

  return ret;

}

    const StatName& name, StatNameTagVectorOptConstRef stat_name_tags) {

  if (scopeRejectsAll()) {

    return parent_.null_counter_;

  }

  TagUtility::TagStatNameJoiner joiner(prefix_.statName(), name, stat_name_tags, symbolTable());

  Stats::StatName final_stat_name = joiner.nameWithTags();

  StatsMatcher::FastResult fast_reject_result = scopeFastRejects(final_stat_name);

  if (fast_reject_result == StatsMatcher::FastResult::Rejects) {

    return parent_.null_counter_;

  }

  StatRefMap<Counter>* tls_cache = nullptr;

  StatNameHashSet* tls_rejected_stats = nullptr;

  if (!parent_.shutting_down_ && parent_.tls_cache_) {

    TlsCacheEntry& entry = parent_.tlsCache().insertScope(this->scope_id_);

    tls_cache = &entry.counters_;

    tls_rejected_stats = &entry.rejected_stats_;

  }

  const CentralCacheEntrySharedPtr& central_cache = centralCacheNoThreadAnalysis();

  return safeMakeStat<Counter>(

      final_stat_name, joiner.tagExtractedName(), stat_name_tags, central_cache->counters_,

      fast_reject_result, central_cache->rejected_stats_,

      [](Allocator& allocator, StatName name, StatName tag_extracted_name,

         const StatNameTagVector& tags) -> CounterSharedPtr {

        return allocator.makeCounter(name, tag_extracted_name, tags);

      },

      tls_cache, tls_rejected_stats, parent_.null_counter_);

}

void ThreadLocalStoreImpl::deliverHistogramToSinks(const Histogram& histogram, uint64_t value) {

  if (shutting_down_) {

    return;

  }

  for (Sink& sink : timer_sinks_) {

    sink.onHistogramComplete(histogram, value);

  }

}

    Gauge::ImportMode import_mode) {

  if (scopeRejectsAll() && import_mode != Gauge::ImportMode::HiddenAccumulate) {

    return parent_.null_gauge_;

  }

  TagUtility::TagStatNameJoiner joiner(prefix_.statName(), name, stat_name_tags, symbolTable());

  StatName final_stat_name = joiner.nameWithTags();

  StatsMatcher::FastResult fast_reject_result;

  if (import_mode != Gauge::ImportMode::HiddenAccumulate) {

    fast_reject_result = scopeFastRejects(final_stat_name);

  } else {

    fast_reject_result = StatsMatcher::FastResult::Matches;

  }

  if (fast_reject_result == StatsMatcher::FastResult::Rejects) {

    return parent_.null_gauge_;

  }

  StatRefMap<Gauge>* tls_cache = nullptr;

  StatNameHashSet* tls_rejected_stats = nullptr;

  if (!parent_.shutting_down_ && parent_.tls_cache_) {

    TlsCacheEntry& entry = parent_.tlsCache().insertScope(this->scope_id_);

    tls_cache = &entry.gauges_;

    tls_rejected_stats = &entry.rejected_stats_;

  }

  const CentralCacheEntrySharedPtr& central_cache = centralCacheNoThreadAnalysis();

  Gauge& gauge = safeMakeStat<Gauge>(

      final_stat_name, joiner.tagExtractedName(), stat_name_tags, central_cache->gauges_,

      fast_reject_result, central_cache->rejected_stats_,

      [import_mode](Allocator& allocator, StatName name, StatName tag_extracted_name,

                    const StatNameTagVector& tags) -> GaugeSharedPtr {

        return allocator.makeGauge(name, tag_extracted_name, tags, import_mode);

      },

      tls_cache, tls_rejected_stats, parent_.null_gauge_);

  gauge.mergeImportMode(import_mode);

  return gauge;

}

    const StatName& name, StatNameTagVectorOptConstRef stat_name_tags, Histogram::Unit unit) {

  if (scopeRejectsAll()) {

    return parent_.null_histogram_;

  }

  TagUtility::TagStatNameJoiner joiner(prefix_.statName(), name, stat_name_tags, symbolTable());

  StatName final_stat_name = joiner.nameWithTags();

  StatsMatcher::FastResult fast_reject_result = scopeFastRejects(final_stat_name);

  if (fast_reject_result == StatsMatcher::FastResult::Rejects) {

    return parent_.null_histogram_;

  }

  StatNameHashMap<ParentHistogramSharedPtr>* tls_cache = nullptr;

  StatNameHashSet* tls_rejected_stats = nullptr;

  if (!parent_.shutting_down_ && parent_.tls_cache_) {

    TlsCacheEntry& entry = parent_.tlsCache().insertScope(this->scope_id_);

    tls_cache = &entry.parent_histograms_;

    auto iter = tls_cache->find(final_stat_name);

    if (iter != tls_cache->end()) {

      return *iter->second;

    }

    tls_rejected_stats = &entry.rejected_stats_;

    if (tls_rejected_stats->find(final_stat_name) != tls_rejected_stats->end()) {

      return parent_.null_histogram_;

    }

  }

  Thread::LockGuard lock(parent_.lock_);

  const CentralCacheEntrySharedPtr& central_cache = centralCacheNoThreadAnalysis();

  auto iter = central_cache->histograms_.find(final_stat_name);

  ParentHistogramImplSharedPtr* central_ref = nullptr;

  if (iter != central_cache->histograms_.end()) {

    central_ref = &iter->second;

  } else if (parent_.checkAndRememberRejection(final_stat_name, fast_reject_result,

                                               central_cache->rejected_stats_, tls_rejected_stats,

                                               effectiveMatcher())) {

    return parent_.null_histogram_;

  } else {

    StatNameTagHelper tag_helper(parent_, joiner.tagExtractedName(), stat_name_tags);

    ConstSupportedBuckets* buckets = nullptr;

    const auto string_stat_name = symbolTable().toString(final_stat_name);

    buckets = &parent_.histogram_settings_->buckets(string_stat_name);

    const auto bins = parent_.histogram_settings_->bins(string_stat_name);

    RefcountPtr<ParentHistogramImpl> stat;

    {

      Thread::LockGuard lock(parent_.hist_mutex_);

      auto iter = parent_.histogram_set_.find(final_stat_name);

      if (iter != parent_.histogram_set_.end()) {

        stat = RefcountPtr<ParentHistogramImpl>(*iter);

      } else {

        if (limits_.max_histograms != 0 &&

            central_cache->histograms_.size() >= limits_.max_histograms) {

          parent_.histograms_overflow_->inc();

          return parent_.null_histogram_;

        }

        stat = new ParentHistogramImpl(final_stat_name, unit, parent_,

                                       tag_helper.tagExtractedName(), tag_helper.statNameTags(),

                                       *buckets, bins, parent_.next_histogram_id_++);

        if (!parent_.shutting_down_) {

          parent_.histogram_set_.insert(stat.get());

          if (parent_.sink_predicates_.has_value() &&

              parent_.sink_predicates_->includeHistogram(*stat)) {

            parent_.sinked_histograms_.insert(stat.get());

          }

        }

      }

    }

    central_ref = &central_cache->histograms_[stat->statName()];

    *central_ref = stat;

  }

  if (tls_cache != nullptr) {

    tls_cache->insert(std::make_pair((*central_ref)->statName(), *central_ref));

  }

  return **central_ref;

}

    const StatName& name, StatNameTagVectorOptConstRef stat_name_tags) {

  if (scopeRejectsAll()) {

    return parent_.null_text_readout_;

  }

  TagUtility::TagStatNameJoiner joiner(prefix_.statName(), name, stat_name_tags, symbolTable());

  Stats::StatName final_stat_name = joiner.nameWithTags();

  StatsMatcher::FastResult fast_reject_result = scopeFastRejects(final_stat_name);

  if (fast_reject_result == StatsMatcher::FastResult::Rejects) {

    return parent_.null_text_readout_;

  }

  StatRefMap<TextReadout>* tls_cache = nullptr;

  StatNameHashSet* tls_rejected_stats = nullptr;

  if (!parent_.shutting_down_ && parent_.tls_cache_) {

    TlsCacheEntry& entry = parent_.tlsCache().insertScope(this->scope_id_);

    tls_cache = &entry.text_readouts_;

    tls_rejected_stats = &entry.rejected_stats_;

  }

  const CentralCacheEntrySharedPtr& central_cache = centralCacheNoThreadAnalysis();

  return safeMakeStat<TextReadout>(

      final_stat_name, joiner.tagExtractedName(), stat_name_tags, central_cache->text_readouts_,

      fast_reject_result, central_cache->rejected_stats_,

      [](Allocator& allocator, StatName name, StatName tag_extracted_name,

         const StatNameTagVector& tags) -> TextReadoutSharedPtr {

        return allocator.makeTextReadout(name, tag_extracted_name, tags);

      },

      tls_cache, tls_rejected_stats, parent_.null_text_readout_);

}

CounterOptConstRef ThreadLocalStoreImpl::ScopeImpl::findCounter(StatName name) const {

  Thread::LockGuard lock(parent_.lock_);

  return findStatLockHeld<Counter>(name, central_cache_->counters_);

}

GaugeOptConstRef ThreadLocalStoreImpl::ScopeImpl::findGauge(StatName name) const {

  Thread::LockGuard lock(parent_.lock_);

  return findStatLockHeld<Gauge>(name, central_cache_->gauges_);

}

HistogramOptConstRef ThreadLocalStoreImpl::ScopeImpl::findHistogram(StatName name) const {

  Thread::LockGuard lock(parent_.lock_);

  return findHistogramLockHeld(name);

}

    ABSL_EXCLUSIVE_LOCKS_REQUIRED(parent_.lock_) {

  auto iter = central_cache_->histograms_.find(name);

  if (iter == central_cache_->histograms_.end()) {

    return absl::nullopt;

  }

  RefcountPtr<Histogram> histogram_ref(iter->second);

  return std::cref(*histogram_ref);

}

TextReadoutOptConstRef ThreadLocalStoreImpl::ScopeImpl::findTextReadout(StatName name) const {

  Thread::LockGuard lock(parent_.lock_);

  return findStatLockHeld<TextReadout>(name, central_cache_->text_readouts_);

}

Histogram& ThreadLocalStoreImpl::tlsHistogram(ParentHistogramImpl& parent, uint64_t id) {

  TlsHistogramSharedPtr* tls_histogram = nullptr;

  if (!shutting_down_ && tls_cache_) {

    tls_histogram = &(tlsCache().tls_histogram_cache_[id]);

    if (*tls_histogram != nullptr) {

      return **tls_histogram;

    }

  }

  StatNameTagHelper tag_helper(*this, parent.statName(), absl::nullopt);

  TlsHistogramSharedPtr hist_tls_ptr(

      new ThreadLocalHistogramImpl(parent.statName(), parent.unit(), tag_helper.tagExtractedName(),

                                   tag_helper.statNameTags(), symbolTable(), parent.bins()));

  parent.addTlsHistogram(hist_tls_ptr);

  if (tls_histogram != nullptr) {

    *tls_histogram = hist_tls_ptr;

  }

  return *hist_tls_ptr;

}

    : HistogramImplHelper(name, tag_extracted_name, stat_name_tags, symbol_table), unit_(unit),

      used_(false), created_thread_id_(std::this_thread::get_id()), symbol_table_(symbol_table) {

  histograms_[0] = bins ? hist_alloc_nbins(bins.value()) : hist_alloc();

  histograms_[1] = bins ? hist_alloc_nbins(bins.value()) : hist_alloc();

}

ThreadLocalHistogramImpl::~ThreadLocalHistogramImpl() {

  MetricImpl::clear(symbol_table_);

  hist_free(histograms_[0]);

  hist_free(histograms_[1]);

}

void ThreadLocalHistogramImpl::recordValue(uint64_t value) {

  ASSERT(std::this_thread::get_id() == created_thread_id_);

  hist_insert_intscale(histograms_[current_active_], value, 0, 1);

  used_ = true;

}

void ThreadLocalHistogramImpl::merge(histogram_t* target) {

  histogram_t** other_histogram = &histograms_[otherHistogramIndex()];

  hist_accumulate(target, other_histogram, 1);

  hist_clear(*other_histogram);

}

    : MetricImpl(name, tag_extracted_name, stat_name_tags, thread_local_store.symbolTable()),

      unit_(unit), bins_(bins), thread_local_store_(thread_local_store),

      interval_histogram_(hist_alloc()), cumulative_histogram_(hist_alloc()),

      interval_statistics_(interval_histogram_, unit, supported_buckets),

      cumulative_statistics_(cumulative_histogram_, unit, supported_buckets), id_(id) {}

ParentHistogramImpl::~ParentHistogramImpl() {

  thread_local_store_.releaseHistogramCrossThread(id_);

  ASSERT(ref_count_ == 0);

  MetricImpl::clear(thread_local_store_.symbolTable());

  hist_free(interval_histogram_);

  hist_free(cumulative_histogram_);

}

void ParentHistogramImpl::incRefCount() { ++ref_count_; }

bool ParentHistogramImpl::decRefCount() {

  bool ret;

  if (shutting_down_) {

    ret = --ref_count_ == 0;

  } else {

    ret = thread_local_store_.decHistogramRefCount(*this, ref_count_);

  }

  return ret;

}

                                                std::atomic<uint32_t>& ref_count) {

  Thread::LockGuard lock(hist_mutex_);

  ASSERT(ref_count >= 1);

  if (--ref_count == 0) {

    if (!shutting_down_) {

      const size_t count = histogram_set_.erase(hist.statName());

      ASSERT(shutting_down_ || count == 1);

      sinked_histograms_.erase(&hist);

    }

    return true;

  }

  return false;

}

SymbolTable& ParentHistogramImpl::symbolTable() { return thread_local_store_.symbolTable(); }

Histogram::Unit ParentHistogramImpl::unit() const { return unit_; }

void ParentHistogramImpl::recordValue(uint64_t value) {

  Histogram& tls_histogram = thread_local_store_.tlsHistogram(*this, id_);

  tls_histogram.recordValue(value);

  thread_local_store_.deliverHistogramToSinks(*this, value);

}

bool ParentHistogramImpl::used() const {

  return merged_;

}

void ParentHistogramImpl::markUnused() {

  merged_ = false;

  Thread::LockGuard lock(merge_lock_);

  for (const TlsHistogramSharedPtr& tls_histogram : tls_histograms_) {

    tls_histogram->markUnused();

  }

}

bool ParentHistogramImpl::hidden() const { return false; }

void ParentHistogramImpl::merge() {

  Thread::ReleasableLockGuard lock(merge_lock_);

  if (merged_ || usedLockHeld()) {

    hist_clear(interval_histogram_);

    for (const TlsHistogramSharedPtr& tls_histogram : tls_histograms_) {

      tls_histogram->merge(interval_histogram_);

    }

    lock.release();

    hist_accumulate(cumulative_histogram_, &interval_histogram_, 1);

    cumulative_statistics_.refresh(cumulative_histogram_);

    interval_statistics_.refresh(interval_histogram_);

    merged_ = true;

  }

}

std::string ParentHistogramImpl::quantileSummary() const {

  if (used()) {

    std::vector<std::string> summary;

    const std::vector<double>& supported_quantiles_ref = interval_statistics_.supportedQuantiles();

    summary.reserve(supported_quantiles_ref.size());

    for (size_t i = 0; i < supported_quantiles_ref.size(); ++i) {

      summary.push_back(fmt::format("P{:g}({},{})", 100 * supported_quantiles_ref[i],

                                    interval_statistics_.computedQuantiles()[i],

                                    cumulative_statistics_.computedQuantiles()[i]));

    }

    return absl::StrJoin(summary, " ");

  } else {

}

std::string ParentHistogramImpl::bucketSummary() const {

  if (used()) {

    std::vector<std::string> bucket_summary;

    ConstSupportedBuckets& supported_buckets = interval_statistics_.supportedBuckets();

    bucket_summary.reserve(supported_buckets.size());

    for (size_t i = 0; i < supported_buckets.size(); ++i) {

      bucket_summary.push_back(fmt::format("B{:g}({},{})", supported_buckets[i],

                                           interval_statistics_.computedBuckets()[i],

                                           cumulative_statistics_.computedBuckets()[i]));

    }

    return absl::StrJoin(bucket_summary, " ");

  } else {

    return {"No recorded values"};

  }

}

ParentHistogramImpl::detailedlBucketsHelper(const histogram_t& histogram) const {

  const uint32_t num_buckets = hist_num_buckets(&histogram);

  std::vector<Stats::ParentHistogram::Bucket> buckets(num_buckets);

  hist_bucket_t hist_bucket;

  for (uint32_t i = 0; i < num_buckets; ++i) {

    ParentHistogram::Bucket& bucket = buckets[i];

    hist_bucket_idx_bucket(&histogram, i, &hist_bucket, &bucket.count_);

    bucket.lower_bound_ = hist_bucket_to_double(hist_bucket);

    bucket.width_ = hist_bucket_to_double_bin_width(hist_bucket);

    if (unit_ == Histogram::Unit::Percent) {

      constexpr double percent_scale = 1.0 / Histogram::PercentScale;

      bucket.lower_bound_ *= percent_scale;

      bucket.width_ *= percent_scale;

    }

  }

  return buckets;

}

uint64_t ParentHistogramImpl::cumulativeCountLessThanOrEqualToValue(double value) const {

  const double raw_value =

      (unit_ == Histogram::Unit::Percent) ? (value * Histogram::PercentScale) : value;

  return hist_approx_count_below(cumulative_histogram_, raw_value);

}

void ParentHistogramImpl::addTlsHistogram(const TlsHistogramSharedPtr& hist_ptr) {

  Thread::LockGuard lock(merge_lock_);

  tls_histograms_.emplace_back(hist_ptr);

}

bool ParentHistogramImpl::usedLockHeld() const {

  for (const TlsHistogramSharedPtr& tls_histogram : tls_histograms_) {

    if (tls_histogram->used()) {

      return true;

    }

  }

  return false;

}

void ThreadLocalStoreImpl::forEachCounter(SizeFn f_size, StatFn<Counter> f_stat) const {

  alloc_.forEachCounter(f_size, f_stat);

}

void ThreadLocalStoreImpl::forEachGauge(SizeFn f_size, StatFn<Gauge> f_stat) const {

  alloc_.forEachGauge(f_size, f_stat);

}

void ThreadLocalStoreImpl::forEachTextReadout(SizeFn f_size, StatFn<TextReadout> f_stat) const {

  alloc_.forEachTextReadout(f_size, f_stat);

}

void ThreadLocalStoreImpl::forEachHistogram(SizeFn f_size, StatFn<ParentHistogram> f_stat) const {

  Thread::LockGuard lock(hist_mutex_);

  if (f_size != nullptr) {

    f_size(histogram_set_.size());

  }

  for (ParentHistogramImpl* histogram : histogram_set_) {

    f_stat(*histogram);

  }

}

                                        StatFn<const Scope> f_scope) const {

  std::vector<ScopeSharedPtr> scopes;

  iterateScopes([&scopes](const ScopeImplSharedPtr& scope) -> bool {

    scopes.push_back(scope);

    return true;

  });

  if (f_size != nullptr) {

    f_size(scopes.size());

  }

  for (const ScopeSharedPtr& scope : scopes) {

    f_scope(*scope);

  }

}

  explicit MetricBag(uint64_t scope_id) : scope_id_(scope_id) {}

  bool empty() const {

    return counters_.empty() && gauges_.empty() && histograms_.empty() && text_readouts_.empty();

  }

void ThreadLocalStoreImpl::evictUnused() {

  ASSERT_IS_MAIN_OR_TEST_THREAD();

  if (shutting_down_ || !tls_cache_) {

  auto evicted_metrics = std::make_shared<std::vector<MetricBag>>();

  {

    Thread::LockGuard lock(lock_);

    iterateScopesLockHeld([evicted_metrics](const ScopeImplSharedPtr& scope) -> bool {

      if (scope->evictable_) {

        MetricBag metrics(scope->scope_id_);

        CentralCacheEntrySharedPtr& central_cache = scope->centralCacheMutableNoThreadAnalysis();

        auto filter_unused = []<typename T>(StatNameHashMap<T>& unused_metrics) {

          return [&unused_metrics](std::pair<StatName, T> kv) {

            const auto& [name, metric] = kv;

            if (metric->used()) {

              metric->markUnused();

              return false;

            } else {

              unused_metrics.try_emplace(name, metric);

              return true;

            }

          };

        };

        absl::erase_if(central_cache->counters_, filter_unused(metrics.counters_));

        absl::erase_if(central_cache->gauges_, filter_unused(metrics.gauges_));

        absl::erase_if(central_cache->text_readouts_, filter_unused(metrics.text_readouts_));

        absl::erase_if(central_cache->histograms_, filter_unused(metrics.histograms_));

        if (!metrics.empty()) {

          evicted_metrics->push_back(std::move(metrics));

        }

      }

      return true;

    });

  }

  if (!evicted_metrics->empty()) {

    tls_cache_->runOnAllThreads(

        [evicted_metrics](OptRef<TlsCache> tls_cache) {

          for (const auto& metrics : *evicted_metrics) {

            TlsCacheEntry& entry = tls_cache->insertScope(metrics.scope_id_);

            absl::erase_if(entry.counters_,

                           [&](std::pair<StatName, std::reference_wrapper<Counter>> kv) {

                             return metrics.counters_.contains(kv.first);

                           });

            absl::erase_if(entry.gauges_,

                           [&](std::pair<StatName, std::reference_wrapper<Gauge>> kv) {

                             return metrics.gauges_.contains(kv.first);

                           });

            absl::erase_if(entry.text_readouts_,

                           [&](std::pair<StatName, std::reference_wrapper<TextReadout>> kv) {

                             return metrics.text_readouts_.contains(kv.first);

                           });

            absl::erase_if(entry.parent_histograms_,

                           [&](std::pair<StatName, ParentHistogramSharedPtr> kv) {

                             return metrics.histograms_.contains(kv.first);

                           });

          }

        },

        [evicted_metrics]() {

          size_t scopes = 0, counters = 0, gauges = 0, readouts = 0, histograms = 0;

          for (const auto& metrics : *evicted_metrics) {

            scopes += 1;

            counters += metrics.counters_.size();

            gauges += metrics.gauges_.size();

            readouts += metrics.text_readouts_.size();

            histograms += metrics.histograms_.size();

          }

          ENVOY_LOG(debug,

                    "deleted stale {} counters, {} gauges, {} text readouts, {} histograms from "

                    "{} scopes",

                    counters, gauges, readouts, histograms, scopes);

        });

  }

}

    ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock_) {

  for (auto& iter : scopes_) {

    sync_.syncPoint(ThreadLocalStoreImpl::IterateScopeSync);

    const std::weak_ptr<ScopeImpl>& scope = iter.second;

    const ScopeImplSharedPtr& locked = scope.lock();

    if (locked != nullptr && !fn(locked)) {

      return false;

    }

  }

  return true;

}

void ThreadLocalStoreImpl::forEachSinkedCounter(SizeFn f_size, StatFn<Counter> f_stat) const {

  alloc_.forEachSinkedCounter(f_size, f_stat);

}

void ThreadLocalStoreImpl::forEachSinkedGauge(SizeFn f_size, StatFn<Gauge> f_stat) const {

  alloc_.forEachSinkedGauge(f_size, f_stat);

}

                                                    StatFn<TextReadout> f_stat) const {

  alloc_.forEachSinkedTextReadout(f_size, f_stat);

}

                                                  StatFn<ParentHistogram> f_stat) const {

  if (sink_predicates_.has_value()) {

    Thread::LockGuard lock(hist_mutex_);

    if (f_size != nullptr) {

      f_size(sinked_histograms_.size());

    }

    for (auto histogram : sinked_histograms_) {

      f_stat(*histogram);

    }

  } else {

    forEachHistogram(f_size, f_stat);

  }

}

void ThreadLocalStoreImpl::setSinkPredicates(std::unique_ptr<SinkPredicates>&& sink_predicates) {

  ASSERT(sink_predicates != nullptr);

  if (sink_predicates != nullptr) {

    sink_predicates_.emplace(*sink_predicates);

    alloc_.setSinkPredicates(std::move(sink_predicates));

    Thread::LockGuard lock(hist_mutex_);

    sinked_histograms_.clear();

    for (auto& histogram : histogram_set_) {

      if (sink_predicates_->includeHistogram(*histogram)) {

        sinked_histograms_.insert(histogram);

      }

    }

  }

}

                                                StatNameTagVector& stat_tags) {

  extractAndAppendTags(symbolTable().toString(name), pool, stat_tags);

}

                                                StatNameTagVector& stat_tags) {

  TagVector tags;

  tagProducer().produceTags(name, tags);

  for (const auto& tag : tags) {

    stat_tags.emplace_back(pool.add(tag.name_), pool.add(tag.value_));

  }

}

void ThreadLocalStoreImpl::ensureOverflowStats(const ScopeStatsLimitSettings& limits) {

  const bool need_counter_overflow_stat = limits.max_counters != 0;

  const bool need_gauge_overflow_stat = limits.max_gauges != 0;

  const bool need_histogram_overflow_stat = limits.max_histograms != 0;

  if (!need_counter_overflow_stat && !need_gauge_overflow_stat && !need_histogram_overflow_stat) {

    return;

  }

  Thread::LockGuard lock(lock_);

  if (need_counter_overflow_stat && counters_overflow_ == nullptr) {

    StatNamePool pool(symbolTable());

    StatName name = pool.add("server.stats_overflow.counter");

    counters_overflow_ = alloc_.makeCounter(name, name, {});

  }

  if (need_gauge_overflow_stat && gauges_overflow_ == nullptr) {

    StatNamePool pool(symbolTable());

    StatName name = pool.add("server.stats_overflow.gauge");

    gauges_overflow_ = alloc_.makeCounter(name, name, {});

  }

  if (need_histogram_overflow_stat && histograms_overflow_ == nullptr) {

    StatNamePool pool(symbolTable());

    StatName name = pool.add("server.stats_overflow.histogram");

    histograms_overflow_ = alloc_.makeCounter(name, name, {});

  }

}