1
#include "source/common/memory/stats.h"
2

            
3
#include <atomic>
4
#include <cstdint>
5

            
6
#include "source/common/common/assert.h"
7
#include "source/common/common/logger.h"
8

            
9
#if defined(TCMALLOC)
10
#include "tcmalloc/malloc_extension.h"
11
#elif defined(GPERFTOOLS_TCMALLOC)
12
#include "gperftools/malloc_extension.h"
13
#endif
14

            
15
namespace Envoy {
16
namespace Memory {
17

            
18
namespace {
19
std::atomic<uint64_t> max_unfreed_memory_bytes{DEFAULT_MAX_UNFREED_MEMORY_BYTES};
20
} // namespace
21

            
22
5985
uint64_t maxUnfreedMemoryBytes() {
23
5985
  return max_unfreed_memory_bytes.load(std::memory_order_relaxed);
24
5985
}
25

            
26
3
void setMaxUnfreedMemoryBytes(uint64_t value) {
27
3
  max_unfreed_memory_bytes.store(value, std::memory_order_relaxed);
28
3
}
29

            
30
28481
uint64_t Stats::totalCurrentlyAllocated() {
31
#if defined(TCMALLOC)
32
  return tcmalloc::MallocExtension::GetNumericProperty("generic.current_allocated_bytes")
33
      .value_or(0);
34
#elif defined(GPERFTOOLS_TCMALLOC)
35
  size_t value = 0;
36
28481
  MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes", &value);
37
28481
  return value;
38
#else
39
  return 0;
40
#endif
41
28481
}
42

            
43
22370
uint64_t Stats::totalCurrentlyReserved() {
44
#if defined(TCMALLOC)
45
  // In Google's tcmalloc the semantics of generic.heap_size has
46
  // changed: it doesn't include unmapped bytes.
47
  return tcmalloc::MallocExtension::GetNumericProperty("generic.heap_size").value_or(0) +
48
         tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.pageheap_unmapped_bytes")
49
             .value_or(0);
50
#elif defined(GPERFTOOLS_TCMALLOC)
51
  size_t value = 0;
52
22370
  MallocExtension::instance()->GetNumericProperty("generic.heap_size", &value);
53
22370
  return value;
54
#else
55
  return 0;
56
#endif
57
22370
}
58

            
59
1
uint64_t Stats::totalThreadCacheBytes() {
60
#if defined(TCMALLOC)
61
  return tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.current_total_thread_cache_bytes")
62
      .value_or(0);
63
#elif defined(GPERFTOOLS_TCMALLOC)
64
  size_t value = 0;
65
1
  MallocExtension::instance()->GetNumericProperty("tcmalloc.current_total_thread_cache_bytes",
66
1
                                                  &value);
67
1
  return value;
68
#else
69
  return 0;
70
#endif
71
1
}
72

            
73
3
uint64_t Stats::totalPageHeapFree() {
74
#if defined(TCMALLOC)
75
  return tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.pageheap_free_bytes").value_or(0);
76
#elif defined(GPERFTOOLS_TCMALLOC)
77
  size_t value = 0;
78
3
  MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes", &value);
79
3
  return value;
80
#else
81
  return 0;
82
#endif
83
3
}
84

            
85
5
uint64_t Stats::totalPageHeapUnmapped() {
86
#if defined(TCMALLOC)
87
  return tcmalloc::MallocExtension::GetNumericProperty("tcmalloc.pageheap_unmapped_bytes")
88
      .value_or(0);
89
#elif defined(GPERFTOOLS_TCMALLOC)
90
  size_t value = 0;
91
5
  MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes", &value);
92
5
  return value;
93
#else
94
  return 0;
95
#endif
96
5
}
97

            
98
28348
uint64_t Stats::totalPhysicalBytes() {
99
#if defined(TCMALLOC)
100
  return tcmalloc::MallocExtension::GetProperties()["generic.physical_memory_used"].value;
101
#elif defined(GPERFTOOLS_TCMALLOC)
102
  size_t value = 0;
103
28348
  MallocExtension::instance()->GetNumericProperty("generic.total_physical_bytes", &value);
104
28348
  return value;
105
#else
106
  return 0;
107
#endif
108
28348
}
109

            
110
1
void Stats::dumpStatsToLog() {
111
#if defined(TCMALLOC)
112
  ENVOY_LOG_MISC(debug, "TCMalloc stats:\n{}", tcmalloc::MallocExtension::GetStats());
113
#elif defined(GPERFTOOLS_TCMALLOC)
114
  constexpr int buffer_size = 100000;
115
1
  auto buffer = std::make_unique<char[]>(buffer_size);
116
1
  MallocExtension::instance()->GetStats(buffer.get(), buffer_size);
117
1
  ENVOY_LOG_MISC(debug, "TCMalloc stats:\n{}", buffer.get());
118
#else
119
  return;
120
#endif
121
1
}
122

            
123
1
absl::optional<std::string> Stats::dumpStats() {
124
#if defined(TCMALLOC)
125
  return tcmalloc::MallocExtension::GetStats();
126
#elif defined(GPERFTOOLS_TCMALLOC)
127
  constexpr int buffer_size = 100000;
128
1
  std::string buffer(buffer_size, '\0');
129
1
  MallocExtension::instance()->GetStats(buffer.data(), buffer_size);
130
1
  buffer.resize(strlen(buffer.c_str()));
131
1
  return buffer;
132
#else
133
  return absl::nullopt;
134
#endif
135
1
}
136

            
137
namespace {
138

            
139
/**
140
 * Computes the background release rate in bytes per second from the configured bytes_to_release
141
 * and memory_release_interval.
142
 */
143
size_t computeBackgroundReleaseRate(uint64_t bytes_to_release,
144
10678
                                    std::chrono::milliseconds memory_release_interval_msec) {
145
10678
  if (bytes_to_release == 0 || memory_release_interval_msec.count() == 0) {
146
10676
    return 0;
147
10676
  }
148
2
  return static_cast<size_t>(bytes_to_release * 1000 / memory_release_interval_msec.count());
149
10678
}
150

            
151
} // namespace
152

            
153
AllocatorManager::AllocatorManager(
154
    Api::Api& api, const envoy::config::bootstrap::v3::MemoryAllocatorManager& config)
155
10678
    : bytes_to_release_(config.bytes_to_release()),
156
10678
      memory_release_interval_msec_(std::chrono::milliseconds(
157
10678
          PROTOBUF_GET_MS_OR_DEFAULT(config, memory_release_interval, 1000))),
158
      background_release_rate_bytes_per_second_(
159
10678
          computeBackgroundReleaseRate(bytes_to_release_, memory_release_interval_msec_)),
160
10678
      api_(api) {
161
10678
  configureTcmallocOptions(config);
162
10678
  configureBackgroundMemoryRelease();
163
10678
};
164

            
165
10678
AllocatorManager::~AllocatorManager() {
166
#if defined(TCMALLOC)
167
  if (tcmalloc_thread_) {
168
    // Signal the ProcessBackgroundActions loop to exit and wait for the thread to finish.
169
    tcmalloc::MallocExtension::SetBackgroundProcessActionsEnabled(false);
170
    tcmalloc_thread_->join();
171
    tcmalloc_thread_.reset();
172
    // Reset the release rate and re-enable background actions so that a subsequent
173
    // AllocatorManager instance can start fresh.
174
    tcmalloc::MallocExtension::SetBackgroundReleaseRate(
175
        tcmalloc::MallocExtension::BytesPerSecond{0});
176
    tcmalloc::MallocExtension::SetBackgroundProcessActionsEnabled(true);
177
  }
178
#endif
179
10678
}
180

            
181
void AllocatorManager::configureTcmallocOptions(
182
10678
    const envoy::config::bootstrap::v3::MemoryAllocatorManager& config) {
183
10678
  if (config.max_unfreed_memory_bytes() > 0) {
184
1
    setMaxUnfreedMemoryBytes(config.max_unfreed_memory_bytes());
185
1
    ENVOY_LOG_MISC(info, "Set max unfreed memory threshold to {} bytes.",
186
1
                   config.max_unfreed_memory_bytes());
187
1
  }
188
#if defined(TCMALLOC)
189
  if (config.has_soft_memory_limit_bytes()) {
190
    tcmalloc::MallocExtension::SetMemoryLimit(config.soft_memory_limit_bytes().value(),
191
                                              tcmalloc::MallocExtension::LimitKind::kSoft);
192
    ENVOY_LOG_MISC(info, "Set tcmalloc soft memory limit to {} bytes.",
193
                   config.soft_memory_limit_bytes().value());
194
  }
195
  if (config.has_max_per_cpu_cache_size_bytes()) {
196
    tcmalloc::MallocExtension::SetMaxPerCpuCacheSize(config.max_per_cpu_cache_size_bytes().value());
197
    ENVOY_LOG_MISC(info, "Set tcmalloc max per-CPU cache size to {} bytes.",
198
                   config.max_per_cpu_cache_size_bytes().value());
199
  }
200
#else
201
10678
  if (config.has_soft_memory_limit_bytes()) {
202
1
    ENVOY_LOG_MISC(warn, "Soft memory limit is only supported with Google's tcmalloc, ignoring.");
203
1
  }
204
10678
  if (config.has_max_per_cpu_cache_size_bytes()) {
205
1
    ENVOY_LOG_MISC(warn,
206
1
                   "Max per-CPU cache size is only supported with Google's tcmalloc, ignoring.");
207
1
  }
208
10678
#endif
209
10678
}
210

            
211
/**
212
 * Configures tcmalloc to use its native ProcessBackgroundActions for background memory
213
 * maintenance. This enables comprehensive memory management including per-CPU cache reclamation,
214
 * cache shuffling, size class resizing, transfer cache plundering, and memory release at the
215
 * configured rate. If `bytes_to_release_` is `0`, no background processing will be started.
216
 */
217
10678
void AllocatorManager::configureBackgroundMemoryRelease() {
218
10678
#if defined(GPERFTOOLS_TCMALLOC)
219
10678
  if (bytes_to_release_ > 0) {
220
2
    ENVOY_LOG_MISC(error,
221
2
                   "Memory releasing is not supported for gperf tcmalloc, no memory releasing "
222
2
                   "will be configured.");
223
2
  }
224
#elif defined(TCMALLOC)
225
  ENVOY_BUG(!tcmalloc_thread_, "Invalid state, tcmalloc has already been initialised.");
226
  if (bytes_to_release_ > 0) {
227
    if (!tcmalloc::MallocExtension::NeedsProcessBackgroundActions()) {
228
      ENVOY_LOG_MISC(warn, "This platform does not support tcmalloc background actions.");
229
      return;
230
    }
231

            
232
    tcmalloc::MallocExtension::SetBackgroundReleaseRate(
233
        tcmalloc::MallocExtension::BytesPerSecond{background_release_rate_bytes_per_second_});
234

            
235
    tcmalloc_thread_ = api_.threadFactory().createThread(
236
        []() -> void {
237
          ENVOY_LOG_MISC(debug, "Started {}.", TCMALLOC_ROUTINE_THREAD_ID);
238
          // ProcessBackgroundActions runs an infinite loop that handles all tcmalloc background
239
          // maintenance including cache reclamation and memory release. It returns only when
240
          // SetBackgroundProcessActionsEnabled(false) is called.
241
          tcmalloc::MallocExtension::ProcessBackgroundActions();
242
        },
243
        Thread::Options{std::string(TCMALLOC_ROUTINE_THREAD_ID)});
244

            
245
    ENVOY_LOG_MISC(info, "Configured tcmalloc with background release rate: {} bytes per second.",
246
                   background_release_rate_bytes_per_second_);
247
  }
248
#endif
249
10678
}
250

            
251
} // namespace Memory
252
} // namespace Envoy