1
#pragma once
2

            
3
#include <array>
4
#include <cstdint>
5
#include <list>
6
#include <memory>
7
#include <string>
8
#include <type_traits>
9

            
10
#include "envoy/common/optref.h"
11
#include "envoy/config/core/v3/base.pb.h"
12
#include "envoy/http/header_map.h"
13

            
14
#include "source/common/common/compiled_string_map.h"
15
#include "source/common/common/non_copyable.h"
16
#include "source/common/common/utility.h"
17
#include "source/common/http/headers.h"
18

            
19
namespace Envoy {
20
namespace Http {
21

            
22
/**
23
 * These are definitions of all of the inline header access functions described inside header_map.h
24
 */
25
#define DEFINE_INLINE_HEADER_FUNCS(name)                                                           \
26
public:                                                                                            \
27
9984982
  const HeaderEntry* name() const override { return getInline(HeaderHandles::get().name); }        \
28
2502702
  size_t remove##name() override { return removeInline(HeaderHandles::get().name); }               \
29
6539883
  absl::string_view get##name##Value() const override {                                            \
30
6539883
    return getInlineValue(HeaderHandles::get().name);                                              \
31
6539883
  }                                                                                                \
32
2840819
  void set##name(absl::string_view value) override { setInline(HeaderHandles::get().name, value); }
33

            
34
#define DEFINE_INLINE_HEADER_STRING_FUNCS(name)                                                    \
35
  DEFINE_INLINE_HEADER_FUNCS(name)                                                                 \
36
6136
  void append##name(absl::string_view data, absl::string_view delimiter) override {                \
37
6136
    appendInline(HeaderHandles::get().name, data, delimiter);                                      \
38
6136
  }                                                                                                \
39
166709
  void setReference##name(absl::string_view value) override {                                      \
40
166709
    setReferenceInline(HeaderHandles::get().name, value);                                          \
41
166709
  }
42

            
43
#define DEFINE_INLINE_HEADER_NUMERIC_FUNCS(name)                                                   \
44
  DEFINE_INLINE_HEADER_FUNCS(name)                                                                 \
45
132342
  void set##name(uint64_t value) override { setInline(HeaderHandles::get().name, value); }
46

            
47
/**
48
 * Implementation of Http::HeaderMap. This is heavily optimized for performance. Roughly, when
49
 * headers are added to the map by string, we do a trie lookup to see if it's one of the O(1)
50
 * headers. If it is, we store a reference to it that can be accessed later directly via direct
51
 * method access. Most high performance paths use O(1) direct method access. In general, we try to
52
 * copy as little as possible and allocate as little as possible in any of the paths.
53
 */
54
class HeaderMapImpl : NonCopyable {
55
public:
56
  HeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
57
                const uint32_t max_headers_count = UINT32_MAX)
58
1553222
      : max_headers_kb_(max_headers_kb), max_headers_count_(max_headers_count) {}
59
1552301
  virtual ~HeaderMapImpl() = default;
60

            
61
  // The following "constructors" call virtual functions during construction and must use the
62
  // static factory pattern.
63
  static void copyFrom(HeaderMap& lhs, const HeaderMap& rhs);
64
  // The value_type of iterator must be pair, and the first value of them must be LowerCaseString.
65
  // If not, it won't be compiled successfully.
66
37232
  template <class It> static void initFromInitList(HeaderMap& new_header_map, It begin, It end) {
67
78734
    for (auto it = begin; it != end; ++it) {
68
41502
      static_assert(std::is_same<decltype(it->first), LowerCaseString>::value,
69
41502
                    "iterator must be pair and the first value of them must be LowerCaseString");
70
41502
      HeaderString key_string;
71
41502
      key_string.setCopy(it->first.get().c_str(), it->first.get().size());
72
41502
      HeaderString value_string;
73
41502
      value_string.setCopy(it->second.c_str(), it->second.size());
74
41502
      new_header_map.addViaMove(std::move(key_string), std::move(value_string));
75
41502
    }
76
37232
  }
77

            
78
  // Performs a manual byte size count for test verification.
79
  void verifyByteSizeInternalForTest() const;
80

            
81
  // Note: This class does not actually implement Http::HeaderMap to avoid virtual inheritance in
82
  // the derived classes. Instead, it is used as a mix-in class for TypedHeaderMapImpl below. This
83
  // both avoid virtual inheritance and allows the concrete final header maps to use a variable
84
  // length member at the end.
85
  bool operator==(const HeaderMap& rhs) const;
86
  bool operator!=(const HeaderMap& rhs) const;
87
  void addViaMove(HeaderString&& key, HeaderString&& value);
88
  void addReference(const LowerCaseString& key, absl::string_view value);
89
  void addReferenceKey(const LowerCaseString& key, uint64_t value);
90
  void addReferenceKey(const LowerCaseString& key, absl::string_view value);
91
  void addCopy(const LowerCaseString& key, uint64_t value);
92
  void addCopy(const LowerCaseString& key, absl::string_view value);
93
  void appendCopy(const LowerCaseString& key, absl::string_view value);
94
  void setReference(const LowerCaseString& key, absl::string_view value);
95
  void setReferenceKey(const LowerCaseString& key, absl::string_view value);
96
  void setCopy(const LowerCaseString& key, absl::string_view value);
97
  uint64_t byteSize() const;
98
3110
  uint32_t maxHeadersKb() const { return max_headers_kb_; }
99
3899
  uint32_t maxHeadersCount() const { return max_headers_count_; }
100
  HeaderMap::GetResult get(const LowerCaseString& key) const;
101
  void iterate(HeaderMap::ConstIterateCb cb) const;
102
  void iterateReverse(HeaderMap::ConstIterateCb cb) const;
103
  void clear();
104
  size_t remove(const LowerCaseString& key);
105
  size_t removeIf(const HeaderMap::HeaderMatchPredicate& predicate);
106
  size_t removePrefix(const LowerCaseString& key);
107
1306778
  size_t size() const { return headers_.size(); }
108
7654
  bool empty() const { return headers_.empty(); }
109
  void dumpState(std::ostream& os, int indent_level = 0) const;
110
110761
  StatefulHeaderKeyFormatterOptConstRef formatter() const {
111
110761
    return StatefulHeaderKeyFormatterOptConstRef(makeOptRefFromPtr(formatter_.get()));
112
110761
  }
113
433894
  StatefulHeaderKeyFormatterOptRef formatter() { return makeOptRefFromPtr(formatter_.get()); }
114

            
115
protected:
116
  struct HeaderEntryImpl : public HeaderEntry, NonCopyable {
117
    HeaderEntryImpl(const LowerCaseString& key);
118
    HeaderEntryImpl(const LowerCaseString& key, HeaderString&& value);
119
    HeaderEntryImpl(HeaderString&& key, HeaderString&& value);
120

            
121
    // HeaderEntry
122
40891771
    const HeaderString& key() const override { return key_; }
123
    void value(absl::string_view value) override;
124
    void value(uint64_t value) override;
125
    void value(const HeaderEntry& header) override;
126
46906297
    const HeaderString& value() const override { return value_; }
127
6469123
    HeaderString& value() override { return value_; }
128

            
129
    HeaderString key_;
130
    HeaderString value_;
131
    std::list<HeaderEntryImpl>::iterator entry_;
132
  };
133
  using HeaderNode = std::list<HeaderEntryImpl>::iterator;
134

            
135
  /**
136
   * This is the static lookup table that is used to determine whether a header is one of the O(1)
137
   * headers. This uses a trie for lookup time at most equal to the size of the incoming string.
138
   */
139
  struct StaticLookupResponse {
140
    HeaderEntryImpl** entry_;
141
    const LowerCaseString* key_;
142
  };
143

            
144
  /**
145
   * Base class for a static lookup table that converts a string key into an O(1) header.
146
   */
147
  template <class Interface>
148
  struct StaticLookupTable
149
      : public CompiledStringMap<std::function<StaticLookupResponse(HeaderMapImpl&)>> {
150
    StaticLookupTable();
151

            
152
2830
    std::vector<KV> finalizedTable() {
153
2830
      CustomInlineHeaderRegistry::finalize<Interface::header_map_type>();
154
2830
      auto& headers = CustomInlineHeaderRegistry::headers<Interface::header_map_type>();
155
2830
      size_ = headers.size();
156
2830
      std::vector<KV> input;
157
2830
      input.reserve(size_);
158
62899
      for (const auto& header : headers) {
159
2832110
        input.emplace_back(header.first.get(), [&header](HeaderMapImpl& h) -> StaticLookupResponse {
160
2810775
          return {&h.inlineHeaders()[header.second], &header.first};
161
2810775
        });
162
62896
      }
163
2830
      return input;
164
2830
    }
165

            
166
3149776
    static size_t size() {
167
      // The size of the lookup table is finalized when the singleton lookup table is created. This
168
      // allows for late binding of custom headers as well as envoy header prefix changes. This
169
      // does mean that once the first header map is created of this type, no further changes are
170
      // possible.
171
      // TODO(mattklein123): If we decide to keep this implementation, it is conceivable that header
172
      // maps could be created by an API factory that is owned by the listener/HCM, thus making
173
      // O(1) header delivery over xDS possible.
174
3149776
      return ConstSingleton<StaticLookupTable>::get().size_;
175
3149776
    }
176

            
177
    static absl::optional<StaticLookupResponse> lookup(HeaderMapImpl& header_map,
178
4152928
                                                       absl::string_view key) {
179
4152928
      const auto& entry = ConstSingleton<StaticLookupTable>::get().find(key);
180
4152928
      if (entry != nullptr) {
181
2852013
        return entry(header_map);
182
3601813
      } else {
183
1300915
        return absl::nullopt;
184
1300915
      }
185
4152928
    }
186

            
187
    // This is the size of the number of callbacks; in the case of Requests,
188
    // this is one smaller than the number of entries in the lookup table,
189
    // because of legacy `host` mapping to the same thing as `:authority`.
190
    size_t size_;
191
  };
192

            
193
  /**
194
   * List of HeaderEntryImpl that keeps the pseudo headers (key starting with ':') in the front
195
   * of the list (as required by nghttp2) and otherwise maintains insertion order.
196
   * When the list size is greater or equal to 3, all headers are added to a map, to allow fast
197
   * access given a header key. Once the map is initialized, it will be used even
198
   * if the number of headers decreases below the threshold.
199
   *
200
   * Note: the internal iterators held in fields make this unsafe to copy and move, since the
201
   * reference to end() is not preserved across a move (see Notes in
202
   * https://en.cppreference.com/w/cpp/container/list/list). The NonCopyable will suppress both copy
203
   * and move constructors/assignment.
204
   * TODO(htuch): Maybe we want this to movable one day; for now, our header map moves happen on
205
   * HeaderMapPtr, so the performance impact should not be evident.
206
   */
207
  class HeaderList : NonCopyable {
208
  public:
209
    using HeaderNodeVector = absl::InlinedVector<HeaderNode, 1>;
210
    using HeaderLazyMap = absl::flat_hash_map<absl::string_view, HeaderNodeVector>;
211

            
212
1553222
    HeaderList() : pseudo_headers_end_(headers_.end()) {}
213

            
214
973312
    template <class Key> bool isPseudoHeader(const Key& key) {
215
973312
      return !key.getStringView().empty() && key.getStringView()[0] == ':';
216
973312
    }
217

            
218
3832915
    template <class Key, class... Value> HeaderNode insert(Key&& key, Value&&... value) {
219
3832915
      const bool is_pseudo_header = isPseudoHeader(key);
220
3832915
      HeaderNode i = headers_.emplace(is_pseudo_header ? pseudo_headers_end_ : headers_.end(),
221
3832915
                                      std::forward<Key>(key), std::forward<Value>(value)...);
222
3832915
      if (!lazy_map_.empty()) {
223
249951
        lazy_map_[i->key().getStringView()].push_back(i);
224
249951
      }
225
3832915
      if (!is_pseudo_header && pseudo_headers_end_ == headers_.end()) {
226
1325964
        pseudo_headers_end_ = i;
227
1325964
      }
228
3832915
      return i;
229
3832915
    }
230

            
231
12576
    HeaderNode erase(HeaderNode i, bool remove_from_map) {
232
12576
      if (pseudo_headers_end_ == i) {
233
7163
        pseudo_headers_end_++;
234
7163
      }
235
12576
      if (remove_from_map) {
236
11248
        lazy_map_.erase(i->key().getStringView());
237
11248
      }
238
12576
      return headers_.erase(i);
239
12576
    }
240

            
241
11
    template <class UnaryPredicate> void removeIf(UnaryPredicate p) {
242
11
      if (!lazy_map_.empty()) {
243
        // Lazy map is used, iterate over its elements and remove those that satisfy the predicate
244
        // from the map and from the list.
245
8
        for (auto map_it = lazy_map_.begin(); map_it != lazy_map_.end();) {
246
6
          auto& values_vec = map_it->second;
247
6
          ASSERT(!values_vec.empty());
248
          // The following call to std::remove_if removes the elements that satisfy the
249
          // UnaryPredicate and shifts the vector elements, but does not resize the vector.
250
          // The call to erase that follows erases the unneeded cells (from remove_pos to the
251
          // end) and modifies the vector's size.
252
6
          const auto remove_pos =
253
6
              std::remove_if(values_vec.begin(), values_vec.end(), [&](HeaderNode it) {
254
6
                if (p(*(it->entry_))) {
255
                  // Remove the element from the list.
256
2
                  if (pseudo_headers_end_ == it->entry_) {
257
                    pseudo_headers_end_++;
258
                  }
259
2
                  headers_.erase(it);
260
2
                  return true;
261
2
                }
262
4
                return false;
263
6
              });
264
6
          values_vec.erase(remove_pos, values_vec.end());
265

            
266
          // If all elements were removed from the map entry, erase it.
267
6
          if (values_vec.empty()) {
268
2
            lazy_map_.erase(map_it++);
269
4
          } else {
270
4
            map_it++;
271
4
          }
272
6
        }
273
11
      } else {
274
        // The lazy map isn't used, iterate over the list elements and remove elements that satisfy
275
        // the predicate.
276
38
        headers_.remove_if([&](const HeaderEntryImpl& entry) {
277
38
          const bool to_remove = p(entry);
278
38
          if (to_remove) {
279
13
            if (pseudo_headers_end_ == entry.entry_) {
280
6
              pseudo_headers_end_++;
281
6
            }
282
13
          }
283
38
          return to_remove;
284
38
        });
285
9
      }
286
11
    }
287

            
288
    /*
289
     * Creates and populates a map if the number of headers is at least 3.
290
     *
291
     * @return if a map was created.
292
     */
293
    bool maybeMakeMap();
294

            
295
    /*
296
     * Removes a given key and its values from the HeaderList.
297
     *
298
     * @return the number of bytes that were removed.
299
     */
300
    size_t remove(absl::string_view key);
301

            
302
7847
    std::list<HeaderEntryImpl>::iterator begin() { return headers_.begin(); }
303
7847
    std::list<HeaderEntryImpl>::iterator end() { return headers_.end(); }
304
4185402
    std::list<HeaderEntryImpl>::const_iterator begin() const { return headers_.begin(); }
305
4187772
    std::list<HeaderEntryImpl>::const_iterator end() const { return headers_.end(); }
306
1
    std::list<HeaderEntryImpl>::const_reverse_iterator rbegin() const { return headers_.rbegin(); }
307
2
    std::list<HeaderEntryImpl>::const_reverse_iterator rend() const { return headers_.rend(); }
308
312300
    HeaderLazyMap::iterator mapFind(absl::string_view key) { return lazy_map_.find(key); }
309
312299
    HeaderLazyMap::iterator mapEnd() { return lazy_map_.end(); }
310
1327327
    size_t size() const { return headers_.size(); }
311
7654
    bool empty() const { return headers_.empty(); }
312
333
    void clear() {
313
333
      headers_.clear();
314
333
      pseudo_headers_end_ = headers_.end();
315
333
      lazy_map_.clear();
316
333
    }
317

            
318
  private:
319
    std::list<HeaderEntryImpl> headers_;
320
    HeaderNode pseudo_headers_end_;
321
    HeaderLazyMap lazy_map_;
322
  };
323

            
324
  void insertByKey(HeaderString&& key, HeaderString&& value);
325
  static uint64_t appendToHeader(HeaderString& header, absl::string_view data,
326
                                 absl::string_view delimiter = ",");
327
  HeaderEntryImpl& maybeCreateInline(HeaderEntryImpl** entry, const LowerCaseString& key);
328
  HeaderEntryImpl& maybeCreateInline(HeaderEntryImpl** entry, const LowerCaseString& key,
329
                                     HeaderString&& value);
330

            
331
  HeaderMap::NonConstGetResult getExisting(absl::string_view key);
332
  size_t removeExisting(absl::string_view key);
333

            
334
  size_t removeInline(HeaderEntryImpl** entry);
335
  void updateSize(uint64_t from_size, uint64_t to_size);
336
  void addSize(uint64_t size);
337
  void subtractSize(uint64_t size);
338
  virtual absl::optional<StaticLookupResponse> staticLookup(absl::string_view) PURE;
339
  virtual void clearInline() PURE;
340
  virtual HeaderEntryImpl** inlineHeaders() PURE;
341

            
342
  HeaderList headers_;
343
  // TODO(mattklein123): The formatter does not currently get copied when a header map gets
344
  // copied. This may be problematic in certain cases like request shadowing. This is omitted
345
  // on purpose until someone asks for it, at which point a clone() method can be created to
346
  // avoid using extra space/processing for a shared_ptr.
347
  StatefulHeaderKeyFormatterPtr formatter_;
348
  // This holds the internal byte size of the HeaderMap.
349
  uint64_t cached_byte_size_ = 0;
350
  // This holds the max size of the headers in kilobyte in the HeaderMap.
351
  const uint32_t max_headers_kb_ = UINT32_MAX;
352
  // This holds the max count of the headers in the HeaderMap.
353
  const uint32_t max_headers_count_ = UINT32_MAX;
354

            
355
  // For benchmarking to access non-public methods to test staticLookup.
356
  friend class StaticLookupBenchmarker;
357
};
358

            
359
/**
360
 * Typed derived classes for all header map types. This class implements the actual typed
361
 * interface and for the majority of methods just passes through to the HeaderMapImpl mix-in. Per
362
 * above, this avoids virtual inheritance.
363
 */
364
template <class Interface> class TypedHeaderMapImpl : public HeaderMapImpl, public Interface {
365
public:
366
  TypedHeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
367
                     const uint32_t max_headers_count = UINT32_MAX)
368
1553224
      : HeaderMapImpl(max_headers_kb, max_headers_count) {}
369
84282
  void setFormatter(StatefulHeaderKeyFormatterPtr&& formatter) {
370
84282
    formatter_ = std::move(formatter);
371
84282
  }
372

            
373
  // Implementation of Http::HeaderMap that passes through to HeaderMapImpl.
374
672
  bool operator==(const HeaderMap& rhs) const override { return HeaderMapImpl::operator==(rhs); }
375
8
  bool operator!=(const HeaderMap& rhs) const override { return HeaderMapImpl::operator!=(rhs); }
376
1576828
  void addViaMove(HeaderString&& key, HeaderString&& value) override {
377
1576828
    HeaderMapImpl::addViaMove(std::move(key), std::move(value));
378
1576828
  }
379
697
  void addReference(const LowerCaseString& key, absl::string_view value) override {
380
697
    HeaderMapImpl::addReference(key, value);
381
697
  }
382
179
  void addReferenceKey(const LowerCaseString& key, uint64_t value) override {
383
179
    HeaderMapImpl::addReferenceKey(key, value);
384
179
  }
385
6633
  void addReferenceKey(const LowerCaseString& key, absl::string_view value) override {
386
6633
    HeaderMapImpl::addReferenceKey(key, value);
387
6633
  }
388
1470
  void addCopy(const LowerCaseString& key, uint64_t value) override {
389
1470
    HeaderMapImpl::addCopy(key, value);
390
1470
  }
391
1704726
  void addCopy(const LowerCaseString& key, absl::string_view value) override {
392
1704726
    HeaderMapImpl::addCopy(key, value);
393
1704726
  }
394
36304
  void appendCopy(const LowerCaseString& key, absl::string_view value) override {
395
36304
    HeaderMapImpl::appendCopy(key, value);
396
36304
  }
397
739
  void setReference(const LowerCaseString& key, absl::string_view value) override {
398
739
    HeaderMapImpl::setReference(key, value);
399
739
  }
400
1005
  void setReferenceKey(const LowerCaseString& key, absl::string_view value) override {
401
1005
    HeaderMapImpl::setReferenceKey(key, value);
402
1005
  }
403
3486
  void setCopy(const LowerCaseString& key, absl::string_view value) override {
404
3486
    HeaderMapImpl::setCopy(key, value);
405
3486
  }
406
1725466
  uint64_t byteSize() const override { return HeaderMapImpl::byteSize(); }
407
3110
  uint32_t maxHeadersKb() const override { return HeaderMapImpl::maxHeadersKb(); }
408
3899
  uint32_t maxHeadersCount() const override { return HeaderMapImpl::maxHeadersCount(); }
409
810353
  HeaderMap::GetResult get(const LowerCaseString& key) const override {
410
810353
    return HeaderMapImpl::get(key);
411
810353
  }
412
436098
  void iterate(HeaderMap::ConstIterateCb cb) const override { HeaderMapImpl::iterate(cb); }
413
1
  void iterateReverse(HeaderMap::ConstIterateCb cb) const override {
414
1
    HeaderMapImpl::iterateReverse(cb);
415
1
  }
416
333
  void clear() override { HeaderMapImpl::clear(); }
417
5033
  size_t remove(const LowerCaseString& key) override { return HeaderMapImpl::remove(key); }
418
5
  size_t removeIf(const HeaderMap::HeaderMatchPredicate& predicate) override {
419
5
    return HeaderMapImpl::removeIf(predicate);
420
5
  }
421
6
  size_t removePrefix(const LowerCaseString& key) override {
422
6
    return HeaderMapImpl::removePrefix(key);
423
6
  }
424
1306101
  size_t size() const override { return HeaderMapImpl::size(); }
425
7654
  bool empty() const override { return HeaderMapImpl::empty(); }
426
559
  void dumpState(std::ostream& os, int indent_level = 0) const override {
427
559
    HeaderMapImpl::dumpState(os, indent_level);
428
559
  }
429
110761
  StatefulHeaderKeyFormatterOptConstRef formatter() const override {
430
110761
    return HeaderMapImpl::formatter();
431
110761
  }
432
433894
  StatefulHeaderKeyFormatterOptRef formatter() override { return HeaderMapImpl::formatter(); }
433

            
434
  // Generic custom header functions for each fully typed interface. To avoid accidental issues,
435
  // the Handle type is different for each interface, which is why these functions live here vs.
436
  // inside HeaderMapImpl.
437
  using Handle = CustomInlineHeaderRegistry::Handle<Interface::header_map_type>;
438
16529693
  const HeaderEntry* getInline(Handle handle) const override {
439
16529693
    ASSERT(handle.it_->second < inlineHeadersSize());
440
16529693
    return constInlineHeaders()[handle.it_->second];
441
16529693
  }
442
6136
  void appendInline(Handle handle, absl::string_view data, absl::string_view delimiter) override {
443
6136
    ASSERT(handle.it_->second < inlineHeadersSize());
444
6136
    HeaderEntry& entry = maybeCreateInline(&inlineHeaders()[handle.it_->second], handle.it_->first);
445
6136
    addSize(HeaderMapImpl::appendToHeader(entry.value(), data, delimiter));
446
6136
  }
447
167098
  void setReferenceInline(Handle handle, absl::string_view value) override {
448
167098
    ASSERT(handle.it_->second < inlineHeadersSize());
449
167098
    HeaderEntry& entry = maybeCreateInline(&inlineHeaders()[handle.it_->second], handle.it_->first);
450
167098
    updateSize(entry.value().size(), value.size());
451
167098
    entry.value().setReference(value);
452
167098
  }
453
2841498
  void setInline(Handle handle, absl::string_view value) override {
454
2841498
    ASSERT(handle.it_->second < inlineHeadersSize());
455
2841498
    HeaderEntry& entry = maybeCreateInline(&inlineHeaders()[handle.it_->second], handle.it_->first);
456
2841498
    updateSize(entry.value().size(), value.size());
457
2841498
    entry.value().setCopy(value);
458
2841498
  }
459
132343
  void setInline(Handle handle, uint64_t value) override {
460
132343
    ASSERT(handle.it_->second < inlineHeadersSize());
461
132343
    HeaderEntry& entry = maybeCreateInline(&inlineHeaders()[handle.it_->second], handle.it_->first);
462
132343
    subtractSize(entry.value().size());
463
132343
    entry.value().setInteger(value);
464
132343
    addSize(entry.value().size());
465
132343
  }
466
2502880
  size_t removeInline(Handle handle) override {
467
2502880
    ASSERT(handle.it_->second < inlineHeadersSize());
468
2502880
    return HeaderMapImpl::removeInline(&inlineHeaders()[handle.it_->second]);
469
2502880
  }
470
3149776
  static size_t inlineHeadersSize() {
471
3149776
    return StaticLookupTable<Interface>::size() * sizeof(HeaderEntryImpl*);
472
3149776
  }
473

            
474
protected:
475
4152927
  absl::optional<StaticLookupResponse> staticLookup(absl::string_view key) override {
476
4152927
    return StaticLookupTable<Interface>::lookup(*this, key);
477
4152927
  }
478
  virtual const HeaderEntryImpl* const* constInlineHeaders() const PURE;
479
};
480

            
481
#define DEFINE_HEADER_HANDLE(name)                                                                 \
482
  Handle name =                                                                                    \
483
      CustomInlineHeaderRegistry::getInlineHeader<header_map_type>(Headers::get().name).value();
484

            
485
/**
486
 * Concrete implementation of RequestHeaderMap which allows for variable custom registered inline
487
 * headers.
488
 */
489
class RequestHeaderMapImpl final : public TypedHeaderMapImpl<RequestHeaderMap>,
490
                                   public InlineStorage {
491
public:
492
  static std::unique_ptr<RequestHeaderMapImpl>
493
  create(const uint32_t max_headers_kb = UINT32_MAX,
494
1244187
         const uint32_t max_headers_count = UINT32_MAX) {
495
1244187
    return std::unique_ptr<RequestHeaderMapImpl>(
496
1244187
        new (inlineHeadersSize()) RequestHeaderMapImpl(max_headers_kb, max_headers_count));
497
1244187
  }
498

            
499
  INLINE_REQ_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
500
  INLINE_REQ_NUMERIC_HEADERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS)
501
  INLINE_REQ_RESP_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
502
  INLINE_REQ_RESP_NUMERIC_HEADERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS)
503

            
504
protected:
505
  // NOTE: Because inline_headers_ is a variable size member, it must be the last member in the
506
  // most derived class. This forces the definition of the following three functions to also be
507
  // in the most derived class and thus duplicated. There may be a way to consolidate thus but it's
508
  // not clear and can be deferred for now.
509
1244336
  void clearInline() override { memset(inline_headers_, 0, inlineHeadersSize()); }
510
15009983
  const HeaderEntryImpl* const* constInlineHeaders() const override { return inline_headers_; }
511
7576189
  HeaderEntryImpl** inlineHeaders() override { return inline_headers_; }
512

            
513
private:
514
  struct HeaderHandleValues {
515
    INLINE_REQ_STRING_HEADERS(DEFINE_HEADER_HANDLE)
516
    INLINE_REQ_NUMERIC_HEADERS(DEFINE_HEADER_HANDLE)
517
    INLINE_REQ_RESP_STRING_HEADERS(DEFINE_HEADER_HANDLE)
518
    INLINE_REQ_RESP_NUMERIC_HEADERS(DEFINE_HEADER_HANDLE)
519
  };
520

            
521
  using HeaderHandles = ConstSingleton<HeaderHandleValues>;
522

            
523
  RequestHeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
524
                       const uint32_t max_headers_count = UINT32_MAX)
525
1244187
      : TypedHeaderMapImpl<RequestHeaderMap>(max_headers_kb, max_headers_count) {
526
1244187
    clearInline();
527
1244187
  }
528

            
529
  HeaderEntryImpl* inline_headers_[];
530
};
531

            
532
/**
533
 * Concrete implementation of RequestTrailerMap which allows for variable custom registered inline
534
 * headers.
535
 */
536
class RequestTrailerMapImpl final : public TypedHeaderMapImpl<RequestTrailerMap>,
537
                                    public InlineStorage {
538
public:
539
  static std::unique_ptr<RequestTrailerMapImpl>
540
  create(const uint32_t max_headers_kb = UINT32_MAX,
541
16194
         const uint32_t max_headers_count = UINT32_MAX) {
542
16194
    return std::unique_ptr<RequestTrailerMapImpl>(
543
16194
        new (inlineHeadersSize()) RequestTrailerMapImpl(max_headers_kb, max_headers_count));
544
16194
  }
545

            
546
protected:
547
  // See comment in RequestHeaderMapImpl.
548
16194
  void clearInline() override { memset(inline_headers_, 0, inlineHeadersSize()); }
549
  const HeaderEntryImpl* const* constInlineHeaders() const override { return inline_headers_; }
550
  HeaderEntryImpl** inlineHeaders() override { return inline_headers_; }
551

            
552
private:
553
  RequestTrailerMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
554
                        const uint32_t max_headers_count = UINT32_MAX)
555
16194
      : TypedHeaderMapImpl<RequestTrailerMap>(max_headers_kb, max_headers_count) {
556
16194
    clearInline();
557
16194
  }
558

            
559
  HeaderEntryImpl* inline_headers_[];
560
};
561

            
562
/**
563
 * Concrete implementation of ResponseHeaderMap which allows for variable custom registered inline
564
 * headers.
565
 */
566
class ResponseHeaderMapImpl final : public TypedHeaderMapImpl<ResponseHeaderMap>,
567
                                    public InlineStorage {
568
public:
569
  static std::unique_ptr<ResponseHeaderMapImpl>
570
  create(const uint32_t max_headers_kb = UINT32_MAX,
571
252227
         const uint32_t max_headers_count = UINT32_MAX) {
572
252227
    return std::unique_ptr<ResponseHeaderMapImpl>(
573
252227
        new (inlineHeadersSize()) ResponseHeaderMapImpl(max_headers_kb, max_headers_count));
574
252227
  }
575

            
576
  INLINE_RESP_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
577
  INLINE_RESP_NUMERIC_HEADERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS)
578
  INLINE_REQ_RESP_STRING_HEADERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
579
  INLINE_REQ_RESP_NUMERIC_HEADERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS)
580
  INLINE_RESP_STRING_HEADERS_TRAILERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
581
  INLINE_RESP_NUMERIC_HEADERS_TRAILERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS)
582

            
583
protected:
584
  // See comment in RequestHeaderMapImpl.
585
252276
  void clearInline() override { memset(inline_headers_, 0, inlineHeadersSize()); }
586
1516974
  const HeaderEntryImpl* const* constInlineHeaders() const override { return inline_headers_; }
587
920425
  HeaderEntryImpl** inlineHeaders() override { return inline_headers_; }
588

            
589
private:
590
  struct HeaderHandleValues {
591
    INLINE_RESP_STRING_HEADERS(DEFINE_HEADER_HANDLE)
592
    INLINE_RESP_NUMERIC_HEADERS(DEFINE_HEADER_HANDLE)
593
    INLINE_REQ_RESP_STRING_HEADERS(DEFINE_HEADER_HANDLE)
594
    INLINE_REQ_RESP_NUMERIC_HEADERS(DEFINE_HEADER_HANDLE)
595
    INLINE_RESP_STRING_HEADERS_TRAILERS(DEFINE_HEADER_HANDLE)
596
    INLINE_RESP_NUMERIC_HEADERS_TRAILERS(DEFINE_HEADER_HANDLE)
597
  };
598

            
599
  using HeaderHandles = ConstSingleton<HeaderHandleValues>;
600

            
601
  ResponseHeaderMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
602
                        const uint32_t max_headers_count = UINT32_MAX)
603
252227
      : TypedHeaderMapImpl<ResponseHeaderMap>(max_headers_kb, max_headers_count) {
604
252227
    clearInline();
605
252227
  }
606
  HeaderEntryImpl* inline_headers_[];
607
};
608

            
609
/**
610
 * Concrete implementation of ResponseTrailerMap which allows for variable custom registered
611
 * inline headers.
612
 */
613
class ResponseTrailerMapImpl final : public TypedHeaderMapImpl<ResponseTrailerMap>,
614
                                     public InlineStorage {
615
public:
616
  static std::unique_ptr<ResponseTrailerMapImpl>
617
  create(const uint32_t max_headers_kb = UINT32_MAX,
618
40616
         const uint32_t max_headers_count = UINT32_MAX) {
619
40616
    return std::unique_ptr<ResponseTrailerMapImpl>(
620
40616
        new (inlineHeadersSize()) ResponseTrailerMapImpl(max_headers_kb, max_headers_count));
621
40616
  }
622

            
623
  INLINE_RESP_STRING_HEADERS_TRAILERS(DEFINE_INLINE_HEADER_STRING_FUNCS)
624
  INLINE_RESP_NUMERIC_HEADERS_TRAILERS(DEFINE_INLINE_HEADER_NUMERIC_FUNCS)
625

            
626
protected:
627
  // See comment in RequestHeaderMapImpl.
628
40751
  void clearInline() override { memset(inline_headers_, 0, inlineHeadersSize()); }
629
2738
  const HeaderEntryImpl* const* constInlineHeaders() const override { return inline_headers_; }
630
5349
  HeaderEntryImpl** inlineHeaders() override { return inline_headers_; }
631

            
632
private:
633
  struct HeaderHandleValues {
634
    INLINE_RESP_STRING_HEADERS_TRAILERS(DEFINE_HEADER_HANDLE)
635
    INLINE_RESP_NUMERIC_HEADERS_TRAILERS(DEFINE_HEADER_HANDLE)
636
  };
637

            
638
  using HeaderHandles = ConstSingleton<HeaderHandleValues>;
639

            
640
  ResponseTrailerMapImpl(const uint32_t max_headers_kb = UINT32_MAX,
641
                         const uint32_t max_headers_count = UINT32_MAX)
642
40616
      : TypedHeaderMapImpl<ResponseTrailerMap>(max_headers_kb, max_headers_count) {
643
40616
    clearInline();
644
40616
  }
645

            
646
  HeaderEntryImpl* inline_headers_[];
647
};
648

            
649
class TunnelResponseHeadersOrTrailersImpl : public TunnelResponseHeadersOrTrailers {
650
public:
651
  ProtobufTypes::MessagePtr serializeAsProto() const override;
652
};
653

            
654
template <class T>
655
std::unique_ptr<T>
656
37231
createHeaderMap(const std::initializer_list<std::pair<LowerCaseString, std::string>>& values) {
657
37231
  auto new_header_map = T::create();
658
37231
  HeaderMapImpl::initFromInitList(*new_header_map, values.begin(), values.end());
659
37231
  return new_header_map;
660
37231
}
661

            
662
1
template <class T, class It> std::unique_ptr<T> createHeaderMap(It begin, It end) {
663
1
  auto new_header_map = T::create();
664
1
  HeaderMapImpl::initFromInitList(*new_header_map, begin, end);
665
1
  return new_header_map;
666
1
}
667

            
668
57079
template <class T> std::unique_ptr<T> createHeaderMap(const HeaderMap& rhs) {
669
  // TODO(mattklein123): Use of this function allows copying a request header map into a response
670
  // header map, etc. which is probably not what we want. Unfortunately, we do this on purpose in
671
  // a few places when dealing with gRPC headers/trailers conversions so it's not trivial to remove.
672
  // We should revisit this to figure how to make this a bit safer as a non-intentional conversion
673
  // may have surprising results with different O(1) headers, implementations, etc.
674
57079
  auto new_header_map = T::create();
675
57079
  HeaderMapImpl::copyFrom(*new_header_map, rhs);
676
57079
  return new_header_map;
677
57079
}
678

            
679
struct EmptyHeaders {
680
  RequestHeaderMapPtr request_headers = RequestHeaderMapImpl::create();
681
  ResponseHeaderMapPtr response_headers = ResponseHeaderMapImpl::create();
682
  ResponseTrailerMapPtr response_trailers = ResponseTrailerMapImpl::create();
683
};
684

            
685
using StaticEmptyHeaders = ConstSingleton<EmptyHeaders>;
686

            
687
class HeaderMapImplUtility {
688
public:
689
  struct HeaderMapImplInfo {
690
    // Human readable name for the header map used in info logging.
691
    std::string name_;
692
    // The byte size of the header map including both fixed space as well as variable space used
693
    // by the registered custom headers.
694
    size_t size_;
695
    // All registered custom headers for the header map.
696
    std::vector<std::string> registered_headers_;
697
  };
698

            
699
  /**
700
   * Fetch detailed information about each header map implementation for use in logging.
701
   */
702
  static std::vector<HeaderMapImplInfo> getAllHeaderMapImplInfo();
703
};
704

            
705
} // namespace Http
706
} // namespace Envoy