Coverage Report

Created: 2023-11-12 09:30

/proc/self/cwd/source/common/formatter/substitution_formatter.h
Line
Count
Source (jump to first uncovered line)
1
#pragma once
2
3
#include <bitset>
4
#include <functional>
5
#include <list>
6
#include <string>
7
#include <vector>
8
9
#include "envoy/common/time.h"
10
#include "envoy/config/core/v3/base.pb.h"
11
#include "envoy/formatter/substitution_formatter.h"
12
#include "envoy/stream_info/stream_info.h"
13
14
#include "source/common/common/utility.h"
15
#include "source/common/formatter/http_specific_formatter.h"
16
#include "source/common/formatter/stream_info_formatter.h"
17
#include "source/common/json/json_loader.h"
18
19
#include "absl/container/flat_hash_map.h"
20
#include "absl/types/optional.h"
21
22
namespace Envoy {
23
namespace Formatter {
24
25
/**
26
 * Access log format parser.
27
 */
28
class SubstitutionFormatParser {
29
public:
30
  template <class FormatterContext = HttpFormatterContext>
31
  static std::vector<FormatterProviderBasePtr<FormatterContext>>
32
  parse(const std::string& format,
33
403k
        const std::vector<CommandParserBasePtr<FormatterContext>>& command_parsers = {}) {
34
403k
    std::string current_token;
35
403k
    std::vector<FormatterProviderBasePtr<FormatterContext>> formatters;
36
37
8.43M
    for (size_t pos = 0; pos < format.size(); ++pos) {
38
8.03M
      if (format[pos] != '%') {
39
4.77M
        current_token += format[pos];
40
4.77M
        continue;
41
4.77M
      }
42
43
      // escape '%%'
44
3.25M
      if (format.size() > pos + 1) {
45
3.25M
        if (format[pos + 1] == '%') {
46
2.89M
          current_token += '%';
47
2.89M
          pos++;
48
2.89M
          continue;
49
2.89M
        }
50
3.25M
      }
51
52
360k
      if (!current_token.empty()) {
53
208k
        formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{
54
208k
            new PlainStringFormatterBase<FormatterContext>(current_token)});
55
208k
        current_token = "";
56
208k
      }
57
58
360k
      std::smatch m;
59
360k
      const std::string search_space = std::string(format.substr(pos));
60
360k
      if (!std::regex_search(search_space, m, commandWithArgsRegex())) {
61
1.02k
        throwEnvoyExceptionOrPanic(
62
1.02k
            fmt::format("Incorrect configuration: {}. Couldn't find valid command at position {}",
63
1.02k
                        format, pos));
64
1.02k
      }
65
66
359k
      const std::string match = m.str(0);
67
      // command is at at index 1.
68
359k
      const std::string command = m.str(1);
69
      // subcommand is at index 2.
70
359k
      const std::string subcommand = m.str(2);
71
      // optional length is at index 3. If present, validate that it is valid integer.
72
359k
      absl::optional<size_t> max_length;
73
359k
      if (m.str(3).length() != 0) {
74
3.80k
        size_t length_value;
75
3.80k
        if (!absl::SimpleAtoi(m.str(3), &length_value)) {
76
47
          throwEnvoyExceptionOrPanic(absl::StrCat("Length must be an integer, given: ", m.str(3)));
77
47
        }
78
3.75k
        max_length = length_value;
79
3.75k
      }
80
359k
      std::vector<std::string> path;
81
82
359k
      const size_t command_end_position = pos + m.str(0).length() - 1;
83
84
359k
      bool added = false;
85
86
      // First, try the built-in command parsers.
87
359k
      for (const auto& cmd : BuiltInCommandParsersBase<FormatterContext>::commandParsers()) {
88
359k
        auto formatter = cmd->parse(command, subcommand, max_length);
89
359k
        if (formatter) {
90
83.8k
          formatters.push_back(std::move(formatter));
91
83.8k
          added = true;
92
83.8k
          break;
93
83.8k
        }
94
359k
      }
95
96
      // Next, try the command parsers provided by the user.
97
359k
      if (!added) {
98
275k
        for (const auto& cmd : command_parsers) {
99
0
          auto formatter = cmd->parse(command, subcommand, max_length);
100
0
          if (formatter) {
101
0
            formatters.push_back(std::move(formatter));
102
0
            added = true;
103
0
            break;
104
0
          }
105
0
        }
106
275k
      }
107
108
359k
      if (!added) {
109
        // Finally, try the context independent formatters.
110
275k
        formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{
111
275k
            new StreamInfoFormatterBase<FormatterContext>(command, subcommand, max_length)});
112
275k
      }
113
114
359k
      pos = command_end_position;
115
359k
    }
116
117
402k
    if (!current_token.empty() || format.empty()) {
118
      // Create a PlainStringFormatter with the final string literal. If the format string
119
      // was empty, this creates a PlainStringFormatter with an empty string.
120
351k
      formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{
121
351k
          new PlainStringFormatterBase<FormatterContext>(current_token)});
122
351k
    }
123
124
402k
    return formatters;
125
403k
  }
126
127
private:
128
  static const std::regex& commandWithArgsRegex();
129
};
130
131
inline constexpr absl::string_view DefaultUnspecifiedValueStringView = "-";
132
133
/**
134
 * Composite formatter implementation.
135
 */
136
template <class FormatterContext>
137
class CommonFormatterBaseImpl : public FormatterBase<FormatterContext> {
138
public:
139
  using CommandParsers = std::vector<CommandParserBasePtr<FormatterContext>>;
140
141
  CommonFormatterBaseImpl(const std::string& format, bool omit_empty_values = false)
142
      : empty_value_string_(omit_empty_values ? absl::string_view{}
143
383k
                                              : DefaultUnspecifiedValueStringView) {
144
383k
    providers_ = SubstitutionFormatParser::parse<FormatterContext>(format);
145
383k
  }
146
  CommonFormatterBaseImpl(const std::string& format, bool omit_empty_values,
147
                          const CommandParsers& command_parsers)
148
      : empty_value_string_(omit_empty_values ? absl::string_view{}
149
306
                                              : DefaultUnspecifiedValueStringView) {
150
306
    providers_ = SubstitutionFormatParser::parse<FormatterContext>(format, command_parsers);
151
306
  }
152
153
  // FormatterBase
154
  std::string formatWithContext(const FormatterContext& context,
155
453k
                                const StreamInfo::StreamInfo& stream_info) const override {
156
453k
    std::string log_line;
157
453k
    log_line.reserve(256);
158
159
868k
    for (const auto& provider : providers_) {
160
868k
      const auto bit = provider->formatWithContext(context, stream_info);
161
868k
      log_line += bit.value_or(empty_value_string_);
162
868k
    }
163
164
453k
    return log_line;
165
453k
  }
166
167
private:
168
  const std::string empty_value_string_;
169
  std::vector<FormatterProviderBasePtr<FormatterContext>> providers_;
170
};
171
172
template <class FormatterContext>
173
class FormatterBaseImpl : public CommonFormatterBaseImpl<FormatterContext> {
174
public:
175
  using CommonFormatterBaseImpl<FormatterContext>::CommonFormatterBaseImpl;
176
};
177
178
// Helper classes for StructFormatter::StructFormatMapVisitor.
179
template <class... Ts> struct StructFormatMapVisitorHelper : Ts... { using Ts::operator()...; };
180
template <class... Ts> StructFormatMapVisitorHelper(Ts...) -> StructFormatMapVisitorHelper<Ts...>;
181
182
/**
183
 * An formatter for structured log formats, which returns a Struct proto that
184
 * can be converted easily into multiple formats.
185
 */
186
template <class FormatterContext> class StructFormatterBase {
187
public:
188
  using CommandParsers = std::vector<CommandParserBasePtr<FormatterContext>>;
189
  using PlainNumber = PlainNumberFormatterBase<FormatterContext>;
190
  using PlainString = PlainStringFormatterBase<FormatterContext>;
191
192
  StructFormatterBase(const ProtobufWkt::Struct& format_mapping, bool preserve_types,
193
                      bool omit_empty_values, const CommandParsers& commands = {})
194
      : omit_empty_values_(omit_empty_values), preserve_types_(preserve_types),
195
        empty_value_(omit_empty_values_ ? std::string()
196
                                        : std::string(DefaultUnspecifiedValueStringView)),
197
669
        struct_output_format_(FormatBuilder(commands).toFormatMapValue(format_mapping)) {}
198
199
  ProtobufWkt::Struct formatWithContext(const FormatterContext& context,
200
158
                                        const StreamInfo::StreamInfo& info) const {
201
158
    StructFormatMapVisitor visitor{
202
16.8k
        [&](const std::vector<FormatterProviderBasePtr<FormatterContext>>& providers) {
203
16.8k
          return providersCallback(providers, context, info);
204
16.8k
        },
205
18.5k
        [&, this](const StructFormatterBase::StructFormatMapWrapper& format_map) {
206
18.5k
          return structFormatMapCallback(format_map, visitor);
207
18.5k
        },
208
10.0k
        [&, this](const StructFormatterBase::StructFormatListWrapper& format_list) {
209
10.0k
          return structFormatListCallback(format_list, visitor);
210
10.0k
        },
211
158
    };
212
158
    return structFormatMapCallback(struct_output_format_, visitor).struct_value();
213
158
  }
214
215
private:
216
  struct StructFormatMapWrapper;
217
  struct StructFormatListWrapper;
218
  using StructFormatValue =
219
      absl::variant<const std::vector<FormatterProviderBasePtr<FormatterContext>>,
220
                    const StructFormatMapWrapper, const StructFormatListWrapper>;
221
  // Although not required for Struct/JSON, it is nice to have the order of
222
  // properties preserved between the format and the log entry, thus std::map.
223
  using StructFormatMap = std::map<std::string, StructFormatValue>;
224
  using StructFormatMapPtr = std::unique_ptr<StructFormatMap>;
225
  struct StructFormatMapWrapper {
226
    StructFormatMapPtr value_;
227
  };
228
229
  using StructFormatList = std::list<StructFormatValue>;
230
  using StructFormatListPtr = std::unique_ptr<StructFormatList>;
231
  struct StructFormatListWrapper {
232
    StructFormatListPtr value_;
233
  };
234
235
  using StructFormatMapVisitor = StructFormatMapVisitorHelper<
236
      const std::function<ProtobufWkt::Value(
237
          const std::vector<FormatterProviderBasePtr<FormatterContext>>&)>,
238
      const std::function<ProtobufWkt::Value(const StructFormatterBase::StructFormatMapWrapper&)>,
239
      const std::function<ProtobufWkt::Value(const StructFormatterBase::StructFormatListWrapper&)>>;
240
241
  // Methods for building the format map.
242
  class FormatBuilder {
243
  public:
244
669
    explicit FormatBuilder(const CommandParsers& commands) : commands_(commands) {}
245
    std::vector<FormatterProviderBasePtr<FormatterContext>>
246
19.3k
    toFormatStringValue(const std::string& string_format) const {
247
19.3k
      return SubstitutionFormatParser::parse<FormatterContext>(string_format, commands_);
248
19.3k
    }
249
    std::vector<FormatterProviderBasePtr<FormatterContext>>
250
6.60k
    toFormatNumberValue(double value) const {
251
6.60k
      std::vector<FormatterProviderBasePtr<FormatterContext>> formatters;
252
6.60k
      formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{new PlainNumber(value)});
253
6.60k
      return formatters;
254
6.60k
    }
255
26.9k
    StructFormatMapWrapper toFormatMapValue(const ProtobufWkt::Struct& struct_format) const {
256
26.9k
      auto output = std::make_unique<StructFormatMap>();
257
33.7k
      for (const auto& pair : struct_format.fields()) {
258
33.7k
        switch (pair.second.kind_case()) {
259
6.84k
        case ProtobufWkt::Value::kStringValue:
260
6.84k
          output->emplace(pair.first, toFormatStringValue(pair.second.string_value()));
261
6.84k
          break;
262
263
19.5k
        case ProtobufWkt::Value::kStructValue:
264
19.5k
          output->emplace(pair.first, toFormatMapValue(pair.second.struct_value()));
265
19.5k
          break;
266
267
5.24k
        case ProtobufWkt::Value::kListValue:
268
5.24k
          output->emplace(pair.first, toFormatListValue(pair.second.list_value()));
269
5.24k
          break;
270
271
2.13k
        case ProtobufWkt::Value::kNumberValue:
272
2.13k
          output->emplace(pair.first, toFormatNumberValue(pair.second.number_value()));
273
2.13k
          break;
274
47
        default:
275
47
          throwEnvoyExceptionOrPanic(
276
33.7k
              "Only string values, nested structs, list values and number values are "
277
33.7k
              "supported in structured access log format.");
278
33.7k
        }
279
33.7k
      }
280
26.4k
      return {std::move(output)};
281
26.9k
    }
282
    StructFormatListWrapper
283
15.4k
    toFormatListValue(const ProtobufWkt::ListValue& list_value_format) const {
284
15.4k
      auto output = std::make_unique<StructFormatList>();
285
33.9k
      for (const auto& value : list_value_format.values()) {
286
33.9k
        switch (value.kind_case()) {
287
12.4k
        case ProtobufWkt::Value::kStringValue:
288
12.4k
          output->emplace_back(toFormatStringValue(value.string_value()));
289
12.4k
          break;
290
291
6.76k
        case ProtobufWkt::Value::kStructValue:
292
6.76k
          output->emplace_back(toFormatMapValue(value.struct_value()));
293
6.76k
          break;
294
295
10.2k
        case ProtobufWkt::Value::kListValue:
296
10.2k
          output->emplace_back(toFormatListValue(value.list_value()));
297
10.2k
          break;
298
299
4.47k
        case ProtobufWkt::Value::kNumberValue:
300
4.47k
          output->emplace_back(toFormatNumberValue(value.number_value()));
301
4.47k
          break;
302
303
49
        default:
304
49
          throwEnvoyExceptionOrPanic(
305
33.9k
              "Only string values, nested structs, list values and number values are "
306
33.9k
              "supported in structured access log format.");
307
33.9k
        }
308
33.9k
      }
309
14.9k
      return {std::move(output)};
310
15.4k
    }
311
312
  private:
313
    const CommandParsers& commands_;
314
  };
315
316
  // Methods for doing the actual formatting.
317
  ProtobufWkt::Value
318
  providersCallback(const std::vector<FormatterProviderBasePtr<FormatterContext>>& providers,
319
                    const FormatterContext& context,
320
16.8k
                    const StreamInfo::StreamInfo& stream_info) const {
321
16.8k
    ASSERT(!providers.empty());
322
16.8k
    if (providers.size() == 1) {
323
13.6k
      const auto& provider = providers.front();
324
13.6k
      if (preserve_types_) {
325
13.6k
        return provider->formatValueWithContext(context, stream_info);
326
13.6k
      }
327
328
0
      if (omit_empty_values_) {
329
0
        return ValueUtil::optionalStringValue(provider->formatWithContext(context, stream_info));
330
0
      }
331
332
0
      const auto str = provider->formatWithContext(context, stream_info);
333
0
      return ValueUtil::stringValue(str.value_or(empty_value_));
334
0
    }
335
    // Multiple providers forces string output.
336
3.16k
    std::string str;
337
23.4k
    for (const auto& provider : providers) {
338
23.4k
      const auto bit = provider->formatWithContext(context, stream_info);
339
23.4k
      str += bit.value_or(empty_value_);
340
23.4k
    }
341
3.16k
    return ValueUtil::stringValue(str);
342
16.8k
  }
343
  ProtobufWkt::Value
344
  structFormatMapCallback(const StructFormatterBase::StructFormatMapWrapper& format_map,
345
18.6k
                          const StructFormatMapVisitor& visitor) const {
346
18.6k
    ProtobufWkt::Struct output;
347
18.6k
    auto* fields = output.mutable_fields();
348
19.7k
    for (const auto& pair : *format_map.value_) {
349
19.7k
      ProtobufWkt::Value value = absl::visit(visitor, pair.second);
350
19.7k
      if (omit_empty_values_ && value.kind_case() == ProtobufWkt::Value::kNullValue) {
351
73
        continue;
352
73
      }
353
19.6k
      (*fields)[pair.first] = value;
354
19.6k
    }
355
18.6k
    if (omit_empty_values_ && output.fields().empty()) {
356
399
      return ValueUtil::nullValue();
357
399
    }
358
18.2k
    return ValueUtil::structValue(output);
359
18.6k
  }
360
  ProtobufWkt::Value
361
  structFormatListCallback(const StructFormatterBase::StructFormatListWrapper& format_list,
362
10.0k
                           const StructFormatMapVisitor& visitor) const {
363
10.0k
    std::vector<ProtobufWkt::Value> output;
364
25.7k
    for (const auto& val : *format_list.value_) {
365
25.7k
      ProtobufWkt::Value value = absl::visit(visitor, val);
366
25.7k
      if (omit_empty_values_ && value.kind_case() == ProtobufWkt::Value::kNullValue) {
367
326
        continue;
368
326
      }
369
25.3k
      output.push_back(value);
370
25.3k
    }
371
10.0k
    return ValueUtil::listValue(output);
372
10.0k
  }
373
374
  const bool omit_empty_values_;
375
  const bool preserve_types_;
376
  const std::string empty_value_;
377
378
  const StructFormatMapWrapper struct_output_format_;
379
};
380
381
template <class FormatterContext>
382
using StructFormatterBasePtr = std::unique_ptr<StructFormatterBase<FormatterContext>>;
383
384
template <class FormatterContext>
385
class CommonJsonFormatterBaseImpl : public FormatterBase<FormatterContext> {
386
public:
387
  using CommandParsers = std::vector<CommandParserBasePtr<FormatterContext>>;
388
389
  CommonJsonFormatterBaseImpl(const ProtobufWkt::Struct& format_mapping, bool preserve_types,
390
                              bool omit_empty_values, bool sort_properties,
391
                              const CommandParsers& commands = {})
392
      : struct_formatter_(format_mapping, preserve_types, omit_empty_values, commands),
393
669
        sort_properties_(sort_properties) {}
394
395
  // FormatterBase
396
  std::string formatWithContext(const FormatterContext& context,
397
158
                                const StreamInfo::StreamInfo& info) const override {
398
158
    const ProtobufWkt::Struct output_struct = struct_formatter_.formatWithContext(context, info);
399
400
158
    std::string log_line = "";
401
158
#ifdef ENVOY_ENABLE_YAML
402
158
    if (sort_properties_) {
403
123
      log_line = Json::Factory::loadFromProtobufStruct(output_struct)->asJsonString();
404
123
    } else {
405
35
      log_line = MessageUtil::getJsonStringFromMessageOrError(output_struct, false, true);
406
35
    }
407
#else
408
    UNREFERENCED_PARAMETER(sort_properties_);
409
    IS_ENVOY_BUG("Json support compiled out");
410
#endif
411
158
    return absl::StrCat(log_line, "\n");
412
158
  }
413
414
private:
415
  const StructFormatterBase<FormatterContext> struct_formatter_;
416
  const bool sort_properties_;
417
};
418
419
template <class FormatterContext>
420
class JsonFormatterBaseImpl : public CommonJsonFormatterBaseImpl<FormatterContext> {
421
public:
422
  using CommonJsonFormatterBaseImpl<FormatterContext>::CommonJsonFormatterBaseImpl;
423
};
424
425
using StructFormatter = StructFormatterBase<HttpFormatterContext>;
426
using StructFormatterPtr = std::unique_ptr<StructFormatter>;
427
428
// Aliases for backwards compatibility.
429
using FormatterImpl = FormatterBaseImpl<HttpFormatterContext>;
430
using JsonFormatterImpl = JsonFormatterBaseImpl<HttpFormatterContext>;
431
432
} // namespace Formatter
433
} // namespace Envoy