/proc/self/cwd/source/common/formatter/substitution_formatter.h
Line | Count | Source (jump to first uncovered line) |
1 | | #pragma once |
2 | | |
3 | | #include <bitset> |
4 | | #include <functional> |
5 | | #include <list> |
6 | | #include <string> |
7 | | #include <vector> |
8 | | |
9 | | #include "envoy/common/time.h" |
10 | | #include "envoy/config/core/v3/base.pb.h" |
11 | | #include "envoy/formatter/substitution_formatter.h" |
12 | | #include "envoy/stream_info/stream_info.h" |
13 | | |
14 | | #include "source/common/common/utility.h" |
15 | | #include "source/common/formatter/http_specific_formatter.h" |
16 | | #include "source/common/formatter/stream_info_formatter.h" |
17 | | #include "source/common/json/json_loader.h" |
18 | | |
19 | | #include "absl/container/flat_hash_map.h" |
20 | | #include "absl/types/optional.h" |
21 | | |
22 | | namespace Envoy { |
23 | | namespace Formatter { |
24 | | |
25 | | /** |
26 | | * Access log format parser. |
27 | | */ |
28 | | class SubstitutionFormatParser { |
29 | | public: |
30 | | template <class FormatterContext = HttpFormatterContext> |
31 | | static std::vector<FormatterProviderBasePtr<FormatterContext>> |
32 | | parse(const std::string& format, |
33 | 403k | const std::vector<CommandParserBasePtr<FormatterContext>>& command_parsers = {}) { |
34 | 403k | std::string current_token; |
35 | 403k | std::vector<FormatterProviderBasePtr<FormatterContext>> formatters; |
36 | | |
37 | 8.43M | for (size_t pos = 0; pos < format.size(); ++pos) { |
38 | 8.03M | if (format[pos] != '%') { |
39 | 4.77M | current_token += format[pos]; |
40 | 4.77M | continue; |
41 | 4.77M | } |
42 | | |
43 | | // escape '%%' |
44 | 3.25M | if (format.size() > pos + 1) { |
45 | 3.25M | if (format[pos + 1] == '%') { |
46 | 2.89M | current_token += '%'; |
47 | 2.89M | pos++; |
48 | 2.89M | continue; |
49 | 2.89M | } |
50 | 3.25M | } |
51 | | |
52 | 360k | if (!current_token.empty()) { |
53 | 208k | formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{ |
54 | 208k | new PlainStringFormatterBase<FormatterContext>(current_token)}); |
55 | 208k | current_token = ""; |
56 | 208k | } |
57 | | |
58 | 360k | std::smatch m; |
59 | 360k | const std::string search_space = std::string(format.substr(pos)); |
60 | 360k | if (!std::regex_search(search_space, m, commandWithArgsRegex())) { |
61 | 1.02k | throwEnvoyExceptionOrPanic( |
62 | 1.02k | fmt::format("Incorrect configuration: {}. Couldn't find valid command at position {}", |
63 | 1.02k | format, pos)); |
64 | 1.02k | } |
65 | | |
66 | 359k | const std::string match = m.str(0); |
67 | | // command is at at index 1. |
68 | 359k | const std::string command = m.str(1); |
69 | | // subcommand is at index 2. |
70 | 359k | const std::string subcommand = m.str(2); |
71 | | // optional length is at index 3. If present, validate that it is valid integer. |
72 | 359k | absl::optional<size_t> max_length; |
73 | 359k | if (m.str(3).length() != 0) { |
74 | 3.80k | size_t length_value; |
75 | 3.80k | if (!absl::SimpleAtoi(m.str(3), &length_value)) { |
76 | 47 | throwEnvoyExceptionOrPanic(absl::StrCat("Length must be an integer, given: ", m.str(3))); |
77 | 47 | } |
78 | 3.75k | max_length = length_value; |
79 | 3.75k | } |
80 | 359k | std::vector<std::string> path; |
81 | | |
82 | 359k | const size_t command_end_position = pos + m.str(0).length() - 1; |
83 | | |
84 | 359k | bool added = false; |
85 | | |
86 | | // First, try the built-in command parsers. |
87 | 359k | for (const auto& cmd : BuiltInCommandParsersBase<FormatterContext>::commandParsers()) { |
88 | 359k | auto formatter = cmd->parse(command, subcommand, max_length); |
89 | 359k | if (formatter) { |
90 | 83.8k | formatters.push_back(std::move(formatter)); |
91 | 83.8k | added = true; |
92 | 83.8k | break; |
93 | 83.8k | } |
94 | 359k | } |
95 | | |
96 | | // Next, try the command parsers provided by the user. |
97 | 359k | if (!added) { |
98 | 275k | for (const auto& cmd : command_parsers) { |
99 | 0 | auto formatter = cmd->parse(command, subcommand, max_length); |
100 | 0 | if (formatter) { |
101 | 0 | formatters.push_back(std::move(formatter)); |
102 | 0 | added = true; |
103 | 0 | break; |
104 | 0 | } |
105 | 0 | } |
106 | 275k | } |
107 | | |
108 | 359k | if (!added) { |
109 | | // Finally, try the context independent formatters. |
110 | 275k | formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{ |
111 | 275k | new StreamInfoFormatterBase<FormatterContext>(command, subcommand, max_length)}); |
112 | 275k | } |
113 | | |
114 | 359k | pos = command_end_position; |
115 | 359k | } |
116 | | |
117 | 402k | if (!current_token.empty() || format.empty()) { |
118 | | // Create a PlainStringFormatter with the final string literal. If the format string |
119 | | // was empty, this creates a PlainStringFormatter with an empty string. |
120 | 351k | formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{ |
121 | 351k | new PlainStringFormatterBase<FormatterContext>(current_token)}); |
122 | 351k | } |
123 | | |
124 | 402k | return formatters; |
125 | 403k | } |
126 | | |
127 | | private: |
128 | | static const std::regex& commandWithArgsRegex(); |
129 | | }; |
130 | | |
131 | | inline constexpr absl::string_view DefaultUnspecifiedValueStringView = "-"; |
132 | | |
133 | | /** |
134 | | * Composite formatter implementation. |
135 | | */ |
136 | | template <class FormatterContext> |
137 | | class CommonFormatterBaseImpl : public FormatterBase<FormatterContext> { |
138 | | public: |
139 | | using CommandParsers = std::vector<CommandParserBasePtr<FormatterContext>>; |
140 | | |
141 | | CommonFormatterBaseImpl(const std::string& format, bool omit_empty_values = false) |
142 | | : empty_value_string_(omit_empty_values ? absl::string_view{} |
143 | 383k | : DefaultUnspecifiedValueStringView) { |
144 | 383k | providers_ = SubstitutionFormatParser::parse<FormatterContext>(format); |
145 | 383k | } |
146 | | CommonFormatterBaseImpl(const std::string& format, bool omit_empty_values, |
147 | | const CommandParsers& command_parsers) |
148 | | : empty_value_string_(omit_empty_values ? absl::string_view{} |
149 | 306 | : DefaultUnspecifiedValueStringView) { |
150 | 306 | providers_ = SubstitutionFormatParser::parse<FormatterContext>(format, command_parsers); |
151 | 306 | } |
152 | | |
153 | | // FormatterBase |
154 | | std::string formatWithContext(const FormatterContext& context, |
155 | 453k | const StreamInfo::StreamInfo& stream_info) const override { |
156 | 453k | std::string log_line; |
157 | 453k | log_line.reserve(256); |
158 | | |
159 | 868k | for (const auto& provider : providers_) { |
160 | 868k | const auto bit = provider->formatWithContext(context, stream_info); |
161 | 868k | log_line += bit.value_or(empty_value_string_); |
162 | 868k | } |
163 | | |
164 | 453k | return log_line; |
165 | 453k | } |
166 | | |
167 | | private: |
168 | | const std::string empty_value_string_; |
169 | | std::vector<FormatterProviderBasePtr<FormatterContext>> providers_; |
170 | | }; |
171 | | |
172 | | template <class FormatterContext> |
173 | | class FormatterBaseImpl : public CommonFormatterBaseImpl<FormatterContext> { |
174 | | public: |
175 | | using CommonFormatterBaseImpl<FormatterContext>::CommonFormatterBaseImpl; |
176 | | }; |
177 | | |
178 | | // Helper classes for StructFormatter::StructFormatMapVisitor. |
179 | | template <class... Ts> struct StructFormatMapVisitorHelper : Ts... { using Ts::operator()...; }; |
180 | | template <class... Ts> StructFormatMapVisitorHelper(Ts...) -> StructFormatMapVisitorHelper<Ts...>; |
181 | | |
182 | | /** |
183 | | * An formatter for structured log formats, which returns a Struct proto that |
184 | | * can be converted easily into multiple formats. |
185 | | */ |
186 | | template <class FormatterContext> class StructFormatterBase { |
187 | | public: |
188 | | using CommandParsers = std::vector<CommandParserBasePtr<FormatterContext>>; |
189 | | using PlainNumber = PlainNumberFormatterBase<FormatterContext>; |
190 | | using PlainString = PlainStringFormatterBase<FormatterContext>; |
191 | | |
192 | | StructFormatterBase(const ProtobufWkt::Struct& format_mapping, bool preserve_types, |
193 | | bool omit_empty_values, const CommandParsers& commands = {}) |
194 | | : omit_empty_values_(omit_empty_values), preserve_types_(preserve_types), |
195 | | empty_value_(omit_empty_values_ ? std::string() |
196 | | : std::string(DefaultUnspecifiedValueStringView)), |
197 | 669 | struct_output_format_(FormatBuilder(commands).toFormatMapValue(format_mapping)) {} |
198 | | |
199 | | ProtobufWkt::Struct formatWithContext(const FormatterContext& context, |
200 | 158 | const StreamInfo::StreamInfo& info) const { |
201 | 158 | StructFormatMapVisitor visitor{ |
202 | 16.8k | [&](const std::vector<FormatterProviderBasePtr<FormatterContext>>& providers) { |
203 | 16.8k | return providersCallback(providers, context, info); |
204 | 16.8k | }, |
205 | 18.5k | [&, this](const StructFormatterBase::StructFormatMapWrapper& format_map) { |
206 | 18.5k | return structFormatMapCallback(format_map, visitor); |
207 | 18.5k | }, |
208 | 10.0k | [&, this](const StructFormatterBase::StructFormatListWrapper& format_list) { |
209 | 10.0k | return structFormatListCallback(format_list, visitor); |
210 | 10.0k | }, |
211 | 158 | }; |
212 | 158 | return structFormatMapCallback(struct_output_format_, visitor).struct_value(); |
213 | 158 | } |
214 | | |
215 | | private: |
216 | | struct StructFormatMapWrapper; |
217 | | struct StructFormatListWrapper; |
218 | | using StructFormatValue = |
219 | | absl::variant<const std::vector<FormatterProviderBasePtr<FormatterContext>>, |
220 | | const StructFormatMapWrapper, const StructFormatListWrapper>; |
221 | | // Although not required for Struct/JSON, it is nice to have the order of |
222 | | // properties preserved between the format and the log entry, thus std::map. |
223 | | using StructFormatMap = std::map<std::string, StructFormatValue>; |
224 | | using StructFormatMapPtr = std::unique_ptr<StructFormatMap>; |
225 | | struct StructFormatMapWrapper { |
226 | | StructFormatMapPtr value_; |
227 | | }; |
228 | | |
229 | | using StructFormatList = std::list<StructFormatValue>; |
230 | | using StructFormatListPtr = std::unique_ptr<StructFormatList>; |
231 | | struct StructFormatListWrapper { |
232 | | StructFormatListPtr value_; |
233 | | }; |
234 | | |
235 | | using StructFormatMapVisitor = StructFormatMapVisitorHelper< |
236 | | const std::function<ProtobufWkt::Value( |
237 | | const std::vector<FormatterProviderBasePtr<FormatterContext>>&)>, |
238 | | const std::function<ProtobufWkt::Value(const StructFormatterBase::StructFormatMapWrapper&)>, |
239 | | const std::function<ProtobufWkt::Value(const StructFormatterBase::StructFormatListWrapper&)>>; |
240 | | |
241 | | // Methods for building the format map. |
242 | | class FormatBuilder { |
243 | | public: |
244 | 669 | explicit FormatBuilder(const CommandParsers& commands) : commands_(commands) {} |
245 | | std::vector<FormatterProviderBasePtr<FormatterContext>> |
246 | 19.3k | toFormatStringValue(const std::string& string_format) const { |
247 | 19.3k | return SubstitutionFormatParser::parse<FormatterContext>(string_format, commands_); |
248 | 19.3k | } |
249 | | std::vector<FormatterProviderBasePtr<FormatterContext>> |
250 | 6.60k | toFormatNumberValue(double value) const { |
251 | 6.60k | std::vector<FormatterProviderBasePtr<FormatterContext>> formatters; |
252 | 6.60k | formatters.emplace_back(FormatterProviderBasePtr<FormatterContext>{new PlainNumber(value)}); |
253 | 6.60k | return formatters; |
254 | 6.60k | } |
255 | 26.9k | StructFormatMapWrapper toFormatMapValue(const ProtobufWkt::Struct& struct_format) const { |
256 | 26.9k | auto output = std::make_unique<StructFormatMap>(); |
257 | 33.7k | for (const auto& pair : struct_format.fields()) { |
258 | 33.7k | switch (pair.second.kind_case()) { |
259 | 6.84k | case ProtobufWkt::Value::kStringValue: |
260 | 6.84k | output->emplace(pair.first, toFormatStringValue(pair.second.string_value())); |
261 | 6.84k | break; |
262 | | |
263 | 19.5k | case ProtobufWkt::Value::kStructValue: |
264 | 19.5k | output->emplace(pair.first, toFormatMapValue(pair.second.struct_value())); |
265 | 19.5k | break; |
266 | | |
267 | 5.24k | case ProtobufWkt::Value::kListValue: |
268 | 5.24k | output->emplace(pair.first, toFormatListValue(pair.second.list_value())); |
269 | 5.24k | break; |
270 | | |
271 | 2.13k | case ProtobufWkt::Value::kNumberValue: |
272 | 2.13k | output->emplace(pair.first, toFormatNumberValue(pair.second.number_value())); |
273 | 2.13k | break; |
274 | 47 | default: |
275 | 47 | throwEnvoyExceptionOrPanic( |
276 | 33.7k | "Only string values, nested structs, list values and number values are " |
277 | 33.7k | "supported in structured access log format."); |
278 | 33.7k | } |
279 | 33.7k | } |
280 | 26.4k | return {std::move(output)}; |
281 | 26.9k | } |
282 | | StructFormatListWrapper |
283 | 15.4k | toFormatListValue(const ProtobufWkt::ListValue& list_value_format) const { |
284 | 15.4k | auto output = std::make_unique<StructFormatList>(); |
285 | 33.9k | for (const auto& value : list_value_format.values()) { |
286 | 33.9k | switch (value.kind_case()) { |
287 | 12.4k | case ProtobufWkt::Value::kStringValue: |
288 | 12.4k | output->emplace_back(toFormatStringValue(value.string_value())); |
289 | 12.4k | break; |
290 | | |
291 | 6.76k | case ProtobufWkt::Value::kStructValue: |
292 | 6.76k | output->emplace_back(toFormatMapValue(value.struct_value())); |
293 | 6.76k | break; |
294 | | |
295 | 10.2k | case ProtobufWkt::Value::kListValue: |
296 | 10.2k | output->emplace_back(toFormatListValue(value.list_value())); |
297 | 10.2k | break; |
298 | | |
299 | 4.47k | case ProtobufWkt::Value::kNumberValue: |
300 | 4.47k | output->emplace_back(toFormatNumberValue(value.number_value())); |
301 | 4.47k | break; |
302 | | |
303 | 49 | default: |
304 | 49 | throwEnvoyExceptionOrPanic( |
305 | 33.9k | "Only string values, nested structs, list values and number values are " |
306 | 33.9k | "supported in structured access log format."); |
307 | 33.9k | } |
308 | 33.9k | } |
309 | 14.9k | return {std::move(output)}; |
310 | 15.4k | } |
311 | | |
312 | | private: |
313 | | const CommandParsers& commands_; |
314 | | }; |
315 | | |
316 | | // Methods for doing the actual formatting. |
317 | | ProtobufWkt::Value |
318 | | providersCallback(const std::vector<FormatterProviderBasePtr<FormatterContext>>& providers, |
319 | | const FormatterContext& context, |
320 | 16.8k | const StreamInfo::StreamInfo& stream_info) const { |
321 | 16.8k | ASSERT(!providers.empty()); |
322 | 16.8k | if (providers.size() == 1) { |
323 | 13.6k | const auto& provider = providers.front(); |
324 | 13.6k | if (preserve_types_) { |
325 | 13.6k | return provider->formatValueWithContext(context, stream_info); |
326 | 13.6k | } |
327 | | |
328 | 0 | if (omit_empty_values_) { |
329 | 0 | return ValueUtil::optionalStringValue(provider->formatWithContext(context, stream_info)); |
330 | 0 | } |
331 | | |
332 | 0 | const auto str = provider->formatWithContext(context, stream_info); |
333 | 0 | return ValueUtil::stringValue(str.value_or(empty_value_)); |
334 | 0 | } |
335 | | // Multiple providers forces string output. |
336 | 3.16k | std::string str; |
337 | 23.4k | for (const auto& provider : providers) { |
338 | 23.4k | const auto bit = provider->formatWithContext(context, stream_info); |
339 | 23.4k | str += bit.value_or(empty_value_); |
340 | 23.4k | } |
341 | 3.16k | return ValueUtil::stringValue(str); |
342 | 16.8k | } |
343 | | ProtobufWkt::Value |
344 | | structFormatMapCallback(const StructFormatterBase::StructFormatMapWrapper& format_map, |
345 | 18.6k | const StructFormatMapVisitor& visitor) const { |
346 | 18.6k | ProtobufWkt::Struct output; |
347 | 18.6k | auto* fields = output.mutable_fields(); |
348 | 19.7k | for (const auto& pair : *format_map.value_) { |
349 | 19.7k | ProtobufWkt::Value value = absl::visit(visitor, pair.second); |
350 | 19.7k | if (omit_empty_values_ && value.kind_case() == ProtobufWkt::Value::kNullValue) { |
351 | 73 | continue; |
352 | 73 | } |
353 | 19.6k | (*fields)[pair.first] = value; |
354 | 19.6k | } |
355 | 18.6k | if (omit_empty_values_ && output.fields().empty()) { |
356 | 399 | return ValueUtil::nullValue(); |
357 | 399 | } |
358 | 18.2k | return ValueUtil::structValue(output); |
359 | 18.6k | } |
360 | | ProtobufWkt::Value |
361 | | structFormatListCallback(const StructFormatterBase::StructFormatListWrapper& format_list, |
362 | 10.0k | const StructFormatMapVisitor& visitor) const { |
363 | 10.0k | std::vector<ProtobufWkt::Value> output; |
364 | 25.7k | for (const auto& val : *format_list.value_) { |
365 | 25.7k | ProtobufWkt::Value value = absl::visit(visitor, val); |
366 | 25.7k | if (omit_empty_values_ && value.kind_case() == ProtobufWkt::Value::kNullValue) { |
367 | 326 | continue; |
368 | 326 | } |
369 | 25.3k | output.push_back(value); |
370 | 25.3k | } |
371 | 10.0k | return ValueUtil::listValue(output); |
372 | 10.0k | } |
373 | | |
374 | | const bool omit_empty_values_; |
375 | | const bool preserve_types_; |
376 | | const std::string empty_value_; |
377 | | |
378 | | const StructFormatMapWrapper struct_output_format_; |
379 | | }; |
380 | | |
381 | | template <class FormatterContext> |
382 | | using StructFormatterBasePtr = std::unique_ptr<StructFormatterBase<FormatterContext>>; |
383 | | |
384 | | template <class FormatterContext> |
385 | | class CommonJsonFormatterBaseImpl : public FormatterBase<FormatterContext> { |
386 | | public: |
387 | | using CommandParsers = std::vector<CommandParserBasePtr<FormatterContext>>; |
388 | | |
389 | | CommonJsonFormatterBaseImpl(const ProtobufWkt::Struct& format_mapping, bool preserve_types, |
390 | | bool omit_empty_values, bool sort_properties, |
391 | | const CommandParsers& commands = {}) |
392 | | : struct_formatter_(format_mapping, preserve_types, omit_empty_values, commands), |
393 | 669 | sort_properties_(sort_properties) {} |
394 | | |
395 | | // FormatterBase |
396 | | std::string formatWithContext(const FormatterContext& context, |
397 | 158 | const StreamInfo::StreamInfo& info) const override { |
398 | 158 | const ProtobufWkt::Struct output_struct = struct_formatter_.formatWithContext(context, info); |
399 | | |
400 | 158 | std::string log_line = ""; |
401 | 158 | #ifdef ENVOY_ENABLE_YAML |
402 | 158 | if (sort_properties_) { |
403 | 123 | log_line = Json::Factory::loadFromProtobufStruct(output_struct)->asJsonString(); |
404 | 123 | } else { |
405 | 35 | log_line = MessageUtil::getJsonStringFromMessageOrError(output_struct, false, true); |
406 | 35 | } |
407 | | #else |
408 | | UNREFERENCED_PARAMETER(sort_properties_); |
409 | | IS_ENVOY_BUG("Json support compiled out"); |
410 | | #endif |
411 | 158 | return absl::StrCat(log_line, "\n"); |
412 | 158 | } |
413 | | |
414 | | private: |
415 | | const StructFormatterBase<FormatterContext> struct_formatter_; |
416 | | const bool sort_properties_; |
417 | | }; |
418 | | |
419 | | template <class FormatterContext> |
420 | | class JsonFormatterBaseImpl : public CommonJsonFormatterBaseImpl<FormatterContext> { |
421 | | public: |
422 | | using CommonJsonFormatterBaseImpl<FormatterContext>::CommonJsonFormatterBaseImpl; |
423 | | }; |
424 | | |
425 | | using StructFormatter = StructFormatterBase<HttpFormatterContext>; |
426 | | using StructFormatterPtr = std::unique_ptr<StructFormatter>; |
427 | | |
428 | | // Aliases for backwards compatibility. |
429 | | using FormatterImpl = FormatterBaseImpl<HttpFormatterContext>; |
430 | | using JsonFormatterImpl = JsonFormatterBaseImpl<HttpFormatterContext>; |
431 | | |
432 | | } // namespace Formatter |
433 | | } // namespace Envoy |