Line data Source code
1 : #include "source/common/protobuf/utility.h"
2 :
3 : #include <limits>
4 : #include <numeric>
5 :
6 : #include "envoy/annotations/deprecation.pb.h"
7 : #include "envoy/protobuf/message_validator.h"
8 : #include "envoy/type/v3/percent.pb.h"
9 :
10 : #include "source/common/common/assert.h"
11 : #include "source/common/common/documentation_url.h"
12 : #include "source/common/common/fmt.h"
13 : #include "source/common/protobuf/deterministic_hash.h"
14 : #include "source/common/protobuf/message_validator_impl.h"
15 : #include "source/common/protobuf/protobuf.h"
16 : #include "source/common/protobuf/visitor.h"
17 : #include "source/common/runtime/runtime_features.h"
18 :
19 : #include "absl/strings/match.h"
20 : #include "udpa/annotations/sensitive.pb.h"
21 : #include "udpa/annotations/status.pb.h"
22 : #include "validate/validate.h"
23 : #include "xds/annotations/v3/status.pb.h"
24 :
25 : using namespace std::chrono_literals;
26 :
27 : namespace Envoy {
28 : namespace {
29 :
30 1327604 : absl::string_view filenameFromPath(absl::string_view full_path) {
31 1327604 : size_t index = full_path.rfind('/');
32 1327604 : if (index == std::string::npos || index == full_path.size()) {
33 0 : return full_path;
34 0 : }
35 1327604 : return full_path.substr(index + 1, full_path.size());
36 1327604 : }
37 :
38 : // Logs a warning for use of a deprecated field or runtime-overridden use of an
39 : // otherwise fatal field. Throws a warning on use of a fatal by default field.
40 : void deprecatedFieldHelper(Runtime::Loader* runtime, bool proto_annotated_as_deprecated,
41 : bool proto_annotated_as_disallowed, const std::string& feature_name,
42 : std::string error, const Protobuf::Message& message,
43 908 : ProtobufMessage::ValidationVisitor& validation_visitor) {
44 : // This option is for Envoy builds with --define deprecated_features=disabled
45 : // The build options CI then verifies that as Envoy developers deprecate fields,
46 : // that they update canonical configs and unit tests to not use those deprecated
47 : // fields, by making their use fatal in the build options CI.
48 : #ifdef ENVOY_DISABLE_DEPRECATED_FEATURES
49 : bool warn_only = false;
50 : #else
51 908 : bool warn_only = true;
52 908 : #endif
53 908 : if (runtime &&
54 908 : runtime->snapshot().getBoolean("envoy.features.fail_on_any_deprecated_feature", false)) {
55 0 : warn_only = false;
56 0 : }
57 908 : bool warn_default = warn_only;
58 : // Allow runtime to be null both to not crash if this is called before server initialization,
59 : // and so proto validation works in context where runtime singleton is not set up (e.g.
60 : // standalone config validation utilities)
61 908 : if (runtime && proto_annotated_as_deprecated) {
62 : // This is set here, rather than above, so that in the absence of a
63 : // registry (i.e. test) the default for if a feature is allowed or not is
64 : // based on ENVOY_DISABLE_DEPRECATED_FEATURES.
65 0 : warn_only &= !proto_annotated_as_disallowed;
66 0 : warn_default = warn_only;
67 0 : warn_only = runtime->snapshot().deprecatedFeatureEnabled(feature_name, warn_only);
68 0 : }
69 : // Note this only checks if the runtime override has an actual effect. It
70 : // does not change the logged warning if someone "allows" a deprecated but not
71 : // yet fatal field.
72 908 : const bool runtime_overridden = (warn_default == false && warn_only == true);
73 :
74 908 : std::string with_overridden = fmt::format(
75 908 : fmt::runtime(error),
76 908 : (runtime_overridden ? "runtime overrides to continue using now fatal-by-default " : ""));
77 :
78 908 : validation_visitor.onDeprecatedField("type " + message.GetTypeName() + " " + with_overridden,
79 908 : warn_only);
80 908 : }
81 :
82 : } // namespace
83 :
84 : namespace ProtobufPercentHelper {
85 :
86 627 : uint64_t checkAndReturnDefault(uint64_t default_value, uint64_t max_value) {
87 627 : ASSERT(default_value <= max_value);
88 627 : return default_value;
89 627 : }
90 :
91 29 : uint64_t convertPercent(double percent, uint64_t max_value) {
92 : // Checked by schema.
93 29 : ASSERT(percent >= 0.0 && percent <= 100.0);
94 29 : return max_value * (percent / 100.0);
95 29 : }
96 :
97 136 : bool evaluateFractionalPercent(envoy::type::v3::FractionalPercent percent, uint64_t random_value) {
98 136 : return random_value % fractionalPercentDenominatorToInt(percent.denominator()) <
99 136 : percent.numerator();
100 136 : }
101 :
102 : uint64_t fractionalPercentDenominatorToInt(
103 272 : const envoy::type::v3::FractionalPercent::DenominatorType& denominator) {
104 272 : switch (denominator) {
105 0 : PANIC_ON_PROTO_ENUM_SENTINEL_VALUES;
106 272 : case envoy::type::v3::FractionalPercent::HUNDRED:
107 272 : return 100;
108 0 : case envoy::type::v3::FractionalPercent::TEN_THOUSAND:
109 0 : return 10000;
110 0 : case envoy::type::v3::FractionalPercent::MILLION:
111 0 : return 1000000;
112 272 : }
113 0 : PANIC_DUE_TO_CORRUPT_ENUM
114 0 : }
115 :
116 : } // namespace ProtobufPercentHelper
117 :
118 : void ProtoExceptionUtil::throwMissingFieldException(const std::string& field_name,
119 3 : const Protobuf::Message& message) {
120 3 : std::string error =
121 3 : fmt::format("Field '{}' is missing in: {}", field_name, message.DebugString());
122 3 : throwEnvoyExceptionOrPanic(error);
123 3 : }
124 :
125 : void ProtoExceptionUtil::throwProtoValidationException(const std::string& validation_error,
126 3925 : const Protobuf::Message& message) {
127 3925 : std::string error = fmt::format("Proto constraint validation failed ({}): {}", validation_error,
128 3925 : message.DebugString());
129 3925 : throwEnvoyExceptionOrPanic(error);
130 3925 : }
131 :
132 1604 : size_t MessageUtil::hash(const Protobuf::Message& message) {
133 1604 : #if defined(ENVOY_ENABLE_FULL_PROTOS)
134 1604 : if (Runtime::runtimeFeatureEnabled("envoy.restart_features.use_fast_protobuf_hash")) {
135 0 : return DeterministicProtoHash::hash(message);
136 1604 : } else {
137 1604 : std::string text_format;
138 1604 : Protobuf::TextFormat::Printer printer;
139 1604 : printer.SetExpandAny(true);
140 1604 : printer.SetUseFieldNumber(true);
141 1604 : printer.SetSingleLineMode(true);
142 1604 : printer.SetHideUnknownFields(true);
143 1604 : printer.PrintToString(message, &text_format);
144 1604 : return HashUtil::xxHash64(text_format);
145 1604 : }
146 : #else
147 : return HashUtil::xxHash64(message.SerializeAsString());
148 : #endif
149 1604 : }
150 :
151 : #if !defined(ENVOY_ENABLE_FULL_PROTOS)
152 : // NOLINTNEXTLINE(readability-identifier-naming)
153 : bool MessageLiteDifferencer::Equals(const Protobuf::Message& message1,
154 : const Protobuf::Message& message2) {
155 : return MessageUtil::hash(message1) == MessageUtil::hash(message2);
156 : }
157 :
158 : // NOLINTNEXTLINE(readability-identifier-naming)
159 : bool MessageLiteDifferencer::Equivalent(const Protobuf::Message& message1,
160 : const Protobuf::Message& message2) {
161 : return Equals(message1, message2);
162 : }
163 : #endif
164 :
165 : namespace {
166 :
167 : void checkForDeprecatedNonRepeatedEnumValue(
168 : const Protobuf::Message& message, absl::string_view filename,
169 : const Protobuf::FieldDescriptor* field, const Protobuf::Reflection* reflection,
170 1327604 : Runtime::Loader* runtime, ProtobufMessage::ValidationVisitor& validation_visitor) {
171 : // Repeated fields will be handled by recursion in checkForUnexpectedFields.
172 1327604 : if (field->is_repeated() || field->cpp_type() != Protobuf::FieldDescriptor::CPPTYPE_ENUM) {
173 1283319 : return;
174 1283319 : }
175 :
176 44285 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);
177 44285 : bool default_value = !reflection->HasField(*reflectable_message, field);
178 :
179 44285 : const Protobuf::EnumValueDescriptor* enum_value_descriptor =
180 44285 : reflection->GetEnum(*reflectable_message, field);
181 44285 : if (!enum_value_descriptor->options().deprecated()) {
182 43772 : return;
183 43772 : }
184 :
185 513 : const std::string error =
186 513 : absl::StrCat("Using {}", (default_value ? "the default now-" : ""), "deprecated value ",
187 513 : enum_value_descriptor->name(), " for enum '", field->full_name(), "' from file ",
188 513 : filename, ". This enum value will be removed from Envoy soon",
189 513 : (default_value ? " so a non-default value must now be explicitly set" : ""),
190 513 : ". Please see " ENVOY_DOC_URL_VERSION_HISTORY " for details.");
191 513 : deprecatedFieldHelper(
192 513 : runtime, true /*deprecated*/,
193 513 : enum_value_descriptor->options().GetExtension(envoy::annotations::disallowed_by_default_enum),
194 513 : absl::StrCat("envoy.deprecated_features:", enum_value_descriptor->full_name()), error,
195 513 : message, validation_visitor);
196 513 : }
197 :
198 : constexpr absl::string_view WipWarning =
199 : "API features marked as work-in-progress are not considered stable, are not covered by the "
200 : "threat model, are not supported by the security team, and are subject to breaking changes. Do "
201 : "not use this feature without understanding each of the previous points.";
202 :
203 : class UnexpectedFieldProtoVisitor : public ProtobufMessage::ConstProtoVisitor {
204 : public:
205 : UnexpectedFieldProtoVisitor(ProtobufMessage::ValidationVisitor& validation_visitor,
206 : Runtime::Loader* runtime)
207 10823 : : validation_visitor_(validation_visitor), runtime_(runtime) {}
208 :
209 1327604 : void onField(const Protobuf::Message& message, const Protobuf::FieldDescriptor& field) override {
210 1327604 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);
211 1327604 : const Protobuf::Reflection* reflection = reflectable_message->GetReflection();
212 1327604 : absl::string_view filename = filenameFromPath(field.file()->name());
213 :
214 : // Before we check to see if the field is in use, see if there's a
215 : // deprecated default enum value.
216 1327604 : checkForDeprecatedNonRepeatedEnumValue(message, filename, &field, reflection, runtime_,
217 1327604 : validation_visitor_);
218 :
219 : // If this field is not in use, continue.
220 1327604 : if ((field.is_repeated() && reflection->FieldSize(*reflectable_message, &field) == 0) ||
221 1327604 : (!field.is_repeated() && !reflection->HasField(*reflectable_message, &field))) {
222 1078947 : return;
223 1078947 : }
224 :
225 248657 : const auto& field_status = field.options().GetExtension(xds::annotations::v3::field_status);
226 248657 : if (field_status.work_in_progress()) {
227 146 : validation_visitor_.onWorkInProgress(fmt::format(
228 146 : "field '{}' is marked as work-in-progress. {}", field.full_name(), WipWarning));
229 146 : }
230 :
231 : // If this field is deprecated, warn or throw an error.
232 248657 : if (field.options().deprecated()) {
233 395 : const std::string warning =
234 395 : absl::StrCat("Using {}deprecated option '", field.full_name(), "' from file ", filename,
235 395 : ". This configuration will be removed from "
236 395 : "Envoy soon. Please see " ENVOY_DOC_URL_VERSION_HISTORY " for details.");
237 :
238 395 : deprecatedFieldHelper(runtime_, true /*deprecated*/,
239 395 : field.options().GetExtension(envoy::annotations::disallowed_by_default),
240 395 : absl::StrCat("envoy.deprecated_features:", field.full_name()), warning,
241 395 : message, validation_visitor_);
242 395 : }
243 248657 : }
244 :
245 : void onMessage(const Protobuf::Message& message,
246 170664 : absl::Span<const Protobuf::Message* const> parents, bool) override {
247 170664 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);
248 170664 : if (reflectable_message->GetDescriptor()
249 170664 : ->options()
250 170664 : .GetExtension(xds::annotations::v3::message_status)
251 170664 : .work_in_progress()) {
252 3 : validation_visitor_.onWorkInProgress(fmt::format(
253 3 : "message '{}' is marked as work-in-progress. {}", message.GetTypeName(), WipWarning));
254 3 : }
255 :
256 170664 : const auto& udpa_file_options =
257 170664 : reflectable_message->GetDescriptor()->file()->options().GetExtension(
258 170664 : udpa::annotations::file_status);
259 170664 : const auto& xds_file_options =
260 170664 : reflectable_message->GetDescriptor()->file()->options().GetExtension(
261 170664 : xds::annotations::v3::file_status);
262 170664 : if (udpa_file_options.work_in_progress() || xds_file_options.work_in_progress()) {
263 43 : validation_visitor_.onWorkInProgress(fmt::format(
264 43 : "message '{}' is contained in proto file '{}' marked as work-in-progress. {}",
265 43 : message.GetTypeName(), reflectable_message->GetDescriptor()->file()->name(), WipWarning));
266 43 : }
267 :
268 : // Reject unknown fields.
269 170664 : const auto& unknown_fields =
270 170664 : reflectable_message->GetReflection()->GetUnknownFields(*reflectable_message);
271 170664 : if (!unknown_fields.empty()) {
272 22 : std::string error_msg;
273 51 : for (int n = 0; n < unknown_fields.field_count(); ++n) {
274 29 : absl::StrAppend(&error_msg, n > 0 ? ", " : "", unknown_fields.field(n).number());
275 29 : }
276 22 : if (!error_msg.empty()) {
277 22 : validation_visitor_.onUnknownField(
278 22 : fmt::format("type {}({}) with unknown field set {{{}}}", message.GetTypeName(),
279 22 : !parents.empty()
280 22 : ? absl::StrJoin(parents, "::",
281 10 : [](std::string* out, const Protobuf::Message* const m) {
282 10 : absl::StrAppend(out, m->GetTypeName());
283 10 : })
284 22 : : "root",
285 22 : error_msg));
286 22 : }
287 22 : }
288 170664 : }
289 :
290 : private:
291 : ProtobufMessage::ValidationVisitor& validation_visitor_;
292 : Runtime::Loader* runtime_;
293 : };
294 :
295 : } // namespace
296 :
297 : void MessageUtil::checkForUnexpectedFields(const Protobuf::Message& message,
298 : ProtobufMessage::ValidationVisitor& validation_visitor,
299 10823 : bool recurse_into_any) {
300 10823 : Runtime::Loader* runtime = validation_visitor.runtime().has_value()
301 10823 : ? &validation_visitor.runtime().value().get()
302 10823 : : nullptr;
303 10823 : UnexpectedFieldProtoVisitor unexpected_field_visitor(validation_visitor, runtime);
304 10823 : ProtobufMessage::traverseMessage(unexpected_field_visitor, message, recurse_into_any);
305 10823 : }
306 :
307 : namespace {
308 :
309 : class PgvCheckVisitor : public ProtobufMessage::ConstProtoVisitor {
310 : public:
311 : void onMessage(const Protobuf::Message& message, absl::Span<const Protobuf::Message* const>,
312 0 : bool was_any_or_top_level) override {
313 0 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(message);
314 0 : std::string err;
315 : // PGV verification is itself recursive up to the point at which it hits an Any message. As
316 : // such, to avoid N^2 checking of the tree, we only perform an additional check at the point
317 : // at which PGV would have stopped because it does not itself check within Any messages.
318 0 : if (was_any_or_top_level &&
319 0 : !pgv::BaseValidator::AbstractCheckMessage(*reflectable_message, &err)) {
320 0 : ProtoExceptionUtil::throwProtoValidationException(err, *reflectable_message);
321 0 : }
322 0 : }
323 :
324 0 : void onField(const Protobuf::Message&, const Protobuf::FieldDescriptor&) override {}
325 : };
326 :
327 : } // namespace
328 :
329 0 : void MessageUtil::recursivePgvCheck(const Protobuf::Message& message) {
330 0 : PgvCheckVisitor visitor;
331 0 : ProtobufMessage::traverseMessage(visitor, message, true);
332 0 : }
333 :
334 106 : void MessageUtil::packFrom(ProtobufWkt::Any& any_message, const Protobuf::Message& message) {
335 106 : #if defined(ENVOY_ENABLE_FULL_PROTOS)
336 106 : any_message.PackFrom(message);
337 : #else
338 : any_message.set_type_url(message.GetTypeName());
339 : any_message.set_value(message.SerializeAsString());
340 : #endif
341 106 : }
342 :
343 1902 : void MessageUtil::unpackTo(const ProtobufWkt::Any& any_message, Protobuf::Message& message) {
344 1902 : #if defined(ENVOY_ENABLE_FULL_PROTOS)
345 1902 : if (!any_message.UnpackTo(&message)) {
346 17 : throwEnvoyExceptionOrPanic(fmt::format("Unable to unpack as {}: {}",
347 17 : message.GetDescriptor()->full_name(),
348 17 : any_message.DebugString()));
349 : #else
350 : if (!message.ParseFromString(any_message.value())) {
351 : throwEnvoyExceptionOrPanic(
352 : fmt::format("Unable to unpack as {}: {}", message.GetTypeName(), any_message.type_url()));
353 : #endif
354 17 : }
355 1902 : }
356 :
357 : absl::Status MessageUtil::unpackToNoThrow(const ProtobufWkt::Any& any_message,
358 0 : Protobuf::Message& message) {
359 0 : #if defined(ENVOY_ENABLE_FULL_PROTOS)
360 0 : if (!any_message.UnpackTo(&message)) {
361 0 : return absl::InternalError(absl::StrCat("Unable to unpack as ",
362 0 : message.GetDescriptor()->full_name(), ": ",
363 0 : any_message.DebugString()));
364 : #else
365 : if (!message.ParseFromString(any_message.value())) {
366 : return absl::InternalError(
367 : absl::StrCat("Unable to unpack as ", message.GetTypeName(), ": ", any_message.type_url()));
368 : #endif
369 0 : }
370 : // Ok Status is returned if `UnpackTo` succeeded.
371 0 : return absl::OkStatus();
372 0 : }
373 :
374 : std::string MessageUtil::convertToStringForLogs(const Protobuf::Message& message, bool pretty_print,
375 0 : bool always_print_primitive_fields) {
376 0 : #ifdef ENVOY_ENABLE_YAML
377 0 : return getJsonStringFromMessageOrError(message, pretty_print, always_print_primitive_fields);
378 : #else
379 : UNREFERENCED_PARAMETER(pretty_print);
380 : UNREFERENCED_PARAMETER(always_print_primitive_fields);
381 : return message.DebugString();
382 : #endif
383 0 : }
384 :
385 0 : ProtobufWkt::Struct MessageUtil::keyValueStruct(const std::string& key, const std::string& value) {
386 0 : ProtobufWkt::Struct struct_obj;
387 0 : ProtobufWkt::Value val;
388 0 : val.set_string_value(value);
389 0 : (*struct_obj.mutable_fields())[key] = val;
390 0 : return struct_obj;
391 0 : }
392 :
393 13 : ProtobufWkt::Struct MessageUtil::keyValueStruct(const std::map<std::string, std::string>& fields) {
394 13 : ProtobufWkt::Struct struct_obj;
395 13 : ProtobufWkt::Value val;
396 65 : for (const auto& pair : fields) {
397 65 : val.set_string_value(pair.second);
398 65 : (*struct_obj.mutable_fields())[pair.first] = val;
399 65 : }
400 13 : return struct_obj;
401 13 : }
402 :
403 0 : std::string MessageUtil::codeEnumToString(absl::StatusCode code) {
404 0 : std::string result = absl::StatusCodeToString(code);
405 : // This preserves the behavior of the `ProtobufUtil::Status(code, "").ToString();`
406 0 : return !result.empty() ? result : "UNKNOWN: ";
407 0 : }
408 :
409 : namespace {
410 :
411 : // Forward declaration for mutually-recursive helper functions.
412 : void redact(Protobuf::Message* message, bool ancestor_is_sensitive);
413 :
414 : using Transform = std::function<void(Protobuf::Message*, const Protobuf::Reflection*,
415 : const Protobuf::FieldDescriptor*)>;
416 :
417 : // To redact opaque types, namely `Any` and `TypedStruct`, we have to reify them to the concrete
418 : // message types specified by their `type_url` before we can redact their contents. This is mostly
419 : // identical between `Any` and `TypedStruct`, the only difference being how they are packed and
420 : // unpacked. Note that we have to use reflection on the opaque type here, rather than downcasting
421 : // to `Any` or `TypedStruct`, because any message we might be handling could have originated from
422 : // a `DynamicMessageFactory`.
423 : bool redactOpaque(Protobuf::Message* message, bool ancestor_is_sensitive,
424 59372 : absl::string_view opaque_type_name, Transform unpack, Transform repack) {
425 : // Ensure this message has the opaque type we're expecting.
426 59372 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);
427 59372 : const auto* opaque_descriptor = reflectable_message->GetDescriptor();
428 59372 : if (opaque_descriptor->full_name() != opaque_type_name) {
429 58004 : return false;
430 58004 : }
431 :
432 : // Find descriptors for the `type_url` and `value` fields. The `type_url` field must not be
433 : // empty, but `value` may be (in which case our work is done).
434 1368 : const auto* reflection = reflectable_message->GetReflection();
435 1368 : const auto* type_url_field_descriptor = opaque_descriptor->FindFieldByName("type_url");
436 1368 : const auto* value_field_descriptor = opaque_descriptor->FindFieldByName("value");
437 1368 : ASSERT(type_url_field_descriptor != nullptr && value_field_descriptor != nullptr);
438 1368 : if (!reflection->HasField(*reflectable_message, type_url_field_descriptor) &&
439 1368 : !reflection->HasField(*reflectable_message, value_field_descriptor)) {
440 0 : return true;
441 0 : }
442 1368 : if (!reflection->HasField(*reflectable_message, type_url_field_descriptor) ||
443 1368 : !reflection->HasField(*reflectable_message, value_field_descriptor)) {
444 196 : return false;
445 196 : }
446 :
447 : // Try to find a descriptor for `type_url` in the pool and instantiate a new message of the
448 : // correct concrete type.
449 1172 : const std::string type_url(
450 1172 : reflection->GetString(*reflectable_message, type_url_field_descriptor));
451 1172 : const std::string concrete_type_name(TypeUtil::typeUrlToDescriptorFullName(type_url));
452 1172 : const auto* concrete_descriptor =
453 1172 : Protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName(concrete_type_name);
454 1172 : if (concrete_descriptor == nullptr) {
455 : // If the type URL doesn't correspond to a known proto, don't try to reify it, just treat it
456 : // like any other message. See the documented limitation on `MessageUtil::redact()` for more
457 : // context.
458 0 : ENVOY_LOG_MISC(warn, "Could not reify {} with unknown type URL {}", opaque_type_name, type_url);
459 0 : return false;
460 0 : }
461 1172 : Protobuf::DynamicMessageFactory message_factory;
462 1172 : std::unique_ptr<Protobuf::Message> typed_message(
463 1172 : message_factory.GetPrototype(concrete_descriptor)->New());
464 :
465 : // Finally we can unpack, redact, and repack the opaque message using the provided callbacks.
466 :
467 : // Note: the content of opaque types may contain illegal content that mismatches the type_url
468 : // which may cause unpacking to fail. We catch the exception here to avoid crashing Envoy.
469 1172 : TRY_ASSERT_MAIN_THREAD { unpack(typed_message.get(), reflection, value_field_descriptor); }
470 1172 : END_TRY CATCH(const EnvoyException& e, {
471 1172 : ENVOY_LOG_MISC(warn, "Could not unpack {} with type URL {}: {}", opaque_type_name, type_url,
472 1172 : e.what());
473 1172 : return false;
474 1172 : });
475 1172 : redact(typed_message.get(), ancestor_is_sensitive);
476 1172 : repack(typed_message.get(), reflection, value_field_descriptor);
477 1172 : return true;
478 1172 : }
479 :
480 20572 : bool redactAny(Protobuf::Message* message, bool ancestor_is_sensitive) {
481 20572 : return redactOpaque(
482 20572 : message, ancestor_is_sensitive, "google.protobuf.Any",
483 20572 : [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
484 20572 : const Protobuf::FieldDescriptor* field_descriptor) {
485 1172 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);
486 : // To unpack an `Any`, parse the serialized proto.
487 1172 : typed_message->ParseFromString(
488 1172 : reflection->GetString(*reflectable_message, field_descriptor));
489 1172 : },
490 20572 : [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
491 20572 : const Protobuf::FieldDescriptor* field_descriptor) {
492 1172 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);
493 : // To repack an `Any`, reserialize its proto.
494 1172 : reflection->SetString(&(*reflectable_message), field_descriptor,
495 1172 : typed_message->SerializeAsString());
496 1172 : });
497 20572 : }
498 :
499 : // To redact a `TypedStruct`, we have to reify it based on its `type_url` to redact it.
500 : bool redactTypedStruct(Protobuf::Message* message, const char* typed_struct_type,
501 38800 : bool ancestor_is_sensitive) {
502 38800 : return redactOpaque(
503 38800 : message, ancestor_is_sensitive, typed_struct_type,
504 38800 : [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
505 38800 : const Protobuf::FieldDescriptor* field_descriptor) {
506 0 : #ifdef ENVOY_ENABLE_YAML
507 : // To unpack a `TypedStruct`, convert the struct from JSON.
508 0 : MessageUtil::jsonConvert(reflection->GetMessage(*message, field_descriptor),
509 0 : *typed_message);
510 : #else
511 : UNREFERENCED_PARAMETER(message);
512 : UNREFERENCED_PARAMETER(typed_message);
513 : UNREFERENCED_PARAMETER(reflection);
514 : UNREFERENCED_PARAMETER(field_descriptor);
515 : IS_ENVOY_BUG("redaction requested with JSON/YAML support removed");
516 : #endif
517 0 : },
518 38800 : [message](Protobuf::Message* typed_message, const Protobuf::Reflection* reflection,
519 38800 : const Protobuf::FieldDescriptor* field_descriptor) {
520 : // To repack a `TypedStruct`, convert the message back to JSON.
521 0 : #ifdef ENVOY_ENABLE_YAML
522 0 : MessageUtil::jsonConvert(*typed_message,
523 0 : *(reflection->MutableMessage(message, field_descriptor)));
524 : #else
525 : UNREFERENCED_PARAMETER(message);
526 : UNREFERENCED_PARAMETER(typed_message);
527 : UNREFERENCED_PARAMETER(reflection);
528 : UNREFERENCED_PARAMETER(field_descriptor);
529 : IS_ENVOY_BUG("redaction requested with JSON/YAML support removed");
530 : #endif
531 0 : });
532 38800 : }
533 :
534 : // Recursive helper method for MessageUtil::redact() below.
535 20572 : void redact(Protobuf::Message* message, bool ancestor_is_sensitive) {
536 20572 : if (redactAny(message, ancestor_is_sensitive) ||
537 20572 : redactTypedStruct(message, "xds.type.v3.TypedStruct", ancestor_is_sensitive) ||
538 20572 : redactTypedStruct(message, "udpa.type.v1.TypedStruct", ancestor_is_sensitive)) {
539 1172 : return;
540 1172 : }
541 :
542 19400 : Protobuf::ReflectableMessage reflectable_message = createReflectableMessage(*message);
543 19400 : const auto* descriptor = reflectable_message->GetDescriptor();
544 19400 : const auto* reflection = reflectable_message->GetReflection();
545 151091 : for (int i = 0; i < descriptor->field_count(); ++i) {
546 131691 : const auto* field_descriptor = descriptor->field(i);
547 :
548 : // Redact if this field or any of its ancestors have the `sensitive` option set.
549 131691 : const bool sensitive = ancestor_is_sensitive ||
550 131691 : field_descriptor->options().GetExtension(udpa::annotations::sensitive);
551 :
552 131691 : if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {
553 : // Recursive case: traverse message fields.
554 57511 : if (field_descriptor->is_map()) {
555 : // Redact values of maps only. Redacting both leaves the map with multiple "[redacted]"
556 : // keys.
557 1336 : const int field_size = reflection->FieldSize(*reflectable_message, field_descriptor);
558 2134 : for (int i = 0; i < field_size; ++i) {
559 798 : Protobuf::Message* map_pair_base =
560 798 : reflection->MutableRepeatedMessage(&(*reflectable_message), field_descriptor, i);
561 798 : Protobuf::ReflectableMessage map_pair = createReflectableMessage(*map_pair_base);
562 798 : auto* value_field_desc = map_pair->GetDescriptor()->FindFieldByName("value");
563 798 : if (sensitive && (value_field_desc->type() == Protobuf::FieldDescriptor::TYPE_STRING ||
564 0 : value_field_desc->type() == Protobuf::FieldDescriptor::TYPE_BYTES)) {
565 0 : map_pair->GetReflection()->SetString(&(*map_pair), value_field_desc, "[redacted]");
566 798 : } else if (value_field_desc->type() == Protobuf::FieldDescriptor::TYPE_MESSAGE) {
567 798 : redact(map_pair->GetReflection()->MutableMessage(&(*map_pair), value_field_desc),
568 798 : sensitive);
569 798 : } else if (sensitive) {
570 0 : map_pair->GetReflection()->ClearField(&(*map_pair), value_field_desc);
571 0 : }
572 798 : }
573 56175 : } else if (field_descriptor->is_repeated()) {
574 8102 : const int field_size = reflection->FieldSize(*reflectable_message, field_descriptor);
575 20463 : for (int i = 0; i < field_size; ++i) {
576 12361 : redact(reflection->MutableRepeatedMessage(&(*reflectable_message), field_descriptor, i),
577 12361 : sensitive);
578 12361 : }
579 48073 : } else if (reflection->HasField(*reflectable_message, field_descriptor)) {
580 6073 : redact(reflection->MutableMessage(&(*reflectable_message), field_descriptor), sensitive);
581 6073 : }
582 74180 : } else if (sensitive) {
583 : // Base case: replace strings and bytes with "[redacted]" and clear all others.
584 1680 : if (field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_STRING ||
585 1680 : field_descriptor->type() == Protobuf::FieldDescriptor::TYPE_BYTES) {
586 1680 : if (field_descriptor->is_repeated()) {
587 0 : const int field_size = reflection->FieldSize(*reflectable_message, field_descriptor);
588 0 : for (int i = 0; i < field_size; ++i) {
589 0 : reflection->SetRepeatedString(&(*reflectable_message), field_descriptor, i,
590 0 : "[redacted]");
591 0 : }
592 1680 : } else if (reflection->HasField(*reflectable_message, field_descriptor)) {
593 420 : reflection->SetString(&(*reflectable_message), field_descriptor, "[redacted]");
594 420 : }
595 1680 : } else {
596 0 : reflection->ClearField(&(*reflectable_message), field_descriptor);
597 0 : }
598 1680 : }
599 131691 : }
600 19400 : }
601 :
602 : } // namespace
603 :
604 168 : void MessageUtil::redact(Protobuf::Message& message) {
605 168 : ::Envoy::redact(&message, /* ancestor_is_sensitive = */ false);
606 168 : }
607 :
608 0 : void MessageUtil::wireCast(const Protobuf::Message& src, Protobuf::Message& dst) {
609 : // This should should generally succeed, but if there are malformed UTF-8 strings in a message,
610 : // this can fail.
611 0 : if (!dst.ParseFromString(src.SerializeAsString())) {
612 0 : throwEnvoyExceptionOrPanic("Unable to deserialize during wireCast()");
613 0 : }
614 0 : }
615 :
616 2667 : bool ValueUtil::equal(const ProtobufWkt::Value& v1, const ProtobufWkt::Value& v2) {
617 2667 : ProtobufWkt::Value::KindCase kind = v1.kind_case();
618 2667 : if (kind != v2.kind_case()) {
619 0 : return false;
620 0 : }
621 :
622 2667 : switch (kind) {
623 652 : case ProtobufWkt::Value::KIND_NOT_SET:
624 652 : return v2.kind_case() == ProtobufWkt::Value::KIND_NOT_SET;
625 :
626 219 : case ProtobufWkt::Value::kNullValue:
627 219 : return true;
628 :
629 242 : case ProtobufWkt::Value::kNumberValue:
630 242 : return v1.number_value() == v2.number_value();
631 :
632 227 : case ProtobufWkt::Value::kStringValue:
633 227 : return v1.string_value() == v2.string_value();
634 :
635 179 : case ProtobufWkt::Value::kBoolValue:
636 179 : return v1.bool_value() == v2.bool_value();
637 :
638 881 : case ProtobufWkt::Value::kStructValue: {
639 881 : const ProtobufWkt::Struct& s1 = v1.struct_value();
640 881 : const ProtobufWkt::Struct& s2 = v2.struct_value();
641 881 : if (s1.fields_size() != s2.fields_size()) {
642 0 : return false;
643 0 : }
644 1961 : for (const auto& it1 : s1.fields()) {
645 1961 : const auto& it2 = s2.fields().find(it1.first);
646 1961 : if (it2 == s2.fields().end()) {
647 0 : return false;
648 0 : }
649 :
650 1961 : if (!equal(it1.second, it2->second)) {
651 17 : return false;
652 17 : }
653 1961 : }
654 864 : return true;
655 881 : }
656 :
657 267 : case ProtobufWkt::Value::kListValue: {
658 267 : const ProtobufWkt::ListValue& l1 = v1.list_value();
659 267 : const ProtobufWkt::ListValue& l2 = v2.list_value();
660 267 : if (l1.values_size() != l2.values_size()) {
661 0 : return false;
662 0 : }
663 672 : for (int i = 0; i < l1.values_size(); i++) {
664 414 : if (!equal(l1.values(i), l2.values(i))) {
665 9 : return false;
666 9 : }
667 414 : }
668 258 : return true;
669 267 : }
670 2667 : }
671 0 : return false;
672 2667 : }
673 :
674 36 : const ProtobufWkt::Value& ValueUtil::nullValue() {
675 36 : static const auto* v = []() -> ProtobufWkt::Value* {
676 2 : auto* vv = new ProtobufWkt::Value();
677 2 : vv->set_null_value(ProtobufWkt::NULL_VALUE);
678 2 : return vv;
679 2 : }();
680 36 : return *v;
681 36 : }
682 :
683 1696 : ProtobufWkt::Value ValueUtil::stringValue(const std::string& str) {
684 1696 : ProtobufWkt::Value val;
685 1696 : val.set_string_value(str);
686 1696 : return val;
687 1696 : }
688 :
689 0 : ProtobufWkt::Value ValueUtil::optionalStringValue(const absl::optional<std::string>& str) {
690 0 : if (str.has_value()) {
691 0 : return ValueUtil::stringValue(str.value());
692 0 : }
693 0 : return ValueUtil::nullValue();
694 0 : }
695 :
696 126 : ProtobufWkt::Value ValueUtil::boolValue(bool b) {
697 126 : ProtobufWkt::Value val;
698 126 : val.set_bool_value(b);
699 126 : return val;
700 126 : }
701 :
702 1 : ProtobufWkt::Value ValueUtil::structValue(const ProtobufWkt::Struct& obj) {
703 1 : ProtobufWkt::Value val;
704 1 : (*val.mutable_struct_value()) = obj;
705 1 : return val;
706 1 : }
707 :
708 0 : ProtobufWkt::Value ValueUtil::listValue(const std::vector<ProtobufWkt::Value>& values) {
709 0 : auto list = std::make_unique<ProtobufWkt::ListValue>();
710 0 : for (const auto& value : values) {
711 0 : *list->add_values() = value;
712 0 : }
713 0 : ProtobufWkt::Value val;
714 0 : val.set_allocated_list_value(list.release());
715 0 : return val;
716 0 : }
717 :
718 : namespace {
719 :
720 : absl::Status validateDurationNoThrow(const ProtobufWkt::Duration& duration,
721 948 : int64_t max_seconds_value) {
722 948 : if (duration.seconds() < 0 || duration.nanos() < 0) {
723 2 : return absl::OutOfRangeError(
724 2 : fmt::format("Expected positive duration: {}", duration.DebugString()));
725 2 : }
726 946 : if (duration.nanos() > 999999999 || duration.seconds() > max_seconds_value) {
727 4 : return absl::OutOfRangeError(fmt::format("Duration out-of-range: {}", duration.DebugString()));
728 4 : }
729 942 : return absl::OkStatus();
730 946 : }
731 :
732 917 : void validateDuration(const ProtobufWkt::Duration& duration, int64_t max_seconds_value) {
733 917 : const auto result = validateDurationNoThrow(duration, max_seconds_value);
734 917 : if (!result.ok()) {
735 6 : throwEnvoyExceptionOrPanic(std::string(result.message()));
736 6 : }
737 917 : }
738 :
739 0 : void validateDuration(const ProtobufWkt::Duration& duration) {
740 0 : validateDuration(duration, Protobuf::util::TimeUtil::kDurationMaxSeconds);
741 0 : }
742 :
743 917 : void validateDurationAsMilliseconds(const ProtobufWkt::Duration& duration) {
744 : // Apply stricter max boundary to the `seconds` value to avoid overflow.
745 : // Note that protobuf internally converts to nanoseconds.
746 : // The kMaxInt64Nanoseconds = 9223372036, which is about 300 years.
747 917 : constexpr int64_t kMaxInt64Nanoseconds =
748 917 : std::numeric_limits<int64_t>::max() / (1000 * 1000 * 1000);
749 917 : validateDuration(duration, kMaxInt64Nanoseconds);
750 917 : }
751 :
752 31 : absl::Status validateDurationAsMillisecondsNoThrow(const ProtobufWkt::Duration& duration) {
753 31 : constexpr int64_t kMaxInt64Nanoseconds =
754 31 : std::numeric_limits<int64_t>::max() / (1000 * 1000 * 1000);
755 31 : return validateDurationNoThrow(duration, kMaxInt64Nanoseconds);
756 31 : }
757 :
758 : } // namespace
759 :
760 917 : uint64_t DurationUtil::durationToMilliseconds(const ProtobufWkt::Duration& duration) {
761 917 : validateDurationAsMilliseconds(duration);
762 917 : return Protobuf::util::TimeUtil::DurationToMilliseconds(duration);
763 917 : }
764 :
765 : absl::StatusOr<uint64_t>
766 31 : DurationUtil::durationToMillisecondsNoThrow(const ProtobufWkt::Duration& duration) {
767 31 : const auto result = validateDurationAsMillisecondsNoThrow(duration);
768 31 : if (!result.ok()) {
769 0 : return result;
770 0 : }
771 31 : return Protobuf::util::TimeUtil::DurationToMilliseconds(duration);
772 31 : }
773 :
774 0 : uint64_t DurationUtil::durationToSeconds(const ProtobufWkt::Duration& duration) {
775 0 : validateDuration(duration);
776 0 : return Protobuf::util::TimeUtil::DurationToSeconds(duration);
777 0 : }
778 :
779 : void TimestampUtil::systemClockToTimestamp(const SystemTime system_clock_time,
780 469 : ProtobufWkt::Timestamp& timestamp) {
781 : // Converts to millisecond-precision Timestamp by explicitly casting to millisecond-precision
782 : // time_point.
783 469 : timestamp.MergeFrom(Protobuf::util::TimeUtil::MillisecondsToTimestamp(
784 469 : std::chrono::time_point_cast<std::chrono::milliseconds>(system_clock_time)
785 469 : .time_since_epoch()
786 469 : .count()));
787 469 : }
788 :
789 4432 : absl::string_view TypeUtil::typeUrlToDescriptorFullName(absl::string_view type_url) {
790 4432 : const size_t pos = type_url.rfind('/');
791 4432 : if (pos != absl::string_view::npos) {
792 4411 : type_url = type_url.substr(pos + 1);
793 4411 : }
794 4432 : return type_url;
795 4432 : }
796 :
797 0 : std::string TypeUtil::descriptorFullNameToTypeUrl(absl::string_view type) {
798 0 : return "type.googleapis.com/" + std::string(type);
799 0 : }
800 :
801 0 : void StructUtil::update(ProtobufWkt::Struct& obj, const ProtobufWkt::Struct& with) {
802 0 : auto& obj_fields = *obj.mutable_fields();
803 :
804 0 : for (const auto& [key, val] : with.fields()) {
805 0 : auto& obj_key = obj_fields[key];
806 :
807 : // If the types are different, the last one wins.
808 0 : const auto val_kind = val.kind_case();
809 0 : if (val_kind != obj_key.kind_case()) {
810 0 : obj_key = val;
811 0 : continue;
812 0 : }
813 :
814 : // Otherwise, the strategy depends on the value kind.
815 0 : switch (val.kind_case()) {
816 : // For scalars, the last one wins.
817 0 : case ProtobufWkt::Value::kNullValue:
818 0 : case ProtobufWkt::Value::kNumberValue:
819 0 : case ProtobufWkt::Value::kStringValue:
820 0 : case ProtobufWkt::Value::kBoolValue:
821 0 : obj_key = val;
822 0 : break;
823 : // If we got a structure, recursively update.
824 0 : case ProtobufWkt::Value::kStructValue:
825 0 : update(*obj_key.mutable_struct_value(), val.struct_value());
826 0 : break;
827 : // For lists, append the new values.
828 0 : case ProtobufWkt::Value::kListValue: {
829 0 : auto& obj_key_vec = *obj_key.mutable_list_value()->mutable_values();
830 0 : const auto& vals = val.list_value().values();
831 0 : obj_key_vec.MergeFrom(vals);
832 0 : break;
833 0 : }
834 0 : case ProtobufWkt::Value::KIND_NOT_SET:
835 0 : break;
836 0 : }
837 0 : }
838 0 : }
839 :
840 : } // namespace Envoy
|
