Line data Source code
1 : #include <limits>
2 : #include <numeric>
3 :
4 : #include "envoy/annotations/deprecation.pb.h"
5 : #include "envoy/protobuf/message_validator.h"
6 : #include "envoy/type/v3/percent.pb.h"
7 :
8 : #include "source/common/common/assert.h"
9 : #include "source/common/common/documentation_url.h"
10 : #include "source/common/common/fmt.h"
11 : #include "source/common/protobuf/message_validator_impl.h"
12 : #include "source/common/protobuf/protobuf.h"
13 : #include "source/common/protobuf/utility.h"
14 : #include "source/common/protobuf/visitor.h"
15 : #include "source/common/runtime/runtime_features.h"
16 :
17 : #include "absl/strings/match.h"
18 : #include "udpa/annotations/sensitive.pb.h"
19 : #include "udpa/annotations/status.pb.h"
20 : #include "utf8_validity.h"
21 : #include "validate/validate.h"
22 : #include "xds/annotations/v3/status.pb.h"
23 : #include "yaml-cpp/yaml.h"
24 :
25 : using namespace std::chrono_literals;
26 :
27 : namespace Envoy {
28 : namespace {
29 :
30 2122 : void blockFormat(YAML::Node node) {
31 2122 : node.SetStyle(YAML::EmitterStyle::Block);
32 :
33 2122 : if (node.Type() == YAML::NodeType::Sequence) {
34 338 : for (const auto& it : node) {
35 174 : blockFormat(it);
36 174 : }
37 338 : }
38 2122 : if (node.Type() == YAML::NodeType::Map) {
39 1244 : for (const auto& it : node) {
40 1244 : blockFormat(it.second);
41 1244 : }
42 918 : }
43 2122 : }
44 :
45 12484 : ProtobufWkt::Value parseYamlNode(const YAML::Node& node) {
46 12484 : ProtobufWkt::Value value;
47 12484 : switch (node.Type()) {
48 34 : case YAML::NodeType::Null:
49 34 : value.set_null_value(ProtobufWkt::NULL_VALUE);
50 34 : break;
51 5558 : case YAML::NodeType::Scalar: {
52 5558 : if (node.Tag() == "!") {
53 1106 : value.set_string_value(node.as<std::string>());
54 1106 : break;
55 1106 : }
56 4452 : bool bool_value;
57 4452 : if (YAML::convert<bool>::decode(node, bool_value)) {
58 726 : value.set_bool_value(bool_value);
59 726 : break;
60 726 : }
61 3726 : int64_t int_value;
62 3726 : if (YAML::convert<int64_t>::decode(node, int_value)) {
63 614 : if (std::numeric_limits<int32_t>::min() <= int_value &&
64 614 : std::numeric_limits<int32_t>::max() >= int_value) {
65 : // We could convert all integer values to string but it will break some stuff relying on
66 : // ProtobufWkt::Struct itself, only convert small numbers into number_value here.
67 610 : value.set_number_value(int_value);
68 610 : } else {
69 : // Proto3 JSON mapping allows use string for integer, this still has to be converted from
70 : // int_value to support hexadecimal and octal literals.
71 4 : value.set_string_value(std::to_string(int_value));
72 4 : }
73 614 : break;
74 614 : }
75 : // Fall back on string, including float/double case. When protobuf parse the JSON into a message
76 : // it will convert based on the type in the message definition.
77 3112 : value.set_string_value(node.as<std::string>());
78 3112 : break;
79 3726 : }
80 1228 : case YAML::NodeType::Sequence: {
81 1228 : auto& list_values = *value.mutable_list_value()->mutable_values();
82 1228 : for (const auto& it : node) {
83 1064 : *list_values.Add() = parseYamlNode(it);
84 1064 : }
85 1228 : break;
86 3726 : }
87 5664 : case YAML::NodeType::Map: {
88 5664 : auto& struct_fields = *value.mutable_struct_value()->mutable_fields();
89 9793 : for (const auto& it : node) {
90 9793 : if (it.first.Tag() != "!ignore") {
91 9793 : struct_fields[it.first.as<std::string>()] = parseYamlNode(it.second);
92 9793 : }
93 9793 : }
94 5664 : break;
95 3726 : }
96 0 : case YAML::NodeType::Undefined:
97 0 : throw EnvoyException("Undefined YAML value");
98 12484 : }
99 12484 : return value;
100 12484 : }
101 :
102 : void jsonConvertInternal(const Protobuf::Message& source,
103 : ProtobufMessage::ValidationVisitor& validation_visitor,
104 961 : Protobuf::Message& dest) {
105 961 : Protobuf::util::JsonPrintOptions json_options;
106 961 : json_options.preserve_proto_field_names = true;
107 961 : std::string json;
108 961 : const auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);
109 961 : if (!status.ok()) {
110 0 : throw EnvoyException(fmt::format("Unable to convert protobuf message to JSON string: {} {}",
111 0 : status.ToString(), source.DebugString()));
112 0 : }
113 961 : MessageUtil::loadFromJson(json, dest, validation_visitor);
114 961 : }
115 :
116 : } // namespace
117 :
118 : void MessageUtil::loadFromJson(const std::string& json, Protobuf::Message& message,
119 2089 : ProtobufMessage::ValidationVisitor& validation_visitor) {
120 2089 : bool has_unknown_field;
121 2089 : auto status = loadFromJsonNoThrow(json, message, has_unknown_field);
122 2089 : if (status.ok()) {
123 1845 : return;
124 1845 : }
125 244 : if (has_unknown_field) {
126 : // If the parsing failure is caused by the unknown fields.
127 0 : validation_visitor.onUnknownField("type " + message.GetTypeName() + " reason " +
128 0 : status.ToString());
129 244 : } else {
130 : // If the error has nothing to do with unknown field.
131 244 : throw EnvoyException("Unable to parse JSON as proto (" + status.ToString() + "): " + json);
132 244 : }
133 244 : }
134 :
135 : absl::Status MessageUtil::loadFromJsonNoThrow(const std::string& json, Protobuf::Message& message,
136 2089 : bool& has_unknown_fileld) {
137 2089 : has_unknown_fileld = false;
138 2089 : Protobuf::util::JsonParseOptions options;
139 2089 : options.case_insensitive_enum_parsing = true;
140 : // Let's first try and get a clean parse when checking for unknown fields;
141 : // this should be the common case.
142 2089 : options.ignore_unknown_fields = false;
143 : // Clear existing values (if any) from the destination message before serialization.
144 2089 : message.Clear();
145 2089 : const auto strict_status = Protobuf::util::JsonStringToMessage(json, &message, options);
146 2089 : if (strict_status.ok()) {
147 : // Success, no need to do any extra work.
148 1845 : return strict_status;
149 1845 : }
150 : // If we fail, we see if we get a clean parse when allowing unknown fields.
151 : // This is essentially a workaround
152 : // for https://github.com/protocolbuffers/protobuf/issues/5967.
153 : // TODO(htuch): clean this up when protobuf supports JSON/YAML unknown field
154 : // detection directly.
155 244 : options.ignore_unknown_fields = true;
156 244 : const auto relaxed_status = Protobuf::util::JsonStringToMessage(json, &message, options);
157 : // If we still fail with relaxed unknown field checking, the error has nothing
158 : // to do with unknown fields.
159 244 : if (relaxed_status.ok()) {
160 0 : has_unknown_fileld = true;
161 0 : return strict_status;
162 0 : }
163 244 : return relaxed_status;
164 244 : }
165 :
166 1058 : void MessageUtil::loadFromJson(const std::string& json, ProtobufWkt::Struct& message) {
167 : // No need to validate if converting to a Struct, since there are no unknown
168 : // fields possible.
169 1058 : loadFromJson(json, message, ProtobufMessage::getNullValidationVisitor());
170 1058 : }
171 :
172 : void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& message,
173 961 : ProtobufMessage::ValidationVisitor& validation_visitor) {
174 961 : ProtobufWkt::Value value = ValueUtil::loadFromYaml(yaml);
175 961 : if (value.kind_case() == ProtobufWkt::Value::kStructValue ||
176 961 : value.kind_case() == ProtobufWkt::Value::kListValue) {
177 961 : jsonConvertInternal(value, validation_visitor, message);
178 961 : return;
179 961 : }
180 0 : throw EnvoyException("Unable to convert YAML as JSON: " + yaml);
181 961 : }
182 :
183 : void MessageUtil::loadFromFile(const std::string& path, Protobuf::Message& message,
184 : ProtobufMessage::ValidationVisitor& validation_visitor,
185 525 : Api::Api& api) {
186 525 : auto file_or_error = api.fileSystem().fileReadToEnd(path);
187 525 : THROW_IF_STATUS_NOT_OK(file_or_error, throw);
188 525 : const std::string contents = file_or_error.value();
189 : // If the filename ends with .pb, attempt to parse it as a binary proto.
190 525 : if (absl::EndsWithIgnoreCase(path, FileExtensions::get().ProtoBinary)) {
191 : // Attempt to parse the binary format.
192 98 : if (message.ParseFromString(contents)) {
193 98 : MessageUtil::checkForUnexpectedFields(message, validation_visitor);
194 98 : }
195 98 : return;
196 98 : }
197 :
198 : // If the filename ends with .pb_text, attempt to parse it as a text proto.
199 427 : if (absl::EndsWithIgnoreCase(path, FileExtensions::get().ProtoText)) {
200 357 : if (Protobuf::TextFormat::ParseFromString(contents, &message)) {
201 355 : return;
202 355 : }
203 2 : throw EnvoyException("Unable to parse file \"" + path + "\" as a text protobuf (type " +
204 2 : message.GetTypeName() + ")");
205 357 : }
206 70 : if (absl::EndsWithIgnoreCase(path, FileExtensions::get().Yaml) ||
207 70 : absl::EndsWithIgnoreCase(path, FileExtensions::get().Yml)) {
208 0 : loadFromYaml(contents, message, validation_visitor);
209 70 : } else {
210 70 : loadFromJson(contents, message, validation_visitor);
211 70 : }
212 70 : }
213 :
214 : std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& message,
215 : const bool block_print,
216 759 : const bool always_print_primitive_fields) {
217 :
218 759 : auto json_or_error = getJsonStringFromMessage(message, false, always_print_primitive_fields);
219 759 : if (!json_or_error.ok()) {
220 0 : throw EnvoyException(json_or_error.status().ToString());
221 0 : }
222 759 : YAML::Node node;
223 759 : TRY_ASSERT_MAIN_THREAD { node = YAML::Load(json_or_error.value()); }
224 759 : END_TRY
225 759 : catch (YAML::ParserException& e) {
226 55 : throw EnvoyException(e.what());
227 55 : }
228 759 : catch (YAML::BadConversion& e) {
229 0 : throw EnvoyException(e.what());
230 0 : }
231 759 : catch (std::exception& e) {
232 : // Catch unknown YAML parsing exceptions.
233 0 : throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));
234 0 : }
235 704 : if (block_print) {
236 704 : blockFormat(node);
237 704 : }
238 704 : YAML::Emitter out;
239 704 : out << node;
240 704 : return out.c_str();
241 759 : }
242 :
243 : absl::StatusOr<std::string>
244 : MessageUtil::getJsonStringFromMessage(const Protobuf::Message& message, const bool pretty_print,
245 3037 : const bool always_print_primitive_fields) {
246 3037 : Protobuf::util::JsonPrintOptions json_options;
247 : // By default, proto field names are converted to camelCase when the message is converted to JSON.
248 : // Setting this option makes debugging easier because it keeps field names consistent in JSON
249 : // printouts.
250 3037 : json_options.preserve_proto_field_names = true;
251 3037 : if (pretty_print) {
252 98 : json_options.add_whitespace = true;
253 98 : }
254 : // Primitive types such as int32s and enums will not be serialized if they have the default value.
255 : // This flag disables that behavior.
256 3037 : if (always_print_primitive_fields) {
257 1698 : json_options.always_print_primitive_fields = true;
258 1698 : }
259 3037 : std::string json;
260 3037 : if (auto status = Protobuf::util::MessageToJsonString(message, &json, json_options);
261 3037 : !status.ok()) {
262 0 : return status;
263 0 : }
264 3037 : return json;
265 3037 : }
266 :
267 : std::string MessageUtil::getJsonStringFromMessageOrError(const Protobuf::Message& message,
268 : bool pretty_print,
269 2277 : bool always_print_primitive_fields) {
270 2277 : auto json_or_error =
271 2277 : getJsonStringFromMessage(message, pretty_print, always_print_primitive_fields);
272 2277 : return json_or_error.ok() ? std::move(json_or_error).value()
273 2277 : : fmt::format("Failed to convert protobuf message to JSON string: {}",
274 0 : json_or_error.status().ToString());
275 2277 : }
276 :
277 0 : void MessageUtil::jsonConvert(const Protobuf::Message& source, Protobuf::Message& dest) {
278 0 : jsonConvertInternal(source, ProtobufMessage::getNullValidationVisitor(), dest);
279 0 : }
280 :
281 0 : void MessageUtil::jsonConvert(const Protobuf::Message& source, ProtobufWkt::Struct& dest) {
282 : // Any proto3 message can be transformed to Struct, so there is no need to check for unknown
283 : // fields. There is one catch; Duration/Timestamp etc. which have non-object canonical JSON
284 : // representations don't work.
285 0 : jsonConvertInternal(source, ProtobufMessage::getNullValidationVisitor(), dest);
286 0 : }
287 :
288 : void MessageUtil::jsonConvert(const ProtobufWkt::Struct& source,
289 : ProtobufMessage::ValidationVisitor& validation_visitor,
290 0 : Protobuf::Message& dest) {
291 0 : jsonConvertInternal(source, validation_visitor, dest);
292 0 : }
293 :
294 0 : bool MessageUtil::jsonConvertValue(const Protobuf::Message& source, ProtobufWkt::Value& dest) {
295 0 : Protobuf::util::JsonPrintOptions json_options;
296 0 : json_options.preserve_proto_field_names = true;
297 0 : std::string json;
298 0 : auto status = Protobuf::util::MessageToJsonString(source, &json, json_options);
299 0 : if (!status.ok()) {
300 0 : return false;
301 0 : }
302 0 : bool has_unknow_field;
303 0 : status = MessageUtil::loadFromJsonNoThrow(json, dest, has_unknow_field);
304 0 : if (status.ok() || has_unknow_field) {
305 0 : return true;
306 0 : }
307 0 : return false;
308 0 : }
309 :
310 1627 : ProtobufWkt::Value ValueUtil::loadFromYaml(const std::string& yaml) {
311 1627 : TRY_ASSERT_MAIN_THREAD { return parseYamlNode(YAML::Load(yaml)); }
312 1627 : END_TRY
313 1627 : catch (YAML::ParserException& e) {
314 0 : throw EnvoyException(e.what());
315 0 : }
316 1627 : catch (YAML::BadConversion& e) {
317 0 : throw EnvoyException(e.what());
318 0 : }
319 1627 : catch (std::exception& e) {
320 : // There is a potentially wide space of exceptions thrown by the YAML parser,
321 : // and enumerating them all may be difficult. Envoy doesn't work well with
322 : // unhandled exceptions, so we capture them and record the exception name in
323 : // the Envoy Exception text.
324 0 : throw EnvoyException(fmt::format("Unexpected YAML exception: {}", +e.what()));
325 0 : }
326 1627 : }
327 :
328 : namespace {
329 :
330 : // This is a modified copy of the UTF8CoerceToStructurallyValid from the protobuf code.
331 : // A copy was needed after if was removed from the protobuf.
332 210 : std::string utf8CoerceToStructurallyValid(absl::string_view str, const char replace_char) {
333 210 : std::string result(str);
334 210 : auto replace_pos = result.begin();
335 210 : while (!str.empty()) {
336 210 : size_t n_valid_bytes = utf8_range::SpanStructurallyValid(str);
337 210 : if (n_valid_bytes == str.size()) {
338 210 : break;
339 210 : }
340 0 : replace_pos += n_valid_bytes;
341 0 : *replace_pos++ = replace_char; // replace one bad byte
342 0 : str.remove_prefix(n_valid_bytes + 1);
343 0 : }
344 210 : return result;
345 210 : }
346 :
347 : } // namespace
348 :
349 210 : std::string MessageUtil::sanitizeUtf8String(absl::string_view input) {
350 : // The choice of '!' is somewhat arbitrary, but we wanted to avoid any character that has
351 : // special semantic meaning in URLs or similar.
352 210 : return utf8CoerceToStructurallyValid(input, '!');
353 210 : }
354 :
355 : } // namespace Envoy
|
