Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc. All rights reserved.
3#
4# Use of this source code is governed by a BSD-style
5# license that can be found in the LICENSE file or at
6# https://developers.google.com/open-source/licenses/bsd
7"""Contains routines for printing protocol messages in JSON format.
9Simple usage example:
11 # Create a proto object and serialize it to a json format string.
12 message = my_proto_pb2.MyMessage(foo='bar')
13 json_string = json_format.MessageToJson(message)
15 # Parse a json format string to proto object.
16 message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
17"""
19__author__ = 'jieluo@google.com (Jie Luo)'
21import base64
22from collections import OrderedDict
23import json
24import math
25from operator import methodcaller
26import re
28from google.protobuf import descriptor
29from google.protobuf import descriptor_pool
30from google.protobuf import message_factory
31from google.protobuf import symbol_database
32from google.protobuf.internal import type_checkers
34_INT_TYPES = frozenset([
35 descriptor.FieldDescriptor.CPPTYPE_INT32,
36 descriptor.FieldDescriptor.CPPTYPE_UINT32,
37 descriptor.FieldDescriptor.CPPTYPE_INT64,
38 descriptor.FieldDescriptor.CPPTYPE_UINT64,
39])
40_INT64_TYPES = frozenset([
41 descriptor.FieldDescriptor.CPPTYPE_INT64,
42 descriptor.FieldDescriptor.CPPTYPE_UINT64,
43])
44_FLOAT_TYPES = frozenset([
45 descriptor.FieldDescriptor.CPPTYPE_FLOAT,
46 descriptor.FieldDescriptor.CPPTYPE_DOUBLE,
47])
48_INFINITY = 'Infinity'
49_NEG_INFINITY = '-Infinity'
50_NAN = 'NaN'
52_UNPAIRED_SURROGATE_PATTERN = re.compile(
53 '[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
54)
56_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
59class Error(Exception):
60 """Top-level module error for json_format."""
63class SerializeToJsonError(Error):
64 """Thrown if serialization to JSON fails."""
67class ParseError(Error):
68 """Thrown in case of parsing error."""
71class EnumStringValueParseError(ParseError):
72 """Thrown if unknown string enum value is encountered.
74 This exception is suppressed if ignore_unknown_fields is set.
75 """
78def MessageToJson(
79 message,
80 preserving_proto_field_name=False,
81 indent=2,
82 sort_keys=False,
83 use_integers_for_enums=False,
84 descriptor_pool=None,
85 ensure_ascii=True,
86 always_print_fields_with_no_presence=False,
87 *,
88 unquote_int64_if_possible=False,
89):
90 """Converts protobuf message to JSON format.
92 Args:
93 message: The protocol buffers message instance to serialize.
94 always_print_fields_with_no_presence: If True, fields without presence
95 (implicit presence scalars, repeated fields, and map fields) will always
96 be serialized. Any field that supports presence is not affected by this
97 option (including singular message fields and oneof fields).
98 preserving_proto_field_name: If True, use the original proto field names as
99 defined in the .proto file. If False, convert the field names to
100 lowerCamelCase.
101 indent: The JSON object will be pretty-printed with this indent level. An
102 indent level of 0 or negative will only insert newlines. If the indent
103 level is None, no newlines will be inserted.
104 sort_keys: If True, then the output will be sorted by field names.
105 use_integers_for_enums: If true, print integers instead of enum names.
106 descriptor_pool: A Descriptor Pool for resolving types. If None use the
107 default.
108 ensure_ascii: If True, strings with non-ASCII characters are escaped. If
109 False, Unicode strings are returned unchanged.
110 unquote_int64_if_possible: If True, unquote int64 fields for values that are
111 safe to emit as numbers (all values smaller than 2^53 and a sparse set of
112 values that are larger).
114 Returns:
115 A string containing the JSON formatted protocol buffer message.
116 """
117 printer = _Printer(
118 preserving_proto_field_name,
119 use_integers_for_enums,
120 descriptor_pool,
121 always_print_fields_with_no_presence,
122 unquote_int64_if_possible=unquote_int64_if_possible,
123 )
124 return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
127def MessageToDict(
128 message,
129 always_print_fields_with_no_presence=False,
130 preserving_proto_field_name=False,
131 use_integers_for_enums=False,
132 descriptor_pool=None,
133 *,
134 unquote_int64_if_possible=False,
135):
136 """Converts protobuf message to a dictionary.
138 When the dictionary is encoded to JSON, it conforms to ProtoJSON spec.
140 Args:
141 message: The protocol buffers message instance to serialize.
142 always_print_fields_with_no_presence: If True, fields without presence
143 (implicit presence scalars, repeated fields, and map fields) will always
144 be serialized. Any field that supports presence is not affected by this
145 option (including singular message fields and oneof fields).
146 preserving_proto_field_name: If True, use the original proto field names as
147 defined in the .proto file. If False, convert the field names to
148 lowerCamelCase.
149 use_integers_for_enums: If true, print integers instead of enum names.
150 descriptor_pool: A Descriptor Pool for resolving types. If None use the
151 default.
152 unquote_int64_if_possible: If True, unquote int64 fields for values that are
153 safe to emit as numbers (all values smaller than 2^53 and a sparse set of
154 values that are larger).
156 Returns:
157 A dict representation of the protocol buffer message.
158 """
159 printer = _Printer(
160 preserving_proto_field_name,
161 use_integers_for_enums,
162 descriptor_pool,
163 always_print_fields_with_no_presence,
164 unquote_int64_if_possible=unquote_int64_if_possible,
165 )
166 # pylint: disable=protected-access
167 return printer._MessageToJsonObject(message)
170def _IsMapEntry(field):
171 return (
172 field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
173 and field.message_type.has_options
174 and field.message_type.GetOptions().map_entry
175 )
178class _Printer(object):
179 """JSON format printer for protocol message."""
181 def __init__(
182 self,
183 preserving_proto_field_name=False,
184 use_integers_for_enums=False,
185 descriptor_pool=None,
186 always_print_fields_with_no_presence=False,
187 *,
188 unquote_int64_if_possible=False,
189 ):
190 self.always_print_fields_with_no_presence = (
191 always_print_fields_with_no_presence
192 )
193 self.preserving_proto_field_name = preserving_proto_field_name
194 self.use_integers_for_enums = use_integers_for_enums
195 self.descriptor_pool = descriptor_pool
196 self.unquote_int64_if_possible = unquote_int64_if_possible
198 def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
199 js = self._MessageToJsonObject(message)
200 return json.dumps(
201 js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii
202 )
204 def _MessageToJsonObject(self, message):
205 """Converts message to an object according to ProtoJSON Specification."""
206 message_descriptor = message.DESCRIPTOR
207 full_name = message_descriptor.full_name
208 if _IsWrapperMessage(message_descriptor):
209 return self._WrapperMessageToJsonObject(message)
210 if full_name in _WKTJSONMETHODS:
211 return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self)
212 js = {}
213 return self._RegularMessageToJsonObject(message, js)
215 def _RegularMessageToJsonObject(self, message, js):
216 """Converts normal message according to ProtoJSON Specification."""
217 fields = message.ListFields()
219 try:
220 for field, value in fields:
221 if field.is_extension:
222 name = '[%s]' % field.full_name
223 elif self.preserving_proto_field_name:
224 name = field.name
225 else:
226 name = field.json_name
228 if _IsMapEntry(field):
229 # Convert a map field.
230 v_field = field.message_type.fields_by_name['value']
231 js_map = {}
232 for key in value:
233 if isinstance(key, bool):
234 if key:
235 recorded_key = 'true'
236 else:
237 recorded_key = 'false'
238 else:
239 recorded_key = str(key)
240 js_map[recorded_key] = self._FieldToJsonObject(v_field, value[key])
241 js[name] = js_map
242 elif field.is_repeated:
243 # Convert a repeated field.
244 js[name] = [self._FieldToJsonObject(field, k) for k in value]
245 else:
246 js[name] = self._FieldToJsonObject(field, value)
248 # Serialize default value if including_default_value_fields is True.
249 if (
250 self.always_print_fields_with_no_presence
251 ):
252 message_descriptor = message.DESCRIPTOR
253 for field in message_descriptor.fields:
255 # always_print_fields_with_no_presence doesn't apply to
256 # any field which supports presence.
257 if self.always_print_fields_with_no_presence and field.has_presence:
258 continue
260 if self.preserving_proto_field_name:
261 name = field.name
262 else:
263 name = field.json_name
264 if name in js:
265 # Skip the field which has been serialized already.
266 continue
267 if _IsMapEntry(field):
268 js[name] = {}
269 elif field.is_repeated:
270 js[name] = []
271 else:
272 js[name] = self._FieldToJsonObject(field, field.default_value)
274 except ValueError as e:
275 raise SerializeToJsonError(
276 'Failed to serialize {0} field: {1}.'.format(field.name, e)
277 ) from e
279 return js
281 def _FieldToJsonObject(self, field, value):
282 """Converts field value according to ProtoJSON Specification."""
283 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
284 return self._MessageToJsonObject(value)
285 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
286 if self.use_integers_for_enums:
287 return value
288 if field.enum_type.full_name == 'google.protobuf.NullValue':
289 return None
290 enum_value = field.enum_type.values_by_number.get(value, None)
291 if enum_value is not None:
292 option = _GetJsonEnumValueOption(enum_value)
293 if option is not None:
294 return option.string
295 return enum_value.name
296 else:
297 if field.enum_type.is_closed:
298 raise SerializeToJsonError(
299 'Enum field contains an integer value '
300 'which can not mapped to an enum value.'
301 )
302 else:
303 return value
304 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
305 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
306 # Use base64 Data encoding for bytes
307 return base64.b64encode(value).decode('utf-8')
308 else:
309 return str(value)
310 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
311 return bool(value)
312 elif field.cpp_type in _INT64_TYPES:
313 if self.unquote_int64_if_possible and float(value) == value:
314 return value
315 else:
316 return str(value)
317 elif field.cpp_type in _FLOAT_TYPES:
318 if math.isinf(value):
319 if value < 0.0:
320 return _NEG_INFINITY
321 else:
322 return _INFINITY
323 if math.isnan(value):
324 return _NAN
325 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
326 return type_checkers.ToShortestFloat(value)
328 return value
330 def _AnyMessageToJsonObject(self, message):
331 """Converts Any message according to ProtoJSON Specification."""
332 if not message.ListFields():
333 return {}
334 # Must print @type first, use OrderedDict instead of {}
335 js = OrderedDict()
336 type_url = message.type_url
337 js['@type'] = type_url
338 sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
339 sub_message.ParseFromString(message.value)
340 message_descriptor = sub_message.DESCRIPTOR
341 full_name = message_descriptor.full_name
342 if _IsWrapperMessage(message_descriptor):
343 js['value'] = self._WrapperMessageToJsonObject(sub_message)
344 return js
345 if full_name in _WKTJSONMETHODS:
346 js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], sub_message)(
347 self
348 )
349 return js
350 return self._RegularMessageToJsonObject(sub_message, js)
352 def _GenericMessageToJsonObject(self, message):
353 """Converts message according to ProtoJSON Specification."""
354 # Duration, Timestamp and FieldMask have ToJsonString method to do the
355 # convert. Users can also call the method directly.
356 return message.ToJsonString()
358 def _ValueMessageToJsonObject(self, message):
359 """Converts Value message according to ProtoJSON Specification."""
360 which = message.WhichOneof('kind')
361 # If the Value message is not set treat as null_value when serialize
362 # to JSON. The parse back result will be different from original message.
363 if which is None or which == 'null_value':
364 return None
365 if which == 'list_value':
366 return self._ListValueMessageToJsonObject(message.list_value)
367 if which == 'number_value':
368 value = message.number_value
369 if math.isinf(value):
370 raise ValueError(
371 'Fail to serialize Infinity for Value.number_value, '
372 'which would parse as string_value'
373 )
374 if math.isnan(value):
375 raise ValueError(
376 'Fail to serialize NaN for Value.number_value, '
377 'which would parse as string_value'
378 )
379 else:
380 value = getattr(message, which)
381 oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
382 return self._FieldToJsonObject(oneof_descriptor, value)
384 def _ListValueMessageToJsonObject(self, message):
385 """Converts ListValue message according to ProtoJSON Specification."""
386 return [self._ValueMessageToJsonObject(value) for value in message.values]
388 def _StructMessageToJsonObject(self, message):
389 """Converts Struct message according to ProtoJSON Specification."""
390 fields = message.fields
391 ret = {}
392 for key in fields:
393 ret[key] = self._ValueMessageToJsonObject(fields[key])
394 return ret
396 def _WrapperMessageToJsonObject(self, message):
397 return self._FieldToJsonObject(
398 message.DESCRIPTOR.fields_by_name['value'], message.value
399 )
402def _IsWrapperMessage(message_descriptor):
403 return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
406def _DuplicateChecker(js):
407 result = {}
408 for name, value in js:
409 if name in result:
410 raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
411 result[name] = value
412 return result
415def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
416 """Creates a message from a type URL."""
417 db = symbol_database.Default()
418 pool = db.pool if descriptor_pool is None else descriptor_pool
419 type_name = type_url.split('/')[-1]
420 try:
421 message_descriptor = pool.FindMessageTypeByName(type_name)
422 except KeyError as e:
423 raise TypeError(
424 'Can not find message descriptor by type_url: {0}'.format(type_url)
425 ) from e
426 message_class = message_factory.GetMessageClass(message_descriptor)
427 return message_class()
430def Parse(
431 text,
432 message,
433 ignore_unknown_fields=False,
434 descriptor_pool=None,
435 max_recursion_depth=100,
436):
437 """Parses a JSON representation of a protocol message into a message.
439 Args:
440 text: Message JSON representation.
441 message: A protocol buffer message to merge into.
442 ignore_unknown_fields: If True, do not raise errors for unknown fields.
443 descriptor_pool: A Descriptor Pool for resolving types. If None use the
444 default.
445 max_recursion_depth: max recursion depth of JSON message to be deserialized.
446 JSON messages over this depth will fail to be deserialized. Default value
447 is 100.
449 Returns:
450 The same message passed as argument.
452 Raises::
453 ParseError: On JSON parsing problems.
454 """
455 if not isinstance(text, str):
456 text = text.decode('utf-8')
458 try:
459 js = json.loads(text, object_pairs_hook=_DuplicateChecker)
460 except Exception as e:
461 raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e
463 try:
464 return ParseDict(
465 js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth
466 )
467 except ParseError as e:
468 raise e
469 except Exception as e:
470 raise ParseError(
471 'Failed to parse JSON: {0}: {1}.'.format(type(e).__name__, str(e))
472 ) from e
475def ParseDict(
476 js_dict,
477 message,
478 ignore_unknown_fields=False,
479 descriptor_pool=None,
480 max_recursion_depth=100,
481):
482 """Parses a JSON dictionary representation into a message.
484 Args:
485 js_dict: Dict representation of a JSON message.
486 message: A protocol buffer message to merge into.
487 ignore_unknown_fields: If True, do not raise errors for unknown fields.
488 descriptor_pool: A Descriptor Pool for resolving types. If None use the
489 default.
490 max_recursion_depth: max recursion depth of JSON message to be deserialized.
491 JSON messages over this depth will fail to be deserialized. Default value
492 is 100.
494 Returns:
495 The same message passed as argument.
496 """
497 parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
498 parser.ConvertMessage(js_dict, message, '')
499 return message
502_INT_OR_FLOAT = (int, float)
503_LIST_LIKE = (list, tuple)
506class _Parser(object):
507 """JSON format parser for protocol message."""
509 def __init__(
510 self, ignore_unknown_fields, descriptor_pool, max_recursion_depth
511 ):
512 self.ignore_unknown_fields = ignore_unknown_fields
513 self.descriptor_pool = descriptor_pool
514 self.max_recursion_depth = max_recursion_depth
515 self.recursion_depth = 0
517 def ConvertMessage(self, value, message, path):
518 """Convert a JSON object into a message.
520 Args:
521 value: A JSON object.
522 message: A WKT or regular protocol message to record the data.
523 path: parent path to log parse error info.
525 Raises:
526 ParseError: In case of convert problems.
527 """
528 # Increment recursion depth at message entry. The max_recursion_depth limit
529 # is exclusive: a depth value equal to max_recursion_depth will trigger an
530 # error. For example, with max_recursion_depth=5, nesting up to depth 4 is
531 # allowed, but attempting depth 5 raises ParseError.
532 self.recursion_depth += 1
533 if self.recursion_depth > self.max_recursion_depth:
534 raise ParseError(
535 'Message too deep. Max recursion depth is {0}'.format(
536 self.max_recursion_depth
537 )
538 )
539 message_descriptor = message.DESCRIPTOR
540 full_name = message_descriptor.full_name
541 if not path:
542 path = message_descriptor.name
543 if _IsWrapperMessage(message_descriptor):
544 self._ConvertWrapperMessage(value, message, path)
545 elif full_name in _WKTJSONMETHODS:
546 methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)
547 else:
548 self._ConvertFieldValuePair(value, message, path)
549 self.recursion_depth -= 1
551 def _ConvertFieldValuePair(self, js, message, path):
552 """Convert field value pairs into regular message.
554 Args:
555 js: A JSON object to convert the field value pairs.
556 message: A regular protocol message to record the data.
557 path: parent path to log parse error info.
559 Raises:
560 ParseError: In case of problems converting.
561 """
562 names = []
563 message_descriptor = message.DESCRIPTOR
564 fields_by_json_name = dict(
565 (f.json_name, f) for f in message_descriptor.fields
566 )
568 def _ClearFieldOrExtension(message, field):
569 if field.is_extension:
570 message.ClearExtension(field)
571 else:
572 message.ClearField(field.name)
574 def _GetFieldOrExtension(message, field):
575 if field.is_extension:
576 return message.Extensions[field]
577 else:
578 return getattr(message, field.name)
580 def _SetFieldOrExtension(message, field, value):
581 if field.is_extension:
582 message.Extensions[field] = value
583 else:
584 setattr(message, field.name, value)
586 for name in js:
587 try:
588 field = fields_by_json_name.get(name, None)
589 if not field:
590 field = message_descriptor.fields_by_name.get(name, None)
591 if not field and _VALID_EXTENSION_NAME.match(name):
592 if not message_descriptor.is_extendable:
593 raise ParseError(
594 'Message type {0} does not have extensions at {1}'.format(
595 message_descriptor.full_name, path
596 )
597 )
598 identifier = name[1:-1] # strip [] brackets
599 # pylint: disable=protected-access
600 field = message.Extensions._FindExtensionByName(identifier)
601 # pylint: enable=protected-access
602 if not field:
603 # Try looking for extension by the message type name, dropping the
604 # field name following the final . separator in full_name.
605 identifier = '.'.join(identifier.split('.')[:-1])
606 # pylint: disable=protected-access
607 field = message.Extensions._FindExtensionByName(identifier)
608 # pylint: enable=protected-access
609 if not field:
610 if self.ignore_unknown_fields:
611 continue
612 raise ParseError(
613 (
614 'Message type "{0}" has no field named "{1}" at "{2}".\n'
615 ' Available Fields(except extensions): "{3}"'
616 ).format(
617 message_descriptor.full_name,
618 name,
619 path,
620 [f.json_name for f in message_descriptor.fields],
621 )
622 )
623 if name in names:
624 raise ParseError(
625 'Message type "{0}" should not have multiple '
626 '"{1}" fields at "{2}".'.format(
627 message.DESCRIPTOR.full_name, name, path
628 )
629 )
630 names.append(name)
631 value = js[name]
632 # Check no other oneof field is parsed.
633 if field.containing_oneof is not None and value is not None:
634 oneof_name = field.containing_oneof.name
635 if oneof_name in names:
636 raise ParseError(
637 'Message type "{0}" should not have multiple '
638 '"{1}" oneof fields at "{2}".'.format(
639 message.DESCRIPTOR.full_name, oneof_name, path
640 )
641 )
642 names.append(oneof_name)
644 if value is None:
645 if (
646 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
647 and field.message_type.full_name == 'google.protobuf.Value'
648 ):
649 sub_message = _GetFieldOrExtension(message, field)
650 sub_message.null_value = 0
651 elif (
652 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM
653 and field.enum_type.full_name == 'google.protobuf.NullValue'
654 ):
655 _SetFieldOrExtension(message, field, 0)
656 else:
657 _ClearFieldOrExtension(message, field)
658 continue
660 # Parse field value.
661 if _IsMapEntry(field):
662 _ClearFieldOrExtension(message, field)
663 self._ConvertMapFieldValue(
664 value, message, field, '{0}.{1}'.format(path, name)
665 )
666 elif field.is_repeated:
667 _ClearFieldOrExtension(message, field)
668 if not isinstance(value, _LIST_LIKE):
669 raise ParseError(
670 'repeated field {0} must be in [] which is {1} at {2}'.format(
671 name, value, path
672 )
673 )
674 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
675 # Repeated message field.
676 for index, item in enumerate(value):
677 sub_message = _GetFieldOrExtension(message, field).add()
678 # None is a null_value in Value.
679 if (
680 item is None
681 and sub_message.DESCRIPTOR.full_name
682 != 'google.protobuf.Value'
683 ):
684 raise ParseError(
685 'null is not allowed to be used as an element'
686 ' in a repeated field at {0}.{1}[{2}]'.format(
687 path, name, index
688 )
689 )
690 self.ConvertMessage(
691 item, sub_message, '{0}.{1}[{2}]'.format(path, name, index)
692 )
693 else:
694 # Repeated scalar field.
695 for index, item in enumerate(value):
696 if item is None:
697 raise ParseError(
698 'null is not allowed to be used as an element'
699 ' in a repeated field at {0}.{1}[{2}]'.format(
700 path, name, index
701 )
702 )
703 self._ConvertAndAppendScalar(
704 message, field, item, '{0}.{1}[{2}]'.format(path, name, index)
705 )
706 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
707 sub_message = _GetFieldOrExtension(message, field)
708 sub_message.SetInParent()
709 self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))
710 else:
711 self._ConvertAndSetScalar(
712 message, field, value, '{0}.{1}'.format(path, name)
713 )
714 except ParseError as e:
715 if field and field.containing_oneof is None:
716 raise ParseError(
717 'Failed to parse {0} field: {1}.'.format(name, e)
718 ) from e
719 else:
720 raise ParseError(str(e)) from e
721 except ValueError as e:
722 raise ParseError(
723 'Failed to parse {0} field: {1}.'.format(name, e)
724 ) from e
725 except TypeError as e:
726 raise ParseError(
727 'Failed to parse {0} field: {1}.'.format(name, e)
728 ) from e
730 def _ConvertAnyMessage(self, value, message, path):
731 """Convert a JSON representation into Any message."""
732 if isinstance(value, dict) and not value:
733 return
734 try:
735 type_url = value['@type']
736 except KeyError as e:
737 raise ParseError(
738 '@type is missing when parsing any message at {0}'.format(path)
739 ) from e
741 try:
742 sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
743 except TypeError as e:
744 raise ParseError('{0} at {1}'.format(e, path)) from e
745 message_descriptor = sub_message.DESCRIPTOR
746 full_name = message_descriptor.full_name
747 if _IsWrapperMessage(message_descriptor):
748 self._ConvertWrapperMessage(
749 value['value'], sub_message, '{0}.value'.format(path)
750 )
751 elif full_name in _WKTJSONMETHODS:
752 # For well-known types (including nested Any), use ConvertMessage
753 # to ensure recursion depth is properly tracked
754 self.ConvertMessage(value['value'], sub_message, '{0}.value'.format(path))
755 else:
756 del value['@type']
757 try:
758 self._ConvertFieldValuePair(value, sub_message, path)
759 finally:
760 value['@type'] = type_url
761 # Sets Any message
762 message.value = sub_message.SerializeToString()
763 message.type_url = type_url
765 def _ConvertGenericMessage(self, value, message, path):
766 """Convert a JSON representation into message with FromJsonString."""
767 # Duration, Timestamp, FieldMask have a FromJsonString method to do the
768 # conversion. Users can also call the method directly.
769 try:
770 message.FromJsonString(value)
771 except ValueError as e:
772 raise ParseError('{0} at {1}'.format(e, path)) from e
774 def _ConvertValueMessage(self, value, message, path):
775 """Convert a JSON representation into Value message."""
776 if isinstance(value, dict):
777 self.ConvertMessage(value, message.struct_value, path)
778 elif isinstance(value, _LIST_LIKE):
779 self.ConvertMessage(value, message.list_value, path)
780 elif value is None:
781 message.null_value = 0
782 elif isinstance(value, bool):
783 message.bool_value = value
784 elif isinstance(value, str):
785 message.string_value = value
786 elif isinstance(value, _INT_OR_FLOAT):
787 message.number_value = value
788 else:
789 raise ParseError(
790 'Value {0} has unexpected type {1} at {2}'.format(
791 value, type(value), path
792 )
793 )
795 def _ConvertListOrTupleValueMessage(self, value, message, path):
796 """Convert a JSON representation into ListValue message."""
797 if not isinstance(value, _LIST_LIKE):
798 raise ParseError(
799 'ListValue must be in [] which is {0} at {1}'.format(value, path)
800 )
801 message.ClearField('values')
802 for index, item in enumerate(value):
803 self.ConvertMessage(
804 item, message.values.add(), '{0}[{1}]'.format(path, index)
805 )
807 def _ConvertStructMessage(self, value, message, path):
808 """Convert a JSON representation into Struct message."""
809 if not isinstance(value, dict):
810 raise ParseError(
811 'Struct must be in a dict which is {0} at {1}'.format(value, path)
812 )
813 # Clear will mark the struct as modified so it will be created even if
814 # there are no values.
815 message.Clear()
816 for key in value:
817 self.ConvertMessage(
818 value[key], message.fields[key], '{0}.{1}'.format(path, key)
819 )
820 return
822 def _ConvertWrapperMessage(self, value, message, path):
823 """Convert a JSON representation into Wrapper message."""
824 field = message.DESCRIPTOR.fields_by_name['value']
825 self._ConvertAndSetScalar(
826 message, field, value, path='{0}.value'.format(path)
827 )
829 def _ConvertMapFieldValue(self, value, message, field, path):
830 """Convert map field value for a message map field.
832 Args:
833 value: A JSON object to convert the map field value.
834 message: A protocol message to record the converted data.
835 field: The descriptor of the map field to be converted.
836 path: parent path to log parse error info.
838 Raises:
839 ParseError: In case of convert problems.
840 """
841 if not isinstance(value, dict):
842 raise ParseError(
843 'Map field {0} must be in a dict which is {1} at {2}'.format(
844 field.name, value, path
845 )
846 )
847 key_field = field.message_type.fields_by_name['key']
848 value_field = field.message_type.fields_by_name['value']
849 for key in value:
850 key_value = _ConvertScalarFieldValue(
851 key, key_field, '{0}.key'.format(path), True
852 )
853 if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
854 self.ConvertMessage(
855 value[key],
856 getattr(message, field.name)[key_value],
857 '{0}[{1}]'.format(path, key_value),
858 )
859 else:
860 self._ConvertAndSetScalarToMapKey(
861 message,
862 field,
863 key_value,
864 value[key],
865 path='{0}[{1}]'.format(path, key_value),
866 )
868 def _ConvertAndSetScalar(self, message, field, js_value, path):
869 """Convert scalar from js_value and assign it to message.field."""
870 try:
871 value = _ConvertScalarFieldValue(js_value, field, path)
872 if field.is_extension:
873 message.Extensions[field] = value
874 else:
875 setattr(message, field.name, value)
876 except EnumStringValueParseError:
877 if not self.ignore_unknown_fields:
878 raise
880 def _ConvertAndAppendScalar(self, message, repeated_field, js_value, path):
881 """Convert scalar from js_value and append it to message.repeated_field."""
882 try:
883 if repeated_field.is_extension:
884 repeated = message.Extensions[repeated_field]
885 else:
886 repeated = getattr(message, repeated_field.name)
887 value = _ConvertScalarFieldValue(js_value, repeated_field, path)
888 repeated.append(value)
889 except EnumStringValueParseError:
890 if not self.ignore_unknown_fields:
891 raise
893 def _ConvertAndSetScalarToMapKey(
894 self, message, map_field, converted_key, js_value, path
895 ):
896 """Convert scalar from 'js_value' and add it to message.map_field[converted_key]."""
897 try:
898 getattr(message, map_field.name)[converted_key] = (
899 _ConvertScalarFieldValue(
900 js_value,
901 map_field.message_type.fields_by_name['value'],
902 path,
903 )
904 )
905 except EnumStringValueParseError:
906 if not self.ignore_unknown_fields:
907 raise
909def _GetJsonEnumValueOption(ev):
910 """Helper to get the JsonEnumValueOptions for an enum value.
912 Args:
913 ev: The EnumValueDescriptor.
915 Returns:
916 The JsonEnumValueOptions message if the extension is present,
917 otherwise None.
918 """
919 try:
920 extension_descriptor = descriptor_pool.Default().FindExtensionByName(
921 'pb.enumvalue.json'
922 )
923 except KeyError:
924 return None
925 if ev.GetOptions().HasExtension(extension_descriptor):
926 return ev.GetOptions().Extensions[extension_descriptor]
927 return None
929def _ConvertScalarFieldValue(value, field, path, require_str=False):
930 """Convert a single scalar field value.
932 Args:
933 value: A scalar value to convert the scalar field value.
934 field: The descriptor of the field to convert.
935 path: parent path to log parse error info.
936 require_str: If True, the field value must be a str.
938 Returns:
939 The converted scalar field value
941 Raises:
942 ParseError: In case of convert problems.
943 EnumStringValueParseError: In case of unknown enum string value.
944 """
945 try:
946 if field.cpp_type in _INT_TYPES:
947 return _ConvertInteger(value)
948 elif field.cpp_type in _FLOAT_TYPES:
949 return _ConvertFloat(value, field)
950 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
951 return _ConvertBool(value, require_str)
952 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
953 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
954 if isinstance(value, str):
955 encoded = value.encode('utf-8')
956 else:
957 encoded = value
958 # Add extra padding '='
959 padded_value = encoded + b'=' * (4 - len(encoded) % 4)
960 return base64.urlsafe_b64decode(padded_value)
961 else:
962 # Checking for unpaired surrogates appears to be unreliable,
963 # depending on the specific Python version, so we check manually.
964 if _UNPAIRED_SURROGATE_PATTERN.search(value):
965 raise ParseError('Unpaired surrogate')
966 return value
967 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
968 # Convert an enum value.
969 enum_value = field.enum_type.values_by_name.get(value, None)
970 # First check to see if we have a custom enum string.
971 if enum_value is None:
972 for ev in field.enum_type.values:
973 option = _GetJsonEnumValueOption(ev)
974 if option is not None and option.string == value:
975 enum_value = ev
976 break
977 # If not, try parsing it as an integer.
978 if enum_value is None:
979 try:
980 number = int(value)
981 enum_value = field.enum_type.values_by_number.get(number, None)
982 except ValueError as e:
983 # Since parsing to integer failed and lookup in values_by_name didn't
984 # find this name, we have an enum string value which is unknown.
985 raise EnumStringValueParseError(
986 'Invalid enum value {0} for enum type {1}'.format(
987 value, field.enum_type.full_name
988 )
989 ) from e
990 if enum_value is None:
991 if field.enum_type.is_closed:
992 raise ParseError(
993 'Invalid enum value {0} for enum type {1}'.format(
994 value, field.enum_type.full_name
995 )
996 )
997 else:
998 return number
999 return enum_value.number
1000 except EnumStringValueParseError as e:
1001 raise EnumStringValueParseError('{0} at {1}'.format(e, path)) from e
1002 except ParseError as e:
1003 raise ParseError('{0} at {1}'.format(e, path)) from e
1006def _ConvertInteger(value):
1007 """Convert an integer.
1009 Args:
1010 value: A scalar value to convert.
1012 Returns:
1013 The integer value.
1015 Raises:
1016 ParseError: If an integer couldn't be consumed.
1017 """
1018 if isinstance(value, float) and not value.is_integer():
1019 raise ParseError("Couldn't parse integer: {0}".format(value))
1021 if isinstance(value, str) and value.find(' ') != -1:
1022 raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
1024 if isinstance(value, bool):
1025 raise ParseError(
1026 'Bool value {0} is not acceptable for integer field'.format(value)
1027 )
1029 try:
1030 return int(value)
1031 except ValueError as e:
1032 # Attempt to parse as an integer-valued float.
1033 try:
1034 f = float(value)
1035 except ValueError:
1036 # Raise the original exception for the int parse.
1037 raise e # pylint: disable=raise-missing-from
1038 if not f.is_integer():
1039 raise ParseError(
1040 'Couldn\'t parse non-integer string: "{0}"'.format(value)
1041 ) from e
1042 return int(f)
1045def _ConvertFloat(value, field):
1046 """Convert an floating point number."""
1047 if isinstance(value, float):
1048 if math.isnan(value):
1049 raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
1050 if math.isinf(value):
1051 if value > 0:
1052 raise ParseError(
1053 "Couldn't parse Infinity or value too large, "
1054 'use quoted "Infinity" instead'
1055 )
1056 else:
1057 raise ParseError(
1058 "Couldn't parse -Infinity or value too small, "
1059 'use quoted "-Infinity" instead'
1060 )
1061 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
1062 # pylint: disable=protected-access
1063 if value > type_checkers._FLOAT_MAX:
1064 raise ParseError('Float value too large')
1065 # pylint: disable=protected-access
1066 if value < type_checkers._FLOAT_MIN:
1067 raise ParseError('Float value too small')
1068 if value == 'nan':
1069 raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')
1070 try:
1071 # Assume Python compatible syntax.
1072 return float(value)
1073 except ValueError as e:
1074 # Check alternative spellings.
1075 if value == _NEG_INFINITY:
1076 return float('-inf')
1077 elif value == _INFINITY:
1078 return float('inf')
1079 elif value == _NAN:
1080 return float('nan')
1081 else:
1082 raise ParseError("Couldn't parse float: {0}".format(value)) from e
1085def _ConvertBool(value, require_str):
1086 """Convert a boolean value.
1088 Args:
1089 value: A scalar value to convert.
1090 require_str: If True, value must be a str.
1092 Returns:
1093 The bool parsed.
1095 Raises:
1096 ParseError: If a boolean value couldn't be consumed.
1097 """
1098 if require_str:
1099 if value == 'true':
1100 return True
1101 elif value == 'false':
1102 return False
1103 else:
1104 raise ParseError('Expected "true" or "false", not {0}'.format(value))
1106 if not isinstance(value, bool):
1107 raise ParseError('Expected true or false without quotes')
1108 return value
1111_WKTJSONMETHODS = {
1112 'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'],
1113 'google.protobuf.Duration': [
1114 '_GenericMessageToJsonObject',
1115 '_ConvertGenericMessage',
1116 ],
1117 'google.protobuf.FieldMask': [
1118 '_GenericMessageToJsonObject',
1119 '_ConvertGenericMessage',
1120 ],
1121 'google.protobuf.ListValue': [
1122 '_ListValueMessageToJsonObject',
1123 '_ConvertListOrTupleValueMessage',
1124 ],
1125 'google.protobuf.Struct': [
1126 '_StructMessageToJsonObject',
1127 '_ConvertStructMessage',
1128 ],
1129 'google.protobuf.Timestamp': [
1130 '_GenericMessageToJsonObject',
1131 '_ConvertGenericMessage',
1132 ],
1133 'google.protobuf.Value': [
1134 '_ValueMessageToJsonObject',
1135 '_ConvertValueMessage',
1136 ],
1137}