Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc. All rights reserved.
3#
4# Use of this source code is governed by a BSD-style
5# license that can be found in the LICENSE file or at
6# https://developers.google.com/open-source/licenses/bsd
7"""Contains routines for printing protocol messages in JSON format.
9Simple usage example:
11 # Create a proto object and serialize it to a json format string.
12 message = my_proto_pb2.MyMessage(foo='bar')
13 json_string = json_format.MessageToJson(message)
15 # Parse a json format string to proto object.
16 message = json_format.Parse(json_string, my_proto_pb2.MyMessage())
17"""
19__author__ = 'jieluo@google.com (Jie Luo)'
21import base64
22from collections import OrderedDict
23import json
24import math
25from operator import methodcaller
26import re
28from google.protobuf import descriptor
29from google.protobuf import descriptor_pool
30from google.protobuf import message_factory
31from google.protobuf import symbol_database
32from google.protobuf.internal import type_checkers
34_INT_TYPES = frozenset([
35 descriptor.FieldDescriptor.CPPTYPE_INT32,
36 descriptor.FieldDescriptor.CPPTYPE_UINT32,
37 descriptor.FieldDescriptor.CPPTYPE_INT64,
38 descriptor.FieldDescriptor.CPPTYPE_UINT64,
39])
40_INT64_TYPES = frozenset([
41 descriptor.FieldDescriptor.CPPTYPE_INT64,
42 descriptor.FieldDescriptor.CPPTYPE_UINT64,
43])
44_FLOAT_TYPES = frozenset([
45 descriptor.FieldDescriptor.CPPTYPE_FLOAT,
46 descriptor.FieldDescriptor.CPPTYPE_DOUBLE,
47])
48_INFINITY = 'Infinity'
49_NEG_INFINITY = '-Infinity'
50_NAN = 'NaN'
52_UNPAIRED_SURROGATE_PATTERN = re.compile(
53 '[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]'
54)
56_VALID_EXTENSION_NAME = re.compile(r'\[[a-zA-Z0-9\._]*\]$')
59class Error(Exception):
60 """Top-level module error for json_format."""
63class SerializeToJsonError(Error):
64 """Thrown if serialization to JSON fails."""
67class ParseError(Error):
68 """Thrown in case of parsing error."""
71class EnumStringValueParseError(ParseError):
72 """Thrown if unknown string enum value is encountered.
74 This exception is suppressed if ignore_unknown_fields is set.
75 """
78def MessageToJson(
79 message,
80 preserving_proto_field_name=False,
81 indent=2,
82 sort_keys=False,
83 use_integers_for_enums=False,
84 descriptor_pool=None,
85 ensure_ascii=True,
86 always_print_fields_with_no_presence=False,
87 *,
88 unquote_int64_if_possible=False,
89):
90 """Converts protobuf message to JSON format.
92 Args:
93 message: The protocol buffers message instance to serialize.
94 always_print_fields_with_no_presence: If True, fields without presence
95 (implicit presence scalars, repeated fields, and map fields) will always
96 be serialized. Any field that supports presence is not affected by this
97 option (including singular message fields and oneof fields).
98 preserving_proto_field_name: If True, use the original proto field names as
99 defined in the .proto file. If False, convert the field names to
100 lowerCamelCase.
101 indent: The JSON object will be pretty-printed with this indent level. An
102 indent level of 0 or negative will only insert newlines. If the indent
103 level is None, no newlines will be inserted.
104 sort_keys: If True, then the output will be sorted by field names.
105 use_integers_for_enums: If true, print integers instead of enum names.
106 descriptor_pool: A Descriptor Pool for resolving types. If None use the
107 default.
108 ensure_ascii: If True, strings with non-ASCII characters are escaped. If
109 False, Unicode strings are returned unchanged.
110 unquote_int64_if_possible: If True, unquote int64 fields for values that are
111 safe to emit as numbers (all values smaller than 2^53 and a sparse set of
112 values that are larger).
114 Returns:
115 A string containing the JSON formatted protocol buffer message.
116 """
117 printer = _Printer(
118 preserving_proto_field_name,
119 use_integers_for_enums,
120 descriptor_pool,
121 always_print_fields_with_no_presence,
122 unquote_int64_if_possible=unquote_int64_if_possible,
123 )
124 return printer.ToJsonString(message, indent, sort_keys, ensure_ascii)
127def MessageToDict(
128 message,
129 always_print_fields_with_no_presence=False,
130 preserving_proto_field_name=False,
131 use_integers_for_enums=False,
132 descriptor_pool=None,
133 *,
134 unquote_int64_if_possible=False,
135):
136 """Converts protobuf message to a dictionary.
138 When the dictionary is encoded to JSON, it conforms to ProtoJSON spec.
140 Args:
141 message: The protocol buffers message instance to serialize.
142 always_print_fields_with_no_presence: If True, fields without presence
143 (implicit presence scalars, repeated fields, and map fields) will always
144 be serialized. Any field that supports presence is not affected by this
145 option (including singular message fields and oneof fields).
146 preserving_proto_field_name: If True, use the original proto field names as
147 defined in the .proto file. If False, convert the field names to
148 lowerCamelCase.
149 use_integers_for_enums: If true, print integers instead of enum names.
150 descriptor_pool: A Descriptor Pool for resolving types. If None use the
151 default.
152 unquote_int64_if_possible: If True, unquote int64 fields for values that are
153 safe to emit as numbers (all values smaller than 2^53 and a sparse set of
154 values that are larger).
156 Returns:
157 A dict representation of the protocol buffer message.
158 """
159 printer = _Printer(
160 preserving_proto_field_name,
161 use_integers_for_enums,
162 descriptor_pool,
163 always_print_fields_with_no_presence,
164 unquote_int64_if_possible=unquote_int64_if_possible,
165 )
166 # pylint: disable=protected-access
167 return printer._MessageToJsonObject(message)
170def _IsMapEntry(field):
171 return (
172 field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
173 and field.message_type.has_options
174 and field.message_type.GetOptions().map_entry
175 )
178class _Printer(object):
179 """JSON format printer for protocol message."""
181 def __init__(
182 self,
183 preserving_proto_field_name=False,
184 use_integers_for_enums=False,
185 descriptor_pool=None,
186 always_print_fields_with_no_presence=False,
187 *,
188 unquote_int64_if_possible=False,
189 ):
190 self.always_print_fields_with_no_presence = (
191 always_print_fields_with_no_presence
192 )
193 self.preserving_proto_field_name = preserving_proto_field_name
194 self.use_integers_for_enums = use_integers_for_enums
195 self.descriptor_pool = descriptor_pool
196 self.unquote_int64_if_possible = unquote_int64_if_possible
197 self._enumvalue_json_extension = None
199 def _GetEnumValueJsonExtension(self):
200 if self._enumvalue_json_extension is None:
201 # Options are always put on the default pool, so we only search the
202 # default pool.
203 try:
204 # Using reflection to FindExtensionByName is quite expensive, hence the
205 # gymnastics to cache it into the instance attribute
206 # _enumvalue_json_extension.
207 # TODO: b/551998570 - Over the longer term, we can consider putting this
208 # information in bootstrap files so that we don't have to rely on using
209 # reflection to perform this lookup at all.
210 self._enumvalue_json_extension = (
211 descriptor_pool.Default().FindExtensionByName('pb.enumvalue.json')
212 )
213 except KeyError:
214 self._enumvalue_json_extension = {}
215 return self._enumvalue_json_extension or None
217 def _GetJsonEnumValueOption(self, ev):
218 """Helper to get the JsonEnumValueOptions for an enum value."""
219 extension_descriptor = self._GetEnumValueJsonExtension()
220 if extension_descriptor is None:
221 return None
222 return _GetJsonEnumValueOption(ev, extension_descriptor)
224 def ToJsonString(self, message, indent, sort_keys, ensure_ascii):
225 js = self._MessageToJsonObject(message)
226 return json.dumps(
227 js, indent=indent, sort_keys=sort_keys, ensure_ascii=ensure_ascii
228 )
230 def _MessageToJsonObject(self, message):
231 """Converts message to an object according to ProtoJSON Specification."""
232 message_descriptor = message.DESCRIPTOR
233 full_name = message_descriptor.full_name
234 if _IsWrapperMessage(message_descriptor):
235 return self._WrapperMessageToJsonObject(message)
236 if full_name in _WKTJSONMETHODS:
237 return methodcaller(_WKTJSONMETHODS[full_name][0], message)(self)
238 js = {}
239 return self._RegularMessageToJsonObject(message, js)
241 def _RegularMessageToJsonObject(self, message, js):
242 """Converts normal message according to ProtoJSON Specification."""
243 fields = message.ListFields()
245 try:
246 for field, value in fields:
247 if field.is_extension:
248 name = '[%s]' % field.full_name
249 elif self.preserving_proto_field_name:
250 name = field.name
251 else:
252 name = field.json_name
254 if _IsMapEntry(field):
255 # Convert a map field.
256 v_field = field.message_type.fields_by_name['value']
257 js_map = {}
258 for key in value:
259 if isinstance(key, bool):
260 if key:
261 recorded_key = 'true'
262 else:
263 recorded_key = 'false'
264 else:
265 recorded_key = str(key)
266 js_map[recorded_key] = self._FieldToJsonObject(v_field, value[key])
267 js[name] = js_map
268 elif field.is_repeated:
269 # Convert a repeated field.
270 js[name] = [self._FieldToJsonObject(field, k) for k in value]
271 else:
272 js[name] = self._FieldToJsonObject(field, value)
274 # Serialize default value if including_default_value_fields is True.
275 if (
276 self.always_print_fields_with_no_presence
277 ):
278 message_descriptor = message.DESCRIPTOR
279 for field in message_descriptor.fields:
281 # always_print_fields_with_no_presence doesn't apply to
282 # any field which supports presence.
283 if self.always_print_fields_with_no_presence and field.has_presence:
284 continue
286 if self.preserving_proto_field_name:
287 name = field.name
288 else:
289 name = field.json_name
290 if name in js:
291 # Skip the field which has been serialized already.
292 continue
293 if _IsMapEntry(field):
294 js[name] = {}
295 elif field.is_repeated:
296 js[name] = []
297 else:
298 js[name] = self._FieldToJsonObject(field, field.default_value)
300 except ValueError as e:
301 raise SerializeToJsonError(
302 'Failed to serialize {0} field: {1}.'.format(field.name, e)
303 ) from e
305 return js
307 def _FieldToJsonObject(self, field, value):
308 """Converts field value according to ProtoJSON Specification."""
309 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
310 return self._MessageToJsonObject(value)
311 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
312 if self.use_integers_for_enums:
313 return value
314 if field.enum_type.full_name == 'google.protobuf.NullValue':
315 return None
316 enum_value = field.enum_type.values_by_number.get(value, None)
317 if enum_value is not None:
318 option = self._GetJsonEnumValueOption(enum_value)
319 if option is not None:
320 return option.string
321 return enum_value.name
322 else:
323 if field.enum_type.is_closed:
324 raise SerializeToJsonError(
325 'Enum field contains an integer value '
326 'which can not mapped to an enum value.'
327 )
328 else:
329 return value
330 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
331 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
332 # Use base64 Data encoding for bytes
333 return base64.b64encode(value).decode('utf-8')
334 else:
335 return str(value)
336 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
337 return bool(value)
338 elif field.cpp_type in _INT64_TYPES:
339 if self.unquote_int64_if_possible and float(value) == value:
340 return value
341 else:
342 return str(value)
343 elif field.cpp_type in _FLOAT_TYPES:
344 if math.isinf(value):
345 if value < 0.0:
346 return _NEG_INFINITY
347 else:
348 return _INFINITY
349 if math.isnan(value):
350 return _NAN
351 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
352 return type_checkers.ToShortestFloat(value)
354 return value
356 def _AnyMessageToJsonObject(self, message):
357 """Converts Any message according to ProtoJSON Specification."""
358 if not message.ListFields():
359 return {}
360 # Must print @type first, use OrderedDict instead of {}
361 js = OrderedDict()
362 type_url = message.type_url
363 js['@type'] = type_url
364 sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
365 sub_message.ParseFromString(message.value)
366 message_descriptor = sub_message.DESCRIPTOR
367 full_name = message_descriptor.full_name
368 if _IsWrapperMessage(message_descriptor):
369 js['value'] = self._WrapperMessageToJsonObject(sub_message)
370 return js
371 if full_name in _WKTJSONMETHODS:
372 js['value'] = methodcaller(_WKTJSONMETHODS[full_name][0], sub_message)(
373 self
374 )
375 return js
376 return self._RegularMessageToJsonObject(sub_message, js)
378 def _GenericMessageToJsonObject(self, message):
379 """Converts message according to ProtoJSON Specification."""
380 # Duration, Timestamp and FieldMask have ToJsonString method to do the
381 # convert. Users can also call the method directly.
382 return message.ToJsonString()
384 def _ValueMessageToJsonObject(self, message):
385 """Converts Value message according to ProtoJSON Specification."""
386 which = message.WhichOneof('kind')
387 # If the Value message is not set treat as null_value when serialize
388 # to JSON. The parse back result will be different from original message.
389 if which is None or which == 'null_value':
390 return None
391 if which == 'list_value':
392 return self._ListValueMessageToJsonObject(message.list_value)
393 if which == 'number_value':
394 value = message.number_value
395 if math.isinf(value):
396 raise ValueError(
397 'Fail to serialize Infinity for Value.number_value, '
398 'which would parse as string_value'
399 )
400 if math.isnan(value):
401 raise ValueError(
402 'Fail to serialize NaN for Value.number_value, '
403 'which would parse as string_value'
404 )
405 else:
406 value = getattr(message, which)
407 oneof_descriptor = message.DESCRIPTOR.fields_by_name[which]
408 return self._FieldToJsonObject(oneof_descriptor, value)
410 def _ListValueMessageToJsonObject(self, message):
411 """Converts ListValue message according to ProtoJSON Specification."""
412 return [self._ValueMessageToJsonObject(value) for value in message.values]
414 def _StructMessageToJsonObject(self, message):
415 """Converts Struct message according to ProtoJSON Specification."""
416 fields = message.fields
417 ret = {}
418 for key in fields:
419 ret[key] = self._ValueMessageToJsonObject(fields[key])
420 return ret
422 def _WrapperMessageToJsonObject(self, message):
423 return self._FieldToJsonObject(
424 message.DESCRIPTOR.fields_by_name['value'], message.value
425 )
428def _IsWrapperMessage(message_descriptor):
429 return message_descriptor.file.name == 'google/protobuf/wrappers.proto'
432def _DuplicateChecker(js):
433 result = {}
434 for name, value in js:
435 if name in result:
436 raise ParseError('Failed to load JSON: duplicate key {0}.'.format(name))
437 result[name] = value
438 return result
441def _CreateMessageFromTypeUrl(type_url, descriptor_pool):
442 """Creates a message from a type URL."""
443 db = symbol_database.Default()
444 pool = db.pool if descriptor_pool is None else descriptor_pool
445 type_name = type_url.split('/')[-1]
446 try:
447 message_descriptor = pool.FindMessageTypeByName(type_name)
448 except KeyError as e:
449 raise TypeError(
450 'Can not find message descriptor by type_url: {0}'.format(type_url)
451 ) from e
452 message_class = message_factory.GetMessageClass(message_descriptor)
453 return message_class()
456def Parse(
457 text,
458 message,
459 ignore_unknown_fields=False,
460 descriptor_pool=None,
461 max_recursion_depth=100,
462):
463 """Parses a JSON representation of a protocol message into a message.
465 Args:
466 text: Message JSON representation.
467 message: A protocol buffer message to merge into.
468 ignore_unknown_fields: If True, do not raise errors for unknown fields.
469 descriptor_pool: A Descriptor Pool for resolving types. If None use the
470 default.
471 max_recursion_depth: max recursion depth of JSON message to be deserialized.
472 JSON messages over this depth will fail to be deserialized. Default value
473 is 100.
475 Returns:
476 The same message passed as argument.
478 Raises::
479 ParseError: On JSON parsing problems.
480 """
481 if not isinstance(text, str):
482 text = text.decode('utf-8')
484 try:
485 js = json.loads(text, object_pairs_hook=_DuplicateChecker)
486 except Exception as e:
487 raise ParseError('Failed to load JSON: {0}.'.format(str(e))) from e
489 try:
490 return ParseDict(
491 js, message, ignore_unknown_fields, descriptor_pool, max_recursion_depth
492 )
493 except ParseError as e:
494 raise e
495 except Exception as e:
496 raise ParseError(
497 'Failed to parse JSON: {0}: {1}.'.format(type(e).__name__, str(e))
498 ) from e
501def ParseDict(
502 js_dict,
503 message,
504 ignore_unknown_fields=False,
505 descriptor_pool=None,
506 max_recursion_depth=100,
507):
508 """Parses a JSON dictionary representation into a message.
510 Args:
511 js_dict: Dict representation of a JSON message.
512 message: A protocol buffer message to merge into.
513 ignore_unknown_fields: If True, do not raise errors for unknown fields.
514 descriptor_pool: A Descriptor Pool for resolving types. If None use the
515 default.
516 max_recursion_depth: max recursion depth of JSON message to be deserialized.
517 JSON messages over this depth will fail to be deserialized. Default value
518 is 100.
520 Returns:
521 The same message passed as argument.
522 """
523 parser = _Parser(ignore_unknown_fields, descriptor_pool, max_recursion_depth)
524 parser.ConvertMessage(js_dict, message, '')
525 return message
528_INT_OR_FLOAT = (int, float)
529_LIST_LIKE = (list, tuple)
532class _Parser(object):
533 """JSON format parser for protocol message."""
535 def __init__(
536 self, ignore_unknown_fields, descriptor_pool, max_recursion_depth
537 ):
538 self.ignore_unknown_fields = ignore_unknown_fields
539 self.descriptor_pool = descriptor_pool
540 self.max_recursion_depth = max_recursion_depth
541 self.recursion_depth = 0
542 self._custom_enum_names_cache = {}
543 self._enumvalue_json_extension = None
545 def _GetEnumValueJsonExtension(self):
546 if self._enumvalue_json_extension is None:
547 # Options are always put on the default pool, so we only search the
548 # default pool.
549 try:
550 self._enumvalue_json_extension = (
551 descriptor_pool.Default().FindExtensionByName('pb.enumvalue.json')
552 )
553 except KeyError:
554 self._enumvalue_json_extension = {}
555 return self._enumvalue_json_extension or None
557 def ConvertMessage(self, value, message, path):
558 """Convert a JSON object into a message.
560 Args:
561 value: A JSON object.
562 message: A WKT or regular protocol message to record the data.
563 path: parent path to log parse error info.
565 Raises:
566 ParseError: In case of convert problems.
567 """
568 # Increment recursion depth at message entry. The max_recursion_depth limit
569 # is exclusive: a depth value equal to max_recursion_depth will trigger an
570 # error. For example, with max_recursion_depth=5, nesting up to depth 4 is
571 # allowed, but attempting depth 5 raises ParseError.
572 self.recursion_depth += 1
573 if self.recursion_depth > self.max_recursion_depth:
574 raise ParseError(
575 'Message too deep. Max recursion depth is {0}'.format(
576 self.max_recursion_depth
577 )
578 )
579 message_descriptor = message.DESCRIPTOR
580 full_name = message_descriptor.full_name
581 if not path:
582 path = message_descriptor.name
583 if _IsWrapperMessage(message_descriptor):
584 self._ConvertWrapperMessage(value, message, path)
585 elif full_name in _WKTJSONMETHODS:
586 methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self)
587 else:
588 self._ConvertFieldValuePair(value, message, path)
589 self.recursion_depth -= 1
591 def _ConvertFieldValuePair(self, js, message, path):
592 """Convert field value pairs into regular message.
594 Args:
595 js: A JSON object to convert the field value pairs.
596 message: A regular protocol message to record the data.
597 path: parent path to log parse error info.
599 Raises:
600 ParseError: In case of problems converting.
601 """
602 names = []
603 message_descriptor = message.DESCRIPTOR
604 fields_by_json_name = dict(
605 (f.json_name, f) for f in message_descriptor.fields
606 )
608 def _ClearFieldOrExtension(message, field):
609 if field.is_extension:
610 message.ClearExtension(field)
611 else:
612 message.ClearField(field.name)
614 def _GetFieldOrExtension(message, field):
615 if field.is_extension:
616 return message.Extensions[field]
617 else:
618 return getattr(message, field.name)
620 def _SetFieldOrExtension(message, field, value):
621 if field.is_extension:
622 message.Extensions[field] = value
623 else:
624 setattr(message, field.name, value)
626 for name in js:
627 try:
628 field = fields_by_json_name.get(name, None)
629 if not field:
630 field = message_descriptor.fields_by_name.get(name, None)
631 if not field and _VALID_EXTENSION_NAME.match(name):
632 if not message_descriptor.is_extendable:
633 raise ParseError(
634 'Message type {0} does not have extensions at {1}'.format(
635 message_descriptor.full_name, path
636 )
637 )
638 identifier = name[1:-1] # strip [] brackets
639 # pylint: disable=protected-access
640 field = message.Extensions._FindExtensionByName(identifier)
641 # pylint: enable=protected-access
642 if not field:
643 # Try looking for extension by the message type name, dropping the
644 # field name following the final . separator in full_name.
645 identifier = '.'.join(identifier.split('.')[:-1])
646 # pylint: disable=protected-access
647 field = message.Extensions._FindExtensionByName(identifier)
648 # pylint: enable=protected-access
649 if not field:
650 if self.ignore_unknown_fields:
651 continue
652 raise ParseError(
653 (
654 'Message type "{0}" has no field named "{1}" at "{2}".\n'
655 ' Available Fields(except extensions): "{3}"'
656 ).format(
657 message_descriptor.full_name,
658 name,
659 path,
660 [f.json_name for f in message_descriptor.fields],
661 )
662 )
663 if name in names:
664 raise ParseError(
665 'Message type "{0}" should not have multiple '
666 '"{1}" fields at "{2}".'.format(
667 message.DESCRIPTOR.full_name, name, path
668 )
669 )
670 names.append(name)
671 value = js[name]
672 # Check no other oneof field is parsed.
673 if field.containing_oneof is not None and value is not None:
674 oneof_name = field.containing_oneof.name
675 if oneof_name in names:
676 raise ParseError(
677 'Message type "{0}" should not have multiple '
678 '"{1}" oneof fields at "{2}".'.format(
679 message.DESCRIPTOR.full_name, oneof_name, path
680 )
681 )
682 names.append(oneof_name)
684 if value is None:
685 if (
686 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE
687 and field.message_type.full_name == 'google.protobuf.Value'
688 ):
689 sub_message = _GetFieldOrExtension(message, field)
690 sub_message.null_value = 0
691 elif (
692 field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM
693 and field.enum_type.full_name == 'google.protobuf.NullValue'
694 ):
695 _SetFieldOrExtension(message, field, 0)
696 else:
697 _ClearFieldOrExtension(message, field)
698 continue
700 # Parse field value.
701 if _IsMapEntry(field):
702 _ClearFieldOrExtension(message, field)
703 self._ConvertMapFieldValue(
704 value, message, field, '{0}.{1}'.format(path, name)
705 )
706 elif field.is_repeated:
707 _ClearFieldOrExtension(message, field)
708 if not isinstance(value, _LIST_LIKE):
709 raise ParseError(
710 'repeated field {0} must be in [] which is {1} at {2}'.format(
711 name, value, path
712 )
713 )
714 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
715 # Repeated message field.
716 for index, item in enumerate(value):
717 sub_message = _GetFieldOrExtension(message, field).add()
718 # None is a null_value in Value.
719 if (
720 item is None
721 and sub_message.DESCRIPTOR.full_name
722 != 'google.protobuf.Value'
723 ):
724 raise ParseError(
725 'null is not allowed to be used as an element'
726 ' in a repeated field at {0}.{1}[{2}]'.format(
727 path, name, index
728 )
729 )
730 self.ConvertMessage(
731 item, sub_message, '{0}.{1}[{2}]'.format(path, name, index)
732 )
733 else:
734 # Repeated scalar field.
735 for index, item in enumerate(value):
736 if item is None:
737 raise ParseError(
738 'null is not allowed to be used as an element'
739 ' in a repeated field at {0}.{1}[{2}]'.format(
740 path, name, index
741 )
742 )
743 self._ConvertAndAppendScalar(
744 message, field, item, '{0}.{1}[{2}]'.format(path, name, index)
745 )
746 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
747 sub_message = _GetFieldOrExtension(message, field)
748 sub_message.SetInParent()
749 self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name))
750 else:
751 self._ConvertAndSetScalar(
752 message, field, value, '{0}.{1}'.format(path, name)
753 )
754 except ParseError as e:
755 if field and field.containing_oneof is None:
756 raise ParseError(
757 'Failed to parse {0} field: {1}.'.format(name, e)
758 ) from e
759 else:
760 raise ParseError(str(e)) from e
761 except ValueError as e:
762 raise ParseError(
763 'Failed to parse {0} field: {1}.'.format(name, e)
764 ) from e
765 except TypeError as e:
766 raise ParseError(
767 'Failed to parse {0} field: {1}.'.format(name, e)
768 ) from e
770 def _ConvertAnyMessage(self, value, message, path):
771 """Convert a JSON representation into Any message."""
772 if isinstance(value, dict) and not value:
773 return
774 try:
775 type_url = value['@type']
776 except KeyError as e:
777 raise ParseError(
778 '@type is missing when parsing any message at {0}'.format(path)
779 ) from e
781 try:
782 sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool)
783 except TypeError as e:
784 raise ParseError('{0} at {1}'.format(e, path)) from e
785 message_descriptor = sub_message.DESCRIPTOR
786 full_name = message_descriptor.full_name
787 if _IsWrapperMessage(message_descriptor):
788 self._ConvertWrapperMessage(
789 value['value'], sub_message, '{0}.value'.format(path)
790 )
791 elif full_name in _WKTJSONMETHODS:
792 # For well-known types (including nested Any), use ConvertMessage
793 # to ensure recursion depth is properly tracked
794 self.ConvertMessage(value['value'], sub_message, '{0}.value'.format(path))
795 else:
796 del value['@type']
797 try:
798 self._ConvertFieldValuePair(value, sub_message, path)
799 finally:
800 value['@type'] = type_url
801 # Sets Any message
802 message.value = sub_message.SerializeToString()
803 message.type_url = type_url
805 def _ConvertGenericMessage(self, value, message, path):
806 """Convert a JSON representation into message with FromJsonString."""
807 # Duration, Timestamp, FieldMask have a FromJsonString method to do the
808 # conversion. Users can also call the method directly.
809 try:
810 message.FromJsonString(value)
811 except ValueError as e:
812 raise ParseError('{0} at {1}'.format(e, path)) from e
814 def _ConvertValueMessage(self, value, message, path):
815 """Convert a JSON representation into Value message."""
816 if isinstance(value, dict):
817 self.ConvertMessage(value, message.struct_value, path)
818 elif isinstance(value, _LIST_LIKE):
819 self.ConvertMessage(value, message.list_value, path)
820 elif value is None:
821 message.null_value = 0
822 elif isinstance(value, bool):
823 message.bool_value = value
824 elif isinstance(value, str):
825 message.string_value = value
826 elif isinstance(value, _INT_OR_FLOAT):
827 message.number_value = value
828 else:
829 raise ParseError(
830 'Value {0} has unexpected type {1} at {2}'.format(
831 value, type(value), path
832 )
833 )
835 def _ConvertListOrTupleValueMessage(self, value, message, path):
836 """Convert a JSON representation into ListValue message."""
837 if not isinstance(value, _LIST_LIKE):
838 raise ParseError(
839 'ListValue must be in [] which is {0} at {1}'.format(value, path)
840 )
841 message.ClearField('values')
842 for index, item in enumerate(value):
843 self.ConvertMessage(
844 item, message.values.add(), '{0}[{1}]'.format(path, index)
845 )
847 def _ConvertStructMessage(self, value, message, path):
848 """Convert a JSON representation into Struct message."""
849 if not isinstance(value, dict):
850 raise ParseError(
851 'Struct must be in a dict which is {0} at {1}'.format(value, path)
852 )
853 # Clear will mark the struct as modified so it will be created even if
854 # there are no values.
855 message.Clear()
856 for key in value:
857 self.ConvertMessage(
858 value[key], message.fields[key], '{0}.{1}'.format(path, key)
859 )
860 return
862 def _ConvertWrapperMessage(self, value, message, path):
863 """Convert a JSON representation into Wrapper message."""
864 field = message.DESCRIPTOR.fields_by_name['value']
865 self._ConvertAndSetScalar(
866 message, field, value, path='{0}.value'.format(path)
867 )
869 def _ConvertMapFieldValue(self, value, message, field, path):
870 """Convert map field value for a message map field.
872 Args:
873 value: A JSON object to convert the map field value.
874 message: A protocol message to record the converted data.
875 field: The descriptor of the map field to be converted.
876 path: parent path to log parse error info.
878 Raises:
879 ParseError: In case of convert problems.
880 """
881 if not isinstance(value, dict):
882 raise ParseError(
883 'Map field {0} must be in a dict which is {1} at {2}'.format(
884 field.name, value, path
885 )
886 )
887 key_field = field.message_type.fields_by_name['key']
888 value_field = field.message_type.fields_by_name['value']
889 for key in value:
890 key_value = _ConvertScalarFieldValue(
891 key,
892 key_field,
893 '{0}.key'.format(path),
894 self._custom_enum_names_cache,
895 self._GetEnumValueJsonExtension(),
896 require_str=True,
897 )
898 if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
899 self.ConvertMessage(
900 value[key],
901 getattr(message, field.name)[key_value],
902 '{0}[{1}]'.format(path, key_value),
903 )
904 else:
905 self._ConvertAndSetScalarToMapKey(
906 message,
907 field,
908 key_value,
909 value[key],
910 path='{0}[{1}]'.format(path, key_value),
911 )
913 def _ConvertAndSetScalar(self, message, field, js_value, path):
914 """Convert scalar from js_value and assign it to message.field."""
915 try:
916 value = _ConvertScalarFieldValue(
917 js_value,
918 field,
919 path,
920 self._custom_enum_names_cache,
921 self._GetEnumValueJsonExtension(),
922 )
923 if field.is_extension:
924 message.Extensions[field] = value
925 else:
926 setattr(message, field.name, value)
927 except EnumStringValueParseError:
928 if not self.ignore_unknown_fields:
929 raise
931 def _ConvertAndAppendScalar(self, message, repeated_field, js_value, path):
932 """Convert scalar from js_value and append it to message.repeated_field."""
933 try:
934 if repeated_field.is_extension:
935 repeated = message.Extensions[repeated_field]
936 else:
937 repeated = getattr(message, repeated_field.name)
938 value = _ConvertScalarFieldValue(
939 js_value,
940 repeated_field,
941 path,
942 self._custom_enum_names_cache,
943 self._GetEnumValueJsonExtension(),
944 )
945 repeated.append(value)
946 except EnumStringValueParseError:
947 if not self.ignore_unknown_fields:
948 raise
950 def _ConvertAndSetScalarToMapKey(
951 self, message, map_field, converted_key, js_value, path
952 ):
953 """Convert scalar from 'js_value' and add it to message.map_field[converted_key]."""
954 try:
955 getattr(message, map_field.name)[converted_key] = (
956 _ConvertScalarFieldValue(
957 js_value,
958 map_field.message_type.fields_by_name['value'],
959 path,
960 self._custom_enum_names_cache,
961 self._GetEnumValueJsonExtension(),
962 )
963 )
964 except EnumStringValueParseError:
965 if not self.ignore_unknown_fields:
966 raise
969def _GetJsonEnumValueOption(ev, extension_descriptor):
970 """Helper to get the JsonEnumValueOptions for an enum value.
972 Args:
973 ev: The EnumValueDescriptor.
974 extension_descriptor: The extension descriptor for 'pb.enumvalue.json'.
976 Returns:
977 The JsonEnumValueOptions message if the extension is present,
978 otherwise None.
979 """
980 if ev.GetOptions().HasExtension(extension_descriptor):
981 return ev.GetOptions().Extensions[extension_descriptor]
982 return None
985def _GetCustomJsonEnumNames(
986 enum_type, custom_enum_names_cache, enumvalue_json_extension=None
987):
988 """Helper to get a mapping from custom JSON name to EnumValueDescriptor.
990 Args:
991 enum_type: The EnumDescriptor.
992 custom_enum_names_cache: A dict to store/lookup the cached map.
993 enumvalue_json_extension: The extension descriptor for 'pb.enumvalue.json',
994 or None to look it up in the default descriptor pool.
996 Returns:
997 A dict mapping custom JSON name strings to EnumValueDescriptors.
998 """
999 if enum_type in custom_enum_names_cache:
1000 return custom_enum_names_cache[enum_type]
1002 custom_names = {}
1003 if enumvalue_json_extension is None:
1004 # Options are always put on the default pool, so we only search the default pool.
1005 try:
1006 enumvalue_json_extension = descriptor_pool.Default().FindExtensionByName(
1007 'pb.enumvalue.json'
1008 )
1009 except KeyError:
1010 enumvalue_json_extension = None
1012 if enumvalue_json_extension is not None:
1013 for ev in enum_type.values:
1014 options = ev.GetOptions()
1015 if options.HasExtension(enumvalue_json_extension):
1016 option = options.Extensions[enumvalue_json_extension]
1017 if option.HasField('string'):
1018 custom_names[option.string] = ev
1020 custom_enum_names_cache[enum_type] = custom_names
1021 return custom_names
1024def _ConvertScalarFieldValue(
1025 value,
1026 field,
1027 path,
1028 custom_enum_names_cache,
1029 enumvalue_json_extension=None,
1030 require_str=False,
1031):
1032 """Convert a single scalar field value.
1034 Args:
1035 value: A scalar value to convert the scalar field value.
1036 field: The descriptor of the field to convert.
1037 path: parent path to log parse error info.
1038 custom_enum_names_cache: A dict to store/lookup custom enum names.
1039 enumvalue_json_extension: The extension descriptor for 'pb.enumvalue.json',
1040 or None if not loaded.
1041 require_str: If True, the field value must be a str.
1043 Returns:
1044 The converted scalar field value
1046 Raises:
1047 ParseError: In case of convert problems.
1048 EnumStringValueParseError: In case of unknown enum string value.
1049 """
1050 try:
1051 if field.cpp_type in _INT_TYPES:
1052 return _ConvertInteger(value)
1053 elif field.cpp_type in _FLOAT_TYPES:
1054 return _ConvertFloat(value, field)
1055 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
1056 return _ConvertBool(value, require_str)
1057 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
1058 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
1059 if isinstance(value, str):
1060 encoded = value.encode('utf-8')
1061 else:
1062 encoded = value
1063 # Add extra padding '='
1064 padded_value = encoded + b'=' * (4 - len(encoded) % 4)
1065 return base64.urlsafe_b64decode(padded_value)
1066 else:
1067 # Checking for unpaired surrogates appears to be unreliable,
1068 # depending on the specific Python version, so we check manually.
1069 if _UNPAIRED_SURROGATE_PATTERN.search(value):
1070 raise ParseError('Unpaired surrogate')
1071 return value
1072 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
1073 # Convert an enum value.
1074 enum_value = field.enum_type.values_by_name.get(value, None)
1075 # First check to see if we have a custom enum string.
1076 if (
1077 enum_value is None
1078 and isinstance(value, str)
1079 and enumvalue_json_extension is not None
1080 ):
1081 custom_names = _GetCustomJsonEnumNames(
1082 field.enum_type,
1083 custom_enum_names_cache,
1084 enumvalue_json_extension,
1085 )
1086 enum_value = custom_names.get(value, None)
1087 # If not, try parsing it as an integer.
1088 if enum_value is None:
1089 try:
1090 number = int(value)
1091 enum_value = field.enum_type.values_by_number.get(number, None)
1092 except ValueError as e:
1093 # Since parsing to integer failed and lookup in values_by_name didn't
1094 # find this name, we have an enum string value which is unknown.
1095 raise EnumStringValueParseError(
1096 'Invalid enum value {0} for enum type {1}'.format(
1097 value, field.enum_type.full_name
1098 )
1099 ) from e
1100 if enum_value is None:
1101 if field.enum_type.is_closed:
1102 raise ParseError(
1103 'Invalid enum value {0} for enum type {1}'.format(
1104 value, field.enum_type.full_name
1105 )
1106 )
1107 else:
1108 return number
1109 return enum_value.number
1110 except EnumStringValueParseError as e:
1111 raise EnumStringValueParseError('{0} at {1}'.format(e, path)) from e
1112 except ParseError as e:
1113 raise ParseError('{0} at {1}'.format(e, path)) from e
1116def _ConvertInteger(value):
1117 """Convert an integer.
1119 Args:
1120 value: A scalar value to convert.
1122 Returns:
1123 The integer value.
1125 Raises:
1126 ParseError: If an integer couldn't be consumed.
1127 """
1128 if isinstance(value, float) and not value.is_integer():
1129 raise ParseError("Couldn't parse integer: {0}".format(value))
1131 if isinstance(value, str) and value.find(' ') != -1:
1132 raise ParseError('Couldn\'t parse integer: "{0}"'.format(value))
1134 if isinstance(value, bool):
1135 raise ParseError(
1136 'Bool value {0} is not acceptable for integer field'.format(value)
1137 )
1139 try:
1140 return int(value)
1141 except ValueError as e:
1142 # Attempt to parse as an integer-valued float.
1143 try:
1144 f = float(value)
1145 except ValueError:
1146 # Raise the original exception for the int parse.
1147 raise e # pylint: disable=raise-missing-from
1148 if not f.is_integer():
1149 raise ParseError(
1150 'Couldn\'t parse non-integer string: "{0}"'.format(value)
1151 ) from e
1152 return int(f)
1155def _ConvertFloat(value, field):
1156 """Convert an floating point number."""
1157 if isinstance(value, float):
1158 if math.isnan(value):
1159 raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead')
1160 if math.isinf(value):
1161 if value > 0:
1162 raise ParseError(
1163 "Couldn't parse Infinity or value too large, "
1164 'use quoted "Infinity" instead'
1165 )
1166 else:
1167 raise ParseError(
1168 "Couldn't parse -Infinity or value too small, "
1169 'use quoted "-Infinity" instead'
1170 )
1171 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
1172 # pylint: disable=protected-access
1173 if value > type_checkers._FLOAT_MAX:
1174 raise ParseError('Float value too large')
1175 # pylint: disable=protected-access
1176 if value < type_checkers._FLOAT_MIN:
1177 raise ParseError('Float value too small')
1178 if value == 'nan':
1179 raise ParseError('Couldn\'t parse float "nan", use "NaN" instead')
1180 try:
1181 # Assume Python compatible syntax.
1182 return float(value)
1183 except ValueError as e:
1184 # Check alternative spellings.
1185 if value == _NEG_INFINITY:
1186 return float('-inf')
1187 elif value == _INFINITY:
1188 return float('inf')
1189 elif value == _NAN:
1190 return float('nan')
1191 else:
1192 raise ParseError("Couldn't parse float: {0}".format(value)) from e
1195def _ConvertBool(value, require_str):
1196 """Convert a boolean value.
1198 Args:
1199 value: A scalar value to convert.
1200 require_str: If True, value must be a str.
1202 Returns:
1203 The bool parsed.
1205 Raises:
1206 ParseError: If a boolean value couldn't be consumed.
1207 """
1208 if require_str:
1209 if value == 'true':
1210 return True
1211 elif value == 'false':
1212 return False
1213 else:
1214 raise ParseError('Expected "true" or "false", not {0}'.format(value))
1216 if not isinstance(value, bool):
1217 raise ParseError('Expected true or false without quotes')
1218 return value
1221_WKTJSONMETHODS = {
1222 'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'],
1223 'google.protobuf.Duration': [
1224 '_GenericMessageToJsonObject',
1225 '_ConvertGenericMessage',
1226 ],
1227 'google.protobuf.FieldMask': [
1228 '_GenericMessageToJsonObject',
1229 '_ConvertGenericMessage',
1230 ],
1231 'google.protobuf.ListValue': [
1232 '_ListValueMessageToJsonObject',
1233 '_ConvertListOrTupleValueMessage',
1234 ],
1235 'google.protobuf.Struct': [
1236 '_StructMessageToJsonObject',
1237 '_ConvertStructMessage',
1238 ],
1239 'google.protobuf.Timestamp': [
1240 '_GenericMessageToJsonObject',
1241 '_ConvertGenericMessage',
1242 ],
1243 'google.protobuf.Value': [
1244 '_ValueMessageToJsonObject',
1245 '_ConvertValueMessage',
1246 ],
1247}