Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc. All rights reserved.
3#
4# Use of this source code is governed by a BSD-style
5# license that can be found in the LICENSE file or at
6# https://developers.google.com/open-source/licenses/bsd
7"""Code for decoding protocol buffer primitives.
9This code is very similar to encoder.py -- read the docs for that module first.
11A "decoder" is a function with the signature:
12 Decode(buffer, pos, end, message, field_dict)
13The arguments are:
14 buffer: The string containing the encoded message.
15 pos: The current position in the string.
16 end: The position in the string where the current message ends. May be
17 less than len(buffer) if we're reading a sub-message.
18 message: The message object into which we're parsing.
19 field_dict: message._fields (avoids a hashtable lookup).
20The decoder reads the field and stores it into field_dict, returning the new
21buffer position. A decoder for a repeated field may proactively decode all of
22the elements of that field, if they appear consecutively.
24Note that decoders may throw any of the following:
25 IndexError: Indicates a truncated message.
26 struct.error: Unpacking of a fixed-width field failed.
27 message.DecodeError: Other errors.
29Decoders are expected to raise an exception if they are called with pos > end.
30This allows callers to be lax about bounds checking: it's fineto read past
31"end" as long as you are sure that someone else will notice and throw an
32exception later on.
34Something up the call stack is expected to catch IndexError and struct.error
35and convert them to message.DecodeError.
37Decoders are constructed using decoder constructors with the signature:
38 MakeDecoder(field_number, is_repeated, is_packed, key, new_default)
39The arguments are:
40 field_number: The field number of the field we want to decode.
41 is_repeated: Is the field a repeated field? (bool)
42 is_packed: Is the field a packed field? (bool)
43 key: The key to use when looking up the field within field_dict.
44 (This is actually the FieldDescriptor but nothing in this
45 file should depend on that.)
46 new_default: A function which takes a message object as a parameter and
47 returns a new instance of the default value for this field.
48 (This is called for repeated fields and sub-messages, when an
49 instance does not already exist.)
51As with encoders, we define a decoder constructor for every type of field.
52Then, for every field of every message class we construct an actual decoder.
53That decoder goes into a dict indexed by tag, so when we decode a message
54we repeatedly read a tag, look up the corresponding decoder, and invoke it.
55"""
57__author__ = 'kenton@google.com (Kenton Varda)'
59import math
60import numbers
61import struct
63from google.protobuf import message
64from google.protobuf.internal import containers
65from google.protobuf.internal import encoder
66from google.protobuf.internal import wire_format
68# This is not for optimization, but rather to avoid conflicts with local
69# variables named "message".
70_DecodeError = message.DecodeError
73def IsDefaultScalarValue(value):
74 """Returns whether or not a scalar value is the default value of its type.
76 Specifically, this should be used to determine presence of implicit-presence
77 fields, where we disallow custom defaults.
79 Args:
80 value: A scalar value to check.
82 Returns:
83 True if the value is equivalent to a default value, False otherwise.
84 """
85 if isinstance(value, numbers.Number) and math.copysign(1.0, value) < 0:
86 # Special case for negative zero, where "truthiness" fails to give the right
87 # answer.
88 return False
90 # Normally, we can just use Python's boolean conversion.
91 return not value
94def _VarintDecoder(mask, result_type):
95 """Return an encoder for a basic varint value (does not include tag).
97 Decoded values will be bitwise-anded with the given mask before being
98 returned, e.g. to limit them to 32 bits. The returned decoder does not
99 take the usual "end" parameter -- the caller is expected to do bounds checking
100 after the fact (often the caller can defer such checking until later). The
101 decoder returns a (value, new_pos) pair.
102 """
104 def DecodeVarint(buffer, pos: int = None):
105 result = 0
106 shift = 0
107 while 1:
108 if pos is None:
109 # Read from BytesIO
110 try:
111 b = buffer.read(1)[0]
112 except IndexError as e:
113 if shift == 0:
114 # End of BytesIO.
115 return None
116 else:
117 raise ValueError('Fail to read varint %s' % str(e))
118 else:
119 b = buffer[pos]
120 pos += 1
121 result |= (b & 0x7F) << shift
122 if not (b & 0x80):
123 result &= mask
124 result = result_type(result)
125 return result if pos is None else (result, pos)
126 shift += 7
127 if shift >= 64:
128 raise _DecodeError('Too many bytes when decoding varint.')
130 return DecodeVarint
133def _SignedVarintDecoder(bits, result_type):
134 """Like _VarintDecoder() but decodes signed values."""
136 signbit = 1 << (bits - 1)
137 mask = (1 << bits) - 1
139 def DecodeVarint(buffer, pos):
140 result = 0
141 shift = 0
142 while 1:
143 b = buffer[pos]
144 result |= (b & 0x7F) << shift
145 pos += 1
146 if not (b & 0x80):
147 result &= mask
148 result = (result ^ signbit) - signbit
149 result = result_type(result)
150 return (result, pos)
151 shift += 7
152 if shift >= 64:
153 raise _DecodeError('Too many bytes when decoding varint.')
155 return DecodeVarint
158# All 32-bit and 64-bit values are represented as int.
159_DecodeVarint = _VarintDecoder((1 << 64) - 1, int)
160_DecodeSignedVarint = _SignedVarintDecoder(64, int)
162# Use these versions for values which must be limited to 32 bits.
163_DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int)
164_DecodeSignedVarint32 = _SignedVarintDecoder(32, int)
167def ReadTag(buffer, pos):
168 """Read a tag from the memoryview, and return a (tag_bytes, new_pos) tuple.
170 We return the raw bytes of the tag rather than decoding them. The raw
171 bytes can then be used to look up the proper decoder. This effectively allows
172 us to trade some work that would be done in pure-python (decoding a varint)
173 for work that is done in C (searching for a byte string in a hash table).
174 In a low-level language it would be much cheaper to decode the varint and
175 use that, but not in Python.
177 Args:
178 buffer: memoryview object of the encoded bytes
179 pos: int of the current position to start from
181 Returns:
182 Tuple[bytes, int] of the tag data and new position.
183 """
184 start = pos
185 while buffer[pos] & 0x80:
186 pos += 1
187 pos += 1
189 tag_bytes = buffer[start:pos].tobytes()
190 return tag_bytes, pos
193def DecodeTag(tag_bytes):
194 """Decode a tag from the bytes.
196 Args:
197 tag_bytes: the bytes of the tag
199 Returns:
200 Tuple[int, int] of the tag field number and wire type.
201 """
202 tag, _ = _DecodeVarint(tag_bytes, 0)
203 return wire_format.UnpackTag(tag)
206# --------------------------------------------------------------------
209def _SimpleDecoder(wire_type, decode_value):
210 """Return a constructor for a decoder for fields of a particular type.
212 Args:
213 wire_type: The field's wire type.
214 decode_value: A function which decodes an individual value, e.g.
215 _DecodeVarint()
216 """
218 def SpecificDecoder(
219 field_number,
220 is_repeated,
221 is_packed,
222 key,
223 new_default,
224 clear_if_default=False,
225 ):
226 if is_packed:
227 local_DecodeVarint = _DecodeVarint
229 def DecodePackedField(
230 buffer, pos, end, message, field_dict, current_depth=0
231 ):
232 del current_depth # unused
233 value = field_dict.get(key)
234 if value is None:
235 value = field_dict.setdefault(key, new_default(message))
236 endpoint, pos = local_DecodeVarint(buffer, pos)
237 endpoint += pos
238 if endpoint > end:
239 raise _DecodeError('Truncated message.')
240 while pos < endpoint:
241 element, pos = decode_value(buffer, pos)
242 value.append(element)
243 if pos > endpoint:
244 del value[-1] # Discard corrupt value.
245 raise _DecodeError('Packed element was truncated.')
246 return pos
248 return DecodePackedField
249 elif is_repeated:
250 tag_bytes = encoder.TagBytes(field_number, wire_type)
251 tag_len = len(tag_bytes)
253 def DecodeRepeatedField(
254 buffer, pos, end, message, field_dict, current_depth=0
255 ):
256 del current_depth # unused
257 value = field_dict.get(key)
258 if value is None:
259 value = field_dict.setdefault(key, new_default(message))
260 while 1:
261 element, new_pos = decode_value(buffer, pos)
262 value.append(element)
263 # Predict that the next tag is another copy of the same repeated
264 # field.
265 pos = new_pos + tag_len
266 if buffer[new_pos:pos] != tag_bytes or new_pos >= end:
267 # Prediction failed. Return.
268 if new_pos > end:
269 raise _DecodeError('Truncated message.')
270 return new_pos
272 return DecodeRepeatedField
273 else:
275 def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
276 del current_depth # unused
277 new_value, pos = decode_value(buffer, pos)
278 if pos > end:
279 raise _DecodeError('Truncated message.')
280 if clear_if_default and IsDefaultScalarValue(new_value):
281 field_dict.pop(key, None)
282 else:
283 field_dict[key] = new_value
284 return pos
286 return DecodeField
288 return SpecificDecoder
291def _ModifiedDecoder(wire_type, decode_value, modify_value):
292 """Like SimpleDecoder but additionally invokes modify_value on every value
294 before storing it. Usually modify_value is ZigZagDecode.
295 """
297 # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but
298 # not enough to make a significant difference.
300 def InnerDecode(buffer, pos):
301 result, new_pos = decode_value(buffer, pos)
302 return (modify_value(result), new_pos)
304 return _SimpleDecoder(wire_type, InnerDecode)
307def _StructPackDecoder(wire_type, format):
308 """Return a constructor for a decoder for a fixed-width field.
310 Args:
311 wire_type: The field's wire type.
312 format: The format string to pass to struct.unpack().
313 """
315 value_size = struct.calcsize(format)
316 local_unpack = struct.unpack
318 # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but
319 # not enough to make a significant difference.
321 # Note that we expect someone up-stack to catch struct.error and convert
322 # it to _DecodeError -- this way we don't have to set up exception-
323 # handling blocks every time we parse one value.
325 def InnerDecode(buffer, pos):
326 new_pos = pos + value_size
327 result = local_unpack(format, buffer[pos:new_pos])[0]
328 return (result, new_pos)
330 return _SimpleDecoder(wire_type, InnerDecode)
333def _FloatDecoder():
334 """Returns a decoder for a float field.
336 This code works around a bug in struct.unpack for non-finite 32-bit
337 floating-point values.
338 """
340 local_unpack = struct.unpack
342 def InnerDecode(buffer, pos):
343 """Decode serialized float to a float and new position.
345 Args:
346 buffer: memoryview of the serialized bytes
347 pos: int, position in the memory view to start at.
349 Returns:
350 Tuple[float, int] of the deserialized float value and new position
351 in the serialized data.
352 """
353 # We expect a 32-bit value in little-endian byte order. Bit 1 is the sign
354 # bit, bits 2-9 represent the exponent, and bits 10-32 are the significand.
355 new_pos = pos + 4
356 float_bytes = buffer[pos:new_pos].tobytes()
358 # If this value has all its exponent bits set, then it's non-finite.
359 # In Python 2.4, struct.unpack will convert it to a finite 64-bit value.
360 # To avoid that, we parse it specially.
361 if float_bytes[3:4] in b'\x7F\xFF' and float_bytes[2:3] >= b'\x80':
362 # If at least one significand bit is set...
363 if float_bytes[0:3] != b'\x00\x00\x80':
364 return (math.nan, new_pos)
365 # If sign bit is set...
366 if float_bytes[3:4] == b'\xFF':
367 return (-math.inf, new_pos)
368 return (math.inf, new_pos)
370 # Note that we expect someone up-stack to catch struct.error and convert
371 # it to _DecodeError -- this way we don't have to set up exception-
372 # handling blocks every time we parse one value.
373 result = local_unpack('<f', float_bytes)[0]
374 return (result, new_pos)
376 return _SimpleDecoder(wire_format.WIRETYPE_FIXED32, InnerDecode)
379def _DoubleDecoder():
380 """Returns a decoder for a double field.
382 This code works around a bug in struct.unpack for not-a-number.
383 """
385 local_unpack = struct.unpack
387 def InnerDecode(buffer, pos):
388 """Decode serialized double to a double and new position.
390 Args:
391 buffer: memoryview of the serialized bytes.
392 pos: int, position in the memory view to start at.
394 Returns:
395 Tuple[float, int] of the decoded double value and new position
396 in the serialized data.
397 """
398 # We expect a 64-bit value in little-endian byte order. Bit 1 is the sign
399 # bit, bits 2-12 represent the exponent, and bits 13-64 are the significand.
400 new_pos = pos + 8
401 double_bytes = buffer[pos:new_pos].tobytes()
403 # If this value has all its exponent bits set and at least one significand
404 # bit set, it's not a number. In Python 2.4, struct.unpack will treat it
405 # as inf or -inf. To avoid that, we treat it specially.
406 if (
407 (double_bytes[7:8] in b'\x7F\xFF')
408 and (double_bytes[6:7] >= b'\xF0')
409 and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')
410 ):
411 return (math.nan, new_pos)
413 # Note that we expect someone up-stack to catch struct.error and convert
414 # it to _DecodeError -- this way we don't have to set up exception-
415 # handling blocks every time we parse one value.
416 result = local_unpack('<d', double_bytes)[0]
417 return (result, new_pos)
419 return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode)
422def EnumDecoder(
423 field_number,
424 is_repeated,
425 is_packed,
426 key,
427 new_default,
428 clear_if_default=False,
429):
430 """Returns a decoder for enum field."""
431 enum_type = key.enum_type
432 if is_packed:
433 local_DecodeVarint = _DecodeVarint
435 def DecodePackedField(
436 buffer, pos, end, message, field_dict, current_depth=0
437 ):
438 """Decode serialized packed enum to its value and a new position.
440 Args:
441 buffer: memoryview of the serialized bytes.
442 pos: int, position in the memory view to start at.
443 end: int, end position of serialized data
444 message: Message object to store unknown fields in
445 field_dict: Map[Descriptor, Any] to store decoded values in.
447 Returns:
448 int, new position in serialized data.
449 """
450 del current_depth # unused
451 value = field_dict.get(key)
452 if value is None:
453 value = field_dict.setdefault(key, new_default(message))
454 endpoint, pos = local_DecodeVarint(buffer, pos)
455 endpoint += pos
456 if endpoint > end:
457 raise _DecodeError('Truncated message.')
458 while pos < endpoint:
459 value_start_pos = pos
460 element, pos = _DecodeSignedVarint32(buffer, pos)
461 # pylint: disable=protected-access
462 if element in enum_type.values_by_number:
463 value.append(element)
464 else:
465 if not message._unknown_fields:
466 message._unknown_fields = []
467 tag_bytes = encoder.TagBytes(
468 field_number, wire_format.WIRETYPE_VARINT
469 )
471 message._unknown_fields.append(
472 (tag_bytes, buffer[value_start_pos:pos].tobytes())
473 )
474 # pylint: enable=protected-access
475 if pos > endpoint:
476 if element in enum_type.values_by_number:
477 del value[-1] # Discard corrupt value.
478 else:
479 del message._unknown_fields[-1]
480 # pylint: enable=protected-access
481 raise _DecodeError('Packed element was truncated.')
482 return pos
484 return DecodePackedField
485 elif is_repeated:
486 tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT)
487 tag_len = len(tag_bytes)
489 def DecodeRepeatedField(
490 buffer, pos, end, message, field_dict, current_depth=0
491 ):
492 """Decode serialized repeated enum to its value and a new position.
494 Args:
495 buffer: memoryview of the serialized bytes.
496 pos: int, position in the memory view to start at.
497 end: int, end position of serialized data
498 message: Message object to store unknown fields in
499 field_dict: Map[Descriptor, Any] to store decoded values in.
501 Returns:
502 int, new position in serialized data.
503 """
504 del current_depth # unused
505 value = field_dict.get(key)
506 if value is None:
507 value = field_dict.setdefault(key, new_default(message))
508 while 1:
509 element, new_pos = _DecodeSignedVarint32(buffer, pos)
510 # pylint: disable=protected-access
511 if element in enum_type.values_by_number:
512 value.append(element)
513 else:
514 if not message._unknown_fields:
515 message._unknown_fields = []
516 message._unknown_fields.append(
517 (tag_bytes, buffer[pos:new_pos].tobytes())
518 )
519 # pylint: enable=protected-access
520 # Predict that the next tag is another copy of the same repeated
521 # field.
522 pos = new_pos + tag_len
523 if buffer[new_pos:pos] != tag_bytes or new_pos >= end:
524 # Prediction failed. Return.
525 if new_pos > end:
526 raise _DecodeError('Truncated message.')
527 return new_pos
529 return DecodeRepeatedField
530 else:
532 def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
533 """Decode serialized repeated enum to its value and a new position.
535 Args:
536 buffer: memoryview of the serialized bytes.
537 pos: int, position in the memory view to start at.
538 end: int, end position of serialized data
539 message: Message object to store unknown fields in
540 field_dict: Map[Descriptor, Any] to store decoded values in.
542 Returns:
543 int, new position in serialized data.
544 """
545 del current_depth # unused
546 value_start_pos = pos
547 enum_value, pos = _DecodeSignedVarint32(buffer, pos)
548 if pos > end:
549 raise _DecodeError('Truncated message.')
550 if clear_if_default and IsDefaultScalarValue(enum_value):
551 field_dict.pop(key, None)
552 return pos
553 # pylint: disable=protected-access
554 if enum_value in enum_type.values_by_number:
555 field_dict[key] = enum_value
556 else:
557 if not message._unknown_fields:
558 message._unknown_fields = []
559 tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT)
560 message._unknown_fields.append(
561 (tag_bytes, buffer[value_start_pos:pos].tobytes())
562 )
563 # pylint: enable=protected-access
564 return pos
566 return DecodeField
569# --------------------------------------------------------------------
571Int32Decoder = _SimpleDecoder(
572 wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32
573)
575Int64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeSignedVarint)
577UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32)
578UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint)
580SInt32Decoder = _ModifiedDecoder(
581 wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode
582)
583SInt64Decoder = _ModifiedDecoder(
584 wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode
585)
587# Note that Python conveniently guarantees that when using the '<' prefix on
588# formats, they will also have the same size across all platforms (as opposed
589# to without the prefix, where their sizes depend on the C compiler's basic
590# type sizes).
591Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<I')
592Fixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<Q')
593SFixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<i')
594SFixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<q')
595FloatDecoder = _FloatDecoder()
596DoubleDecoder = _DoubleDecoder()
598BoolDecoder = _ModifiedDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint, bool)
601def StringDecoder(
602 field_number,
603 is_repeated,
604 is_packed,
605 key,
606 new_default,
607 clear_if_default=False,
608):
609 """Returns a decoder for a string field."""
611 local_DecodeVarint = _DecodeVarint
613 def _ConvertToUnicode(memview):
614 """Convert byte to unicode."""
615 byte_str = memview.tobytes()
616 try:
617 value = str(byte_str, 'utf-8')
618 except UnicodeDecodeError as e:
619 # add more information to the error message and re-raise it.
620 e.reason = '%s in field: %s' % (e, key.full_name)
621 raise
623 return value
625 assert not is_packed
626 if is_repeated:
627 tag_bytes = encoder.TagBytes(
628 field_number, wire_format.WIRETYPE_LENGTH_DELIMITED
629 )
630 tag_len = len(tag_bytes)
632 def DecodeRepeatedField(
633 buffer, pos, end, message, field_dict, current_depth=0
634 ):
635 del current_depth # unused
636 value = field_dict.get(key)
637 if value is None:
638 value = field_dict.setdefault(key, new_default(message))
639 while 1:
640 size, pos = local_DecodeVarint(buffer, pos)
641 new_pos = pos + size
642 if new_pos > end:
643 raise _DecodeError('Truncated string.')
644 value.append(_ConvertToUnicode(buffer[pos:new_pos]))
645 # Predict that the next tag is another copy of the same repeated field.
646 pos = new_pos + tag_len
647 if buffer[new_pos:pos] != tag_bytes or new_pos == end:
648 # Prediction failed. Return.
649 return new_pos
651 return DecodeRepeatedField
652 else:
654 def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
655 del current_depth # unused
656 size, pos = local_DecodeVarint(buffer, pos)
657 new_pos = pos + size
658 if new_pos > end:
659 raise _DecodeError('Truncated string.')
660 if clear_if_default and IsDefaultScalarValue(size):
661 field_dict.pop(key, None)
662 else:
663 field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos])
664 return new_pos
666 return DecodeField
669def BytesDecoder(
670 field_number,
671 is_repeated,
672 is_packed,
673 key,
674 new_default,
675 clear_if_default=False,
676):
677 """Returns a decoder for a bytes field."""
679 local_DecodeVarint = _DecodeVarint
681 assert not is_packed
682 if is_repeated:
683 tag_bytes = encoder.TagBytes(
684 field_number, wire_format.WIRETYPE_LENGTH_DELIMITED
685 )
686 tag_len = len(tag_bytes)
688 def DecodeRepeatedField(
689 buffer, pos, end, message, field_dict, current_depth=0
690 ):
691 del current_depth # unused
692 value = field_dict.get(key)
693 if value is None:
694 value = field_dict.setdefault(key, new_default(message))
695 while 1:
696 size, pos = local_DecodeVarint(buffer, pos)
697 new_pos = pos + size
698 if new_pos > end:
699 raise _DecodeError('Truncated string.')
700 value.append(buffer[pos:new_pos].tobytes())
701 # Predict that the next tag is another copy of the same repeated field.
702 pos = new_pos + tag_len
703 if buffer[new_pos:pos] != tag_bytes or new_pos == end:
704 # Prediction failed. Return.
705 return new_pos
707 return DecodeRepeatedField
708 else:
710 def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
711 del current_depth # unused
712 size, pos = local_DecodeVarint(buffer, pos)
713 new_pos = pos + size
714 if new_pos > end:
715 raise _DecodeError('Truncated string.')
716 if clear_if_default and IsDefaultScalarValue(size):
717 field_dict.pop(key, None)
718 else:
719 field_dict[key] = buffer[pos:new_pos].tobytes()
720 return new_pos
722 return DecodeField
725def GroupDecoder(field_number, is_repeated, is_packed, key, new_default):
726 """Returns a decoder for a group field."""
728 end_tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_END_GROUP)
729 end_tag_len = len(end_tag_bytes)
731 assert not is_packed
732 if is_repeated:
733 tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_START_GROUP)
734 tag_len = len(tag_bytes)
736 def DecodeRepeatedField(
737 buffer, pos, end, message, field_dict, current_depth=0
738 ):
739 value = field_dict.get(key)
740 if value is None:
741 value = field_dict.setdefault(key, new_default(message))
742 while 1:
743 value = field_dict.get(key)
744 if value is None:
745 value = field_dict.setdefault(key, new_default(message))
746 # Read sub-message.
747 current_depth += 1
748 if current_depth > _recursion_limit:
749 raise _DecodeError(
750 'Error parsing message: too many levels of nesting.'
751 )
752 pos = value.add()._InternalParse(buffer, pos, end, current_depth)
753 current_depth -= 1
754 # Read end tag.
755 new_pos = pos + end_tag_len
756 if buffer[pos:new_pos] != end_tag_bytes or new_pos > end:
757 raise _DecodeError('Missing group end tag.')
758 # Predict that the next tag is another copy of the same repeated field.
759 pos = new_pos + tag_len
760 if buffer[new_pos:pos] != tag_bytes or new_pos == end:
761 # Prediction failed. Return.
762 return new_pos
764 return DecodeRepeatedField
765 else:
767 def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
768 value = field_dict.get(key)
769 if value is None:
770 value = field_dict.setdefault(key, new_default(message))
771 # Read sub-message.
772 current_depth += 1
773 if current_depth > _recursion_limit:
774 raise _DecodeError('Error parsing message: too many levels of nesting.')
775 pos = value._InternalParse(buffer, pos, end, current_depth)
776 current_depth -= 1
777 # Read end tag.
778 new_pos = pos + end_tag_len
779 if buffer[pos:new_pos] != end_tag_bytes or new_pos > end:
780 raise _DecodeError('Missing group end tag.')
781 return new_pos
783 return DecodeField
786def MessageDecoder(field_number, is_repeated, is_packed, key, new_default):
787 """Returns a decoder for a message field."""
789 local_DecodeVarint = _DecodeVarint
791 assert not is_packed
792 if is_repeated:
793 tag_bytes = encoder.TagBytes(
794 field_number, wire_format.WIRETYPE_LENGTH_DELIMITED
795 )
796 tag_len = len(tag_bytes)
798 def DecodeRepeatedField(
799 buffer, pos, end, message, field_dict, current_depth=0
800 ):
801 value = field_dict.get(key)
802 if value is None:
803 value = field_dict.setdefault(key, new_default(message))
804 while 1:
805 # Read length.
806 size, pos = local_DecodeVarint(buffer, pos)
807 new_pos = pos + size
808 if new_pos > end:
809 raise _DecodeError('Truncated message.')
810 # Read sub-message.
811 current_depth += 1
812 if current_depth > _recursion_limit:
813 raise _DecodeError(
814 'Error parsing message: too many levels of nesting.'
815 )
816 if (
817 value.add()._InternalParse(buffer, pos, new_pos, current_depth)
818 != new_pos
819 ):
820 # The only reason _InternalParse would return early is if it
821 # encountered an end-group tag.
822 raise _DecodeError('Unexpected end-group tag.')
823 current_depth -= 1
824 # Predict that the next tag is another copy of the same repeated field.
825 pos = new_pos + tag_len
826 if buffer[new_pos:pos] != tag_bytes or new_pos == end:
827 # Prediction failed. Return.
828 return new_pos
830 return DecodeRepeatedField
831 else:
833 def DecodeField(buffer, pos, end, message, field_dict, current_depth=0):
834 value = field_dict.get(key)
835 if value is None:
836 value = field_dict.setdefault(key, new_default(message))
837 # Read length.
838 size, pos = local_DecodeVarint(buffer, pos)
839 new_pos = pos + size
840 if new_pos > end:
841 raise _DecodeError('Truncated message.')
842 # Read sub-message.
843 current_depth += 1
844 if current_depth > _recursion_limit:
845 raise _DecodeError('Error parsing message: too many levels of nesting.')
846 if value._InternalParse(buffer, pos, new_pos, current_depth) != new_pos:
847 # The only reason _InternalParse would return early is if it encountered
848 # an end-group tag.
849 raise _DecodeError('Unexpected end-group tag.')
850 current_depth -= 1
851 return new_pos
853 return DecodeField
856# --------------------------------------------------------------------
858MESSAGE_SET_ITEM_TAG = encoder.TagBytes(1, wire_format.WIRETYPE_START_GROUP)
861def MessageSetItemDecoder(descriptor):
862 """Returns a decoder for a MessageSet item.
864 The parameter is the message Descriptor.
866 The message set message looks like this:
867 message MessageSet {
868 repeated group Item = 1 {
869 required int32 type_id = 2;
870 required string message = 3;
871 }
872 }
873 """
875 type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT)
876 message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)
877 item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP)
879 local_ReadTag = ReadTag
880 local_DecodeVarint = _DecodeVarint
882 def DecodeItem(buffer, pos, end, message, field_dict, current_depth=0):
883 """Decode serialized message set to its value and new position.
885 Args:
886 buffer: memoryview of the serialized bytes.
887 pos: int, position in the memory view to start at.
888 end: int, end position of serialized data
889 message: Message object to store unknown fields in
890 field_dict: Map[Descriptor, Any] to store decoded values in.
892 Returns:
893 int, new position in serialized data.
894 """
895 message_set_item_start = pos
896 type_id = -1
897 message_start = -1
898 message_end = -1
900 # Technically, type_id and message can appear in any order, so we need
901 # a little loop here.
902 while 1:
903 tag_bytes, pos = local_ReadTag(buffer, pos)
904 if tag_bytes == type_id_tag_bytes:
905 type_id, pos = local_DecodeVarint(buffer, pos)
906 elif tag_bytes == message_tag_bytes:
907 size, message_start = local_DecodeVarint(buffer, pos)
908 pos = message_end = message_start + size
909 elif tag_bytes == item_end_tag_bytes:
910 break
911 else:
912 field_number, wire_type = DecodeTag(tag_bytes)
913 _, pos = _DecodeUnknownField(buffer, pos, end, field_number, wire_type)
914 if pos == -1:
915 raise _DecodeError('Unexpected end-group tag.')
917 if pos > end:
918 raise _DecodeError('Truncated message.')
920 if type_id == -1:
921 raise _DecodeError('MessageSet item missing type_id.')
922 if message_start == -1:
923 raise _DecodeError('MessageSet item missing message.')
925 extension = message.Extensions._FindExtensionByNumber(type_id)
926 # pylint: disable=protected-access
927 if extension is not None:
928 value = field_dict.get(extension)
929 if value is None:
930 message_type = extension.message_type
931 if not hasattr(message_type, '_concrete_class'):
932 message_factory.GetMessageClass(message_type)
933 value = field_dict.setdefault(extension, message_type._concrete_class())
934 current_depth += 1
935 if current_depth > _recursion_limit:
936 raise _DecodeError('Error parsing message: too many levels of nesting.')
937 if (
938 value._InternalParse(
939 buffer, message_start, message_end, current_depth
940 )
941 != message_end
942 ):
943 # The only reason _InternalParse would return early is if it encountered
944 # an end-group tag.
945 raise _DecodeError('Unexpected end-group tag.')
946 current_depth -= 1
947 else:
948 if not message._unknown_fields:
949 message._unknown_fields = []
950 message._unknown_fields.append(
951 (MESSAGE_SET_ITEM_TAG, buffer[message_set_item_start:pos].tobytes())
952 )
953 # pylint: enable=protected-access
955 return pos
957 return DecodeItem
960def UnknownMessageSetItemDecoder():
961 """Returns a decoder for a Unknown MessageSet item."""
963 type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT)
964 message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)
965 item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP)
967 def DecodeUnknownItem(buffer):
968 pos = 0
969 end = len(buffer)
970 message_start = -1
971 message_end = -1
972 while 1:
973 tag_bytes, pos = ReadTag(buffer, pos)
974 if tag_bytes == type_id_tag_bytes:
975 type_id, pos = _DecodeVarint(buffer, pos)
976 elif tag_bytes == message_tag_bytes:
977 size, message_start = _DecodeVarint(buffer, pos)
978 pos = message_end = message_start + size
979 elif tag_bytes == item_end_tag_bytes:
980 break
981 else:
982 field_number, wire_type = DecodeTag(tag_bytes)
983 _, pos = _DecodeUnknownField(buffer, pos, end, field_number, wire_type)
984 if pos == -1:
985 raise _DecodeError('Unexpected end-group tag.')
987 if pos > end:
988 raise _DecodeError('Truncated message.')
990 if type_id == -1:
991 raise _DecodeError('MessageSet item missing type_id.')
992 if message_start == -1:
993 raise _DecodeError('MessageSet item missing message.')
995 return (type_id, buffer[message_start:message_end].tobytes())
997 return DecodeUnknownItem
1000# --------------------------------------------------------------------
1003def MapDecoder(field_descriptor, new_default, is_message_map):
1004 """Returns a decoder for a map field."""
1006 key = field_descriptor
1007 tag_bytes = encoder.TagBytes(
1008 field_descriptor.number, wire_format.WIRETYPE_LENGTH_DELIMITED
1009 )
1010 tag_len = len(tag_bytes)
1011 local_DecodeVarint = _DecodeVarint
1012 # Can't read _concrete_class yet; might not be initialized.
1013 message_type = field_descriptor.message_type
1015 def DecodeMap(buffer, pos, end, message, field_dict, current_depth=0):
1016 submsg = message_type._concrete_class()
1017 value = field_dict.get(key)
1018 if value is None:
1019 value = field_dict.setdefault(key, new_default(message))
1020 while 1:
1021 # Read length.
1022 size, pos = local_DecodeVarint(buffer, pos)
1023 new_pos = pos + size
1024 if new_pos > end:
1025 raise _DecodeError('Truncated message.')
1026 # Read sub-message.
1027 submsg.Clear()
1028 current_depth += 1
1029 if current_depth > _recursion_limit:
1030 raise _DecodeError('Error parsing message: too many levels of nesting.')
1031 if submsg._InternalParse(buffer, pos, new_pos, current_depth) != new_pos:
1032 # The only reason _InternalParse would return early is if it
1033 # encountered an end-group tag.
1034 raise _DecodeError('Unexpected end-group tag.')
1035 current_depth -= 1
1037 if is_message_map:
1038 value[submsg.key].CopyFrom(submsg.value)
1039 else:
1040 value[submsg.key] = submsg.value
1042 # Predict that the next tag is another copy of the same repeated field.
1043 pos = new_pos + tag_len
1044 if buffer[new_pos:pos] != tag_bytes or new_pos == end:
1045 # Prediction failed. Return.
1046 return new_pos
1048 return DecodeMap
1051def _DecodeFixed64(buffer, pos):
1052 """Decode a fixed64."""
1053 new_pos = pos + 8
1054 return (struct.unpack('<Q', buffer[pos:new_pos])[0], new_pos)
1057def _DecodeFixed32(buffer, pos):
1058 """Decode a fixed32."""
1060 new_pos = pos + 4
1061 return (struct.unpack('<I', buffer[pos:new_pos])[0], new_pos)
1064DEFAULT_RECURSION_LIMIT = 100
1065_recursion_limit = DEFAULT_RECURSION_LIMIT
1068def SetRecursionLimit(new_limit):
1069 global _recursion_limit
1070 _recursion_limit = new_limit
1073def _DecodeUnknownFieldSet(buffer, pos, end_pos=None, current_depth=0):
1074 """Decode UnknownFieldSet. Returns the UnknownFieldSet and new position."""
1076 unknown_field_set = containers.UnknownFieldSet()
1077 while end_pos is None or pos < end_pos:
1078 tag_bytes, pos = ReadTag(buffer, pos)
1079 tag, _ = _DecodeVarint(tag_bytes, 0)
1080 field_number, wire_type = wire_format.UnpackTag(tag)
1081 if wire_type == wire_format.WIRETYPE_END_GROUP:
1082 break
1083 data, pos = _DecodeUnknownField(
1084 buffer, pos, end_pos, field_number, wire_type, current_depth
1085 )
1086 # pylint: disable=protected-access
1087 unknown_field_set._add(field_number, wire_type, data)
1089 return (unknown_field_set, pos)
1092def _DecodeUnknownField(
1093 buffer, pos, end_pos, field_number, wire_type, current_depth=0
1094):
1095 """Decode a unknown field. Returns the UnknownField and new position."""
1097 if wire_type == wire_format.WIRETYPE_VARINT:
1098 data, pos = _DecodeVarint(buffer, pos)
1099 elif wire_type == wire_format.WIRETYPE_FIXED64:
1100 data, pos = _DecodeFixed64(buffer, pos)
1101 elif wire_type == wire_format.WIRETYPE_FIXED32:
1102 data, pos = _DecodeFixed32(buffer, pos)
1103 elif wire_type == wire_format.WIRETYPE_LENGTH_DELIMITED:
1104 size, pos = _DecodeVarint(buffer, pos)
1105 data = buffer[pos : pos + size].tobytes()
1106 pos += size
1107 elif wire_type == wire_format.WIRETYPE_START_GROUP:
1108 end_tag_bytes = encoder.TagBytes(
1109 field_number, wire_format.WIRETYPE_END_GROUP
1110 )
1111 current_depth += 1
1112 if current_depth >= _recursion_limit:
1113 raise _DecodeError('Error parsing message: too many levels of nesting.')
1114 data, pos = _DecodeUnknownFieldSet(buffer, pos, end_pos, current_depth)
1115 current_depth -= 1
1116 # Check end tag.
1117 if buffer[pos - len(end_tag_bytes) : pos] != end_tag_bytes:
1118 raise _DecodeError('Missing group end tag.')
1119 elif wire_type == wire_format.WIRETYPE_END_GROUP:
1120 return (0, -1)
1121 else:
1122 raise _DecodeError('Wrong wire type in tag.')
1124 if pos > end_pos:
1125 raise _DecodeError('Truncated message.')
1127 return (data, pos)