Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc. All rights reserved.
3#
4# Use of this source code is governed by a BSD-style
5# license that can be found in the LICENSE file or at
6# https://developers.google.com/open-source/licenses/bsd
7"""Contains routines for printing messages in Protobuf Text Format.
9Printing and parsing messages in Text Format is useful for debugging
10and human editing of messages.
12Unlike the Binary and ProtoJSON formats, Text Format is not designed to be
13used as a wire format; instead it is intended for human-in-the-loop
14configuration use-cases.
16Systems processing untrusted inputs should strongly prefer to use Binary format
17instead. If a textual format of untrusted inputs is required, consider using
18ProtoJSON format instead.
20Simple usage example::
22 # Create a proto object and serialize it to a text proto string.
23 message = my_proto_pb2.MyMessage(foo='bar')
24 text_proto = text_format.MessageToString(message)
26 # Parse a text proto string.
27 message = text_format.Parse(text_proto, my_proto_pb2.MyMessage())
28"""
30__author__ = 'kenton@google.com (Kenton Varda)'
32# TODO Import thread contention leads to test failures.
33import encodings.raw_unicode_escape # pylint: disable=unused-import
34import encodings.unicode_escape # pylint: disable=unused-import
35import io
36import math
37import re
38import warnings
40from google.protobuf.internal import decoder
41from google.protobuf.internal import type_checkers
42from google.protobuf import descriptor
44from google.protobuf import text_encoding
45from google.protobuf import unknown_fields
47# pylint: disable=g-import-not-at-top
48__all__ = [
49 'MessageToString',
50 'Parse',
51 'PrintMessage',
52 'PrintField',
53 'PrintFieldValue',
54 'Merge',
55 'MessageToBytes',
56]
58_INTEGER_CHECKERS = (
59 type_checkers.Uint32ValueChecker(),
60 type_checkers.Int32ValueChecker(),
61 type_checkers.Uint64ValueChecker(),
62 type_checkers.Int64ValueChecker(),
63)
64_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?$', re.IGNORECASE)
65_FLOAT_NAN = re.compile('nanf?$', re.IGNORECASE)
66_FLOAT_OCTAL_PREFIX = re.compile('-?0[0-9]+')
67_PERCENT_ENCODING = re.compile(r'^%[\da-fA-F][\da-fA-F]$')
68_TYPE_NAME = re.compile(r'^[^\d\W]\w*(\.[^\d\W]\w*)*$')
69_QUOTES = frozenset(("'", '"'))
70_ANY_FULL_TYPE_NAME = 'google.protobuf.Any'
71_DEBUG_STRING_SILENT_MARKER = '\t '
73_as_utf8_default = True
76class Error(Exception):
77 """Top-level module error for text_format."""
80class ParseError(Error):
81 """Thrown in case of text parsing or tokenizing error."""
83 def __init__(self, message=None, line=None, column=None):
84 if message is not None and line is not None:
85 loc = str(line)
86 if column is not None:
87 loc += ':{0}'.format(column)
88 message = '{0} : {1}'.format(loc, message)
89 if message is not None:
90 super(ParseError, self).__init__(message)
91 else:
92 super(ParseError, self).__init__()
93 self._line = line
94 self._column = column
96 def GetLine(self):
97 return self._line
99 def GetColumn(self):
100 return self._column
103class TextWriter(object):
105 def __init__(self, as_utf8):
106 self._writer = io.StringIO()
108 def write(self, val):
109 return self._writer.write(val)
111 def close(self):
112 return self._writer.close()
114 def getvalue(self):
115 return self._writer.getvalue()
118def MessageToString(
119 message,
120 as_utf8=_as_utf8_default,
121 as_one_line=False,
122 use_short_repeated_primitives=False,
123 pointy_brackets=False,
124 use_index_order=False,
125 use_field_number=False,
126 descriptor_pool=None,
127 indent=0,
128 message_formatter=None,
129 print_unknown_fields=False,
130 force_colon=False,
131) -> str:
132 """Convert protobuf message to text format.
134 Args:
135 message: The protocol buffers message.
136 as_utf8: Return unescaped Unicode for non-ASCII characters.
137 as_one_line: Don't introduce newlines between fields.
138 use_short_repeated_primitives: Use short repeated format for primitives.
139 pointy_brackets: If True, use angle brackets instead of curly braces for
140 nesting.
141 use_index_order: If True, fields of a proto message will be printed using
142 the order defined in source code instead of the field number, extensions
143 will be printed at the end of the message and their relative order is
144 determined by the extension number. By default, use the field number
145 order.
146 use_field_number: If True, print field numbers instead of names.
147 descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.
148 indent (int): The initial indent level, in terms of spaces, for pretty
149 print.
150 message_formatter (function(message, indent, as_one_line) -> unicode|None):
151 Custom formatter for selected sub-messages (usually based on message
152 type). Use to pretty print parts of the protobuf for easier diffing.
153 print_unknown_fields: If True, unknown fields will be printed.
154 force_colon: If set, a colon will be added after the field name even if the
155 field is a proto message.
157 Returns:
158 str: A string of the text formatted protocol buffer message.
159 """
160 out = TextWriter(as_utf8)
161 printer = _Printer(
162 out=out,
163 indent=indent,
164 as_utf8=as_utf8,
165 as_one_line=as_one_line,
166 use_short_repeated_primitives=use_short_repeated_primitives,
167 pointy_brackets=pointy_brackets,
168 use_index_order=use_index_order,
169 use_field_number=use_field_number,
170 descriptor_pool=descriptor_pool,
171 message_formatter=message_formatter,
172 print_unknown_fields=print_unknown_fields,
173 force_colon=force_colon,
174 )
175 printer.PrintMessage(message)
176 result = out.getvalue()
177 out.close()
178 if as_one_line:
179 return result.rstrip()
180 return result
183def MessageToBytes(message, **kwargs) -> bytes:
184 """Convert protobuf message to encoded text format. See MessageToString."""
185 text = MessageToString(message, **kwargs)
186 if isinstance(text, bytes):
187 return text
188 codec = 'utf-8' if kwargs.get('as_utf8') else 'ascii'
189 return text.encode(codec)
192def _IsMapEntry(field):
193 return (
194 field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
195 and field.message_type.has_options
196 and field.message_type.GetOptions().map_entry
197 )
200def _IsGroupLike(field):
201 """Determines if a field is consistent with a proto2 group.
203 Args:
204 field: The field descriptor.
206 Returns:
207 True if this field is group-like, false otherwise.
208 """
209 # Groups are always tag-delimited.
210 if field.type != descriptor.FieldDescriptor.TYPE_GROUP:
211 return False
213 # Group fields always are always the lowercase type name.
214 if field.name != field.message_type.name.lower():
215 return False
217 if field.message_type.file != field.file:
218 return False
220 # Group messages are always defined in the same scope as the field. File
221 # level extensions will compare NULL == NULL here, which is why the file
222 # comparison above is necessary to ensure both come from the same file.
223 return (
224 field.message_type.containing_type == field.extension_scope
225 if field.is_extension
226 else field.message_type.containing_type == field.containing_type
227 )
230def PrintMessage(
231 message,
232 out,
233 indent=0,
234 as_utf8=_as_utf8_default,
235 as_one_line=False,
236 use_short_repeated_primitives=False,
237 pointy_brackets=False,
238 use_index_order=False,
239 use_field_number=False,
240 descriptor_pool=None,
241 message_formatter=None,
242 print_unknown_fields=False,
243 force_colon=False,
244):
245 """Convert the message to text format and write it to the out stream.
247 Args:
248 message: The Message object to convert to text format.
249 out: A file handle to write the message to.
250 indent: The initial indent level for pretty print.
251 as_utf8: Return unescaped Unicode for non-ASCII characters.
252 as_one_line: Don't introduce newlines between fields.
253 use_short_repeated_primitives: Use short repeated format for primitives.
254 pointy_brackets: If True, use angle brackets instead of curly braces for
255 nesting.
256 use_index_order: If True, print fields of a proto message using the order
257 defined in source code instead of the field number. By default, use the
258 field number order.
259 use_field_number: If True, print field numbers instead of names.
260 descriptor_pool: A DescriptorPool used to resolve Any types.
261 message_formatter: A function(message, indent, as_one_line): unicode|None to
262 custom format selected sub-messages (usually based on message type). Use
263 to pretty print parts of the protobuf for easier diffing.
264 print_unknown_fields: If True, unknown fields will be printed.
265 force_colon: If set, a colon will be added after the field name even if the
266 field is a proto message.
267 """
268 printer = _Printer(
269 out=out,
270 indent=indent,
271 as_utf8=as_utf8,
272 as_one_line=as_one_line,
273 use_short_repeated_primitives=use_short_repeated_primitives,
274 pointy_brackets=pointy_brackets,
275 use_index_order=use_index_order,
276 use_field_number=use_field_number,
277 descriptor_pool=descriptor_pool,
278 message_formatter=message_formatter,
279 print_unknown_fields=print_unknown_fields,
280 force_colon=force_colon,
281 )
282 printer.PrintMessage(message)
285def PrintField(
286 field,
287 value,
288 out,
289 indent=0,
290 as_utf8=_as_utf8_default,
291 as_one_line=False,
292 use_short_repeated_primitives=False,
293 pointy_brackets=False,
294 use_index_order=False,
295 message_formatter=None,
296 print_unknown_fields=False,
297 force_colon=False,
298):
299 """Print a single field name/value pair."""
300 printer = _Printer(
301 out,
302 indent,
303 as_utf8,
304 as_one_line,
305 use_short_repeated_primitives,
306 pointy_brackets,
307 use_index_order,
308 message_formatter=message_formatter,
309 print_unknown_fields=print_unknown_fields,
310 force_colon=force_colon,
311 )
312 printer.PrintField(field, value)
315def PrintFieldValue(
316 field,
317 value,
318 out,
319 indent=0,
320 as_utf8=_as_utf8_default,
321 as_one_line=False,
322 use_short_repeated_primitives=False,
323 pointy_brackets=False,
324 use_index_order=False,
325 message_formatter=None,
326 print_unknown_fields=False,
327 force_colon=False,
328):
329 """Print a single field value (not including name)."""
330 printer = _Printer(
331 out,
332 indent,
333 as_utf8,
334 as_one_line,
335 use_short_repeated_primitives,
336 pointy_brackets,
337 use_index_order,
338 message_formatter=message_formatter,
339 print_unknown_fields=print_unknown_fields,
340 force_colon=force_colon,
341 )
342 printer.PrintFieldValue(field, value)
345def _BuildMessageFromTypeName(type_name, descriptor_pool):
346 """Returns a protobuf message instance.
348 Args:
349 type_name: Fully-qualified protobuf message type name string.
350 descriptor_pool: DescriptorPool instance.
352 Returns:
353 A Message instance of type matching type_name, or None if the a Descriptor
354 wasn't found matching type_name.
355 """
356 # pylint: disable=g-import-not-at-top
357 if descriptor_pool is None:
358 from google.protobuf import descriptor_pool as pool_mod
360 descriptor_pool = pool_mod.Default()
361 from google.protobuf import message_factory
363 try:
364 message_descriptor = descriptor_pool.FindMessageTypeByName(type_name)
365 except KeyError:
366 return None
367 message_type = message_factory.GetMessageClass(message_descriptor)
368 return message_type()
371# These values must match WireType enum in //google/protobuf/wire_format.h.
372WIRETYPE_LENGTH_DELIMITED = 2
373WIRETYPE_START_GROUP = 3
376class _Printer(object):
377 """Text format printer for protocol message."""
379 def __init__(
380 self,
381 out,
382 indent=0,
383 as_utf8=_as_utf8_default,
384 as_one_line=False,
385 use_short_repeated_primitives=False,
386 pointy_brackets=False,
387 use_index_order=False,
388 use_field_number=False,
389 descriptor_pool=None,
390 message_formatter=None,
391 print_unknown_fields=False,
392 force_colon=False,
393 ):
394 """Initialize the Printer.
396 Args:
397 out: To record the text format result.
398 indent: The initial indent level for pretty print.
399 as_utf8: Return unescaped Unicode for non-ASCII characters.
400 as_one_line: Don't introduce newlines between fields.
401 use_short_repeated_primitives: Use short repeated format for primitives.
402 pointy_brackets: If True, use angle brackets instead of curly braces for
403 nesting.
404 use_index_order: If True, print fields of a proto message using the order
405 defined in source code instead of the field number. By default, use the
406 field number order.
407 use_field_number: If True, print field numbers instead of names.
408 descriptor_pool: A DescriptorPool used to resolve Any types.
409 message_formatter: A function(message, indent, as_one_line): unicode|None
410 to custom format selected sub-messages (usually based on message type).
411 Use to pretty print parts of the protobuf for easier diffing.
412 print_unknown_fields: If True, unknown fields will be printed.
413 force_colon: If set, a colon will be added after the field name even if
414 the field is a proto message.
415 """
416 self.out = out
417 self.indent = indent
418 self.as_utf8 = as_utf8
419 self.as_one_line = as_one_line
420 self.use_short_repeated_primitives = use_short_repeated_primitives
421 self.pointy_brackets = pointy_brackets
422 self.use_index_order = use_index_order
423 self.use_field_number = use_field_number
424 self.descriptor_pool = descriptor_pool
425 self.message_formatter = message_formatter
426 self.print_unknown_fields = print_unknown_fields
427 self.force_colon = force_colon
429 def _TryPrintAsAnyMessage(self, message):
430 """Serializes if message is a google.protobuf.Any field."""
431 if '/' not in message.type_url:
432 return False
433 packed_message = _BuildMessageFromTypeName(
434 message.TypeName(), self.descriptor_pool
435 )
436 if packed_message is not None:
437 packed_message.MergeFromString(message.value)
438 colon = ':' if self.force_colon else ''
439 self.out.write('%s[%s]%s ' % (self.indent * ' ', message.type_url, colon))
440 self._PrintMessageFieldValue(packed_message)
441 self.out.write(' ' if self.as_one_line else '\n')
442 return True
443 else:
444 return False
446 def _TryCustomFormatMessage(self, message):
447 formatted = self.message_formatter(message, self.indent, self.as_one_line)
448 if formatted is None:
449 return False
451 out = self.out
452 out.write(' ' * self.indent)
453 out.write(formatted)
454 out.write(' ' if self.as_one_line else '\n')
455 return True
457 def PrintMessage(self, message):
458 """Convert protobuf message to text format.
460 Args:
461 message: The protocol buffers message.
462 """
463 if self.message_formatter and self._TryCustomFormatMessage(message):
464 return
465 if (
466 message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME
467 and self._TryPrintAsAnyMessage(message)
468 ):
469 return
470 fields = message.ListFields()
471 if self.use_index_order:
472 fields.sort(
473 key=lambda x: x[0].number if x[0].is_extension else x[0].index
474 )
475 for field, value in fields:
476 if _IsMapEntry(field):
477 for key in sorted(value):
478 # This is slow for maps with submessage entries because it copies the
479 # entire tree. Unfortunately this would take significant refactoring
480 # of this file to work around.
481 #
482 # TODO: refactor and optimize if this becomes an issue.
483 entry_submsg = value.GetEntryClass()(key=key, value=value[key])
484 self.PrintField(field, entry_submsg)
485 elif field.is_repeated:
486 if (
487 self.use_short_repeated_primitives
488 and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE
489 and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_STRING
490 ):
491 self._PrintShortRepeatedPrimitivesValue(field, value)
492 else:
493 for element in value:
494 self.PrintField(field, element)
495 else:
496 self.PrintField(field, value)
498 if self.print_unknown_fields:
499 self._PrintUnknownFields(unknown_fields.UnknownFieldSet(message))
501 def _PrintUnknownFields(self, unknown_field_set):
502 """Print unknown fields."""
503 out = self.out
504 for field in unknown_field_set:
505 out.write(' ' * self.indent)
506 out.write(str(field.field_number))
507 if field.wire_type == WIRETYPE_START_GROUP:
508 if self.as_one_line:
509 out.write(' { ')
510 else:
511 out.write(' {\n')
512 self.indent += 2
514 self._PrintUnknownFields(field.data)
516 if self.as_one_line:
517 out.write('} ')
518 else:
519 self.indent -= 2
520 out.write(' ' * self.indent + '}\n')
521 elif field.wire_type == WIRETYPE_LENGTH_DELIMITED:
522 try:
523 # If this field is parseable as a Message, it is probably
524 # an embedded message.
525 # pylint: disable=protected-access
526 embedded_unknown_message, pos = decoder._DecodeUnknownFieldSet(
527 memoryview(field.data), 0, len(field.data)
528 )
529 except Exception: # pylint: disable=broad-except
530 pos = 0
532 if pos == len(field.data):
533 if self.as_one_line:
534 out.write(' { ')
535 else:
536 out.write(' {\n')
537 self.indent += 2
539 self._PrintUnknownFields(embedded_unknown_message)
541 if self.as_one_line:
542 out.write('} ')
543 else:
544 self.indent -= 2
545 out.write(' ' * self.indent + '}\n')
546 else:
547 # A string or bytes field. self.as_utf8 may not work.
548 out.write(': "')
549 out.write(text_encoding.CEscape(field.data, False))
550 out.write('" ' if self.as_one_line else '"\n')
551 else:
552 # varint, fixed32, fixed64
553 out.write(': ')
554 out.write(str(field.data))
555 out.write(' ' if self.as_one_line else '\n')
557 def _PrintFieldName(self, field):
558 """Print field name."""
559 out = self.out
560 out.write(' ' * self.indent)
561 if self.use_field_number:
562 out.write(str(field.number))
563 else:
564 if field.is_extension:
565 out.write('[')
566 if (
567 field.containing_type.GetOptions().message_set_wire_format
568 and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
569 and not field.is_required
570 and not field.is_repeated
571 ):
572 out.write(field.message_type.full_name)
573 else:
574 out.write(field.full_name)
575 out.write(']')
576 elif _IsGroupLike(field):
577 # For groups, use the capitalized name.
578 out.write(field.message_type.name)
579 else:
580 out.write(field.name)
582 if (
583 self.force_colon
584 or field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE
585 ):
586 # The colon is optional in this case, but our cross-language golden files
587 # don't include it. Here, the colon is only included if force_colon is
588 # set to True
589 out.write(':')
591 def PrintField(self, field, value):
592 """Print a single field name/value pair."""
593 self._PrintFieldName(field)
594 self.out.write(' ')
595 self.PrintFieldValue(field, value)
596 self.out.write(' ' if self.as_one_line else '\n')
598 def _PrintShortRepeatedPrimitivesValue(self, field, value):
599 """ "Prints short repeated primitives value."""
600 # Note: this is called only when value has at least one element.
601 self._PrintFieldName(field)
602 self.out.write(' [')
603 for i in range(len(value) - 1):
604 self.PrintFieldValue(field, value[i])
605 self.out.write(', ')
606 self.PrintFieldValue(field, value[-1])
607 self.out.write(']')
608 self.out.write(' ' if self.as_one_line else '\n')
610 def _PrintMessageFieldValue(self, value):
611 if self.pointy_brackets:
612 openb = '<'
613 closeb = '>'
614 else:
615 openb = '{'
616 closeb = '}'
618 if self.as_one_line:
619 self.out.write('%s ' % openb)
620 self.PrintMessage(value)
621 self.out.write(closeb)
622 else:
623 self.out.write('%s\n' % openb)
624 self.indent += 2
625 self.PrintMessage(value)
626 self.indent -= 2
627 self.out.write(' ' * self.indent + closeb)
629 def PrintFieldValue(self, field, value):
630 """Print a single field value (not including name).
632 For repeated fields, the value should be a single element.
634 Args:
635 field: The descriptor of the field to be printed.
636 value: The value of the field.
637 """
638 out = self.out
639 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
640 self._PrintMessageFieldValue(value)
641 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM:
642 enum_value = field.enum_type.values_by_number.get(value, None)
643 if enum_value is not None:
644 out.write(enum_value.name)
645 else:
646 out.write(str(value))
647 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING:
648 out.write('"')
649 if isinstance(value, str) and not self.as_utf8:
650 out_value = value.encode('utf-8')
651 else:
652 out_value = value
653 if field.type == descriptor.FieldDescriptor.TYPE_BYTES:
654 # We always need to escape all binary data in TYPE_BYTES fields.
655 out_as_utf8 = False
656 else:
657 out_as_utf8 = self.as_utf8
658 out.write(text_encoding.CEscape(out_value, out_as_utf8))
659 out.write('"')
660 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL:
661 if value:
662 out.write('true')
663 else:
664 out.write('false')
665 elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT:
666 if math.isnan(value):
667 out.write(str(value))
668 else:
669 out.write(str(type_checkers.ToShortestFloat(value)))
670 else:
671 out.write(str(value))
674def Parse(
675 text,
676 message,
677 allow_unknown_extension=False,
678 allow_field_number=False,
679 descriptor_pool=None,
680 allow_unknown_field=False,
681 max_recursion_depth=None,
682):
683 """Parses a text representation of a protocol message into a message.
685 NOTE: for historical reasons this function does not clear the input
686 message. This is different from what the binary msg.ParseFrom(...) does.
687 If text contains a field already set in message, the value is appended if the
688 field is repeated. Otherwise, an error is raised.
690 Example::
692 a = MyProto()
693 a.repeated_field.append('test')
694 b = MyProto()
696 # Repeated fields are combined
697 text_format.Parse(repr(a), b)
698 text_format.Parse(repr(a), b) # repeated_field contains ["test", "test"]
700 # Non-repeated fields cannot be overwritten
701 a.singular_field = 1
702 b.singular_field = 2
703 text_format.Parse(repr(a), b) # ParseError
705 # Binary version:
706 b.ParseFromString(a.SerializeToString()) # repeated_field is now "test"
708 Caller is responsible for clearing the message as needed.
710 Args:
711 text (str): Message text representation.
712 message (Message): A protocol buffer message to merge into.
713 allow_unknown_extension: if True, skip over missing extensions and keep
714 parsing
715 allow_field_number: if True, both field number and field name are allowed.
716 descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.
717 allow_unknown_field: if True, skip over unknown field and keep parsing.
718 Avoid to use this option if possible. It may hide some errors (e.g.
719 spelling error on field name)
720 max_recursion_depth: Optional maximum recursion depth of the message to be
721 parsed: Text Format inputs over this depth will fail to parse. ``None``
722 means no additional limit (the Python runtime will enforce some limit
723 due to call stack limits). As Text Format is primarily intended to be
724 used on trusted configuration inputs, and to maintain backwards
725 compatibility, the default of ``None`` (unbounded) is intentional. For
726 better consistency with what messages will successfully round trip
727 through binary wire format, or for the discouraged case of processing
728 untrusted Text Format inputs, setting a limit of 100 is recommended.
730 Returns:
731 Message: The same message passed as argument.
733 Raises:
734 ParseError: On text parsing problems.
735 """
736 return ParseLines(
737 text.split(b'\n' if isinstance(text, bytes) else '\n'),
738 message,
739 allow_unknown_extension,
740 allow_field_number,
741 descriptor_pool=descriptor_pool,
742 allow_unknown_field=allow_unknown_field,
743 max_recursion_depth=max_recursion_depth,
744 )
747def Merge(
748 text,
749 message,
750 allow_unknown_extension=False,
751 allow_field_number=False,
752 descriptor_pool=None,
753 allow_unknown_field=False,
754 max_recursion_depth=None,
755):
756 """Parses a text representation of a protocol message into a message.
758 Like Parse(), but allows repeated values for a non-repeated field, and uses
759 the last one. This means any non-repeated, top-level fields specified in text
760 replace those in the message.
762 Args:
763 text (str): Message text representation.
764 message (Message): A protocol buffer message to merge into.
765 allow_unknown_extension: if True, skip over missing extensions and keep
766 parsing
767 allow_field_number: if True, both field number and field name are allowed.
768 descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types.
769 allow_unknown_field: if True, skip over unknown field and keep parsing.
770 Avoid to use this option if possible. It may hide some errors (e.g.
771 spelling error on field name)
772 max_recursion_depth: Optional maximum recursion depth of the message to be
773 parsed: Text Format inputs over this depth will fail to parse. ``None``
774 means no additional limit (the Python runtime will enforce some limit
775 due to call stack limits). As Text Format is primarily intended to be
776 used on trusted configuration inputs, and to maintain backwards
777 compatibility, the default of ``None`` (unbounded) is intentional. For
778 better consistency with what messages will successfully round trip
779 through binary wire format, or for the discouraged case of processing
780 untrusted Text Format inputs, setting a limit of 100 is recommended.
782 Returns:
783 Message: The same message passed as argument.
785 Raises:
786 ParseError: On text parsing problems.
787 """
788 return MergeLines(
789 text.split(b'\n' if isinstance(text, bytes) else '\n'),
790 message,
791 allow_unknown_extension,
792 allow_field_number,
793 descriptor_pool=descriptor_pool,
794 allow_unknown_field=allow_unknown_field,
795 max_recursion_depth=max_recursion_depth,
796 )
799def ParseLines(
800 lines,
801 message,
802 allow_unknown_extension=False,
803 allow_field_number=False,
804 descriptor_pool=None,
805 allow_unknown_field=False,
806 max_recursion_depth=None,
807):
808 """Parses a text representation of a protocol message into a message.
810 See Parse() for caveats.
812 Args:
813 lines: An iterable of lines of a message's text representation.
814 message: A protocol buffer message to merge into.
815 allow_unknown_extension: if True, skip over missing extensions and keep
816 parsing
817 allow_field_number: if True, both field number and field name are allowed.
818 descriptor_pool: A DescriptorPool used to resolve Any types.
819 allow_unknown_field: if True, skip over unknown field and keep parsing.
820 Avoid to use this option if possible. It may hide some errors (e.g.
821 spelling error on field name)
822 max_recursion_depth: Optional maximum recursion depth of the message to be
823 parsed: Text Format inputs over this depth will fail to parse. ``None``
824 means no additional limit (the Python runtime will enforce some limit
825 due to call stack limits). As Text Format is primarily intended to be
826 used on trusted configuration inputs, and to maintain backwards
827 compatibility, the default of ``None`` (unbounded) is intentional. For
828 better consistency with what messages will successfully round trip
829 through binary wire format, or for the discouraged case of processing
830 untrusted Text Format inputs, setting a limit of 100 is recommended.
832 Returns:
833 The same message passed as argument.
835 Raises:
836 ParseError: On text parsing problems.
837 """
838 parser = _Parser(
839 allow_unknown_extension,
840 allow_field_number,
841 descriptor_pool=descriptor_pool,
842 allow_unknown_field=allow_unknown_field,
843 max_recursion_depth=max_recursion_depth,
844 )
845 return parser.ParseLines(lines, message)
848def MergeLines(
849 lines,
850 message,
851 allow_unknown_extension=False,
852 allow_field_number=False,
853 descriptor_pool=None,
854 allow_unknown_field=False,
855 max_recursion_depth=None,
856):
857 """Parses a text representation of a protocol message into a message.
859 See Merge() for more details.
861 Args:
862 lines: An iterable of lines of a message's text representation.
863 message: A protocol buffer message to merge into.
864 allow_unknown_extension: if True, skip over missing extensions and keep
865 parsing
866 allow_field_number: if True, both field number and field name are allowed.
867 descriptor_pool: A DescriptorPool used to resolve Any types.
868 allow_unknown_field: if True, skip over unknown field and keep parsing.
869 Avoid to use this option if possible. It may hide some errors (e.g.
870 spelling error on field name)
871 max_recursion_depth: Optional maximum recursion depth of the message to be
872 parsed: Text Format inputs over this depth will fail to parse. ``None``
873 means no additional limit (the Python runtime will enforce some limit
874 due to call stack limits). As Text Format is primarily intended to be
875 used on trusted configuration inputs, and to maintain backwards
876 compatibility, the default of ``None`` (unbounded) is intentional. For
877 better consistency with what messages will successfully round trip
878 through binary wire format, or for the discouraged case of processing
879 untrusted Text Format inputs, setting a limit of 100 is recommended.
881 Returns:
882 The same message passed as argument.
884 Raises:
885 ParseError: On text parsing problems.
886 """
887 parser = _Parser(
888 allow_unknown_extension,
889 allow_field_number,
890 descriptor_pool=descriptor_pool,
891 allow_unknown_field=allow_unknown_field,
892 max_recursion_depth=max_recursion_depth,
893 )
894 return parser.MergeLines(lines, message)
897class _Parser(object):
898 """Text format parser for protocol message."""
900 def __init__(
901 self,
902 allow_unknown_extension=False,
903 allow_field_number=False,
904 descriptor_pool=None,
905 allow_unknown_field=False,
906 max_recursion_depth=None,
907 ):
908 self.allow_unknown_extension = allow_unknown_extension
909 self.allow_field_number = allow_field_number
910 self.descriptor_pool = descriptor_pool
911 self.allow_unknown_field = allow_unknown_field
912 self.max_recursion_depth = max_recursion_depth
913 self.recursion_depth = 0
915 def ParseLines(self, lines, message):
916 """Parses a text representation of a protocol message into a message."""
917 self._allow_multiple_scalars = False
918 self._ParseOrMerge(lines, message)
919 return message
921 def MergeLines(self, lines, message):
922 """Merges a text representation of a protocol message into a message."""
923 self._allow_multiple_scalars = True
924 self._ParseOrMerge(lines, message)
925 return message
927 def _ParseOrMerge(self, lines, message):
928 """Converts a text representation of a protocol message into a message.
930 Args:
931 lines: Lines of a message's text representation.
932 message: A protocol buffer message to merge into.
934 Raises:
935 ParseError: On text parsing problems.
936 """
937 # Tokenize expects native str lines.
938 try:
939 str_lines = (
940 line if isinstance(line, str) else line.decode('utf-8')
941 for line in lines
942 )
943 tokenizer = Tokenizer(str_lines)
944 except UnicodeDecodeError as e:
945 raise ParseError from e
946 if message:
947 self.root_type = message.DESCRIPTOR.full_name
948 self.recursion_depth += 1
949 if (
950 self.max_recursion_depth is not None
951 and self.recursion_depth > self.max_recursion_depth
952 ):
953 raise ParseError(
954 'Message too deep. Max recursion depth is {0}'.format(
955 self.max_recursion_depth
956 )
957 )
958 while not tokenizer.AtEnd():
959 self._MergeField(tokenizer, message)
960 self.recursion_depth -= 1
962 def _MergeMessage(self, tokenizer, message, end_token):
963 self.recursion_depth += 1
964 if (
965 self.max_recursion_depth is not None
966 and self.recursion_depth > self.max_recursion_depth
967 ):
968 raise ParseError(
969 'Message too deep. Max recursion depth is {0}'.format(
970 self.max_recursion_depth
971 )
972 )
973 while not tokenizer.TryConsume(end_token):
974 if tokenizer.AtEnd():
975 raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,))
976 self._MergeField(tokenizer, message)
977 self.recursion_depth -= 1
979 def _MergeField(self, tokenizer, message):
980 """Merges a single protocol message field into a message.
982 Args:
983 tokenizer: A tokenizer to parse the field name and values.
984 message: A protocol message to record the data.
986 Raises:
987 ParseError: In case of text parsing problems.
988 """
989 message_descriptor = message.DESCRIPTOR
990 if (
991 message_descriptor.full_name == _ANY_FULL_TYPE_NAME
992 and tokenizer.TryConsume('[')
993 ):
994 type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer)
995 tokenizer.TryConsume(':')
996 self._DetectSilentMarker(
997 tokenizer,
998 message_descriptor.full_name,
999 type_url_prefix + '/' + packed_type_name,
1000 )
1001 if tokenizer.TryConsume('<'):
1002 expanded_any_end_token = '>'
1003 else:
1004 tokenizer.Consume('{')
1005 expanded_any_end_token = '}'
1006 expanded_any_sub_message = _BuildMessageFromTypeName(
1007 packed_type_name, self.descriptor_pool
1008 )
1009 # Direct comparison with None is used instead of implicit bool conversion
1010 # to avoid false positives with falsy initial values, e.g. for
1011 # google.protobuf.ListValue.
1012 if expanded_any_sub_message is None:
1013 raise ParseError(
1014 'Type %s not found in descriptor pool' % packed_type_name
1015 )
1016 self._MergeMessage(
1017 tokenizer, expanded_any_sub_message, expanded_any_end_token
1018 )
1019 deterministic = False
1021 message.Pack(
1022 expanded_any_sub_message,
1023 type_url_prefix=type_url_prefix + '/',
1024 deterministic=deterministic,
1025 )
1026 return
1028 if tokenizer.TryConsume('['):
1029 name = [tokenizer.ConsumeIdentifier()]
1030 while tokenizer.TryConsume('.'):
1031 name.append(tokenizer.ConsumeIdentifier())
1032 name = '.'.join(name)
1034 if not message_descriptor.is_extendable:
1035 raise tokenizer.ParseErrorPreviousToken(
1036 'Message type "%s" does not have extensions.'
1037 % message_descriptor.full_name
1038 )
1039 # pylint: disable=protected-access
1040 field = message.Extensions._FindExtensionByName(name)
1041 # pylint: enable=protected-access
1042 if not field:
1043 if self.allow_unknown_extension:
1044 field = None
1045 else:
1046 raise tokenizer.ParseErrorPreviousToken(
1047 'Extension "%s" not registered. '
1048 'Did you import the _pb2 module which defines it? '
1049 'If you are trying to place the extension in the MessageSet '
1050 'field of another message that is in an Any or MessageSet field, '
1051 "that message's _pb2 module must be imported as well" % name
1052 )
1053 elif message_descriptor != field.containing_type:
1054 raise tokenizer.ParseErrorPreviousToken(
1055 'Extension "%s" does not extend message type "%s".'
1056 % (name, message_descriptor.full_name)
1057 )
1059 tokenizer.Consume(']')
1061 else:
1062 name = tokenizer.ConsumeIdentifierOrNumber()
1063 if self.allow_field_number and name.isdigit():
1064 number = ParseInteger(name, True, True)
1065 field = message_descriptor.fields_by_number.get(number, None)
1066 if not field and message_descriptor.is_extendable:
1067 field = message.Extensions._FindExtensionByNumber(number)
1068 else:
1069 field = message_descriptor.fields_by_name.get(name, None)
1071 # Group names are expected to be capitalized as they appear in the
1072 # .proto file, which actually matches their type names, not their field
1073 # names.
1074 if not field:
1075 field = message_descriptor.fields_by_name.get(name.lower(), None)
1076 if field and not _IsGroupLike(field):
1077 field = None
1078 if field and field.message_type.name != name:
1079 field = None
1081 if not field and not self.allow_unknown_field:
1082 raise tokenizer.ParseErrorPreviousToken(
1083 'Message type "%s" has no field named "%s".'
1084 % (message_descriptor.full_name, name)
1085 )
1087 if field:
1088 if not self._allow_multiple_scalars and field.containing_oneof:
1089 # Check if there's a different field set in this oneof.
1090 # Note that we ignore the case if the same field was set before, and we
1091 # apply _allow_multiple_scalars to non-scalar fields as well.
1092 which_oneof = message.WhichOneof(field.containing_oneof.name)
1093 if which_oneof is not None and which_oneof != field.name:
1094 raise tokenizer.ParseErrorPreviousToken(
1095 'Field "%s" is specified along with field "%s", another member '
1096 'of oneof "%s" for message type "%s".'
1097 % (
1098 field.name,
1099 which_oneof,
1100 field.containing_oneof.name,
1101 message_descriptor.full_name,
1102 )
1103 )
1105 if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
1106 tokenizer.TryConsume(':')
1107 self._DetectSilentMarker(
1108 tokenizer, message_descriptor.full_name, field.full_name
1109 )
1110 merger = self._MergeMessageField
1111 else:
1112 tokenizer.Consume(':')
1113 self._DetectSilentMarker(
1114 tokenizer, message_descriptor.full_name, field.full_name
1115 )
1116 merger = self._MergeScalarField
1118 if field.is_repeated and tokenizer.TryConsume('['):
1119 # Short repeated format, e.g. "foo: [1, 2, 3]"
1120 if not tokenizer.TryConsume(']'):
1121 while True:
1122 merger(tokenizer, message, field)
1123 if tokenizer.TryConsume(']'):
1124 break
1125 tokenizer.Consume(',')
1127 else:
1128 merger(tokenizer, message, field)
1130 else: # Proto field is unknown.
1131 assert self.allow_unknown_extension or self.allow_unknown_field
1132 self._SkipFieldContents(tokenizer, name, message_descriptor.full_name)
1134 # For historical reasons, fields may optionally be separated by commas or
1135 # semicolons.
1136 if not tokenizer.TryConsume(','):
1137 tokenizer.TryConsume(';')
1139 def _LogSilentMarker(self, immediate_message_type, field_name):
1140 pass
1142 def _DetectSilentMarker(self, tokenizer, immediate_message_type, field_name):
1143 if tokenizer.contains_silent_marker_before_current_token:
1144 self._LogSilentMarker(immediate_message_type, field_name)
1146 def _ConsumeAnyTypeUrl(self, tokenizer):
1147 """Consumes a google.protobuf.Any type URL.
1149 Assumes the caller has already consumed the opening [ and consumes up to the
1150 closing ].
1152 Args:
1153 tokenizer: A tokenizer to parse the type URL.
1155 Returns:
1156 A tuple of type URL prefix (without trailing slash) and type name.
1157 """
1158 # Consume all tokens with valid URL characters until ]. Whitespace and
1159 # comments are ignored/skipped by the Tokenizer.
1160 tokens = []
1161 last_slash = -1
1162 while True:
1163 try:
1164 tokens.append(tokenizer.ConsumeUrlChars())
1165 continue
1166 except ParseError:
1167 pass
1168 if tokenizer.TryConsume('/'):
1169 last_slash = len(tokens)
1170 tokens.append('/')
1171 else:
1172 tokenizer.Consume(']')
1173 break
1175 if last_slash == -1:
1176 raise tokenizer.ParseError('Type URL does not contain "/".')
1178 prefix = ''.join(tokens[:last_slash])
1179 name = ''.join(tokens[last_slash + 1 :])
1181 if not prefix:
1182 raise tokenizer.ParseError('Type URL prefix is empty.')
1183 if prefix.startswith('/'):
1184 raise tokenizer.ParseError('Type URL prefix starts with "/".')
1186 # Check for invalid percent encodings. '%' needs to be followed by exactly
1187 # two valid hexadecimal digits.
1188 for i, char in enumerate(prefix):
1189 if char == '%' and not _PERCENT_ENCODING.match(prefix[i : i + 3]):
1190 raise tokenizer.ParseError(
1191 f'Invalid percent escape, got "{prefix[i : i + 3]}".'
1192 )
1194 # After the last slash we expect a valid type name, not just any sequence of
1195 # URL characters.
1196 if not _TYPE_NAME.match(name):
1197 raise tokenizer.ParseError('Expected type name, got "%s".' % name)
1199 return prefix, name
1201 def _MergeMessageField(self, tokenizer, message, field):
1202 """Merges a single scalar field into a message.
1204 Args:
1205 tokenizer: A tokenizer to parse the field value.
1206 message: The message of which field is a member.
1207 field: The descriptor of the field to be merged.
1209 Raises:
1210 ParseError: In case of text parsing problems.
1211 """
1212 is_map_entry = _IsMapEntry(field)
1214 if tokenizer.TryConsume('<'):
1215 end_token = '>'
1216 else:
1217 tokenizer.Consume('{')
1218 end_token = '}'
1220 if field.is_repeated:
1221 if field.is_extension:
1222 sub_message = message.Extensions[field].add()
1223 elif is_map_entry:
1224 sub_message = getattr(message, field.name).GetEntryClass()()
1225 else:
1226 sub_message = getattr(message, field.name).add()
1227 else:
1228 if field.is_extension:
1229 if not self._allow_multiple_scalars and message.HasExtension(field):
1230 raise tokenizer.ParseErrorPreviousToken(
1231 'Message type "%s" should not have multiple "%s" extensions.'
1232 % (message.DESCRIPTOR.full_name, field.full_name)
1233 )
1234 sub_message = message.Extensions[field]
1235 else:
1236 # Also apply _allow_multiple_scalars to message field.
1237 # TODO: Change to _allow_singular_overwrites.
1238 if not self._allow_multiple_scalars and message.HasField(field.name):
1239 raise tokenizer.ParseErrorPreviousToken(
1240 'Message type "%s" should not have multiple "%s" fields.'
1241 % (message.DESCRIPTOR.full_name, field.name)
1242 )
1243 sub_message = getattr(message, field.name)
1244 sub_message.SetInParent()
1246 self._MergeMessage(tokenizer, sub_message, end_token)
1248 if is_map_entry:
1249 value_cpptype = field.message_type.fields_by_name['value'].cpp_type
1250 if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE:
1251 value = getattr(message, field.name)[sub_message.key]
1252 value.CopyFrom(sub_message.value)
1253 else:
1254 getattr(message, field.name)[sub_message.key] = sub_message.value
1256 def _MergeScalarField(self, tokenizer, message, field):
1257 """Merges a single scalar field into a message.
1259 Args:
1260 tokenizer: A tokenizer to parse the field value.
1261 message: A protocol message to record the data.
1262 field: The descriptor of the field to be merged.
1264 Raises:
1265 ParseError: In case of text parsing problems.
1266 RuntimeError: On runtime errors.
1267 """
1268 _ = self.allow_unknown_extension
1269 value = None
1271 if field.type in (
1272 descriptor.FieldDescriptor.TYPE_INT32,
1273 descriptor.FieldDescriptor.TYPE_SINT32,
1274 descriptor.FieldDescriptor.TYPE_SFIXED32,
1275 ):
1276 value = _ConsumeInt32(tokenizer)
1277 elif field.type in (
1278 descriptor.FieldDescriptor.TYPE_INT64,
1279 descriptor.FieldDescriptor.TYPE_SINT64,
1280 descriptor.FieldDescriptor.TYPE_SFIXED64,
1281 ):
1282 value = _ConsumeInt64(tokenizer)
1283 elif field.type in (
1284 descriptor.FieldDescriptor.TYPE_UINT32,
1285 descriptor.FieldDescriptor.TYPE_FIXED32,
1286 ):
1287 value = _ConsumeUint32(tokenizer)
1288 elif field.type in (
1289 descriptor.FieldDescriptor.TYPE_UINT64,
1290 descriptor.FieldDescriptor.TYPE_FIXED64,
1291 ):
1292 value = _ConsumeUint64(tokenizer)
1293 elif field.type in (
1294 descriptor.FieldDescriptor.TYPE_FLOAT,
1295 descriptor.FieldDescriptor.TYPE_DOUBLE,
1296 ):
1297 value = tokenizer.ConsumeFloat()
1298 elif field.type == descriptor.FieldDescriptor.TYPE_BOOL:
1299 value = tokenizer.ConsumeBool()
1300 elif field.type == descriptor.FieldDescriptor.TYPE_STRING:
1301 value = tokenizer.ConsumeString()
1302 elif field.type == descriptor.FieldDescriptor.TYPE_BYTES:
1303 value = tokenizer.ConsumeByteString()
1304 elif field.type == descriptor.FieldDescriptor.TYPE_ENUM:
1305 value = tokenizer.ConsumeEnum(field)
1306 else:
1307 raise RuntimeError('Unknown field type %d' % field.type)
1309 if field.is_repeated:
1310 if field.is_extension:
1311 message.Extensions[field].append(value)
1312 else:
1313 getattr(message, field.name).append(value)
1314 else:
1315 if field.is_extension:
1316 if (
1317 not self._allow_multiple_scalars
1318 and field.has_presence
1319 and message.HasExtension(field)
1320 ):
1321 raise tokenizer.ParseErrorPreviousToken(
1322 'Message type "%s" should not have multiple "%s" extensions.'
1323 % (message.DESCRIPTOR.full_name, field.full_name)
1324 )
1325 else:
1326 message.Extensions[field] = value
1327 else:
1328 duplicate_error = False
1329 if not self._allow_multiple_scalars:
1330 if field.has_presence:
1331 duplicate_error = message.HasField(field.name)
1332 else:
1333 # For field that doesn't represent presence, try best effort to
1334 # check multiple scalars by compare to default values.
1335 duplicate_error = not decoder.IsDefaultScalarValue(
1336 getattr(message, field.name)
1337 )
1339 if duplicate_error:
1340 raise tokenizer.ParseErrorPreviousToken(
1341 'Message type "%s" should not have multiple "%s" fields.'
1342 % (message.DESCRIPTOR.full_name, field.name)
1343 )
1344 else:
1345 setattr(message, field.name, value)
1347 def _SkipFieldContents(self, tokenizer, field_name, immediate_message_type):
1348 """Skips over contents (value or message) of a field.
1350 Args:
1351 tokenizer: A tokenizer to parse the field name and values.
1352 field_name: The field name currently being parsed.
1353 immediate_message_type: The type of the message immediately containing the
1354 silent marker.
1355 """
1356 # Try to guess the type of this field.
1357 # If this field is not a message, there should be a ":" between the
1358 # field name and the field value and also the field value should not
1359 # start with "{" or "<" which indicates the beginning of a message body.
1360 # If there is no ":" or there is a "{" or "<" after ":", this field has
1361 # to be a message or the input is ill-formed.
1362 if (
1363 tokenizer.TryConsume(':')
1364 and not tokenizer.LookingAt('{')
1365 and not tokenizer.LookingAt('<')
1366 ):
1367 self._DetectSilentMarker(tokenizer, immediate_message_type, field_name)
1368 if tokenizer.LookingAt('['):
1369 self._SkipRepeatedFieldValue(tokenizer, immediate_message_type)
1370 else:
1371 self._SkipFieldValue(tokenizer)
1372 else:
1373 self._DetectSilentMarker(tokenizer, immediate_message_type, field_name)
1374 self._SkipFieldMessage(tokenizer, immediate_message_type)
1376 def _SkipField(self, tokenizer, immediate_message_type):
1377 """Skips over a complete field (name and value/message).
1379 Args:
1380 tokenizer: A tokenizer to parse the field name and values.
1381 immediate_message_type: The type of the message immediately containing the
1382 silent marker.
1383 """
1384 field_name = ''
1385 if tokenizer.TryConsume('['):
1386 # Consume extension or google.protobuf.Any type URL
1387 field_name += '[' + tokenizer.ConsumeIdentifier()
1388 num_identifiers = 1
1389 while tokenizer.TryConsume('.'):
1390 field_name += '.' + tokenizer.ConsumeIdentifier()
1391 num_identifiers += 1
1392 # This is possibly a type URL for an Any message.
1393 if num_identifiers == 3 and tokenizer.TryConsume('/'):
1394 field_name += '/' + tokenizer.ConsumeIdentifier()
1395 while tokenizer.TryConsume('.'):
1396 field_name += '.' + tokenizer.ConsumeIdentifier()
1397 tokenizer.Consume(']')
1398 field_name += ']'
1399 else:
1400 field_name += tokenizer.ConsumeIdentifierOrNumber()
1402 self._SkipFieldContents(tokenizer, field_name, immediate_message_type)
1404 # For historical reasons, fields may optionally be separated by commas or
1405 # semicolons.
1406 if not tokenizer.TryConsume(','):
1407 tokenizer.TryConsume(';')
1409 def _SkipFieldMessage(self, tokenizer, immediate_message_type):
1410 """Skips over a field message.
1412 Args:
1413 tokenizer: A tokenizer to parse the field name and values.
1414 immediate_message_type: The type of the message immediately containing the
1415 silent marker
1416 """
1417 if tokenizer.TryConsume('<'):
1418 delimiter = '>'
1419 else:
1420 tokenizer.Consume('{')
1421 delimiter = '}'
1423 while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'):
1424 self._SkipField(tokenizer, immediate_message_type)
1426 tokenizer.Consume(delimiter)
1428 def _SkipFieldValue(self, tokenizer):
1429 """Skips over a field value.
1431 Args:
1432 tokenizer: A tokenizer to parse the field name and values.
1434 Raises:
1435 ParseError: In case an invalid field value is found.
1436 """
1437 if (
1438 not tokenizer.TryConsumeByteString()
1439 and not tokenizer.TryConsumeIdentifier()
1440 and not _TryConsumeInt64(tokenizer)
1441 and not _TryConsumeUint64(tokenizer)
1442 and not tokenizer.TryConsumeFloat()
1443 ):
1444 raise ParseError('Invalid field value: ' + tokenizer.token)
1446 def _SkipRepeatedFieldValue(self, tokenizer, immediate_message_type):
1447 """Skips over a repeated field value.
1449 Args:
1450 tokenizer: A tokenizer to parse the field value.
1451 """
1452 tokenizer.Consume('[')
1453 if not tokenizer.TryConsume(']'):
1454 while True:
1455 if tokenizer.LookingAt('<') or tokenizer.LookingAt('{'):
1456 self._SkipFieldMessage(tokenizer, immediate_message_type)
1457 else:
1458 self._SkipFieldValue(tokenizer)
1459 if tokenizer.TryConsume(']'):
1460 break
1461 tokenizer.Consume(',')
1464class Tokenizer(object):
1465 """Protocol buffer text representation tokenizer.
1467 This class handles the lower level string parsing by splitting it into
1468 meaningful tokens.
1470 It was directly ported from the Java protocol buffer API.
1471 """
1473 _WHITESPACE = re.compile(r'\s+')
1474 _COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE)
1475 _WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE)
1476 _TOKEN = re.compile(
1477 '|'.join(
1478 [
1479 r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier
1480 r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number
1481 ]
1482 + [ # quoted str for each quote mark
1483 # Avoid backtracking! https://stackoverflow.com/a/844267
1484 r'{qt}[^{qt}\n\\]*((\\.)+[^{qt}\n\\]*)*({qt}|\\?$)'.format(
1485 qt=mark
1486 )
1487 for mark in _QUOTES
1488 ]
1489 )
1490 )
1492 _IDENTIFIER = re.compile(r'[^\d\W]\w*')
1493 _IDENTIFIER_OR_NUMBER = re.compile(r'\w+')
1494 # Accepted URL characters (excluding "/")
1495 _URL_CHARS = re.compile(r'^[0-9a-zA-Z-.~_ !$&()*+,;=%]+$')
1497 def __init__(self, lines, skip_comments=True):
1498 self._position = 0
1499 self._line = -1
1500 self._column = 0
1501 self._token_start = None
1502 self.token = ''
1503 self._lines = iter(lines)
1504 self._current_line = ''
1505 self._previous_line = 0
1506 self._previous_column = 0
1507 self._more_lines = True
1508 self._skip_comments = skip_comments
1509 self._whitespace_pattern = (
1510 skip_comments and self._WHITESPACE_OR_COMMENT or self._WHITESPACE
1511 )
1512 self.contains_silent_marker_before_current_token = False
1514 self._SkipWhitespace()
1515 self.NextToken()
1517 def LookingAt(self, token):
1518 return self.token == token
1520 def AtEnd(self):
1521 """Checks the end of the text was reached.
1523 Returns:
1524 True iff the end was reached.
1525 """
1526 return not self.token
1528 def _PopLine(self):
1529 while len(self._current_line) <= self._column:
1530 try:
1531 self._current_line = next(self._lines)
1532 except StopIteration:
1533 self._current_line = ''
1534 self._more_lines = False
1535 return
1536 else:
1537 self._line += 1
1538 self._column = 0
1540 def _SkipWhitespace(self):
1541 while True:
1542 self._PopLine()
1543 match = self._whitespace_pattern.match(self._current_line, self._column)
1544 if not match:
1545 break
1546 self.contains_silent_marker_before_current_token = match.group(0) == (
1547 ' ' + _DEBUG_STRING_SILENT_MARKER
1548 )
1549 length = len(match.group(0))
1550 self._column += length
1552 def TryConsume(self, token):
1553 """Tries to consume a given piece of text.
1555 Args:
1556 token: Text to consume.
1558 Returns:
1559 True iff the text was consumed.
1560 """
1561 if self.token == token:
1562 self.NextToken()
1563 return True
1564 return False
1566 def Consume(self, token):
1567 """Consumes a piece of text.
1569 Args:
1570 token: Text to consume.
1572 Raises:
1573 ParseError: If the text couldn't be consumed.
1574 """
1575 if not self.TryConsume(token):
1576 raise self.ParseError('Expected "%s".' % token)
1578 def ConsumeComment(self):
1579 result = self.token
1580 if not self._COMMENT.match(result):
1581 raise self.ParseError('Expected comment.')
1582 self.NextToken()
1583 return result
1585 def ConsumeCommentOrTrailingComment(self):
1586 """Consumes a comment, returns a 2-tuple (trailing bool, comment str)."""
1588 # Tokenizer initializes _previous_line and _previous_column to 0. As the
1589 # tokenizer starts, it looks like there is a previous token on the line.
1590 just_started = self._line == 0 and self._column == 0
1592 before_parsing = self._previous_line
1593 comment = self.ConsumeComment()
1595 # A trailing comment is a comment on the same line than the previous token.
1596 trailing = self._previous_line == before_parsing and not just_started
1598 return trailing, comment
1600 def TryConsumeIdentifier(self):
1601 try:
1602 self.ConsumeIdentifier()
1603 return True
1604 except ParseError:
1605 return False
1607 def ConsumeIdentifier(self):
1608 """Consumes protocol message field identifier.
1610 Returns:
1611 Identifier string.
1613 Raises:
1614 ParseError: If an identifier couldn't be consumed.
1615 """
1616 result = self.token
1617 if not self._IDENTIFIER.match(result):
1618 raise self.ParseError('Expected identifier.')
1619 self.NextToken()
1620 return result
1622 def TryConsumeIdentifierOrNumber(self):
1623 try:
1624 self.ConsumeIdentifierOrNumber()
1625 return True
1626 except ParseError:
1627 return False
1629 def ConsumeIdentifierOrNumber(self):
1630 """Consumes protocol message field identifier.
1632 Returns:
1633 Identifier string.
1635 Raises:
1636 ParseError: If an identifier couldn't be consumed.
1637 """
1638 result = self.token
1639 if not self._IDENTIFIER_OR_NUMBER.match(result):
1640 raise self.ParseError('Expected identifier or number, got %s.' % result)
1641 self.NextToken()
1642 return result
1644 def TryConsumeInteger(self):
1645 try:
1646 self.ConsumeInteger()
1647 return True
1648 except ParseError:
1649 return False
1651 def ConsumeInteger(self):
1652 """Consumes an integer number.
1654 Returns:
1655 The integer parsed.
1657 Raises:
1658 ParseError: If an integer couldn't be consumed.
1659 """
1660 try:
1661 result = _ParseAbstractInteger(self.token)
1662 except ValueError as e:
1663 raise self.ParseError(str(e))
1664 self.NextToken()
1665 return result
1667 def TryConsumeFloat(self):
1668 try:
1669 self.ConsumeFloat()
1670 return True
1671 except ParseError:
1672 return False
1674 def ConsumeFloat(self):
1675 """Consumes an floating point number.
1677 Returns:
1678 The number parsed.
1680 Raises:
1681 ParseError: If a floating point number couldn't be consumed.
1682 """
1683 try:
1684 result = ParseFloat(self.token)
1685 except ValueError as e:
1686 raise self.ParseError(str(e))
1687 self.NextToken()
1688 return result
1690 def ConsumeBool(self):
1691 """Consumes a boolean value.
1693 Returns:
1694 The bool parsed.
1696 Raises:
1697 ParseError: If a boolean value couldn't be consumed.
1698 """
1699 try:
1700 result = ParseBool(self.token)
1701 except ValueError as e:
1702 raise self.ParseError(str(e))
1703 self.NextToken()
1704 return result
1706 def TryConsumeByteString(self):
1707 try:
1708 self.ConsumeByteString()
1709 return True
1710 except ParseError:
1711 return False
1713 def ConsumeString(self):
1714 """Consumes a string value.
1716 Returns:
1717 The string parsed.
1719 Raises:
1720 ParseError: If a string value couldn't be consumed.
1721 """
1722 the_bytes = self.ConsumeByteString()
1723 try:
1724 return str(the_bytes, 'utf-8')
1725 except UnicodeDecodeError as e:
1726 raise self._StringParseError(e)
1728 def ConsumeByteString(self):
1729 """Consumes a byte array value.
1731 Returns:
1732 The array parsed (as a string).
1734 Raises:
1735 ParseError: If a byte array value couldn't be consumed.
1736 """
1737 the_list = [self._ConsumeSingleByteString()]
1738 while self.token and self.token[0] in _QUOTES:
1739 the_list.append(self._ConsumeSingleByteString())
1740 return b''.join(the_list)
1742 def _ConsumeSingleByteString(self):
1743 """Consume one token of a string literal.
1745 String literals (whether bytes or text) can come in multiple adjacent
1746 tokens which are automatically concatenated, like in C or Python. This
1747 method only consumes one token.
1749 Returns:
1750 The token parsed.
1751 Raises:
1752 ParseError: When the wrong format data is found.
1753 """
1754 text = self.token
1755 if len(text) < 1 or text[0] not in _QUOTES:
1756 raise self.ParseError('Expected string but found: %r' % (text,))
1758 if len(text) < 2 or text[-1] != text[0]:
1759 raise self.ParseError('String missing ending quote: %r' % (text,))
1761 try:
1762 result = text_encoding.CUnescape(text[1:-1])
1763 except ValueError as e:
1764 raise self.ParseError(str(e))
1765 self.NextToken()
1766 return result
1768 def ConsumeEnum(self, field):
1769 try:
1770 result = ParseEnum(field, self.token)
1771 except ValueError as e:
1772 raise self.ParseError(str(e))
1773 self.NextToken()
1774 return result
1776 def ConsumeUrlChars(self):
1777 """Consumes a token containing valid URL characters.
1779 Excludes '/' so that it can be treated specially as a delimiter.
1781 Returns:
1782 The next token containing one or more URL characters.
1784 Raises:
1785 ParseError: If the next token contains unaccepted URL characters.
1786 """
1787 if not self._URL_CHARS.match(self.token):
1788 raise self.ParseError('Expected URL character(s), got "%s"' % self.token)
1790 result = self.token
1791 self.NextToken()
1792 return result
1794 def TryConsumeUrlChars(self):
1795 try:
1796 self.ConsumeUrlChars()
1797 return True
1798 except ParseError:
1799 return False
1801 def ParseErrorPreviousToken(self, message):
1802 """Creates and *returns* a ParseError for the previously read token.
1804 Args:
1805 message: A message to set for the exception.
1807 Returns:
1808 A ParseError instance.
1809 """
1810 return ParseError(
1811 message, self._previous_line + 1, self._previous_column + 1
1812 )
1814 def ParseError(self, message):
1815 """Creates and *returns* a ParseError for the current token."""
1816 return ParseError(
1817 "'" + self._current_line + "': " + message,
1818 self._line + 1,
1819 self._column + 1,
1820 )
1822 def _StringParseError(self, e):
1823 return self.ParseError("Couldn't parse string: " + str(e))
1825 def NextToken(self):
1826 """Reads the next meaningful token."""
1827 self._previous_line = self._line
1828 self._previous_column = self._column
1829 self.contains_silent_marker_before_current_token = False
1831 self._column += len(self.token)
1832 self._SkipWhitespace()
1834 if not self._more_lines:
1835 self.token = ''
1836 return
1838 match = self._TOKEN.match(self._current_line, self._column)
1839 if not match and not self._skip_comments:
1840 match = self._COMMENT.match(self._current_line, self._column)
1841 if match:
1842 token = match.group(0)
1843 self.token = token
1844 else:
1845 self.token = self._current_line[self._column]
1848# Aliased so it can still be accessed by current visibility violators.
1849# TODO: Migrate violators to textformat_tokenizer.
1850_Tokenizer = Tokenizer # pylint: disable=invalid-name
1853def _ConsumeInt32(tokenizer):
1854 """Consumes a signed 32bit integer number from tokenizer.
1856 Args:
1857 tokenizer: A tokenizer used to parse the number.
1859 Returns:
1860 The integer parsed.
1862 Raises:
1863 ParseError: If a signed 32bit integer couldn't be consumed.
1864 """
1865 return _ConsumeInteger(tokenizer, is_signed=True, is_long=False)
1868def _ConsumeUint32(tokenizer):
1869 """Consumes an unsigned 32bit integer number from tokenizer.
1871 Args:
1872 tokenizer: A tokenizer used to parse the number.
1874 Returns:
1875 The integer parsed.
1877 Raises:
1878 ParseError: If an unsigned 32bit integer couldn't be consumed.
1879 """
1880 return _ConsumeInteger(tokenizer, is_signed=False, is_long=False)
1883def _TryConsumeInt64(tokenizer):
1884 try:
1885 _ConsumeInt64(tokenizer)
1886 return True
1887 except ParseError:
1888 return False
1891def _ConsumeInt64(tokenizer):
1892 """Consumes a signed 32bit integer number from tokenizer.
1894 Args:
1895 tokenizer: A tokenizer used to parse the number.
1897 Returns:
1898 The integer parsed.
1900 Raises:
1901 ParseError: If a signed 32bit integer couldn't be consumed.
1902 """
1903 return _ConsumeInteger(tokenizer, is_signed=True, is_long=True)
1906def _TryConsumeUint64(tokenizer):
1907 try:
1908 _ConsumeUint64(tokenizer)
1909 return True
1910 except ParseError:
1911 return False
1914def _ConsumeUint64(tokenizer):
1915 """Consumes an unsigned 64bit integer number from tokenizer.
1917 Args:
1918 tokenizer: A tokenizer used to parse the number.
1920 Returns:
1921 The integer parsed.
1923 Raises:
1924 ParseError: If an unsigned 64bit integer couldn't be consumed.
1925 """
1926 return _ConsumeInteger(tokenizer, is_signed=False, is_long=True)
1929def _ConsumeInteger(tokenizer, is_signed=False, is_long=False):
1930 """Consumes an integer number from tokenizer.
1932 Args:
1933 tokenizer: A tokenizer used to parse the number.
1934 is_signed: True if a signed integer must be parsed.
1935 is_long: True if a long integer must be parsed.
1937 Returns:
1938 The integer parsed.
1940 Raises:
1941 ParseError: If an integer with given characteristics couldn't be consumed.
1942 """
1943 try:
1944 result = ParseInteger(tokenizer.token, is_signed=is_signed, is_long=is_long)
1945 except ValueError as e:
1946 raise tokenizer.ParseError(str(e))
1947 tokenizer.NextToken()
1948 return result
1951def ParseInteger(text, is_signed=False, is_long=False):
1952 """Parses an integer.
1954 Args:
1955 text: The text to parse.
1956 is_signed: True if a signed integer must be parsed.
1957 is_long: True if a long integer must be parsed.
1959 Returns:
1960 The integer value.
1962 Raises:
1963 ValueError: Thrown Iff the text is not a valid integer.
1964 """
1965 # Do the actual parsing. Exception handling is propagated to caller.
1966 result = _ParseAbstractInteger(text)
1968 # Check if the integer is sane. Exceptions handled by callers.
1969 checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)]
1970 checker.CheckValue(result)
1971 return result
1974def _ParseAbstractInteger(text):
1975 """Parses an integer without checking size/signedness.
1977 Args:
1978 text: The text to parse.
1980 Returns:
1981 The integer value.
1983 Raises:
1984 ValueError: Thrown Iff the text is not a valid integer.
1985 """
1986 # Do the actual parsing. Exception handling is propagated to caller.
1987 orig_text = text
1988 c_octal_match = re.match(r'(-?)0(\d+)$', text)
1989 if c_octal_match:
1990 # Python 3 no longer supports 0755 octal syntax without the 'o', so
1991 # we always use the '0o' prefix for multi-digit numbers starting with 0.
1992 text = c_octal_match.group(1) + '0o' + c_octal_match.group(2)
1993 try:
1994 return int(text, 0)
1995 except ValueError:
1996 raise ValueError("Couldn't parse integer: %s" % orig_text)
1999def ParseFloat(text):
2000 """Parse a floating point number.
2002 Args:
2003 text: Text to parse.
2005 Returns:
2006 The number parsed.
2008 Raises:
2009 ValueError: If a floating point number couldn't be parsed.
2010 """
2011 if _FLOAT_OCTAL_PREFIX.match(text):
2012 raise ValueError('Invalid octal float: %s' % text)
2013 try:
2014 # Assume Python compatible syntax.
2015 return float(text)
2016 except ValueError:
2017 # Check alternative spellings.
2018 if _FLOAT_INFINITY.match(text):
2019 if text[0] == '-':
2020 return float('-inf')
2021 else:
2022 return float('inf')
2023 elif _FLOAT_NAN.match(text):
2024 return float('nan')
2025 else:
2026 # assume '1.0f' format
2027 try:
2028 return float(text.rstrip('fF'))
2029 except ValueError:
2030 raise ValueError("Couldn't parse float: %s" % text)
2033def ParseBool(text):
2034 """Parse a boolean value.
2036 Args:
2037 text: Text to parse.
2039 Returns:
2040 Boolean values parsed
2042 Raises:
2043 ValueError: If text is not a valid boolean.
2044 """
2045 if text in ('true', 't', '1', 'True'):
2046 return True
2047 elif text in ('false', 'f', '0', 'False'):
2048 return False
2049 else:
2050 raise ValueError('Expected "true" or "false".')
2053def ParseEnum(field, value):
2054 """Parse an enum value.
2056 The value can be specified by a number (the enum value), or by
2057 a string literal (the enum name).
2059 Args:
2060 field: Enum field descriptor.
2061 value: String value.
2063 Returns:
2064 Enum value number.
2066 Raises:
2067 ValueError: If the enum value could not be parsed.
2068 """
2069 enum_descriptor = field.enum_type
2070 try:
2071 number = int(value, 0)
2072 except ValueError:
2073 # Identifier.
2074 enum_value = enum_descriptor.values_by_name.get(value, None)
2075 if enum_value is None:
2076 raise ValueError(
2077 'Enum type "%s" has no value named %s.'
2078 % (enum_descriptor.full_name, value)
2079 )
2080 else:
2081 if not field.enum_type.is_closed:
2082 return number
2083 enum_value = enum_descriptor.values_by_number.get(number, None)
2084 if enum_value is None:
2085 raise ValueError(
2086 'Enum type "%s" has no value with number %d.'
2087 % (enum_descriptor.full_name, number)
2088 )
2089 return enum_value.number