Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc. All rights reserved.
3#
4# Use of this source code is governed by a BSD-style
5# license that can be found in the LICENSE file or at
6# https://developers.google.com/open-source/licenses/bsd
7"""Provides DescriptorPool to use as a container for proto2 descriptors.
9The DescriptorPool is used in conjection with a DescriptorDatabase to maintain
10a collection of protocol buffer descriptors for use when dynamically creating
11message types at runtime.
13For most applications protocol buffers should be used via modules generated by
14the protocol buffer compiler tool. This should only be used when the type of
15protocol buffers used in an application or library cannot be predetermined.
17Below is a straightforward example on how to use this class::
19 pool = DescriptorPool()
20 file_descriptor_protos = [ ... ]
21 for file_descriptor_proto in file_descriptor_protos:
22 pool.Add(file_descriptor_proto)
23 my_message_descriptor = pool.FindMessageTypeByName('some.package.MessageType')
25The message descriptor can be used in conjunction with the message_factory
26module in order to create a protocol buffer class that can be encoded and
27decoded.
29If you want to get a Python class for the specified proto, use the
30helper functions inside google.protobuf.message_factory
31directly instead of this class.
32"""
34__author__ = 'matthewtoia@google.com (Matt Toia)'
36import collections
37import threading
38import warnings
40from google.protobuf import descriptor
41from google.protobuf import descriptor_database
42from google.protobuf import text_encoding
43from google.protobuf.internal import python_edition_defaults
44from google.protobuf.internal import python_message
46_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS # pylint: disable=protected-access
49def _NormalizeFullyQualifiedName(name):
50 """Remove leading period from fully-qualified type name.
52 Due to b/13860351 in descriptor_database.py, types in the root namespace are
53 generated with a leading period. This function removes that prefix.
55 Args:
56 name (str): The fully-qualified symbol name.
58 Returns:
59 str: The normalized fully-qualified symbol name.
60 """
61 return name.lstrip('.')
64def _OptionsOrNone(descriptor_proto):
65 """Returns the value of the field `options`, or None if it is not set."""
66 if descriptor_proto.HasField('options'):
67 return descriptor_proto.options
68 else:
69 return None
72def _IsMessageSetExtension(field):
73 return (
74 field.is_extension
75 and field.containing_type.has_options
76 and field.containing_type.GetOptions().message_set_wire_format
77 and field.type == descriptor.FieldDescriptor.TYPE_MESSAGE
78 and not field.is_required
79 and not field.is_repeated
80 )
83_edition_defaults_lock = threading.Lock()
86class DescriptorPool(object):
87 """A collection of protobufs dynamically constructed by descriptor protos."""
89 if _USE_C_DESCRIPTORS:
91 def __new__(cls, descriptor_db=None):
92 # pylint: disable=protected-access
93 return descriptor._message.DescriptorPool(descriptor_db)
95 def __init__(
96 self, descriptor_db=None, use_deprecated_legacy_json_field_conflicts=False
97 ):
98 """Initializes a Pool of proto buffs.
100 The descriptor_db argument to the constructor is provided to allow
101 specialized file descriptor proto lookup code to be triggered on demand. An
102 example would be an implementation which will read and compile a file
103 specified in a call to FindFileByName() and not require the call to Add()
104 at all. Results from this database will be cached internally here as well.
106 Args:
107 descriptor_db: A secondary source of file descriptors.
108 use_deprecated_legacy_json_field_conflicts: Unused, for compatibility with
109 C++.
110 """
112 self._internal_db = descriptor_database.DescriptorDatabase()
113 self._descriptor_db = descriptor_db
114 self._descriptors = {}
115 self._enum_descriptors = {}
116 self._service_descriptors = {}
117 self._file_descriptors = {}
118 self._toplevel_extensions = {}
119 self._top_enum_values = {}
120 # We store extensions in two two-level mappings: The first key is the
121 # descriptor of the message being extended, the second key is the extension
122 # full name or its tag number.
123 self._extensions_by_name = collections.defaultdict(dict)
124 self._extensions_by_number = collections.defaultdict(dict)
125 self._serialized_edition_defaults = (
126 python_edition_defaults._PROTOBUF_INTERNAL_PYTHON_EDITION_DEFAULTS
127 )
128 self._edition_defaults = None
129 self._feature_cache = dict()
131 def _CheckConflictRegister(self, desc, desc_name, file_name):
132 """Check if the descriptor name conflicts with another of the same name.
134 Args:
135 desc: Descriptor of a message, enum, service, extension or enum value.
136 desc_name (str): the full name of desc.
137 file_name (str): The file name of descriptor.
138 """
139 for register, descriptor_type in [
140 (self._descriptors, descriptor.Descriptor),
141 (self._enum_descriptors, descriptor.EnumDescriptor),
142 (self._service_descriptors, descriptor.ServiceDescriptor),
143 (self._toplevel_extensions, descriptor.FieldDescriptor),
144 (self._top_enum_values, descriptor.EnumValueDescriptor),
145 ]:
146 if desc_name in register:
147 old_desc = register[desc_name]
148 if isinstance(old_desc, descriptor.EnumValueDescriptor):
149 old_file = old_desc.type.file.name
150 else:
151 old_file = old_desc.file.name
153 if not isinstance(desc, descriptor_type) or (old_file != file_name):
154 error_msg = (
155 'Conflict register for file "'
156 + file_name
157 + '": '
158 + desc_name
159 + ' is already defined in file "'
160 + old_file
161 + '". Please fix the conflict by adding '
162 'package name on the proto file, or use different '
163 'name for the duplication.'
164 )
165 if isinstance(desc, descriptor.EnumValueDescriptor):
166 error_msg += (
167 '\nNote: enum values appear as '
168 'siblings of the enum type instead of '
169 'children of it.'
170 )
172 raise TypeError(error_msg)
174 return
176 def Add(self, file_desc_proto):
177 """Adds the FileDescriptorProto and its types to this pool.
179 Args:
180 file_desc_proto (FileDescriptorProto): The file descriptor to add.
181 """
183 self._internal_db.Add(file_desc_proto)
185 def AddSerializedFile(self, serialized_file_desc_proto):
186 """Adds the FileDescriptorProto and its types to this pool.
188 Args:
189 serialized_file_desc_proto (bytes): A bytes string, serialization of the
190 :class:`FileDescriptorProto` to add.
192 Returns:
193 FileDescriptor: Descriptor for the added file.
194 """
196 # pylint: disable=g-import-not-at-top
197 from google.protobuf import descriptor_pb2
199 file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString(
200 serialized_file_desc_proto
201 )
202 file_desc = self._ConvertFileProtoToFileDescriptor(file_desc_proto)
203 file_desc.serialized_pb = serialized_file_desc_proto
204 return file_desc
206 # Never call this method. It is for internal usage only.
207 def _AddDescriptor(self, desc):
208 """Adds a Descriptor to the pool, non-recursively.
210 If the Descriptor contains nested messages or enums, the caller must
211 explicitly register them. This method also registers the FileDescriptor
212 associated with the message.
214 Args:
215 desc: A Descriptor.
216 """
217 if not isinstance(desc, descriptor.Descriptor):
218 raise TypeError('Expected instance of descriptor.Descriptor.')
220 self._CheckConflictRegister(desc, desc.full_name, desc.file.name)
222 self._descriptors[desc.full_name] = desc
223 self._AddFileDescriptor(desc.file)
225 # Never call this method. It is for internal usage only.
226 def _AddEnumDescriptor(self, enum_desc):
227 """Adds an EnumDescriptor to the pool.
229 This method also registers the FileDescriptor associated with the enum.
231 Args:
232 enum_desc: An EnumDescriptor.
233 """
235 if not isinstance(enum_desc, descriptor.EnumDescriptor):
236 raise TypeError('Expected instance of descriptor.EnumDescriptor.')
238 file_name = enum_desc.file.name
239 self._CheckConflictRegister(enum_desc, enum_desc.full_name, file_name)
240 self._enum_descriptors[enum_desc.full_name] = enum_desc
242 # Top enum values need to be indexed.
243 # Count the number of dots to see whether the enum is toplevel or nested
244 # in a message. We cannot use enum_desc.containing_type at this stage.
245 if enum_desc.file.package:
246 top_level = (
247 enum_desc.full_name.count('.') - enum_desc.file.package.count('.')
248 == 1
249 )
250 else:
251 top_level = enum_desc.full_name.count('.') == 0
252 if top_level:
253 file_name = enum_desc.file.name
254 package = enum_desc.file.package
255 for enum_value in enum_desc.values:
256 full_name = _NormalizeFullyQualifiedName(
257 '.'.join((package, enum_value.name))
258 )
259 self._CheckConflictRegister(enum_value, full_name, file_name)
260 self._top_enum_values[full_name] = enum_value
261 self._AddFileDescriptor(enum_desc.file)
263 # Never call this method. It is for internal usage only.
264 def _AddServiceDescriptor(self, service_desc):
265 """Adds a ServiceDescriptor to the pool.
267 Args:
268 service_desc: A ServiceDescriptor.
269 """
271 if not isinstance(service_desc, descriptor.ServiceDescriptor):
272 raise TypeError('Expected instance of descriptor.ServiceDescriptor.')
274 self._CheckConflictRegister(
275 service_desc, service_desc.full_name, service_desc.file.name
276 )
277 self._service_descriptors[service_desc.full_name] = service_desc
279 # Never call this method. It is for internal usage only.
280 def _AddExtensionDescriptor(self, extension):
281 """Adds a FieldDescriptor describing an extension to the pool.
283 Args:
284 extension: A FieldDescriptor.
286 Raises:
287 AssertionError: when another extension with the same number extends the
288 same message.
289 TypeError: when the specified extension is not a
290 descriptor.FieldDescriptor.
291 """
292 if not (
293 isinstance(extension, descriptor.FieldDescriptor)
294 and extension.is_extension
295 ):
296 raise TypeError('Expected an extension descriptor.')
298 if extension.extension_scope is None:
299 self._CheckConflictRegister(
300 extension, extension.full_name, extension.file.name
301 )
302 self._toplevel_extensions[extension.full_name] = extension
304 try:
305 existing_desc = self._extensions_by_number[extension.containing_type][
306 extension.number
307 ]
308 except KeyError:
309 pass
310 else:
311 if extension is not existing_desc:
312 raise AssertionError(
313 'Extensions "%s" and "%s" both try to extend message type "%s" '
314 'with field number %d.'
315 % (
316 extension.full_name,
317 existing_desc.full_name,
318 extension.containing_type.full_name,
319 extension.number,
320 )
321 )
323 self._extensions_by_number[extension.containing_type][
324 extension.number
325 ] = extension
326 self._extensions_by_name[extension.containing_type][
327 extension.full_name
328 ] = extension
330 # Also register MessageSet extensions with the type name.
331 if _IsMessageSetExtension(extension):
332 self._extensions_by_name[extension.containing_type][
333 extension.message_type.full_name
334 ] = extension
336 if hasattr(extension.containing_type, '_concrete_class'):
337 python_message._AttachFieldHelpers(
338 extension.containing_type._concrete_class, extension
339 )
341 # Never call this method. It is for internal usage only.
342 def _InternalAddFileDescriptor(self, file_desc):
343 """Adds a FileDescriptor to the pool, non-recursively.
345 If the FileDescriptor contains messages or enums, the caller must explicitly
346 register them.
348 Args:
349 file_desc: A FileDescriptor.
350 """
352 self._AddFileDescriptor(file_desc)
354 def _AddFileDescriptor(self, file_desc):
355 """Adds a FileDescriptor to the pool, non-recursively.
357 If the FileDescriptor contains messages or enums, the caller must explicitly
358 register them.
360 Args:
361 file_desc: A FileDescriptor.
362 """
364 if not isinstance(file_desc, descriptor.FileDescriptor):
365 raise TypeError('Expected instance of descriptor.FileDescriptor.')
366 self._file_descriptors[file_desc.name] = file_desc
368 def FindFileByName(self, file_name):
369 """Gets a FileDescriptor by file name.
371 Args:
372 file_name (str): The path to the file to get a descriptor for.
374 Returns:
375 FileDescriptor: The descriptor for the named file.
377 Raises:
378 KeyError: if the file cannot be found in the pool.
379 """
381 try:
382 return self._file_descriptors[file_name]
383 except KeyError:
384 pass
386 try:
387 file_proto = self._internal_db.FindFileByName(file_name)
388 except KeyError as error:
389 if self._descriptor_db:
390 file_proto = self._descriptor_db.FindFileByName(file_name)
391 else:
392 raise error
393 if not file_proto:
394 raise KeyError('Cannot find a file named %s' % file_name)
395 return self._ConvertFileProtoToFileDescriptor(file_proto)
397 def FindFileContainingSymbol(self, symbol):
398 """Gets the FileDescriptor for the file containing the specified symbol.
400 Args:
401 symbol (str): The name of the symbol to search for.
403 Returns:
404 FileDescriptor: Descriptor for the file that contains the specified
405 symbol.
407 Raises:
408 KeyError: if the file cannot be found in the pool.
409 """
411 symbol = _NormalizeFullyQualifiedName(symbol)
412 try:
413 return self._InternalFindFileContainingSymbol(symbol)
414 except KeyError:
415 pass
417 try:
418 # Try fallback database. Build and find again if possible.
419 self._FindFileContainingSymbolInDb(symbol)
420 return self._InternalFindFileContainingSymbol(symbol)
421 except KeyError:
422 raise KeyError('Cannot find a file containing %s' % symbol)
424 def _InternalFindFileContainingSymbol(self, symbol):
425 """Gets the already built FileDescriptor containing the specified symbol.
427 Args:
428 symbol (str): The name of the symbol to search for.
430 Returns:
431 FileDescriptor: Descriptor for the file that contains the specified
432 symbol.
434 Raises:
435 KeyError: if the file cannot be found in the pool.
436 """
437 try:
438 return self._descriptors[symbol].file
439 except KeyError:
440 pass
442 try:
443 return self._enum_descriptors[symbol].file
444 except KeyError:
445 pass
447 try:
448 return self._service_descriptors[symbol].file
449 except KeyError:
450 pass
452 try:
453 return self._top_enum_values[symbol].type.file
454 except KeyError:
455 pass
457 try:
458 return self._toplevel_extensions[symbol].file
459 except KeyError:
460 pass
462 # Try fields, enum values and nested extensions inside a message.
463 top_name, _, sub_name = symbol.rpartition('.')
464 try:
465 message = self.FindMessageTypeByName(top_name)
466 assert (
467 sub_name in message.extensions_by_name
468 or sub_name in message.fields_by_name
469 or sub_name in message.enum_values_by_name
470 )
471 return message.file
472 except (KeyError, AssertionError):
473 raise KeyError('Cannot find a file containing %s' % symbol)
475 def FindMessageTypeByName(self, full_name):
476 """Loads the named descriptor from the pool.
478 Args:
479 full_name (str): The full name of the descriptor to load.
481 Returns:
482 Descriptor: The descriptor for the named type.
484 Raises:
485 KeyError: if the message cannot be found in the pool.
486 """
488 full_name = _NormalizeFullyQualifiedName(full_name)
489 if full_name not in self._descriptors:
490 self._FindFileContainingSymbolInDb(full_name)
491 return self._descriptors[full_name]
493 def FindEnumTypeByName(self, full_name):
494 """Loads the named enum descriptor from the pool.
496 Args:
497 full_name (str): The full name of the enum descriptor to load.
499 Returns:
500 EnumDescriptor: The enum descriptor for the named type.
502 Raises:
503 KeyError: if the enum cannot be found in the pool.
504 """
506 full_name = _NormalizeFullyQualifiedName(full_name)
507 if full_name not in self._enum_descriptors:
508 self._FindFileContainingSymbolInDb(full_name)
509 return self._enum_descriptors[full_name]
511 def FindFieldByName(self, full_name):
512 """Loads the named field descriptor from the pool.
514 Args:
515 full_name (str): The full name of the field descriptor to load.
517 Returns:
518 FieldDescriptor: The field descriptor for the named field.
520 Raises:
521 KeyError: if the field cannot be found in the pool.
522 """
523 full_name = _NormalizeFullyQualifiedName(full_name)
524 message_name, _, field_name = full_name.rpartition('.')
525 message_descriptor = self.FindMessageTypeByName(message_name)
526 return message_descriptor.fields_by_name[field_name]
528 def FindOneofByName(self, full_name):
529 """Loads the named oneof descriptor from the pool.
531 Args:
532 full_name (str): The full name of the oneof descriptor to load.
534 Returns:
535 OneofDescriptor: The oneof descriptor for the named oneof.
537 Raises:
538 KeyError: if the oneof cannot be found in the pool.
539 """
540 full_name = _NormalizeFullyQualifiedName(full_name)
541 message_name, _, oneof_name = full_name.rpartition('.')
542 message_descriptor = self.FindMessageTypeByName(message_name)
543 return message_descriptor.oneofs_by_name[oneof_name]
545 def FindExtensionByName(self, full_name):
546 """Loads the named extension descriptor from the pool.
548 Args:
549 full_name (str): The full name of the extension descriptor to load.
551 Returns:
552 FieldDescriptor: The field descriptor for the named extension.
554 Raises:
555 KeyError: if the extension cannot be found in the pool.
556 """
557 full_name = _NormalizeFullyQualifiedName(full_name)
558 try:
559 # The proto compiler does not give any link between the FileDescriptor
560 # and top-level extensions unless the FileDescriptorProto is added to
561 # the DescriptorDatabase, but this can impact memory usage.
562 # So we registered these extensions by name explicitly.
563 return self._toplevel_extensions[full_name]
564 except KeyError:
565 pass
566 message_name, _, extension_name = full_name.rpartition('.')
567 try:
568 # Most extensions are nested inside a message.
569 scope = self.FindMessageTypeByName(message_name)
570 except KeyError:
571 # Some extensions are defined at file scope.
572 scope = self._FindFileContainingSymbolInDb(full_name)
573 return scope.extensions_by_name[extension_name]
575 def FindExtensionByNumber(self, message_descriptor, number):
576 """Gets the extension of the specified message with the specified number.
578 Extensions have to be registered to this pool by calling :func:`Add` or
579 :func:`AddExtensionDescriptor`.
581 Args:
582 message_descriptor (Descriptor): descriptor of the extended message.
583 number (int): Number of the extension field.
585 Returns:
586 FieldDescriptor: The descriptor for the extension.
588 Raises:
589 KeyError: when no extension with the given number is known for the
590 specified message.
591 """
592 try:
593 return self._extensions_by_number[message_descriptor][number]
594 except KeyError:
595 self._TryLoadExtensionFromDB(message_descriptor, number)
596 return self._extensions_by_number[message_descriptor][number]
598 def FindAllExtensions(self, message_descriptor):
599 """Gets all the known extensions of a given message.
601 Extensions have to be registered to this pool by build related
602 :func:`Add` or :func:`AddExtensionDescriptor`.
604 Args:
605 message_descriptor (Descriptor): Descriptor of the extended message.
607 Returns:
608 list[FieldDescriptor]: Field descriptors describing the extensions.
609 """
610 # Fallback to descriptor db if FindAllExtensionNumbers is provided.
611 if self._descriptor_db and hasattr(
612 self._descriptor_db, 'FindAllExtensionNumbers'
613 ):
614 full_name = message_descriptor.full_name
615 try:
616 all_numbers = self._descriptor_db.FindAllExtensionNumbers(full_name)
617 except:
618 pass
619 else:
620 if isinstance(all_numbers, list):
621 for number in all_numbers:
622 if number in self._extensions_by_number[message_descriptor]:
623 continue
624 self._TryLoadExtensionFromDB(message_descriptor, number)
625 else:
626 warnings.warn(
627 'FindAllExtensionNumbers() on fall back DB must return a list,'
628 ' not {0}'.format(type(all_numbers))
629 )
631 return list(self._extensions_by_number[message_descriptor].values())
633 def _TryLoadExtensionFromDB(self, message_descriptor, number):
634 """Try to Load extensions from descriptor db.
636 Args:
637 message_descriptor: descriptor of the extended message.
638 number: the extension number that needs to be loaded.
639 """
640 if not self._descriptor_db:
641 return
642 # Only supported when FindFileContainingExtension is provided.
643 if not hasattr(self._descriptor_db, 'FindFileContainingExtension'):
644 return
646 full_name = message_descriptor.full_name
647 file_proto = None
648 try:
649 file_proto = self._descriptor_db.FindFileContainingExtension(
650 full_name, number
651 )
652 except:
653 return
655 if file_proto is None:
656 return
658 try:
659 self._ConvertFileProtoToFileDescriptor(file_proto)
660 except:
661 warn_msg = 'Unable to load proto file %s for extension number %d.' % (
662 file_proto.name,
663 number,
664 )
665 warnings.warn(warn_msg, RuntimeWarning)
667 def FindServiceByName(self, full_name):
668 """Loads the named service descriptor from the pool.
670 Args:
671 full_name (str): The full name of the service descriptor to load.
673 Returns:
674 ServiceDescriptor: The service descriptor for the named service.
676 Raises:
677 KeyError: if the service cannot be found in the pool.
678 """
679 full_name = _NormalizeFullyQualifiedName(full_name)
680 if full_name not in self._service_descriptors:
681 self._FindFileContainingSymbolInDb(full_name)
682 return self._service_descriptors[full_name]
684 def FindMethodByName(self, full_name):
685 """Loads the named service method descriptor from the pool.
687 Args:
688 full_name (str): The full name of the method descriptor to load.
690 Returns:
691 MethodDescriptor: The method descriptor for the service method.
693 Raises:
694 KeyError: if the method cannot be found in the pool.
695 """
696 full_name = _NormalizeFullyQualifiedName(full_name)
697 service_name, _, method_name = full_name.rpartition('.')
698 service_descriptor = self.FindServiceByName(service_name)
699 return service_descriptor.methods_by_name[method_name]
701 def SetFeatureSetDefaults(self, defaults):
702 """Sets the default feature mappings used during the build.
704 Args:
705 defaults: a FeatureSetDefaults message containing the new mappings.
706 """
707 if self._edition_defaults is not None:
708 raise ValueError(
709 "Feature set defaults can't be changed once the pool has started"
710 ' building!'
711 )
713 # pylint: disable=g-import-not-at-top
714 from google.protobuf import descriptor_pb2
716 if not isinstance(defaults, descriptor_pb2.FeatureSetDefaults):
717 raise TypeError('SetFeatureSetDefaults called with invalid type')
719 if defaults.minimum_edition > defaults.maximum_edition:
720 raise ValueError(
721 'Invalid edition range %s to %s'
722 % (
723 descriptor_pb2.Edition.Name(defaults.minimum_edition),
724 descriptor_pb2.Edition.Name(defaults.maximum_edition),
725 )
726 )
728 prev_edition = descriptor_pb2.Edition.EDITION_UNKNOWN
729 for d in defaults.defaults:
730 if d.edition == descriptor_pb2.Edition.EDITION_UNKNOWN:
731 raise ValueError('Invalid edition EDITION_UNKNOWN specified')
732 if prev_edition >= d.edition:
733 raise ValueError(
734 'Feature set defaults are not strictly increasing. %s is greater'
735 ' than or equal to %s'
736 % (
737 descriptor_pb2.Edition.Name(prev_edition),
738 descriptor_pb2.Edition.Name(d.edition),
739 )
740 )
741 prev_edition = d.edition
742 self._edition_defaults = defaults
744 def _CreateDefaultFeatures(self, edition):
745 """Creates a FeatureSet message with defaults for a specific edition.
747 Args:
748 edition: the edition to generate defaults for.
750 Returns:
751 A FeatureSet message with defaults for a specific edition.
752 """
753 # pylint: disable=g-import-not-at-top
754 from google.protobuf import descriptor_pb2
756 with _edition_defaults_lock:
757 if not self._edition_defaults:
758 self._edition_defaults = descriptor_pb2.FeatureSetDefaults()
759 self._edition_defaults.ParseFromString(
760 self._serialized_edition_defaults
761 )
763 if edition < self._edition_defaults.minimum_edition:
764 raise TypeError(
765 'Edition %s is earlier than the minimum supported edition %s!'
766 % (
767 descriptor_pb2.Edition.Name(edition),
768 descriptor_pb2.Edition.Name(
769 self._edition_defaults.minimum_edition
770 ),
771 )
772 )
773 if (
774 edition > self._edition_defaults.maximum_edition
775 and edition != descriptor_pb2.EDITION_UNSTABLE
776 ):
777 raise TypeError(
778 'Edition %s is later than the maximum supported edition %s!'
779 % (
780 descriptor_pb2.Edition.Name(edition),
781 descriptor_pb2.Edition.Name(
782 self._edition_defaults.maximum_edition
783 ),
784 )
785 )
786 found = None
787 for d in self._edition_defaults.defaults:
788 if d.edition > edition:
789 break
790 found = d
791 if found is None:
792 raise TypeError(
793 'No valid default found for edition %s!'
794 % descriptor_pb2.Edition.Name(edition)
795 )
797 defaults = descriptor_pb2.FeatureSet()
798 defaults.CopyFrom(found.fixed_features)
799 defaults.MergeFrom(found.overridable_features)
800 return defaults
802 def _InternFeatures(self, features):
803 serialized = features.SerializeToString()
804 with _edition_defaults_lock:
805 cached = self._feature_cache.get(serialized)
806 if cached is None:
807 self._feature_cache[serialized] = features
808 cached = features
809 return cached
811 def _FindFileContainingSymbolInDb(self, symbol):
812 """Finds the file in descriptor DB containing the specified symbol.
814 Args:
815 symbol (str): The name of the symbol to search for.
817 Returns:
818 FileDescriptor: The file that contains the specified symbol.
820 Raises:
821 KeyError: if the file cannot be found in the descriptor database.
822 """
823 try:
824 file_proto = self._internal_db.FindFileContainingSymbol(symbol)
825 except KeyError as error:
826 if self._descriptor_db:
827 file_proto = self._descriptor_db.FindFileContainingSymbol(symbol)
828 else:
829 raise error
830 if not file_proto:
831 raise KeyError('Cannot find a file containing %s' % symbol)
832 return self._ConvertFileProtoToFileDescriptor(file_proto)
834 def _ConvertFileProtoToFileDescriptor(self, file_proto):
835 """Creates a FileDescriptor from a proto or returns a cached copy.
837 This method also has the side effect of loading all the symbols found in
838 the file into the appropriate dictionaries in the pool.
840 Args:
841 file_proto: The proto to convert.
843 Returns:
844 A FileDescriptor matching the passed in proto.
845 """
846 if file_proto.name not in self._file_descriptors:
847 built_deps = list(self._GetDeps(file_proto.dependency))
848 direct_deps = [self.FindFileByName(n) for n in file_proto.dependency]
849 public_deps = [direct_deps[i] for i in file_proto.public_dependency]
851 # pylint: disable=g-import-not-at-top
852 from google.protobuf import descriptor_pb2
854 file_descriptor = descriptor.FileDescriptor(
855 pool=self,
856 name=file_proto.name,
857 package=file_proto.package,
858 syntax=file_proto.syntax,
859 edition=descriptor_pb2.Edition.Name(file_proto.edition),
860 options=_OptionsOrNone(file_proto),
861 serialized_pb=file_proto.SerializeToString(),
862 dependencies=direct_deps,
863 public_dependencies=public_deps,
864 # pylint: disable=protected-access
865 create_key=descriptor._internal_create_key,
866 )
867 scope = {}
869 # This loop extracts all the message and enum types from all the
870 # dependencies of the file_proto. This is necessary to create the
871 # scope of available message types when defining the passed in
872 # file proto.
873 for dependency in built_deps:
874 scope.update(
875 self._ExtractSymbols(dependency.message_types_by_name.values())
876 )
877 scope.update(
878 (_PrefixWithDot(enum.full_name), enum)
879 for enum in dependency.enum_types_by_name.values()
880 )
882 for message_type in file_proto.message_type:
883 message_desc = self._ConvertMessageDescriptor(
884 message_type,
885 file_proto.package,
886 file_descriptor,
887 scope,
888 file_proto.syntax,
889 )
890 file_descriptor.message_types_by_name[message_desc.name] = message_desc
892 for enum_type in file_proto.enum_type:
893 file_descriptor.enum_types_by_name[enum_type.name] = (
894 self._ConvertEnumDescriptor(
895 enum_type,
896 file_proto.package,
897 file_descriptor,
898 None,
899 scope,
900 True,
901 )
902 )
904 for index, extension_proto in enumerate(file_proto.extension):
905 extension_desc = self._MakeFieldDescriptor(
906 extension_proto,
907 file_proto.package,
908 index,
909 file_descriptor,
910 is_extension=True,
911 )
912 extension_desc.containing_type = self._GetTypeFromScope(
913 file_descriptor.package, extension_proto.extendee, scope
914 )
915 self._SetFieldType(
916 extension_proto, extension_desc, file_descriptor.package, scope
917 )
918 file_descriptor.extensions_by_name[extension_desc.name] = extension_desc
920 for desc_proto in file_proto.message_type:
921 self._SetAllFieldTypes(file_proto.package, desc_proto, scope)
923 if file_proto.package:
924 desc_proto_prefix = _PrefixWithDot(file_proto.package)
925 else:
926 desc_proto_prefix = ''
928 for desc_proto in file_proto.message_type:
929 desc = self._GetTypeFromScope(desc_proto_prefix, desc_proto.name, scope)
930 file_descriptor.message_types_by_name[desc_proto.name] = desc
932 for index, service_proto in enumerate(file_proto.service):
933 file_descriptor.services_by_name[service_proto.name] = (
934 self._MakeServiceDescriptor(
935 service_proto, index, scope, file_proto.package, file_descriptor
936 )
937 )
939 self._file_descriptors[file_proto.name] = file_descriptor
941 # Add extensions to the pool
942 def AddExtensionForNested(message_type):
943 for nested in message_type.nested_types:
944 AddExtensionForNested(nested)
945 for extension in message_type.extensions:
946 self._AddExtensionDescriptor(extension)
948 file_desc = self._file_descriptors[file_proto.name]
949 for extension in file_desc.extensions_by_name.values():
950 self._AddExtensionDescriptor(extension)
951 for message_type in file_desc.message_types_by_name.values():
952 AddExtensionForNested(message_type)
954 return file_desc
956 def _ConvertMessageDescriptor(
957 self, desc_proto, package=None, file_desc=None, scope=None, syntax=None
958 ):
959 """Adds the proto to the pool in the specified package.
961 Args:
962 desc_proto: The descriptor_pb2.DescriptorProto protobuf message.
963 package: The package the proto should be located in.
964 file_desc: The file containing this message.
965 scope: Dict mapping short and full symbols to message and enum types.
966 syntax: string indicating syntax of the file ("proto2" or "proto3")
968 Returns:
969 The added descriptor.
970 """
972 if package:
973 desc_name = '.'.join((package, desc_proto.name))
974 else:
975 desc_name = desc_proto.name
977 if file_desc is None:
978 file_name = None
979 else:
980 file_name = file_desc.name
982 if scope is None:
983 scope = {}
985 nested = [
986 self._ConvertMessageDescriptor(
987 nested, desc_name, file_desc, scope, syntax
988 )
989 for nested in desc_proto.nested_type
990 ]
991 enums = [
992 self._ConvertEnumDescriptor(
993 enum, desc_name, file_desc, None, scope, False
994 )
995 for enum in desc_proto.enum_type
996 ]
997 fields = [
998 self._MakeFieldDescriptor(field, desc_name, index, file_desc)
999 for index, field in enumerate(desc_proto.field)
1000 ]
1001 extensions = [
1002 self._MakeFieldDescriptor(
1003 extension, desc_name, index, file_desc, is_extension=True
1004 )
1005 for index, extension in enumerate(desc_proto.extension)
1006 ]
1007 oneofs = [
1008 # pylint: disable=g-complex-comprehension
1009 descriptor.OneofDescriptor(
1010 desc.name,
1011 '.'.join((desc_name, desc.name)),
1012 index,
1013 None,
1014 [],
1015 _OptionsOrNone(desc),
1016 # pylint: disable=protected-access
1017 create_key=descriptor._internal_create_key,
1018 )
1019 for index, desc in enumerate(desc_proto.oneof_decl)
1020 ]
1021 extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range]
1022 if extension_ranges:
1023 is_extendable = True
1024 else:
1025 is_extendable = False
1026 desc = descriptor.Descriptor(
1027 name=desc_proto.name,
1028 full_name=desc_name,
1029 filename=file_name,
1030 containing_type=None,
1031 fields=fields,
1032 oneofs=oneofs,
1033 nested_types=nested,
1034 enum_types=enums,
1035 extensions=extensions,
1036 options=_OptionsOrNone(desc_proto),
1037 is_extendable=is_extendable,
1038 extension_ranges=extension_ranges,
1039 file=file_desc,
1040 serialized_start=None,
1041 serialized_end=None,
1042 is_map_entry=desc_proto.options.map_entry,
1043 # pylint: disable=protected-access
1044 create_key=descriptor._internal_create_key,
1045 )
1046 for nested in desc.nested_types:
1047 nested.containing_type = desc
1048 for enum in desc.enum_types:
1049 enum.containing_type = desc
1050 for field_index, field_desc in enumerate(desc_proto.field):
1051 if field_desc.HasField('oneof_index'):
1052 oneof_index = field_desc.oneof_index
1053 oneofs[oneof_index].fields.append(fields[field_index])
1054 fields[field_index].containing_oneof = oneofs[oneof_index]
1056 scope[_PrefixWithDot(desc_name)] = desc
1057 self._CheckConflictRegister(desc, desc.full_name, desc.file.name)
1058 self._descriptors[desc_name] = desc
1059 return desc
1061 def _ConvertEnumDescriptor(
1062 self,
1063 enum_proto,
1064 package=None,
1065 file_desc=None,
1066 containing_type=None,
1067 scope=None,
1068 top_level=False,
1069 ):
1070 """Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf.
1072 Args:
1073 enum_proto: The descriptor_pb2.EnumDescriptorProto protobuf message.
1074 package: Optional package name for the new message EnumDescriptor.
1075 file_desc: The file containing the enum descriptor.
1076 containing_type: The type containing this enum.
1077 scope: Scope containing available types.
1078 top_level: If True, the enum is a top level symbol. If False, the enum is
1079 defined inside a message.
1081 Returns:
1082 The added descriptor
1083 """
1085 if package:
1086 enum_name = '.'.join((package, enum_proto.name))
1087 else:
1088 enum_name = enum_proto.name
1090 if file_desc is None:
1091 file_name = None
1092 else:
1093 file_name = file_desc.name
1095 values = [
1096 self._MakeEnumValueDescriptor(value, index)
1097 for index, value in enumerate(enum_proto.value)
1098 ]
1099 desc = descriptor.EnumDescriptor(
1100 name=enum_proto.name,
1101 full_name=enum_name,
1102 filename=file_name,
1103 file=file_desc,
1104 values=values,
1105 containing_type=containing_type,
1106 options=_OptionsOrNone(enum_proto),
1107 # pylint: disable=protected-access
1108 create_key=descriptor._internal_create_key,
1109 )
1110 scope['.%s' % enum_name] = desc
1111 self._CheckConflictRegister(desc, desc.full_name, desc.file.name)
1112 self._enum_descriptors[enum_name] = desc
1114 # Add top level enum values.
1115 if top_level:
1116 for value in values:
1117 full_name = _NormalizeFullyQualifiedName(
1118 '.'.join((package, value.name))
1119 )
1120 self._CheckConflictRegister(value, full_name, file_name)
1121 self._top_enum_values[full_name] = value
1123 return desc
1125 def _MakeFieldDescriptor(
1126 self, field_proto, message_name, index, file_desc, is_extension=False
1127 ):
1128 """Creates a field descriptor from a FieldDescriptorProto.
1130 For message and enum type fields, this method will do a look up
1131 in the pool for the appropriate descriptor for that type. If it
1132 is unavailable, it will fall back to the _source function to
1133 create it. If this type is still unavailable, construction will
1134 fail.
1136 Args:
1137 field_proto: The proto describing the field.
1138 message_name: The name of the containing message.
1139 index: Index of the field
1140 file_desc: The file containing the field descriptor.
1141 is_extension: Indication that this field is for an extension.
1143 Returns:
1144 An initialized FieldDescriptor object
1145 """
1147 if message_name:
1148 full_name = '.'.join((message_name, field_proto.name))
1149 else:
1150 full_name = field_proto.name
1152 if field_proto.json_name:
1153 json_name = field_proto.json_name
1154 else:
1155 json_name = None
1157 return descriptor.FieldDescriptor(
1158 name=field_proto.name,
1159 full_name=full_name,
1160 index=index,
1161 number=field_proto.number,
1162 type=field_proto.type,
1163 cpp_type=None,
1164 message_type=None,
1165 enum_type=None,
1166 containing_type=None,
1167 label=field_proto.label,
1168 has_default_value=False,
1169 default_value=None,
1170 is_extension=is_extension,
1171 extension_scope=None,
1172 options=_OptionsOrNone(field_proto),
1173 json_name=json_name,
1174 file=file_desc,
1175 # pylint: disable=protected-access
1176 create_key=descriptor._internal_create_key,
1177 )
1179 def _SetAllFieldTypes(self, package, desc_proto, scope):
1180 """Sets all the descriptor's fields's types.
1182 This method also sets the containing types on any extensions.
1184 Args:
1185 package: The current package of desc_proto.
1186 desc_proto: The message descriptor to update.
1187 scope: Enclosing scope of available types.
1188 """
1190 package = _PrefixWithDot(package)
1192 main_desc = self._GetTypeFromScope(package, desc_proto.name, scope)
1194 if package == '.':
1195 nested_package = _PrefixWithDot(desc_proto.name)
1196 else:
1197 nested_package = '.'.join([package, desc_proto.name])
1199 for field_proto, field_desc in zip(desc_proto.field, main_desc.fields):
1200 self._SetFieldType(field_proto, field_desc, nested_package, scope)
1202 for extension_proto, extension_desc in zip(
1203 desc_proto.extension, main_desc.extensions
1204 ):
1205 extension_desc.containing_type = self._GetTypeFromScope(
1206 nested_package, extension_proto.extendee, scope
1207 )
1208 self._SetFieldType(extension_proto, extension_desc, nested_package, scope)
1210 for nested_type in desc_proto.nested_type:
1211 self._SetAllFieldTypes(nested_package, nested_type, scope)
1213 def _SetFieldType(self, field_proto, field_desc, package, scope):
1214 """Sets the field's type, cpp_type, message_type and enum_type.
1216 Args:
1217 field_proto: Data about the field in proto format.
1218 field_desc: The descriptor to modify.
1219 package: The package the field's container is in.
1220 scope: Enclosing scope of available types.
1221 """
1222 if field_proto.type_name:
1223 desc = self._GetTypeFromScope(package, field_proto.type_name, scope)
1224 else:
1225 desc = None
1227 if not field_proto.HasField('type'):
1228 if isinstance(desc, descriptor.Descriptor):
1229 field_proto.type = descriptor.FieldDescriptor.TYPE_MESSAGE
1230 else:
1231 field_proto.type = descriptor.FieldDescriptor.TYPE_ENUM
1233 field_desc.cpp_type = descriptor.FieldDescriptor.ProtoTypeToCppProtoType(
1234 field_proto.type
1235 )
1237 if (
1238 field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE
1239 or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP
1240 ):
1241 field_desc.message_type = desc
1243 if field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
1244 field_desc.enum_type = desc
1246 if field_proto.label == descriptor.FieldDescriptor.LABEL_REPEATED:
1247 field_desc.has_default_value = False
1248 field_desc.default_value = []
1249 elif field_proto.HasField('default_value'):
1250 field_desc.has_default_value = True
1251 if (
1252 field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE
1253 or field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT
1254 ):
1255 field_desc.default_value = float(field_proto.default_value)
1256 elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING:
1257 field_desc.default_value = field_proto.default_value
1258 elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL:
1259 field_desc.default_value = field_proto.default_value.lower() == 'true'
1260 elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
1261 field_desc.default_value = field_desc.enum_type.values_by_name[
1262 field_proto.default_value
1263 ].number
1264 elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES:
1265 field_desc.default_value = text_encoding.CUnescape(
1266 field_proto.default_value
1267 )
1268 elif field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE:
1269 field_desc.default_value = None
1270 else:
1271 # All other types are of the "int" type.
1272 field_desc.default_value = int(field_proto.default_value)
1273 else:
1274 field_desc.has_default_value = False
1275 if (
1276 field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE
1277 or field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT
1278 ):
1279 field_desc.default_value = 0.0
1280 elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING:
1281 field_desc.default_value = ''
1282 elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL:
1283 field_desc.default_value = False
1284 elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM:
1285 field_desc.default_value = field_desc.enum_type.values[0].number
1286 elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES:
1287 field_desc.default_value = b''
1288 elif field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE:
1289 field_desc.default_value = None
1290 elif field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP:
1291 field_desc.default_value = None
1292 else:
1293 # All other types are of the "int" type.
1294 field_desc.default_value = 0
1296 field_desc.type = field_proto.type
1298 def _MakeEnumValueDescriptor(self, value_proto, index):
1299 """Creates a enum value descriptor object from a enum value proto.
1301 Args:
1302 value_proto: The proto describing the enum value.
1303 index: The index of the enum value.
1305 Returns:
1306 An initialized EnumValueDescriptor object.
1307 """
1309 return descriptor.EnumValueDescriptor(
1310 name=value_proto.name,
1311 index=index,
1312 number=value_proto.number,
1313 options=_OptionsOrNone(value_proto),
1314 type=None,
1315 # pylint: disable=protected-access
1316 create_key=descriptor._internal_create_key,
1317 )
1319 def _MakeServiceDescriptor(
1320 self, service_proto, service_index, scope, package, file_desc
1321 ):
1322 """Make a protobuf ServiceDescriptor given a ServiceDescriptorProto.
1324 Args:
1325 service_proto: The descriptor_pb2.ServiceDescriptorProto protobuf message.
1326 service_index: The index of the service in the File.
1327 scope: Dict mapping short and full symbols to message and enum types.
1328 package: Optional package name for the new message EnumDescriptor.
1329 file_desc: The file containing the service descriptor.
1331 Returns:
1332 The added descriptor.
1333 """
1335 if package:
1336 service_name = '.'.join((package, service_proto.name))
1337 else:
1338 service_name = service_proto.name
1340 methods = [
1341 self._MakeMethodDescriptor(
1342 method_proto, service_name, package, scope, index
1343 )
1344 for index, method_proto in enumerate(service_proto.method)
1345 ]
1346 desc = descriptor.ServiceDescriptor(
1347 name=service_proto.name,
1348 full_name=service_name,
1349 index=service_index,
1350 methods=methods,
1351 options=_OptionsOrNone(service_proto),
1352 file=file_desc,
1353 # pylint: disable=protected-access
1354 create_key=descriptor._internal_create_key,
1355 )
1356 self._CheckConflictRegister(desc, desc.full_name, desc.file.name)
1357 self._service_descriptors[service_name] = desc
1358 return desc
1360 def _MakeMethodDescriptor(
1361 self, method_proto, service_name, package, scope, index
1362 ):
1363 """Creates a method descriptor from a MethodDescriptorProto.
1365 Args:
1366 method_proto: The proto describing the method.
1367 service_name: The name of the containing service.
1368 package: Optional package name to look up for types.
1369 scope: Scope containing available types.
1370 index: Index of the method in the service.
1372 Returns:
1373 An initialized MethodDescriptor object.
1374 """
1375 full_name = '.'.join((service_name, method_proto.name))
1376 input_type = self._GetTypeFromScope(package, method_proto.input_type, scope)
1377 output_type = self._GetTypeFromScope(
1378 package, method_proto.output_type, scope
1379 )
1380 return descriptor.MethodDescriptor(
1381 name=method_proto.name,
1382 full_name=full_name,
1383 index=index,
1384 containing_service=None,
1385 input_type=input_type,
1386 output_type=output_type,
1387 client_streaming=method_proto.client_streaming,
1388 server_streaming=method_proto.server_streaming,
1389 options=_OptionsOrNone(method_proto),
1390 # pylint: disable=protected-access
1391 create_key=descriptor._internal_create_key,
1392 )
1394 def _ExtractSymbols(self, descriptors):
1395 """Pulls out all the symbols from descriptor protos.
1397 Args:
1398 descriptors: The messages to extract descriptors from.
1400 Yields:
1401 A two element tuple of the type name and descriptor object.
1402 """
1404 for desc in descriptors:
1405 yield (_PrefixWithDot(desc.full_name), desc)
1406 for symbol in self._ExtractSymbols(desc.nested_types):
1407 yield symbol
1408 for enum in desc.enum_types:
1409 yield (_PrefixWithDot(enum.full_name), enum)
1411 def _GetDeps(self, dependencies, visited=None):
1412 """Recursively finds dependencies for file protos.
1414 Args:
1415 dependencies: The names of the files being depended on.
1416 visited: The names of files already found.
1418 Yields:
1419 Each direct and indirect dependency.
1420 """
1422 visited = visited or set()
1423 for dependency in dependencies:
1424 if dependency not in visited:
1425 visited.add(dependency)
1426 dep_desc = self.FindFileByName(dependency)
1427 yield dep_desc
1428 public_files = [d.name for d in dep_desc.public_dependencies]
1429 yield from self._GetDeps(public_files, visited)
1431 def _GetTypeFromScope(self, package, type_name, scope):
1432 """Finds a given type name in the current scope.
1434 Args:
1435 package: The package the proto should be located in.
1436 type_name: The name of the type to be found in the scope.
1437 scope: Dict mapping short and full symbols to message and enum types.
1439 Returns:
1440 The descriptor for the requested type.
1441 """
1442 if type_name not in scope:
1443 components = _PrefixWithDot(package).split('.')
1444 while components:
1445 possible_match = '.'.join(components + [type_name])
1446 if possible_match in scope:
1447 type_name = possible_match
1448 break
1449 else:
1450 components.pop(-1)
1451 return scope[type_name]
1454def _PrefixWithDot(name):
1455 return name if name.startswith('.') else '.%s' % name
1458if _USE_C_DESCRIPTORS:
1459 # TODO: This pool could be constructed from Python code, when we
1460 # support a flag like 'use_cpp_generated_pool=True'.
1461 # pylint: disable=protected-access
1462 _DEFAULT = descriptor._message.default_pool
1463else:
1464 _DEFAULT = DescriptorPool()
1467def Default():
1468 return _DEFAULT