Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/loaders.py: 90%
156 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:51 +0000
1# Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"). You
4# may not use this file except in compliance with the License. A copy of
5# the License is located at
6#
7# http://aws.amazon.com/apache2.0/
8#
9# or in the "license" file accompanying this file. This file is
10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11# ANY KIND, either express or implied. See the License for the specific
12# language governing permissions and limitations under the License.
13"""Module for loading various model files.
15This module provides the classes that are used to load models used
16by botocore. This can include:
18 * Service models (e.g. the model for EC2, S3, DynamoDB, etc.)
19 * Service model extras which customize the service models
20 * Other models associated with a service (pagination, waiters)
21 * Non service-specific config (Endpoint data, retry config)
23Loading a module is broken down into several steps:
25 * Determining the path to load
26 * Search the data_path for files to load
27 * The mechanics of loading the file
28 * Searching for extras and applying them to the loaded file
30The last item is used so that other faster loading mechanism
31besides the default JSON loader can be used.
33The Search Path
34===============
36Similar to how the PATH environment variable is to finding executables
37and the PYTHONPATH environment variable is to finding python modules
38to import, the botocore loaders have the concept of a data path exposed
39through AWS_DATA_PATH.
41This enables end users to provide additional search paths where we
42will attempt to load models outside of the models we ship with
43botocore. When you create a ``Loader``, there are two paths
44automatically added to the model search path:
46 * <botocore root>/data/
47 * ~/.aws/models
49The first value is the path where all the model files shipped with
50botocore are located.
52The second path is so that users can just drop new model files in
53``~/.aws/models`` without having to mess around with the AWS_DATA_PATH.
55The AWS_DATA_PATH using the platform specific path separator to
56separate entries (typically ``:`` on linux and ``;`` on windows).
59Directory Layout
60================
62The Loader expects a particular directory layout. In order for any
63directory specified in AWS_DATA_PATH to be considered, it must have
64this structure for service models::
66 <root>
67 |
68 |-- servicename1
69 | |-- 2012-10-25
70 | |-- service-2.json
71 |-- ec2
72 | |-- 2014-01-01
73 | | |-- paginators-1.json
74 | | |-- service-2.json
75 | | |-- waiters-2.json
76 | |-- 2015-03-01
77 | |-- paginators-1.json
78 | |-- service-2.json
79 | |-- waiters-2.json
80 | |-- service-2.sdk-extras.json
83That is:
85 * The root directory contains sub directories that are the name
86 of the services.
87 * Within each service directory, there's a sub directory for each
88 available API version.
89 * Within each API version, there are model specific files, including
90 (but not limited to): service-2.json, waiters-2.json, paginators-1.json
92The ``-1`` and ``-2`` suffix at the end of the model files denote which version
93schema is used within the model. Even though this information is available in
94the ``version`` key within the model, this version is also part of the filename
95so that code does not need to load the JSON model in order to determine which
96version to use.
98The ``sdk-extras`` and similar files represent extra data that needs to be
99applied to the model after it is loaded. Data in these files might represent
100information that doesn't quite fit in the original models, but is still needed
101for the sdk. For instance, additional operation parameters might be added here
102which don't represent the actual service api.
103"""
104import logging
105import os
107from botocore import BOTOCORE_ROOT
108from botocore.compat import HAS_GZIP, OrderedDict, json
109from botocore.exceptions import DataNotFoundError, UnknownServiceError
110from botocore.utils import deep_merge
112_JSON_OPEN_METHODS = {
113 '.json': open,
114}
117if HAS_GZIP:
118 from gzip import open as gzip_open
120 _JSON_OPEN_METHODS['.json.gz'] = gzip_open
123logger = logging.getLogger(__name__)
126def instance_cache(func):
127 """Cache the result of a method on a per instance basis.
129 This is not a general purpose caching decorator. In order
130 for this to be used, it must be used on methods on an
131 instance, and that instance *must* provide a
132 ``self._cache`` dictionary.
134 """
136 def _wrapper(self, *args, **kwargs):
137 key = (func.__name__,) + args
138 for pair in sorted(kwargs.items()):
139 key += pair
140 if key in self._cache:
141 return self._cache[key]
142 data = func(self, *args, **kwargs)
143 self._cache[key] = data
144 return data
146 return _wrapper
149class JSONFileLoader:
150 """Loader JSON files.
152 This class can load the default format of models, which is a JSON file.
154 """
156 def exists(self, file_path):
157 """Checks if the file exists.
159 :type file_path: str
160 :param file_path: The full path to the file to load without
161 the '.json' extension.
163 :return: True if file path exists, False otherwise.
165 """
166 for ext in _JSON_OPEN_METHODS:
167 if os.path.isfile(file_path + ext):
168 return True
169 return False
171 def _load_file(self, full_path, open_method):
172 if not os.path.isfile(full_path):
173 return
175 # By default the file will be opened with locale encoding on Python 3.
176 # We specify "utf8" here to ensure the correct behavior.
177 with open_method(full_path, 'rb') as fp:
178 payload = fp.read().decode('utf-8')
180 logger.debug("Loading JSON file: %s", full_path)
181 return json.loads(payload, object_pairs_hook=OrderedDict)
183 def load_file(self, file_path):
184 """Attempt to load the file path.
186 :type file_path: str
187 :param file_path: The full path to the file to load without
188 the '.json' extension.
190 :return: The loaded data if it exists, otherwise None.
192 """
193 for ext, open_method in _JSON_OPEN_METHODS.items():
194 data = self._load_file(file_path + ext, open_method)
195 if data is not None:
196 return data
197 return None
200def create_loader(search_path_string=None):
201 """Create a Loader class.
203 This factory function creates a loader given a search string path.
205 :type search_string_path: str
206 :param search_string_path: The AWS_DATA_PATH value. A string
207 of data path values separated by the ``os.path.pathsep`` value,
208 which is typically ``:`` on POSIX platforms and ``;`` on
209 windows.
211 :return: A ``Loader`` instance.
213 """
214 if search_path_string is None:
215 return Loader()
216 paths = []
217 extra_paths = search_path_string.split(os.pathsep)
218 for path in extra_paths:
219 path = os.path.expanduser(os.path.expandvars(path))
220 paths.append(path)
221 return Loader(extra_search_paths=paths)
224class Loader:
225 """Find and load data models.
227 This class will handle searching for and loading data models.
229 The main method used here is ``load_service_model``, which is a
230 convenience method over ``load_data`` and ``determine_latest_version``.
232 """
234 FILE_LOADER_CLASS = JSONFileLoader
235 # The included models in botocore/data/ that we ship with botocore.
236 BUILTIN_DATA_PATH = os.path.join(BOTOCORE_ROOT, 'data')
237 # For convenience we automatically add ~/.aws/models to the data path.
238 CUSTOMER_DATA_PATH = os.path.join(
239 os.path.expanduser('~'), '.aws', 'models'
240 )
241 BUILTIN_EXTRAS_TYPES = ['sdk']
243 def __init__(
244 self,
245 extra_search_paths=None,
246 file_loader=None,
247 cache=None,
248 include_default_search_paths=True,
249 include_default_extras=True,
250 ):
251 self._cache = {}
252 if file_loader is None:
253 file_loader = self.FILE_LOADER_CLASS()
254 self.file_loader = file_loader
255 if extra_search_paths is not None:
256 self._search_paths = extra_search_paths
257 else:
258 self._search_paths = []
259 if include_default_search_paths:
260 self._search_paths.extend(
261 [self.CUSTOMER_DATA_PATH, self.BUILTIN_DATA_PATH]
262 )
264 self._extras_types = []
265 if include_default_extras:
266 self._extras_types.extend(self.BUILTIN_EXTRAS_TYPES)
268 self._extras_processor = ExtrasProcessor()
270 @property
271 def search_paths(self):
272 return self._search_paths
274 @property
275 def extras_types(self):
276 return self._extras_types
278 @instance_cache
279 def list_available_services(self, type_name):
280 """List all known services.
282 This will traverse the search path and look for all known
283 services.
285 :type type_name: str
286 :param type_name: The type of the service (service-2,
287 paginators-1, waiters-2, etc). This is needed because
288 the list of available services depends on the service
289 type. For example, the latest API version available for
290 a resource-1.json file may not be the latest API version
291 available for a services-2.json file.
293 :return: A list of all services. The list of services will
294 be sorted.
296 """
297 services = set()
298 for possible_path in self._potential_locations():
299 # Any directory in the search path is potentially a service.
300 # We'll collect any initial list of potential services,
301 # but we'll then need to further process these directories
302 # by searching for the corresponding type_name in each
303 # potential directory.
304 possible_services = [
305 d
306 for d in os.listdir(possible_path)
307 if os.path.isdir(os.path.join(possible_path, d))
308 ]
309 for service_name in possible_services:
310 full_dirname = os.path.join(possible_path, service_name)
311 api_versions = os.listdir(full_dirname)
312 for api_version in api_versions:
313 full_load_path = os.path.join(
314 full_dirname, api_version, type_name
315 )
316 if self.file_loader.exists(full_load_path):
317 services.add(service_name)
318 break
319 return sorted(services)
321 @instance_cache
322 def determine_latest_version(self, service_name, type_name):
323 """Find the latest API version available for a service.
325 :type service_name: str
326 :param service_name: The name of the service.
328 :type type_name: str
329 :param type_name: The type of the service (service-2,
330 paginators-1, waiters-2, etc). This is needed because
331 the latest API version available can depend on the service
332 type. For example, the latest API version available for
333 a resource-1.json file may not be the latest API version
334 available for a services-2.json file.
336 :rtype: str
337 :return: The latest API version. If the service does not exist
338 or does not have any available API data, then a
339 ``DataNotFoundError`` exception will be raised.
341 """
342 return max(self.list_api_versions(service_name, type_name))
344 @instance_cache
345 def list_api_versions(self, service_name, type_name):
346 """List all API versions available for a particular service type
348 :type service_name: str
349 :param service_name: The name of the service
351 :type type_name: str
352 :param type_name: The type name for the service (i.e service-2,
353 paginators-1, etc.)
355 :rtype: list
356 :return: A list of API version strings in sorted order.
358 """
359 known_api_versions = set()
360 for possible_path in self._potential_locations(
361 service_name, must_exist=True, is_dir=True
362 ):
363 for dirname in os.listdir(possible_path):
364 full_path = os.path.join(possible_path, dirname, type_name)
365 # Only add to the known_api_versions if the directory
366 # contains a service-2, paginators-1, etc. file corresponding
367 # to the type_name passed in.
368 if self.file_loader.exists(full_path):
369 known_api_versions.add(dirname)
370 if not known_api_versions:
371 raise DataNotFoundError(data_path=service_name)
372 return sorted(known_api_versions)
374 @instance_cache
375 def load_service_model(self, service_name, type_name, api_version=None):
376 """Load a botocore service model
378 This is the main method for loading botocore models (e.g. a service
379 model, pagination configs, waiter configs, etc.).
381 :type service_name: str
382 :param service_name: The name of the service (e.g ``ec2``, ``s3``).
384 :type type_name: str
385 :param type_name: The model type. Valid types include, but are not
386 limited to: ``service-2``, ``paginators-1``, ``waiters-2``.
388 :type api_version: str
389 :param api_version: The API version to load. If this is not
390 provided, then the latest API version will be used.
392 :type load_extras: bool
393 :param load_extras: Whether or not to load the tool extras which
394 contain additional data to be added to the model.
396 :raises: UnknownServiceError if there is no known service with
397 the provided service_name.
399 :raises: DataNotFoundError if no data could be found for the
400 service_name/type_name/api_version.
402 :return: The loaded data, as a python type (e.g. dict, list, etc).
403 """
404 # Wrapper around the load_data. This will calculate the path
405 # to call load_data with.
406 known_services = self.list_available_services(type_name)
407 if service_name not in known_services:
408 raise UnknownServiceError(
409 service_name=service_name,
410 known_service_names=', '.join(sorted(known_services)),
411 )
412 if api_version is None:
413 api_version = self.determine_latest_version(
414 service_name, type_name
415 )
416 full_path = os.path.join(service_name, api_version, type_name)
417 model = self.load_data(full_path)
419 # Load in all the extras
420 extras_data = self._find_extras(service_name, type_name, api_version)
421 self._extras_processor.process(model, extras_data)
423 return model
425 def _find_extras(self, service_name, type_name, api_version):
426 """Creates an iterator over all the extras data."""
427 for extras_type in self.extras_types:
428 extras_name = f'{type_name}.{extras_type}-extras'
429 full_path = os.path.join(service_name, api_version, extras_name)
431 try:
432 yield self.load_data(full_path)
433 except DataNotFoundError:
434 pass
436 @instance_cache
437 def load_data_with_path(self, name):
438 """Same as ``load_data`` but returns file path as second return value.
440 :type name: str
441 :param name: The data path, i.e ``ec2/2015-03-01/service-2``.
443 :return: Tuple of the loaded data and the path to the data file
444 where the data was loaded from. If no data could be found then a
445 DataNotFoundError is raised.
446 """
447 for possible_path in self._potential_locations(name):
448 found = self.file_loader.load_file(possible_path)
449 if found is not None:
450 return found, possible_path
452 # We didn't find anything that matched on any path.
453 raise DataNotFoundError(data_path=name)
455 def load_data(self, name):
456 """Load data given a data path.
458 This is a low level method that will search through the various
459 search paths until it's able to load a value. This is typically
460 only needed to load *non* model files (such as _endpoints and
461 _retry). If you need to load model files, you should prefer
462 ``load_service_model``. Use ``load_data_with_path`` to get the
463 data path of the data file as second return value.
465 :type name: str
466 :param name: The data path, i.e ``ec2/2015-03-01/service-2``.
468 :return: The loaded data. If no data could be found then
469 a DataNotFoundError is raised.
470 """
471 data, _ = self.load_data_with_path(name)
472 return data
474 def _potential_locations(self, name=None, must_exist=False, is_dir=False):
475 # Will give an iterator over the full path of potential locations
476 # according to the search path.
477 for path in self.search_paths:
478 if os.path.isdir(path):
479 full_path = path
480 if name is not None:
481 full_path = os.path.join(path, name)
482 if not must_exist:
483 yield full_path
484 else:
485 if is_dir and os.path.isdir(full_path):
486 yield full_path
487 elif os.path.exists(full_path):
488 yield full_path
490 def is_builtin_path(self, path):
491 """Whether a given path is within the package's data directory.
493 This method can be used together with load_data_with_path(name)
494 to determine if data has been loaded from a file bundled with the
495 package, as opposed to a file in a separate location.
497 :type path: str
498 :param path: The file path to check.
500 :return: Whether the given path is within the package's data directory.
501 """
502 path = os.path.expanduser(os.path.expandvars(path))
503 return path.startswith(self.BUILTIN_DATA_PATH)
506class ExtrasProcessor:
507 """Processes data from extras files into service models."""
509 def process(self, original_model, extra_models):
510 """Processes data from a list of loaded extras files into a model
512 :type original_model: dict
513 :param original_model: The service model to load all the extras into.
515 :type extra_models: iterable of dict
516 :param extra_models: A list of loaded extras models.
517 """
518 for extras in extra_models:
519 self._process(original_model, extras)
521 def _process(self, model, extra_model):
522 """Process a single extras model into a service model."""
523 if 'merge' in extra_model:
524 deep_merge(model, extra_model['merge'])