Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/botocore/loaders.py: 30%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2012-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"). You
4# may not use this file except in compliance with the License. A copy of
5# the License is located at
6#
7# http://aws.amazon.com/apache2.0/
8#
9# or in the "license" file accompanying this file. This file is
10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11# ANY KIND, either express or implied. See the License for the specific
12# language governing permissions and limitations under the License.
13"""Module for loading various model files.
15This module provides the classes that are used to load models used
16by botocore. This can include:
18 * Service models (e.g. the model for EC2, S3, DynamoDB, etc.)
19 * Service model extras which customize the service models
20 * Other models associated with a service (pagination, waiters)
21 * Non service-specific config (Endpoint data, retry config)
23Loading a module is broken down into several steps:
25 * Determining the path to load
26 * Search the data_path for files to load
27 * The mechanics of loading the file
28 * Searching for extras and applying them to the loaded file
30The last item is used so that other faster loading mechanism
31besides the default JSON loader can be used.
33The Search Path
34===============
36Similar to how the PATH environment variable is to finding executables
37and the PYTHONPATH environment variable is to finding python modules
38to import, the botocore loaders have the concept of a data path exposed
39through AWS_DATA_PATH.
41This enables end users to provide additional search paths where we
42will attempt to load models outside of the models we ship with
43botocore. When you create a ``Loader``, there are two paths
44automatically added to the model search path:
46 * <botocore root>/data/
47 * ~/.aws/models
49The first value is the path where all the model files shipped with
50botocore are located.
52The second path is so that users can just drop new model files in
53``~/.aws/models`` without having to mess around with the AWS_DATA_PATH.
55The AWS_DATA_PATH using the platform specific path separator to
56separate entries (typically ``:`` on linux and ``;`` on windows).
59Directory Layout
60================
62The Loader expects a particular directory layout. In order for any
63directory specified in AWS_DATA_PATH to be considered, it must have
64this structure for service models::
66 <root>
67 |
68 |-- servicename1
69 | |-- 2012-10-25
70 | |-- service-2.json
71 |-- ec2
72 | |-- 2014-01-01
73 | | |-- paginators-1.json
74 | | |-- service-2.json
75 | | |-- waiters-2.json
76 | |-- 2015-03-01
77 | |-- paginators-1.json
78 | |-- service-2.json
79 | |-- waiters-2.json
80 | |-- service-2.sdk-extras.json
83That is:
85 * The root directory contains sub directories that are the name
86 of the services.
87 * Within each service directory, there's a sub directory for each
88 available API version.
89 * Within each API version, there are model specific files, including
90 (but not limited to): service-2.json, waiters-2.json, paginators-1.json
92The ``-1`` and ``-2`` suffix at the end of the model files denote which version
93schema is used within the model. Even though this information is available in
94the ``version`` key within the model, this version is also part of the filename
95so that code does not need to load the JSON model in order to determine which
96version to use.
98The ``sdk-extras`` and similar files represent extra data that needs to be
99applied to the model after it is loaded. Data in these files might represent
100information that doesn't quite fit in the original models, but is still needed
101for the sdk. For instance, additional operation parameters might be added here
102which don't represent the actual service api.
103"""
105import logging
106import os
108from botocore import BOTOCORE_ROOT
109from botocore.compat import HAS_GZIP, OrderedDict, json
110from botocore.exceptions import DataNotFoundError, UnknownServiceError
111from botocore.utils import deep_merge
113_JSON_OPEN_METHODS = {
114 '.json': open,
115}
118if HAS_GZIP:
119 from gzip import open as gzip_open
121 _JSON_OPEN_METHODS['.json.gz'] = gzip_open
124logger = logging.getLogger(__name__)
127def instance_cache(func):
128 """Cache the result of a method on a per instance basis.
130 This is not a general purpose caching decorator. In order
131 for this to be used, it must be used on methods on an
132 instance, and that instance *must* provide a
133 ``self._cache`` dictionary.
135 """
137 def _wrapper(self, *args, **kwargs):
138 key = (func.__name__,) + args
139 for pair in sorted(kwargs.items()):
140 key += pair
141 if key in self._cache:
142 return self._cache[key]
143 data = func(self, *args, **kwargs)
144 self._cache[key] = data
145 return data
147 return _wrapper
150class JSONFileLoader:
151 """Loader JSON files.
153 This class can load the default format of models, which is a JSON file.
155 """
157 def exists(self, file_path):
158 """Checks if the file exists.
160 :type file_path: str
161 :param file_path: The full path to the file to load without
162 the '.json' extension.
164 :return: True if file path exists, False otherwise.
166 """
167 for ext in _JSON_OPEN_METHODS:
168 if os.path.isfile(file_path + ext):
169 return True
170 return False
172 def _load_file(self, full_path, open_method):
173 if not os.path.isfile(full_path):
174 return
176 # By default the file will be opened with locale encoding on Python 3.
177 # We specify "utf8" here to ensure the correct behavior.
178 with open_method(full_path, 'rb') as fp:
179 payload = fp.read().decode('utf-8')
181 logger.debug("Loading JSON file: %s", full_path)
182 return json.loads(payload, object_pairs_hook=OrderedDict)
184 def load_file(self, file_path):
185 """Attempt to load the file path.
187 :type file_path: str
188 :param file_path: The full path to the file to load without
189 the '.json' extension.
191 :return: The loaded data if it exists, otherwise None.
193 """
194 for ext, open_method in _JSON_OPEN_METHODS.items():
195 data = self._load_file(file_path + ext, open_method)
196 if data is not None:
197 return data
198 return None
201def create_loader(search_path_string=None):
202 """Create a Loader class.
204 This factory function creates a loader given a search string path.
206 :type search_string_path: str
207 :param search_string_path: The AWS_DATA_PATH value. A string
208 of data path values separated by the ``os.path.pathsep`` value,
209 which is typically ``:`` on POSIX platforms and ``;`` on
210 windows.
212 :return: A ``Loader`` instance.
214 """
215 if search_path_string is None:
216 return Loader()
217 paths = []
218 extra_paths = search_path_string.split(os.pathsep)
219 for path in extra_paths:
220 path = os.path.expanduser(os.path.expandvars(path))
221 paths.append(path)
222 return Loader(extra_search_paths=paths)
225class Loader:
226 """Find and load data models.
228 This class will handle searching for and loading data models.
230 The main method used here is ``load_service_model``, which is a
231 convenience method over ``load_data`` and ``determine_latest_version``.
233 """
235 FILE_LOADER_CLASS = JSONFileLoader
236 # The included models in botocore/data/ that we ship with botocore.
237 BUILTIN_DATA_PATH = os.path.join(BOTOCORE_ROOT, 'data')
238 # For convenience we automatically add ~/.aws/models to the data path.
239 CUSTOMER_DATA_PATH = os.path.join(
240 os.path.expanduser('~'), '.aws', 'models'
241 )
242 BUILTIN_EXTRAS_TYPES = ['sdk']
244 def __init__(
245 self,
246 extra_search_paths=None,
247 file_loader=None,
248 cache=None,
249 include_default_search_paths=True,
250 include_default_extras=True,
251 ):
252 self._cache = {}
253 if file_loader is None:
254 file_loader = self.FILE_LOADER_CLASS()
255 self.file_loader = file_loader
256 if extra_search_paths is not None:
257 self._search_paths = extra_search_paths
258 else:
259 self._search_paths = []
260 if include_default_search_paths:
261 self._search_paths.extend(
262 [self.CUSTOMER_DATA_PATH, self.BUILTIN_DATA_PATH]
263 )
265 self._extras_types = []
266 if include_default_extras:
267 self._extras_types.extend(self.BUILTIN_EXTRAS_TYPES)
269 self._extras_processor = ExtrasProcessor()
271 @property
272 def search_paths(self):
273 return self._search_paths
275 @property
276 def extras_types(self):
277 return self._extras_types
279 @instance_cache
280 def list_available_services(self, type_name):
281 """List all known services.
283 This will traverse the search path and look for all known
284 services.
286 :type type_name: str
287 :param type_name: The type of the service (service-2,
288 paginators-1, waiters-2, etc). This is needed because
289 the list of available services depends on the service
290 type. For example, the latest API version available for
291 a resource-1.json file may not be the latest API version
292 available for a services-2.json file.
294 :return: A list of all services. The list of services will
295 be sorted.
297 """
298 services = set()
299 for possible_path in self._potential_locations():
300 # Any directory in the search path is potentially a service.
301 # We'll collect any initial list of potential services,
302 # but we'll then need to further process these directories
303 # by searching for the corresponding type_name in each
304 # potential directory.
305 possible_services = [
306 d
307 for d in os.listdir(possible_path)
308 if os.path.isdir(os.path.join(possible_path, d))
309 ]
310 for service_name in possible_services:
311 full_dirname = os.path.join(possible_path, service_name)
312 api_versions = os.listdir(full_dirname)
313 for api_version in api_versions:
314 full_load_path = os.path.join(
315 full_dirname, api_version, type_name
316 )
317 if self.file_loader.exists(full_load_path):
318 services.add(service_name)
319 break
320 return sorted(services)
322 @instance_cache
323 def determine_latest_version(self, service_name, type_name):
324 """Find the latest API version available for a service.
326 :type service_name: str
327 :param service_name: The name of the service.
329 :type type_name: str
330 :param type_name: The type of the service (service-2,
331 paginators-1, waiters-2, etc). This is needed because
332 the latest API version available can depend on the service
333 type. For example, the latest API version available for
334 a resource-1.json file may not be the latest API version
335 available for a services-2.json file.
337 :rtype: str
338 :return: The latest API version. If the service does not exist
339 or does not have any available API data, then a
340 ``DataNotFoundError`` exception will be raised.
342 """
343 return max(self.list_api_versions(service_name, type_name))
345 @instance_cache
346 def list_api_versions(self, service_name, type_name):
347 """List all API versions available for a particular service type
349 :type service_name: str
350 :param service_name: The name of the service
352 :type type_name: str
353 :param type_name: The type name for the service (i.e service-2,
354 paginators-1, etc.)
356 :rtype: list
357 :return: A list of API version strings in sorted order.
359 """
360 known_api_versions = set()
361 for possible_path in self._potential_locations(
362 service_name, must_exist=True, is_dir=True
363 ):
364 for dirname in os.listdir(possible_path):
365 full_path = os.path.join(possible_path, dirname, type_name)
366 # Only add to the known_api_versions if the directory
367 # contains a service-2, paginators-1, etc. file corresponding
368 # to the type_name passed in.
369 if self.file_loader.exists(full_path):
370 known_api_versions.add(dirname)
371 if not known_api_versions:
372 raise DataNotFoundError(data_path=service_name)
373 return sorted(known_api_versions)
375 @instance_cache
376 def load_service_model(self, service_name, type_name, api_version=None):
377 """Load a botocore service model
379 This is the main method for loading botocore models (e.g. a service
380 model, pagination configs, waiter configs, etc.).
382 :type service_name: str
383 :param service_name: The name of the service (e.g ``ec2``, ``s3``).
385 :type type_name: str
386 :param type_name: The model type. Valid types include, but are not
387 limited to: ``service-2``, ``paginators-1``, ``waiters-2``.
389 :type api_version: str
390 :param api_version: The API version to load. If this is not
391 provided, then the latest API version will be used.
393 :type load_extras: bool
394 :param load_extras: Whether or not to load the tool extras which
395 contain additional data to be added to the model.
397 :raises: UnknownServiceError if there is no known service with
398 the provided service_name.
400 :raises: DataNotFoundError if no data could be found for the
401 service_name/type_name/api_version.
403 :return: The loaded data, as a python type (e.g. dict, list, etc).
404 """
405 # Wrapper around the load_data. This will calculate the path
406 # to call load_data with.
407 known_services = self.list_available_services(type_name)
408 if service_name not in known_services:
409 raise UnknownServiceError(
410 service_name=service_name,
411 known_service_names=', '.join(sorted(known_services)),
412 )
413 if api_version is None:
414 api_version = self.determine_latest_version(
415 service_name, type_name
416 )
417 full_path = os.path.join(service_name, api_version, type_name)
418 model = self.load_data(full_path)
420 # Load in all the extras
421 extras_data = self._find_extras(service_name, type_name, api_version)
422 self._extras_processor.process(model, extras_data)
424 return model
426 def _find_extras(self, service_name, type_name, api_version):
427 """Creates an iterator over all the extras data."""
428 for extras_type in self.extras_types:
429 extras_name = f'{type_name}.{extras_type}-extras'
430 full_path = os.path.join(service_name, api_version, extras_name)
432 try:
433 yield self.load_data(full_path)
434 except DataNotFoundError:
435 pass
437 @instance_cache
438 def load_data_with_path(self, name):
439 """Same as ``load_data`` but returns file path as second return value.
441 :type name: str
442 :param name: The data path, i.e ``ec2/2015-03-01/service-2``.
444 :return: Tuple of the loaded data and the path to the data file
445 where the data was loaded from. If no data could be found then a
446 DataNotFoundError is raised.
447 """
448 for possible_path in self._potential_locations(name):
449 found = self.file_loader.load_file(possible_path)
450 if found is not None:
451 return found, possible_path
453 # We didn't find anything that matched on any path.
454 raise DataNotFoundError(data_path=name)
456 def load_data(self, name):
457 """Load data given a data path.
459 This is a low level method that will search through the various
460 search paths until it's able to load a value. This is typically
461 only needed to load *non* model files (such as _endpoints and
462 _retry). If you need to load model files, you should prefer
463 ``load_service_model``. Use ``load_data_with_path`` to get the
464 data path of the data file as second return value.
466 :type name: str
467 :param name: The data path, i.e ``ec2/2015-03-01/service-2``.
469 :return: The loaded data. If no data could be found then
470 a DataNotFoundError is raised.
471 """
472 data, _ = self.load_data_with_path(name)
473 return data
475 def _potential_locations(self, name=None, must_exist=False, is_dir=False):
476 # Will give an iterator over the full path of potential locations
477 # according to the search path.
478 for path in self.search_paths:
479 if os.path.isdir(path):
480 full_path = path
481 if name is not None:
482 full_path = os.path.join(path, name)
483 if not must_exist:
484 yield full_path
485 else:
486 if is_dir and os.path.isdir(full_path):
487 yield full_path
488 elif os.path.exists(full_path):
489 yield full_path
491 def is_builtin_path(self, path):
492 """Whether a given path is within the package's data directory.
494 This method can be used together with load_data_with_path(name)
495 to determine if data has been loaded from a file bundled with the
496 package, as opposed to a file in a separate location.
498 :type path: str
499 :param path: The file path to check.
501 :return: Whether the given path is within the package's data directory.
502 """
503 path = os.path.expanduser(os.path.expandvars(path))
504 return path.startswith(self.BUILTIN_DATA_PATH)
507class ExtrasProcessor:
508 """Processes data from extras files into service models."""
510 def process(self, original_model, extra_models):
511 """Processes data from a list of loaded extras files into a model
513 :type original_model: dict
514 :param original_model: The service model to load all the extras into.
516 :type extra_models: iterable of dict
517 :param extra_models: A list of loaded extras models.
518 """
519 for extras in extra_models:
520 self._process(original_model, extras)
522 def _process(self, model, extra_model):
523 """Process a single extras model into a service model."""
524 if 'merge' in extra_model:
525 deep_merge(model, extra_model['merge'])