Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/boto3/resources/collection.py: 29%
133 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-12-08 06:51 +0000
1# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"). You
4# may not use this file except in compliance with the License. A copy of
5# the License is located at
6#
7# https://aws.amazon.com/apache2.0/
8#
9# or in the "license" file accompanying this file. This file is
10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11# ANY KIND, either express or implied. See the License for the specific
12# language governing permissions and limitations under the License.
14import copy
15import logging
17from botocore import xform_name
18from botocore.utils import merge_dicts
20from ..docs import docstring
21from .action import BatchAction
22from .params import create_request_parameters
23from .response import ResourceHandler
25logger = logging.getLogger(__name__)
28class ResourceCollection:
29 """
30 Represents a collection of resources, which can be iterated through,
31 optionally with filtering. Collections automatically handle pagination
32 for you.
34 See :ref:`guide_collections` for a high-level overview of collections,
35 including when remote service requests are performed.
37 :type model: :py:class:`~boto3.resources.model.Collection`
38 :param model: Collection model
39 :type parent: :py:class:`~boto3.resources.base.ServiceResource`
40 :param parent: The collection's parent resource
41 :type handler: :py:class:`~boto3.resources.response.ResourceHandler`
42 :param handler: The resource response handler used to create resource
43 instances
44 """
46 def __init__(self, model, parent, handler, **kwargs):
47 self._model = model
48 self._parent = parent
49 self._py_operation_name = xform_name(model.request.operation)
50 self._handler = handler
51 self._params = copy.deepcopy(kwargs)
53 def __repr__(self):
54 return '{}({}, {})'.format(
55 self.__class__.__name__,
56 self._parent,
57 '{}.{}'.format(
58 self._parent.meta.service_name, self._model.resource.type
59 ),
60 )
62 def __iter__(self):
63 """
64 A generator which yields resource instances after doing the
65 appropriate service operation calls and handling any pagination
66 on your behalf.
68 Page size, item limit, and filter parameters are applied
69 if they have previously been set.
71 >>> bucket = s3.Bucket('boto3')
72 >>> for obj in bucket.objects.all():
73 ... print(obj.key)
74 'key1'
75 'key2'
77 """
78 limit = self._params.get('limit', None)
80 count = 0
81 for page in self.pages():
82 for item in page:
83 yield item
85 # If the limit is set and has been reached, then
86 # we stop processing items here.
87 count += 1
88 if limit is not None and count >= limit:
89 return
91 def _clone(self, **kwargs):
92 """
93 Create a clone of this collection. This is used by the methods
94 below to provide a chainable interface that returns copies
95 rather than the original. This allows things like:
97 >>> base = collection.filter(Param1=1)
98 >>> query1 = base.filter(Param2=2)
99 >>> query2 = base.filter(Param3=3)
100 >>> query1.params
101 {'Param1': 1, 'Param2': 2}
102 >>> query2.params
103 {'Param1': 1, 'Param3': 3}
105 :rtype: :py:class:`ResourceCollection`
106 :return: A clone of this resource collection
107 """
108 params = copy.deepcopy(self._params)
109 merge_dicts(params, kwargs, append_lists=True)
110 clone = self.__class__(
111 self._model, self._parent, self._handler, **params
112 )
113 return clone
115 def pages(self):
116 """
117 A generator which yields pages of resource instances after
118 doing the appropriate service operation calls and handling
119 any pagination on your behalf. Non-paginated calls will
120 return a single page of items.
122 Page size, item limit, and filter parameters are applied
123 if they have previously been set.
125 >>> bucket = s3.Bucket('boto3')
126 >>> for page in bucket.objects.pages():
127 ... for obj in page:
128 ... print(obj.key)
129 'key1'
130 'key2'
132 :rtype: list(:py:class:`~boto3.resources.base.ServiceResource`)
133 :return: List of resource instances
134 """
135 client = self._parent.meta.client
136 cleaned_params = self._params.copy()
137 limit = cleaned_params.pop('limit', None)
138 page_size = cleaned_params.pop('page_size', None)
139 params = create_request_parameters(self._parent, self._model.request)
140 merge_dicts(params, cleaned_params, append_lists=True)
142 # Is this a paginated operation? If so, we need to get an
143 # iterator for the various pages. If not, then we simply
144 # call the operation and return the result as a single
145 # page in a list. For non-paginated results, we just ignore
146 # the page size parameter.
147 if client.can_paginate(self._py_operation_name):
148 logger.debug(
149 'Calling paginated %s:%s with %r',
150 self._parent.meta.service_name,
151 self._py_operation_name,
152 params,
153 )
154 paginator = client.get_paginator(self._py_operation_name)
155 pages = paginator.paginate(
156 PaginationConfig={'MaxItems': limit, 'PageSize': page_size},
157 **params
158 )
159 else:
160 logger.debug(
161 'Calling %s:%s with %r',
162 self._parent.meta.service_name,
163 self._py_operation_name,
164 params,
165 )
166 pages = [getattr(client, self._py_operation_name)(**params)]
168 # Now that we have a page iterator or single page of results
169 # we start processing and yielding individual items.
170 count = 0
171 for page in pages:
172 page_items = []
173 for item in self._handler(self._parent, params, page):
174 page_items.append(item)
176 # If the limit is set and has been reached, then
177 # we stop processing items here.
178 count += 1
179 if limit is not None and count >= limit:
180 break
182 yield page_items
184 # Stop reading pages if we've reached out limit
185 if limit is not None and count >= limit:
186 break
188 def all(self):
189 """
190 Get all items from the collection, optionally with a custom
191 page size and item count limit.
193 This method returns an iterable generator which yields
194 individual resource instances. Example use::
196 # Iterate through items
197 >>> for queue in sqs.queues.all():
198 ... print(queue.url)
199 'https://url1'
200 'https://url2'
202 # Convert to list
203 >>> queues = list(sqs.queues.all())
204 >>> len(queues)
205 2
206 """
207 return self._clone()
209 def filter(self, **kwargs):
210 """
211 Get items from the collection, passing keyword arguments along
212 as parameters to the underlying service operation, which are
213 typically used to filter the results.
215 This method returns an iterable generator which yields
216 individual resource instances. Example use::
218 # Iterate through items
219 >>> for queue in sqs.queues.filter(Param='foo'):
220 ... print(queue.url)
221 'https://url1'
222 'https://url2'
224 # Convert to list
225 >>> queues = list(sqs.queues.filter(Param='foo'))
226 >>> len(queues)
227 2
229 :rtype: :py:class:`ResourceCollection`
230 """
231 return self._clone(**kwargs)
233 def limit(self, count):
234 """
235 Return at most this many resources.
237 >>> for bucket in s3.buckets.limit(5):
238 ... print(bucket.name)
239 'bucket1'
240 'bucket2'
241 'bucket3'
242 'bucket4'
243 'bucket5'
245 :type count: int
246 :param count: Return no more than this many items
247 :rtype: :py:class:`ResourceCollection`
248 """
249 return self._clone(limit=count)
251 def page_size(self, count):
252 """
253 Fetch at most this many resources per service request.
255 >>> for obj in s3.Bucket('boto3').objects.page_size(100):
256 ... print(obj.key)
258 :type count: int
259 :param count: Fetch this many items per request
260 :rtype: :py:class:`ResourceCollection`
261 """
262 return self._clone(page_size=count)
265class CollectionManager:
266 """
267 A collection manager provides access to resource collection instances,
268 which can be iterated and filtered. The manager exposes some
269 convenience functions that are also found on resource collections,
270 such as :py:meth:`~ResourceCollection.all` and
271 :py:meth:`~ResourceCollection.filter`.
273 Get all items::
275 >>> for bucket in s3.buckets.all():
276 ... print(bucket.name)
278 Get only some items via filtering::
280 >>> for queue in sqs.queues.filter(QueueNamePrefix='AWS'):
281 ... print(queue.url)
283 Get whole pages of items:
285 >>> for page in s3.Bucket('boto3').objects.pages():
286 ... for obj in page:
287 ... print(obj.key)
289 A collection manager is not iterable. You **must** call one of the
290 methods that return a :py:class:`ResourceCollection` before trying
291 to iterate, slice, or convert to a list.
293 See the :ref:`guide_collections` guide for a high-level overview
294 of collections, including when remote service requests are performed.
296 :type collection_model: :py:class:`~boto3.resources.model.Collection`
297 :param model: Collection model
299 :type parent: :py:class:`~boto3.resources.base.ServiceResource`
300 :param parent: The collection's parent resource
302 :type factory: :py:class:`~boto3.resources.factory.ResourceFactory`
303 :param factory: The resource factory to create new resources
305 :type service_context: :py:class:`~boto3.utils.ServiceContext`
306 :param service_context: Context about the AWS service
307 """
309 # The class to use when creating an iterator
310 _collection_cls = ResourceCollection
312 def __init__(self, collection_model, parent, factory, service_context):
313 self._model = collection_model
314 operation_name = self._model.request.operation
315 self._parent = parent
317 search_path = collection_model.resource.path
318 self._handler = ResourceHandler(
319 search_path=search_path,
320 factory=factory,
321 resource_model=collection_model.resource,
322 service_context=service_context,
323 operation_name=operation_name,
324 )
326 def __repr__(self):
327 return '{}({}, {})'.format(
328 self.__class__.__name__,
329 self._parent,
330 '{}.{}'.format(
331 self._parent.meta.service_name, self._model.resource.type
332 ),
333 )
335 def iterator(self, **kwargs):
336 """
337 Get a resource collection iterator from this manager.
339 :rtype: :py:class:`ResourceCollection`
340 :return: An iterable representing the collection of resources
341 """
342 return self._collection_cls(
343 self._model, self._parent, self._handler, **kwargs
344 )
346 # Set up some methods to proxy ResourceCollection methods
347 def all(self):
348 return self.iterator()
350 all.__doc__ = ResourceCollection.all.__doc__
352 def filter(self, **kwargs):
353 return self.iterator(**kwargs)
355 filter.__doc__ = ResourceCollection.filter.__doc__
357 def limit(self, count):
358 return self.iterator(limit=count)
360 limit.__doc__ = ResourceCollection.limit.__doc__
362 def page_size(self, count):
363 return self.iterator(page_size=count)
365 page_size.__doc__ = ResourceCollection.page_size.__doc__
367 def pages(self):
368 return self.iterator().pages()
370 pages.__doc__ = ResourceCollection.pages.__doc__
373class CollectionFactory:
374 """
375 A factory to create new
376 :py:class:`CollectionManager` and :py:class:`ResourceCollection`
377 subclasses from a :py:class:`~boto3.resources.model.Collection`
378 model. These subclasses include methods to perform batch operations.
379 """
381 def load_from_definition(
382 self, resource_name, collection_model, service_context, event_emitter
383 ):
384 """
385 Loads a collection from a model, creating a new
386 :py:class:`CollectionManager` subclass
387 with the correct properties and methods, named based on the service
388 and resource name, e.g. ec2.InstanceCollectionManager. It also
389 creates a new :py:class:`ResourceCollection` subclass which is used
390 by the new manager class.
392 :type resource_name: string
393 :param resource_name: Name of the resource to look up. For services,
394 this should match the ``service_name``.
396 :type service_context: :py:class:`~boto3.utils.ServiceContext`
397 :param service_context: Context about the AWS service
399 :type event_emitter: :py:class:`~botocore.hooks.HierarchialEmitter`
400 :param event_emitter: An event emitter
402 :rtype: Subclass of :py:class:`CollectionManager`
403 :return: The collection class.
404 """
405 attrs = {}
406 collection_name = collection_model.name
408 # Create the batch actions for a collection
409 self._load_batch_actions(
410 attrs,
411 resource_name,
412 collection_model,
413 service_context.service_model,
414 event_emitter,
415 )
416 # Add the documentation to the collection class's methods
417 self._load_documented_collection_methods(
418 attrs=attrs,
419 resource_name=resource_name,
420 collection_model=collection_model,
421 service_model=service_context.service_model,
422 event_emitter=event_emitter,
423 base_class=ResourceCollection,
424 )
426 if service_context.service_name == resource_name:
427 cls_name = '{}.{}Collection'.format(
428 service_context.service_name, collection_name
429 )
430 else:
431 cls_name = '{}.{}.{}Collection'.format(
432 service_context.service_name, resource_name, collection_name
433 )
435 collection_cls = type(str(cls_name), (ResourceCollection,), attrs)
437 # Add the documentation to the collection manager's methods
438 self._load_documented_collection_methods(
439 attrs=attrs,
440 resource_name=resource_name,
441 collection_model=collection_model,
442 service_model=service_context.service_model,
443 event_emitter=event_emitter,
444 base_class=CollectionManager,
445 )
446 attrs['_collection_cls'] = collection_cls
447 cls_name += 'Manager'
449 return type(str(cls_name), (CollectionManager,), attrs)
451 def _load_batch_actions(
452 self,
453 attrs,
454 resource_name,
455 collection_model,
456 service_model,
457 event_emitter,
458 ):
459 """
460 Batch actions on the collection become methods on both
461 the collection manager and iterators.
462 """
463 for action_model in collection_model.batch_actions:
464 snake_cased = xform_name(action_model.name)
465 attrs[snake_cased] = self._create_batch_action(
466 resource_name,
467 snake_cased,
468 action_model,
469 collection_model,
470 service_model,
471 event_emitter,
472 )
474 def _load_documented_collection_methods(
475 factory_self,
476 attrs,
477 resource_name,
478 collection_model,
479 service_model,
480 event_emitter,
481 base_class,
482 ):
483 # The base class already has these methods defined. However
484 # the docstrings are generic and not based for a particular service
485 # or resource. So we override these methods by proxying to the
486 # base class's builtin method and adding a docstring
487 # that pertains to the resource.
489 # A collection's all() method.
490 def all(self):
491 return base_class.all(self)
493 all.__doc__ = docstring.CollectionMethodDocstring(
494 resource_name=resource_name,
495 action_name='all',
496 event_emitter=event_emitter,
497 collection_model=collection_model,
498 service_model=service_model,
499 include_signature=False,
500 )
501 attrs['all'] = all
503 # The collection's filter() method.
504 def filter(self, **kwargs):
505 return base_class.filter(self, **kwargs)
507 filter.__doc__ = docstring.CollectionMethodDocstring(
508 resource_name=resource_name,
509 action_name='filter',
510 event_emitter=event_emitter,
511 collection_model=collection_model,
512 service_model=service_model,
513 include_signature=False,
514 )
515 attrs['filter'] = filter
517 # The collection's limit method.
518 def limit(self, count):
519 return base_class.limit(self, count)
521 limit.__doc__ = docstring.CollectionMethodDocstring(
522 resource_name=resource_name,
523 action_name='limit',
524 event_emitter=event_emitter,
525 collection_model=collection_model,
526 service_model=service_model,
527 include_signature=False,
528 )
529 attrs['limit'] = limit
531 # The collection's page_size method.
532 def page_size(self, count):
533 return base_class.page_size(self, count)
535 page_size.__doc__ = docstring.CollectionMethodDocstring(
536 resource_name=resource_name,
537 action_name='page_size',
538 event_emitter=event_emitter,
539 collection_model=collection_model,
540 service_model=service_model,
541 include_signature=False,
542 )
543 attrs['page_size'] = page_size
545 def _create_batch_action(
546 factory_self,
547 resource_name,
548 snake_cased,
549 action_model,
550 collection_model,
551 service_model,
552 event_emitter,
553 ):
554 """
555 Creates a new method which makes a batch operation request
556 to the underlying service API.
557 """
558 action = BatchAction(action_model)
560 def batch_action(self, *args, **kwargs):
561 return action(self, *args, **kwargs)
563 batch_action.__name__ = str(snake_cased)
564 batch_action.__doc__ = docstring.BatchActionDocstring(
565 resource_name=resource_name,
566 event_emitter=event_emitter,
567 batch_action_model=action_model,
568 service_model=service_model,
569 collection_model=collection_model,
570 include_signature=False,
571 )
572 return batch_action