Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"). You
4# may not use this file except in compliance with the License. A copy of
5# the License is located at
6#
7# https://aws.amazon.com/apache2.0/
8#
9# or in the "license" file accompanying this file. This file is
10# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
11# ANY KIND, either express or implied. See the License for the specific
12# language governing permissions and limitations under the License.
14import copy
15import logging
17from botocore import xform_name
18from botocore.utils import merge_dicts
20from ..docs import docstring
21from .action import BatchAction
22from .params import create_request_parameters
23from .response import ResourceHandler
25logger = logging.getLogger(__name__)
28class ResourceCollection:
29 """
30 Represents a collection of resources, which can be iterated through,
31 optionally with filtering. Collections automatically handle pagination
32 for you.
34 See :ref:`guide_collections` for a high-level overview of collections,
35 including when remote service requests are performed.
37 :type model: :py:class:`~boto3.resources.model.Collection`
38 :param model: Collection model
39 :type parent: :py:class:`~boto3.resources.base.ServiceResource`
40 :param parent: The collection's parent resource
41 :type handler: :py:class:`~boto3.resources.response.ResourceHandler`
42 :param handler: The resource response handler used to create resource
43 instances
44 """
46 def __init__(self, model, parent, handler, **kwargs):
47 self._model = model
48 self._parent = parent
49 self._py_operation_name = xform_name(model.request.operation)
50 self._handler = handler
51 self._params = copy.deepcopy(kwargs)
53 def __repr__(self):
54 return '{}({}, {})'.format(
55 self.__class__.__name__,
56 self._parent,
57 f'{self._parent.meta.service_name}.{self._model.resource.type}',
58 )
60 def __iter__(self):
61 """
62 A generator which yields resource instances after doing the
63 appropriate service operation calls and handling any pagination
64 on your behalf.
66 Page size, item limit, and filter parameters are applied
67 if they have previously been set.
69 >>> bucket = s3.Bucket('boto3')
70 >>> for obj in bucket.objects.all():
71 ... print(obj.key)
72 'key1'
73 'key2'
75 """
76 limit = self._params.get('limit', None)
78 count = 0
79 for page in self.pages():
80 for item in page:
81 yield item
83 # If the limit is set and has been reached, then
84 # we stop processing items here.
85 count += 1
86 if limit is not None and count >= limit:
87 return
89 def _clone(self, **kwargs):
90 """
91 Create a clone of this collection. This is used by the methods
92 below to provide a chainable interface that returns copies
93 rather than the original. This allows things like:
95 >>> base = collection.filter(Param1=1)
96 >>> query1 = base.filter(Param2=2)
97 >>> query2 = base.filter(Param3=3)
98 >>> query1.params
99 {'Param1': 1, 'Param2': 2}
100 >>> query2.params
101 {'Param1': 1, 'Param3': 3}
103 :rtype: :py:class:`ResourceCollection`
104 :return: A clone of this resource collection
105 """
106 params = copy.deepcopy(self._params)
107 merge_dicts(params, kwargs, append_lists=True)
108 clone = self.__class__(
109 self._model, self._parent, self._handler, **params
110 )
111 return clone
113 def pages(self):
114 """
115 A generator which yields pages of resource instances after
116 doing the appropriate service operation calls and handling
117 any pagination on your behalf. Non-paginated calls will
118 return a single page of items.
120 Page size, item limit, and filter parameters are applied
121 if they have previously been set.
123 >>> bucket = s3.Bucket('boto3')
124 >>> for page in bucket.objects.pages():
125 ... for obj in page:
126 ... print(obj.key)
127 'key1'
128 'key2'
130 :rtype: list(:py:class:`~boto3.resources.base.ServiceResource`)
131 :return: List of resource instances
132 """
133 client = self._parent.meta.client
134 cleaned_params = self._params.copy()
135 limit = cleaned_params.pop('limit', None)
136 page_size = cleaned_params.pop('page_size', None)
137 params = create_request_parameters(self._parent, self._model.request)
138 merge_dicts(params, cleaned_params, append_lists=True)
140 # Is this a paginated operation? If so, we need to get an
141 # iterator for the various pages. If not, then we simply
142 # call the operation and return the result as a single
143 # page in a list. For non-paginated results, we just ignore
144 # the page size parameter.
145 if client.can_paginate(self._py_operation_name):
146 logger.debug(
147 'Calling paginated %s:%s with %r',
148 self._parent.meta.service_name,
149 self._py_operation_name,
150 params,
151 )
152 paginator = client.get_paginator(self._py_operation_name)
153 pages = paginator.paginate(
154 PaginationConfig={'MaxItems': limit, 'PageSize': page_size},
155 **params,
156 )
157 else:
158 logger.debug(
159 'Calling %s:%s with %r',
160 self._parent.meta.service_name,
161 self._py_operation_name,
162 params,
163 )
164 pages = [getattr(client, self._py_operation_name)(**params)]
166 # Now that we have a page iterator or single page of results
167 # we start processing and yielding individual items.
168 count = 0
169 for page in pages:
170 page_items = []
171 for item in self._handler(self._parent, params, page):
172 page_items.append(item)
174 # If the limit is set and has been reached, then
175 # we stop processing items here.
176 count += 1
177 if limit is not None and count >= limit:
178 break
180 yield page_items
182 # Stop reading pages if we've reached out limit
183 if limit is not None and count >= limit:
184 break
186 def all(self):
187 """
188 Get all items from the collection, optionally with a custom
189 page size and item count limit.
191 This method returns an iterable generator which yields
192 individual resource instances. Example use::
194 # Iterate through items
195 >>> for queue in sqs.queues.all():
196 ... print(queue.url)
197 'https://url1'
198 'https://url2'
200 # Convert to list
201 >>> queues = list(sqs.queues.all())
202 >>> len(queues)
203 2
204 """
205 return self._clone()
207 def filter(self, **kwargs):
208 """
209 Get items from the collection, passing keyword arguments along
210 as parameters to the underlying service operation, which are
211 typically used to filter the results.
213 This method returns an iterable generator which yields
214 individual resource instances. Example use::
216 # Iterate through items
217 >>> for queue in sqs.queues.filter(Param='foo'):
218 ... print(queue.url)
219 'https://url1'
220 'https://url2'
222 # Convert to list
223 >>> queues = list(sqs.queues.filter(Param='foo'))
224 >>> len(queues)
225 2
227 :rtype: :py:class:`ResourceCollection`
228 """
229 return self._clone(**kwargs)
231 def limit(self, count):
232 """
233 Return at most this many resources.
235 >>> for bucket in s3.buckets.limit(5):
236 ... print(bucket.name)
237 'bucket1'
238 'bucket2'
239 'bucket3'
240 'bucket4'
241 'bucket5'
243 :type count: int
244 :param count: Return no more than this many items
245 :rtype: :py:class:`ResourceCollection`
246 """
247 return self._clone(limit=count)
249 def page_size(self, count):
250 """
251 Fetch at most this many resources per service request.
253 >>> for obj in s3.Bucket('boto3').objects.page_size(100):
254 ... print(obj.key)
256 :type count: int
257 :param count: Fetch this many items per request
258 :rtype: :py:class:`ResourceCollection`
259 """
260 return self._clone(page_size=count)
263class CollectionManager:
264 """
265 A collection manager provides access to resource collection instances,
266 which can be iterated and filtered. The manager exposes some
267 convenience functions that are also found on resource collections,
268 such as :py:meth:`~ResourceCollection.all` and
269 :py:meth:`~ResourceCollection.filter`.
271 Get all items::
273 >>> for bucket in s3.buckets.all():
274 ... print(bucket.name)
276 Get only some items via filtering::
278 >>> for queue in sqs.queues.filter(QueueNamePrefix='AWS'):
279 ... print(queue.url)
281 Get whole pages of items:
283 >>> for page in s3.Bucket('boto3').objects.pages():
284 ... for obj in page:
285 ... print(obj.key)
287 A collection manager is not iterable. You **must** call one of the
288 methods that return a :py:class:`ResourceCollection` before trying
289 to iterate, slice, or convert to a list.
291 See the :ref:`guide_collections` guide for a high-level overview
292 of collections, including when remote service requests are performed.
294 :type collection_model: :py:class:`~boto3.resources.model.Collection`
295 :param model: Collection model
297 :type parent: :py:class:`~boto3.resources.base.ServiceResource`
298 :param parent: The collection's parent resource
300 :type factory: :py:class:`~boto3.resources.factory.ResourceFactory`
301 :param factory: The resource factory to create new resources
303 :type service_context: :py:class:`~boto3.utils.ServiceContext`
304 :param service_context: Context about the AWS service
305 """
307 # The class to use when creating an iterator
308 _collection_cls = ResourceCollection
310 def __init__(self, collection_model, parent, factory, service_context):
311 self._model = collection_model
312 operation_name = self._model.request.operation
313 self._parent = parent
315 search_path = collection_model.resource.path
316 self._handler = ResourceHandler(
317 search_path=search_path,
318 factory=factory,
319 resource_model=collection_model.resource,
320 service_context=service_context,
321 operation_name=operation_name,
322 )
324 def __repr__(self):
325 return '{}({}, {})'.format(
326 self.__class__.__name__,
327 self._parent,
328 f'{self._parent.meta.service_name}.{self._model.resource.type}',
329 )
331 def iterator(self, **kwargs):
332 """
333 Get a resource collection iterator from this manager.
335 :rtype: :py:class:`ResourceCollection`
336 :return: An iterable representing the collection of resources
337 """
338 return self._collection_cls(
339 self._model, self._parent, self._handler, **kwargs
340 )
342 # Set up some methods to proxy ResourceCollection methods
343 def all(self):
344 return self.iterator()
346 all.__doc__ = ResourceCollection.all.__doc__
348 def filter(self, **kwargs):
349 return self.iterator(**kwargs)
351 filter.__doc__ = ResourceCollection.filter.__doc__
353 def limit(self, count):
354 return self.iterator(limit=count)
356 limit.__doc__ = ResourceCollection.limit.__doc__
358 def page_size(self, count):
359 return self.iterator(page_size=count)
361 page_size.__doc__ = ResourceCollection.page_size.__doc__
363 def pages(self):
364 return self.iterator().pages()
366 pages.__doc__ = ResourceCollection.pages.__doc__
369class CollectionFactory:
370 """
371 A factory to create new
372 :py:class:`CollectionManager` and :py:class:`ResourceCollection`
373 subclasses from a :py:class:`~boto3.resources.model.Collection`
374 model. These subclasses include methods to perform batch operations.
375 """
377 def load_from_definition(
378 self, resource_name, collection_model, service_context, event_emitter
379 ):
380 """
381 Loads a collection from a model, creating a new
382 :py:class:`CollectionManager` subclass
383 with the correct properties and methods, named based on the service
384 and resource name, e.g. ec2.InstanceCollectionManager. It also
385 creates a new :py:class:`ResourceCollection` subclass which is used
386 by the new manager class.
388 :type resource_name: string
389 :param resource_name: Name of the resource to look up. For services,
390 this should match the ``service_name``.
392 :type service_context: :py:class:`~boto3.utils.ServiceContext`
393 :param service_context: Context about the AWS service
395 :type event_emitter: :py:class:`~botocore.hooks.HierarchialEmitter`
396 :param event_emitter: An event emitter
398 :rtype: Subclass of :py:class:`CollectionManager`
399 :return: The collection class.
400 """
401 attrs = {}
402 collection_name = collection_model.name
404 # Create the batch actions for a collection
405 self._load_batch_actions(
406 attrs,
407 resource_name,
408 collection_model,
409 service_context.service_model,
410 event_emitter,
411 )
412 # Add the documentation to the collection class's methods
413 self._load_documented_collection_methods(
414 attrs=attrs,
415 resource_name=resource_name,
416 collection_model=collection_model,
417 service_model=service_context.service_model,
418 event_emitter=event_emitter,
419 base_class=ResourceCollection,
420 )
422 if service_context.service_name == resource_name:
423 cls_name = (
424 f'{service_context.service_name}.{collection_name}Collection'
425 )
426 else:
427 cls_name = f'{service_context.service_name}.{resource_name}.{collection_name}Collection'
429 collection_cls = type(str(cls_name), (ResourceCollection,), attrs)
431 # Add the documentation to the collection manager's methods
432 self._load_documented_collection_methods(
433 attrs=attrs,
434 resource_name=resource_name,
435 collection_model=collection_model,
436 service_model=service_context.service_model,
437 event_emitter=event_emitter,
438 base_class=CollectionManager,
439 )
440 attrs['_collection_cls'] = collection_cls
441 cls_name += 'Manager'
443 return type(str(cls_name), (CollectionManager,), attrs)
445 def _load_batch_actions(
446 self,
447 attrs,
448 resource_name,
449 collection_model,
450 service_model,
451 event_emitter,
452 ):
453 """
454 Batch actions on the collection become methods on both
455 the collection manager and iterators.
456 """
457 for action_model in collection_model.batch_actions:
458 snake_cased = xform_name(action_model.name)
459 attrs[snake_cased] = self._create_batch_action(
460 resource_name,
461 snake_cased,
462 action_model,
463 collection_model,
464 service_model,
465 event_emitter,
466 )
468 def _load_documented_collection_methods(
469 factory_self,
470 attrs,
471 resource_name,
472 collection_model,
473 service_model,
474 event_emitter,
475 base_class,
476 ):
477 # The base class already has these methods defined. However
478 # the docstrings are generic and not based for a particular service
479 # or resource. So we override these methods by proxying to the
480 # base class's builtin method and adding a docstring
481 # that pertains to the resource.
483 # A collection's all() method.
484 def all(self):
485 return base_class.all(self)
487 all.__doc__ = docstring.CollectionMethodDocstring(
488 resource_name=resource_name,
489 action_name='all',
490 event_emitter=event_emitter,
491 collection_model=collection_model,
492 service_model=service_model,
493 include_signature=False,
494 )
495 attrs['all'] = all
497 # The collection's filter() method.
498 def filter(self, **kwargs):
499 return base_class.filter(self, **kwargs)
501 filter.__doc__ = docstring.CollectionMethodDocstring(
502 resource_name=resource_name,
503 action_name='filter',
504 event_emitter=event_emitter,
505 collection_model=collection_model,
506 service_model=service_model,
507 include_signature=False,
508 )
509 attrs['filter'] = filter
511 # The collection's limit method.
512 def limit(self, count):
513 return base_class.limit(self, count)
515 limit.__doc__ = docstring.CollectionMethodDocstring(
516 resource_name=resource_name,
517 action_name='limit',
518 event_emitter=event_emitter,
519 collection_model=collection_model,
520 service_model=service_model,
521 include_signature=False,
522 )
523 attrs['limit'] = limit
525 # The collection's page_size method.
526 def page_size(self, count):
527 return base_class.page_size(self, count)
529 page_size.__doc__ = docstring.CollectionMethodDocstring(
530 resource_name=resource_name,
531 action_name='page_size',
532 event_emitter=event_emitter,
533 collection_model=collection_model,
534 service_model=service_model,
535 include_signature=False,
536 )
537 attrs['page_size'] = page_size
539 def _create_batch_action(
540 factory_self,
541 resource_name,
542 snake_cased,
543 action_model,
544 collection_model,
545 service_model,
546 event_emitter,
547 ):
548 """
549 Creates a new method which makes a batch operation request
550 to the underlying service API.
551 """
552 action = BatchAction(action_model)
554 def batch_action(self, *args, **kwargs):
555 return action(self, *args, **kwargs)
557 batch_action.__name__ = str(snake_cased)
558 batch_action.__doc__ = docstring.BatchActionDocstring(
559 resource_name=resource_name,
560 event_emitter=event_emitter,
561 batch_action_model=action_model,
562 service_model=service_model,
563 collection_model=collection_model,
564 include_signature=False,
565 )
566 return batch_action