Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/c7n/query.py: 30%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright The Cloud Custodian Authors.
2# SPDX-License-Identifier: Apache-2.0
3"""
4Query capability built on skew metamodel
6tags_spec -> s3, elb, rds
7"""
8from concurrent.futures import as_completed
9import functools
10import itertools
11import json
12from typing import List
14import os
16from c7n.actions import ActionRegistry
17from c7n.exceptions import ClientError, ResourceLimitExceeded, PolicyExecutionError
18from c7n.filters import FilterRegistry, MetricsFilter
19from c7n.manager import ResourceManager
20from c7n.registry import PluginRegistry
21from c7n.tags import register_ec2_tags, register_universal_tags, universal_augment
22from c7n.utils import (
23 local_session, generate_arn, get_retry, chunks, camelResource, jmespath_compile, get_path)
25try:
26 from botocore.paginate import PageIterator, Paginator
27except ImportError:
28 # Likely using another provider in a serverless environment
29 class PageIterator:
30 pass
32 class Paginator:
33 pass
36class ResourceQuery:
38 def __init__(self, session_factory):
39 self.session_factory = session_factory
41 @staticmethod
42 def resolve(resource_type):
43 if not isinstance(resource_type, type):
44 raise ValueError(resource_type)
45 return resource_type
47 def _invoke_client_enum(self, client, enum_op, params, path, retry=None):
48 if client.can_paginate(enum_op):
49 p = client.get_paginator(enum_op)
50 if retry:
51 p.PAGE_ITERATOR_CLS = RetryPageIterator
52 results = p.paginate(**params)
53 data = results.build_full_result()
54 else:
55 op = getattr(client, enum_op)
56 data = op(**params)
58 if path:
59 path = jmespath_compile(path)
60 data = path.search(data)
62 return data
64 def filter(self, resource_manager, **params):
65 """Query a set of resources."""
66 m = self.resolve(resource_manager.resource_type)
67 if resource_manager.get_client:
68 client = resource_manager.get_client()
69 else:
70 client = local_session(self.session_factory).client(
71 m.service, resource_manager.config.region)
72 enum_op, path, extra_args = m.enum_spec
73 if extra_args:
74 params = {**extra_args, **params}
75 return self._invoke_client_enum(
76 client, enum_op, params, path,
77 getattr(resource_manager, 'retry', None)) or []
79 def get(self, resource_manager, identities):
80 """Get resources by identities
81 """
82 m = self.resolve(resource_manager.resource_type)
83 params = {}
84 client_filter = True
86 # Try to formulate server side query in the below two scenarios
87 # else fall back to client side filtering
88 if m.filter_name:
89 if m.filter_type == 'list':
90 params[m.filter_name] = identities
91 client_filter = False
92 elif m.filter_type == 'scalar' and len(identities) == 1:
93 params[m.filter_name] = identities[0]
94 client_filter = False
96 resources = self.filter(resource_manager, **params)
97 if client_filter:
98 # This logic was added to prevent the issue from:
99 # https://github.com/cloud-custodian/cloud-custodian/issues/1398
100 if all(map(lambda r: isinstance(r, str), resources)):
101 resources = [r for r in resources if r in identities]
102 # This logic should fix https://github.com/cloud-custodian/cloud-custodian/issues/7573
103 elif all(map(lambda r: isinstance(r, tuple), resources)):
104 resources = [(p, r) for p, r in resources if r[m.id] in identities]
105 else:
106 resources = [r for r in resources if r[m.id] in identities]
108 return resources
111class ChildResourceQuery(ResourceQuery):
112 """A resource query for resources that must be queried with parent information.
114 Several resource types can only be queried in the context of their
115 parents identifiers. ie. efs mount targets (parent efs), route53 resource
116 records (parent hosted zone), ecs services (ecs cluster).
117 """
119 parent_key = 'c7n:parent-id'
121 def __init__(self, session_factory, manager, capture_parent_id=False):
122 self.session_factory = session_factory
123 self.manager = manager
124 self.capture_parent_id = capture_parent_id
126 def filter(self, resource_manager, parent_ids=None, **params):
127 """Query a set of resources."""
128 m = self.resolve(resource_manager.resource_type)
129 if resource_manager.get_client:
130 client = resource_manager.get_client()
131 else:
132 client = local_session(self.session_factory).client(m.service)
134 enum_op, path, extra_args = m.enum_spec
135 if extra_args:
136 params.update(extra_args)
138 parent_type, parent_key, annotate_parent = m.parent_spec
139 parents = self.manager.get_resource_manager(parent_type)
140 if not parent_ids:
141 parent_ids = []
142 for p in parents.resources(augment=False):
143 if isinstance(p, str):
144 parent_ids.append(p)
145 else:
146 parent_ids.append(p[parents.resource_type.id])
148 # Bail out with no parent ids...
149 existing_param = parent_key in params
150 if not existing_param and len(parent_ids) == 0:
151 return []
153 # Handle a query with parent id
154 if existing_param:
155 return self._invoke_client_enum(client, enum_op, params, path)
157 # Have to query separately for each parent's children.
158 results = []
159 for parent_id in parent_ids:
160 merged_params = self.get_parent_parameters(params, parent_id, parent_key)
161 subset = self._invoke_client_enum(
162 client, enum_op, merged_params, path, retry=self.manager.retry)
163 if annotate_parent:
164 for r in subset:
165 r[self.parent_key] = parent_id
166 if subset:
167 if self.capture_parent_id:
168 results.extend([(parent_id, s) for s in subset])
169 else:
170 results.extend(subset)
171 return results
173 def get_parent_parameters(self, params, parent_id, parent_key):
174 return dict(params, **{parent_key: parent_id})
177class QueryMeta(type):
179 def __new__(cls, name, parents, attrs):
180 if 'resource_type' not in attrs:
181 return super(QueryMeta, cls).__new__(cls, name, parents, attrs)
183 if 'filter_registry' not in attrs:
184 attrs['filter_registry'] = FilterRegistry(
185 '%s.filters' % name.lower())
186 if 'action_registry' not in attrs:
187 attrs['action_registry'] = ActionRegistry(
188 '%s.actions' % name.lower())
190 if attrs['resource_type']:
191 m = ResourceQuery.resolve(attrs['resource_type'])
192 # Generic cloud watch metrics support
193 if m.dimension:
194 attrs['filter_registry'].register('metrics', MetricsFilter)
195 # EC2 Service boilerplate ...
196 if m.service == 'ec2':
197 # Generic ec2 resource tag support
198 if getattr(m, 'taggable', True):
199 register_ec2_tags(
200 attrs['filter_registry'], attrs['action_registry'])
201 if getattr(m, 'universal_taggable', False):
202 compatibility = isinstance(m.universal_taggable, bool) and True or False
203 register_universal_tags(
204 attrs['filter_registry'], attrs['action_registry'],
205 compatibility=compatibility)
207 return super(QueryMeta, cls).__new__(cls, name, parents, attrs)
210def _napi(op_name):
211 return op_name.title().replace('_', '')
214sources = PluginRegistry('sources')
217@sources.register('describe')
218class DescribeSource:
220 resource_query_factory = ResourceQuery
222 def __init__(self, manager):
223 self.manager = manager
224 self.query = self.get_query()
226 def get_resources(self, ids, cache=True):
227 return self.query.get(self.manager, ids)
229 def resources(self, query):
230 return self.query.filter(self.manager, **query)
232 def get_query(self):
233 return self.resource_query_factory(self.manager.session_factory)
235 def get_query_params(self, query_params):
236 return query_params
238 def get_permissions(self):
239 m = self.manager.get_model()
240 prefix = m.permission_prefix or m.service
241 if m.permissions_enum:
242 perms = list(m.permissions_enum)
243 else:
244 perms = ['%s:%s' % (prefix, _napi(m.enum_spec[0]))]
245 if m.universal_taggable is not False:
246 perms.append("tag:GetResources")
247 if m.permissions_augment:
248 perms.extend(m.permissions_augment)
250 if getattr(m, 'detail_spec', None):
251 perms.append("%s:%s" % (prefix, _napi(m.detail_spec[0])))
252 if getattr(m, 'batch_detail_spec', None):
253 perms.append("%s:%s" % (prefix, _napi(m.batch_detail_spec[0])))
254 return perms
256 def augment(self, resources):
257 model = self.manager.get_model()
258 if getattr(model, 'detail_spec', None):
259 detail_spec = getattr(model, 'detail_spec', None)
260 _augment = _scalar_augment
261 elif getattr(model, 'batch_detail_spec', None):
262 detail_spec = getattr(model, 'batch_detail_spec', None)
263 _augment = _batch_augment
264 else:
265 return resources
266 if self.manager.get_client:
267 client = self.manager.get_client()
268 else:
269 client = local_session(self.manager.session_factory).client(
270 model.service, region_name=self.manager.config.region)
271 _augment = functools.partial(
272 _augment, self.manager, model, detail_spec, client)
273 with self.manager.executor_factory(
274 max_workers=self.manager.max_workers) as w:
275 results = list(w.map(
276 _augment, chunks(resources, self.manager.chunk_size)))
277 return list(itertools.chain(*results))
280class DescribeWithResourceTags(DescribeSource):
282 def augment(self, resources):
283 return universal_augment(self.manager, super().augment(resources))
286@sources.register('describe-child')
287class ChildDescribeSource(DescribeSource):
289 resource_query_factory = ChildResourceQuery
291 def get_query(self, capture_parent_id=False):
292 return self.resource_query_factory(
293 self.manager.session_factory, self.manager, capture_parent_id=capture_parent_id)
296@sources.register('config')
297class ConfigSource:
299 retry = staticmethod(get_retry(('ThrottlingException',)))
301 def __init__(self, manager):
302 self.manager = manager
303 self.titleCase = self.manager.resource_type.id[0].isupper()
305 def get_permissions(self):
306 return ["config:GetResourceConfigHistory",
307 "config:ListDiscoveredResources"]
309 def get_resources(self, ids, cache=True):
310 client = local_session(self.manager.session_factory).client('config')
311 results = []
312 m = self.manager.get_model()
313 for i in ids:
314 revisions = self.retry(
315 client.get_resource_config_history,
316 resourceId=i,
317 resourceType=m.config_type,
318 limit=1).get('configurationItems')
319 if not revisions:
320 continue
321 results.append(self.load_resource(revisions[0]))
322 return list(filter(None, results))
324 def get_query_params(self, query):
325 """Parse config select expression from policy and parameter.
327 On policy config supports a full statement being given, or
328 a clause that will be added to the where expression.
330 If no query is specified, a default query is utilized.
332 A valid query should at minimum select fields
333 for configuration, supplementaryConfiguration and
334 must have resourceType qualifier.
335 """
336 if query and not isinstance(query, dict):
337 raise PolicyExecutionError("invalid config source query %s" % (query,))
339 if query is None and 'query' in self.manager.data:
340 _q = [q for q in self.manager.data['query'] if 'expr' in q]
341 if _q:
342 query = _q.pop()
344 if query is None and 'query' in self.manager.data:
345 _c = [q['clause'] for q in self.manager.data['query'] if 'clause' in q]
346 if _c:
347 _c = _c.pop()
348 elif query:
349 return query
350 else:
351 _c = None
353 s = ("select resourceId, configuration, supplementaryConfiguration "
354 "where resourceType = '{}'").format(self.manager.resource_type.config_type)
356 if _c:
357 s += "AND {}".format(_c)
359 return {'expr': s}
361 def load_resource(self, item):
362 item_config = self._load_item_config(item)
363 resource = camelResource(
364 item_config, implicitDate=True, implicitTitle=self.titleCase)
365 self._load_resource_tags(resource, item)
366 return resource
368 def _load_item_config(self, item):
369 if isinstance(item['configuration'], str):
370 item_config = json.loads(item['configuration'])
371 else:
372 item_config = item['configuration']
373 return item_config
375 def _load_resource_tags(self, resource, item):
376 # normalized tag loading across the many variants of config's inconsistencies.
377 if 'Tags' in resource:
378 return
379 elif item.get('tags'):
380 resource['Tags'] = [
381 {u'Key': k, u'Value': v} for k, v in item['tags'].items()]
382 elif item['supplementaryConfiguration'].get('Tags'):
383 stags = item['supplementaryConfiguration']['Tags']
384 if isinstance(stags, str):
385 stags = json.loads(stags)
386 if isinstance(stags, list):
387 resource['Tags'] = [
388 {u'Key': t.get('key', t.get('tagKey')),
389 u'Value': t.get('value', t.get('tagValue'))}
390 for t in stags
391 ]
392 elif isinstance(stags, dict):
393 resource['Tags'] = [{u'Key': k, u'Value': v} for k, v in stags.items()]
395 def get_listed_resources(self, client):
396 # fallback for when config decides to arbitrarily break select
397 # resource for a given resource type.
398 paginator = client.get_paginator('list_discovered_resources')
399 paginator.PAGE_ITERATOR_CLS = RetryPageIterator
400 pages = paginator.paginate(
401 resourceType=self.manager.get_model().config_type)
402 results = []
404 with self.manager.executor_factory(max_workers=2) as w:
405 ridents = pages.build_full_result()
406 resource_ids = [
407 r['resourceId'] for r in ridents.get('resourceIdentifiers', ())]
408 self.manager.log.debug(
409 "querying %d %s resources",
410 len(resource_ids),
411 self.manager.__class__.__name__.lower())
413 for resource_set in chunks(resource_ids, 50):
414 futures = []
415 futures.append(w.submit(self.get_resources, resource_set))
416 for f in as_completed(futures):
417 if f.exception():
418 self.manager.log.error(
419 "Exception getting resources from config \n %s" % (
420 f.exception()))
421 results.extend(f.result())
422 return results
424 def resources(self, query=None):
425 client = local_session(self.manager.session_factory).client('config')
426 query = self.get_query_params(query)
427 pager = Paginator(
428 client.select_resource_config,
429 {'input_token': 'NextToken', 'output_token': 'NextToken',
430 'result_key': 'Results'},
431 client.meta.service_model.operation_model('SelectResourceConfig'))
432 pager.PAGE_ITERATOR_CLS = RetryPageIterator
434 results = []
435 for page in pager.paginate(Expression=query['expr']):
436 results.extend([
437 self.load_resource(json.loads(r)) for r in page['Results']])
439 # Config arbitrarily breaks which resource types its supports for query/select
440 # on any given day, if we don't have a user defined query, then fallback
441 # to iteration mode.
442 if not results and query == self.get_query_params({}):
443 results = self.get_listed_resources(client)
444 return results
446 def augment(self, resources):
447 return resources
450class QueryResourceManager(ResourceManager, metaclass=QueryMeta):
452 resource_type = ""
454 # TODO Check if we can move to describe source
455 max_workers = 3
456 chunk_size = 20
458 _generate_arn = None
460 retry = staticmethod(
461 get_retry((
462 'TooManyRequestsException',
463 'ThrottlingException',
464 'RequestLimitExceeded',
465 'Throttled',
466 'ThrottledException',
467 'Throttling',
468 'Client.RequestLimitExceeded')))
470 source_mapping = sources
472 def __init__(self, ctx, data):
473 super(QueryResourceManager, self).__init__(ctx, data)
474 self.source = self.get_source(self.source_type)
476 @property
477 def source_type(self):
478 return self.data.get('source', 'describe')
480 def get_source(self, source_type):
481 if source_type in self.source_mapping:
482 return self.source_mapping.get(source_type)(self)
483 if source_type in sources:
484 return sources[source_type](self)
485 raise KeyError("Invalid Source %s" % source_type)
487 @classmethod
488 def has_arn(cls):
489 if cls.resource_type.arn is not None:
490 return bool(cls.resource_type.arn)
491 elif getattr(cls.resource_type, 'arn_type', None) is not None:
492 return True
493 elif cls.__dict__.get('get_arns'):
494 return True
495 return False
497 @classmethod
498 def get_model(cls):
499 return ResourceQuery.resolve(cls.resource_type)
501 @classmethod
502 def match_ids(cls, ids):
503 """return ids that match this resource type's id format."""
504 id_prefix = getattr(cls.get_model(), 'id_prefix', None)
505 if id_prefix is not None:
506 return [i for i in ids if i.startswith(id_prefix)]
507 return ids
509 def get_permissions(self):
510 perms = self.source.get_permissions()
511 if getattr(self, 'permissions', None):
512 perms.extend(self.permissions)
513 return perms
515 def get_cache_key(self, query):
516 return {
517 'account': self.account_id,
518 'region': self.config.region,
519 'resource': str(self.__class__.__name__),
520 'source': self.source_type,
521 'q': query
522 }
524 def resources(self, query=None, augment=True) -> List[dict]:
525 query = self.source.get_query_params(query)
526 cache_key = self.get_cache_key(query)
527 resources = None
529 with self._cache:
530 resources = self._cache.get(cache_key)
531 if resources is not None:
532 self.log.debug("Using cached %s: %d" % (
533 "%s.%s" % (self.__class__.__module__, self.__class__.__name__),
534 len(resources)))
536 if resources is None:
537 if query is None:
538 query = {}
539 with self.ctx.tracer.subsegment('resource-fetch'):
540 resources = self.source.resources(query)
541 if augment:
542 with self.ctx.tracer.subsegment('resource-augment'):
543 resources = self.augment(resources)
544 # Don't pollute cache with unaugmented resources.
545 self._cache.save(cache_key, resources)
547 resource_count = len(resources)
548 with self.ctx.tracer.subsegment('filter'):
549 resources = self.filter_resources(resources)
551 # Check if we're out of a policies execution limits.
552 if self.data == self.ctx.policy.data:
553 self.check_resource_limit(len(resources), resource_count)
554 return resources
556 def check_resource_limit(self, selection_count, population_count):
557 """Check if policy's execution affects more resources then its limit.
559 Ideally this would be at a higher level but we've hidden
560 filtering behind the resource manager facade for default usage.
561 """
562 p = self.ctx.policy
563 max_resource_limits = MaxResourceLimit(p, selection_count, population_count)
564 return max_resource_limits.check_resource_limits()
566 def _get_cached_resources(self, ids):
567 key = self.get_cache_key(None)
568 with self._cache:
569 resources = self._cache.get(key)
570 if resources is not None:
571 self.log.debug("Using cached results for get_resources")
572 m = self.get_model()
573 id_set = set(ids)
574 return [r for r in resources if r[m.id] in id_set]
575 return None
577 def get_resources(self, ids, cache=True, augment=True):
578 if not ids:
579 return []
580 if cache:
581 resources = self._get_cached_resources(ids)
582 if resources is not None:
583 return resources
584 try:
585 resources = self.source.get_resources(ids)
586 if augment:
587 resources = self.augment(resources)
588 return resources
589 except ClientError as e:
590 self.log.warning("event ids not resolved: %s error:%s" % (ids, e))
591 return []
593 def augment(self, resources):
594 """subclasses may want to augment resources with additional information.
596 ie. we want tags by default (rds, elb), and policy, location, acl for
597 s3 buckets.
598 """
599 return self.source.augment(resources)
601 @property
602 def account_id(self):
603 """ Return the current account ID.
605 This should now be passed in using the --account-id flag, but for a
606 period of time we will support the old behavior of inferring this from
607 IAM.
608 """
609 return self.config.account_id
611 @property
612 def region(self):
613 """ Return the current region.
614 """
615 return self.config.region
617 def get_arns(self, resources):
618 arns = []
620 m = self.get_model()
621 arn_key = getattr(m, 'arn', None)
622 if arn_key is False:
623 raise ValueError("%s do not have arns" % self.type)
625 for r in resources:
626 if arn_key:
627 arns.append(get_path(arn_key, r))
628 else:
629 _id = get_path(m.id, r)
631 if 'arn' in _id[:3]:
632 arns.append(_id)
633 else:
634 arns.append(self.generate_arn(_id))
636 return arns
638 @property
639 def generate_arn(self):
640 """ Generates generic arn if ID is not already arn format.
641 """
642 if self._generate_arn is None:
643 self._generate_arn = functools.partial(
644 generate_arn,
645 self.resource_type.arn_service or self.resource_type.service,
646 region=not self.resource_type.global_resource and self.config.region or "",
647 account_id=self.account_id,
648 resource_type=self.resource_type.arn_type,
649 separator=self.resource_type.arn_separator)
650 return self._generate_arn
653class MaxResourceLimit:
655 C7N_MAXRES_OP = os.environ.get("C7N_MAXRES_OP", 'or')
657 def __init__(self, policy, selection_count, population_count):
658 self.p = policy
659 self.op = MaxResourceLimit.C7N_MAXRES_OP
660 self.selection_count = selection_count
661 self.population_count = population_count
662 self.amount = None
663 self.percentage_amount = None
664 self.percent = None
665 self._parse_policy()
667 def _parse_policy(self,):
668 if isinstance(self.p.max_resources, dict):
669 self.op = self.p.max_resources.get("op", MaxResourceLimit.C7N_MAXRES_OP).lower()
670 self.percent = self.p.max_resources.get("percent")
671 self.amount = self.p.max_resources.get("amount")
673 if isinstance(self.p.max_resources, int):
674 self.amount = self.p.max_resources
676 if isinstance(self.p.max_resources_percent, (int, float)):
677 self.percent = self.p.max_resources_percent
679 if self.percent:
680 self.percentage_amount = self.population_count * (self.percent / 100.0)
682 def check_resource_limits(self):
683 if self.percentage_amount and self.amount:
684 if (self.selection_count > self.amount and
685 self.selection_count > self.percentage_amount and self.op == "and"):
686 raise ResourceLimitExceeded(
687 ("policy:%s exceeded resource-limit:{limit} and percentage-limit:%s%% "
688 "found:{selection_count} total:{population_count}")
689 % (self.p.name, self.percent), "max-resource and max-percent",
690 self.amount, self.selection_count, self.population_count)
692 if self.amount:
693 if self.selection_count > self.amount and self.op != "and":
694 raise ResourceLimitExceeded(
695 ("policy:%s exceeded resource-limit:{limit} "
696 "found:{selection_count} total: {population_count}") % self.p.name,
697 "max-resource", self.amount, self.selection_count, self.population_count)
699 if self.percentage_amount:
700 if self.selection_count > self.percentage_amount and self.op != "and":
701 raise ResourceLimitExceeded(
702 ("policy:%s exceeded resource-limit:{limit}%% "
703 "found:{selection_count} total:{population_count}") % self.p.name,
704 "max-percent", self.percent, self.selection_count, self.population_count)
707class ChildResourceManager(QueryResourceManager):
709 child_source = 'describe-child'
711 @property
712 def source_type(self):
713 source = self.data.get('source', self.child_source)
714 if source == 'describe':
715 source = self.child_source
716 return source
718 def get_parent_manager(self):
719 return self.get_resource_manager(self.resource_type.parent_spec[0])
722def _batch_augment(manager, model, detail_spec, client, resource_set):
723 detail_op, param_name, param_key, detail_path, detail_args = detail_spec
724 op = getattr(client, detail_op)
725 if manager.retry:
726 args = (op,)
727 op = manager.retry
728 else:
729 args = ()
730 kw = {param_name: [param_key and r[param_key] or r for r in resource_set]}
731 if detail_args:
732 kw.update(detail_args)
733 response = op(*args, **kw)
734 return response[detail_path]
737def _scalar_augment(manager, model, detail_spec, client, resource_set):
738 detail_op, param_name, param_key, detail_path = detail_spec
739 op = getattr(client, detail_op)
740 if manager.retry:
741 args = (op,)
742 op = manager.retry
743 else:
744 args = ()
745 results = []
746 for r in resource_set:
747 kw = {param_name: param_key and r[param_key] or r}
748 try:
749 response = op(*args, **kw)
750 except client.exceptions.ResourceNotFoundException:
751 manager.log.warning("Resource not found: %s using %s" % (detail_op, kw))
752 continue
753 if detail_path:
754 response = response[detail_path]
755 else:
756 response.pop('ResponseMetadata')
757 if param_key is None:
758 response[model.id] = r
759 r = response
760 else:
761 r.update(response)
762 results.append(r)
763 return results
766class RetryPageIterator(PageIterator):
768 retry = staticmethod(QueryResourceManager.retry)
770 def _make_request(self, current_kwargs):
771 return self.retry(self._method, **current_kwargs)
774class TypeMeta(type):
776 def __repr__(cls):
777 if cls.config_type:
778 identifier = cls.config_type
779 elif cls.cfn_type:
780 identifier = cls.cfn_type
781 elif cls.arn_type and cls.service:
782 identifier = "AWS::%s::%s" % (cls.service.title(), cls.arn_type.title())
783 elif cls.enum_spec and cls.service:
784 identifier = "AWS::%s::%s" % (cls.service.title(), cls.enum_spec[1])
785 elif cls.service:
786 identifier = "AWS::%s::%s" % (cls.service.title(), cls.id)
787 else:
788 identifier = cls.__name__
789 return "<TypeInfo %s>" % identifier
792class TypeInfo(metaclass=TypeMeta):
794 """
795 Resource Type Metadata
798 **Required**
800 :param id: For resource types that use QueryResourceManager this field
801 names the field in the enum_spec response that contains the identifier to use
802 in calls to other API's of this service.
803 Therefore, this "id" field might be the "arn" field for some API's but
804 in other API's it's a name or other identifier value.
806 :param name: Defines the name of a field in the resource that contains the "name" of
807 the resource for report purposes.
808 This name value appears in the "report" command output.
809 By default, the id field is automatically included in the report
810 and if name and id fields are the same field name then it's only shown once.
811 example: custodian report --format csv -s . my-policy.yml
814 :param service: Which aws service (per sdk) has the api for this resource.
815 See the "client" info for each service in the boto documentation.
816 https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/index.html #noqa
818 :param enum_spec: Defines the boto3 call used to find at least basic
819 details on all resource instances of the relevant type. The data per
820 resource can be further enriched by a supplying a detail_spec function.
822 enum_spec is also used when we've received an event in which
823 case the results from enum_spec are filtered to include only
824 those identified by the event. If the enum function API allows a filter param to be
825 specified then the filtering can be done on the server
826 side.
828 For instance, ASG uses "describe_auto_scaling_groups"
829 as the enum function and "AutoScalingGroupNames" as a filter
830 param to that function so the API returns only relevant resources.
831 However, it seems that most Cloud Custodian integrations
832 do not use this approach. App mesh list_meshes for instance doesn't
833 support filtering ...
835 https://boto3.amazonaws.com/v1/documentation/reference/services/appmesh/client/list_meshes.html
837 However, if the enum op doesn't support filtering then the enum op must return all
838 instances of the resource and cloud custodian will perform client side filtering.
840 Params to the enum_spec:
841 - enum_op - the aws api operation
842 - path - JMESPATH path to the field in the response that is the collection of result objects
843 - extra_args - optional eg {'maxResults': 100}
845 **Permissions - Optional**
847 :param permission_prefix: Permission string prefix if not service
848 :param permissions_enum: Permissions for resource enumeration/get.
849 Normally we autogen but in some cases we need to specify statically
850 :param permissions_augment: Permissions for resource augment
852 **Arn handling / generation metadata - Optional**
854 :param arn: Defines a field in the resource definition that contains the ARN value, when
855 the resource has an ARM..
857 This value is accessed used by the 'get_arns(..)' fn on the super-class
858 QueryResourceManager. This value must be a simple field name and cannot be a path.
860 If this value is not defined then 'get_arns' contains fallback logic.
861 - First fallback logic is to look at what's defined in the 'id' field of the resource.
862 If the value of the "id" field starts with "arn:" then that value is used as the arn.
863 - Otherwise, an attempt at generating (guessing!) the ARN by assembling it from
864 various fields and runtime values based on a recipe defined in 'generate_arn()' on
865 the super-class QueryResourceManager.
867 If you aren't going to define the "arn" field and can't rely on the "id" to be an
868 ARN then you might get lucky that "generate_arn" works for your resource type.
869 However, failing that then you should override "get_arns" function entirely and
870 implement your own logic.
872 Testing: Whatever approach you use (above) you REALLY SHOULD (!!!) include a unit
873 test that verifies that "get_arns" yields the right shape of ARNs for your resources.
874 This test should be implemented as an additional assertion within the unit tests
875 you'll be already planning to write.
877 :param arn_type: Type, used for arn construction. also required for universal tag augment
878 Only required when you are NOT providing the ARN value directly via the "arn" cfg field.
879 When arn is not provided then QueryResourceManager.generate_arn uses the arn_type value,
880 plus other fields, to construct an ARN; basically, a best guess but not 100% reliable.
881 If generate_arn() isn't good enough for your needs then you should override the
882 QueryResourceManager.get_arn() function and do it yourself.
884 :param arn_separator: How arn type is separated from rest of arn
885 :param arn_service: For services that need custom labeling for arns
887 **Resource retrieval - Optional**
889 :param filter_name: When fetching a single resource via enum_spec this is technically optional,
890 but effectively required for serverless event policies else we have to enumerate the
891 population
892 :param filter_type: filter_type, scalar or list
893 :param detail_spec: Used to enrich the resource descriptions returned by enum_spec.
894 In many cases the enum_spec function is one of the
895 describe style functions that return a fullish spec that
896 is sufficient for the user policy. However, in other cases
897 the enum_spec is a list style function then the
898 response to then enum call will be lacking in detail and
899 might even just be a list of id's. In these cases it is generally
900 necessary to define a "detail_spec" function that may be called for each id returned
901 by the enum_spec which can be used to enrich the values provided by the enum_spec.
903 Params to the detail_spec:
904 - detail_op - the boto api call name
905 - param_name - name of the identifier argument in the boto api call
906 - param_key - name of field in enum_spec response tha that will be pushed into
907 the identifier argument of the boto api call.
908 - detail_path - path to extract from the boto response and merge into the resource model.
909 if not provided then whole response is merged into the results
911 :param batch_detail_spec: Used when the api supports getting resource details enmasse
913 **Misc - Optional**
915 :param default_report_fields: Used for reporting, array of fields
916 :param date: Latest date associated to resource, generally references either create date or
917 modified date. If this field is defined then it will appear in report output
918 such as you would get from ....
919 example: custodian report --format csv -s . my-policy.yml
921 :param dimension: Defines that resource has cloud watch metrics and the resource id can be
922 passed as this value. Further customizations of dimensions require subclass metrics filter
924 :param cfn_type: AWS Cloudformation type.
925 See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html
927 :param config_type: AWS Config Service resource type name.
928 See https://docs.aws.amazon.com/config/latest/developerguide/resource-config-reference.html
929 Typically cfn_type and config_type will have the sane value,
930 but there are some exceptions, so check.
931 The constants defined will be verified by the PolicyMetaLint tests during the build.
934 :param config_id: Resource attribute that maps to the resourceId field in AWS Config. Intended
935 for resources which use one ID attribute for service API calls and a different one for
936 AWS Config (example: IAM resources).
938 :param universal_taggable: Determined whether resource group tagging will be used to
939 augment the resource model, in which case we'll automatically register tag actions/filters.
940 Note:
941 - values of False will disable tag filters/actions,
942 - values of True will register legacy tag filters/actions,
943 - values of object() will just register current standard tag/filters/actions.
945 :param global_resource: Denotes if this resource exists across all regions (iam, cloudfront,
946 r53)
947 :param metrics_namespace: Generally we utilize a service to namespace mapping in the metrics
948 filter. However, some resources have a type specific namespace (ig. ebs)
949 :param id_prefix: Specific to ec2 service resources used to disambiguate a resource by its id
951 """
953 # Required
954 id = None
955 name = None
956 service = None
957 enum_spec = None
959 # Permissions
960 permission_prefix = None
961 permissions_enum = None
962 permissions_augment = None
964 # Arn handling / generation metadata
965 arn = None
966 arn_type = None
967 arn_separator = "/"
968 arn_service = None
970 # Resource retrieval
971 filter_name = None
972 filter_type = None
973 detail_spec = None
974 batch_detail_spec = None
976 # Misc
977 default_report_fields = ()
978 date = None
979 dimension = None
980 cfn_type = None
981 config_type = None
982 config_id = None
983 universal_taggable = False
984 global_resource = False
985 metrics_namespace = None
986 id_prefix = None