Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/c7n/query.py: 30%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Copyright The Cloud Custodian Authors.
2# SPDX-License-Identifier: Apache-2.0
3"""
4Query capability built on skew metamodel
6tags_spec -> s3, elb, rds
7"""
8from concurrent.futures import as_completed
9import functools
10import itertools
11import json
12from typing import List
14import os
16from c7n.actions import ActionRegistry
17from c7n.exceptions import ClientError, ResourceLimitExceeded, PolicyExecutionError
18from c7n.filters import FilterRegistry, MetricsFilter
19from c7n.manager import ResourceManager
20from c7n.registry import PluginRegistry
21from c7n.tags import register_ec2_tags, register_universal_tags, universal_augment
22from c7n.utils import (
23 local_session, generate_arn, get_retry, chunks, camelResource, jmespath_compile, get_path)
25try:
26 from botocore.paginate import PageIterator, Paginator
27except ImportError:
28 # Likely using another provider in a serverless environment
29 class PageIterator:
30 pass
32 class Paginator:
33 pass
36class ResourceQuery:
38 def __init__(self, session_factory):
39 self.session_factory = session_factory
41 @staticmethod
42 def resolve(resource_type):
43 if not isinstance(resource_type, type):
44 raise ValueError(resource_type)
45 return resource_type
47 def _invoke_client_enum(self, client, enum_op, params, path, retry=None):
48 if client.can_paginate(enum_op):
49 p = client.get_paginator(enum_op)
50 if retry:
51 p.PAGE_ITERATOR_CLS = RetryPageIterator
52 results = p.paginate(**params)
53 data = results.build_full_result()
54 else:
55 op = getattr(client, enum_op)
56 data = op(**params)
58 if path:
59 path = jmespath_compile(path)
60 data = path.search(data)
62 return data
64 def filter(self, resource_manager, **params):
65 """Query a set of resources."""
66 m = self.resolve(resource_manager.resource_type)
67 if resource_manager.get_client:
68 client = resource_manager.get_client()
69 else:
70 client = local_session(self.session_factory).client(
71 m.service, resource_manager.config.region)
72 enum_op, path, extra_args = m.enum_spec
73 if extra_args:
74 params = {**extra_args, **params}
75 return self._invoke_client_enum(
76 client, enum_op, params, path,
77 getattr(resource_manager, 'retry', None)) or []
79 def get(self, resource_manager, identities):
80 """Get resources by identities
81 """
82 m = self.resolve(resource_manager.resource_type)
83 params = {}
84 client_filter = True
86 # Try to formulate server side query in the below two scenarios
87 # else fall back to client side filtering
88 if m.filter_name:
89 if m.filter_type == 'list':
90 params[m.filter_name] = identities
91 client_filter = False
92 elif m.filter_type == 'scalar' and len(identities) == 1:
93 params[m.filter_name] = identities[0]
94 client_filter = False
96 resources = self.filter(resource_manager, **params)
97 if client_filter:
98 # This logic was added to prevent the issue from:
99 # https://github.com/cloud-custodian/cloud-custodian/issues/1398
100 if all(map(lambda r: isinstance(r, str), resources)):
101 resources = [r for r in resources if r in identities]
102 # This logic should fix https://github.com/cloud-custodian/cloud-custodian/issues/7573
103 elif all(map(lambda r: isinstance(r, tuple), resources)):
104 resources = [(p, r) for p, r in resources if r[m.id] in identities]
105 else:
106 resources = [r for r in resources if r[m.id] in identities]
108 return resources
111class ChildResourceQuery(ResourceQuery):
112 """A resource query for resources that must be queried with parent information.
114 Several resource types can only be queried in the context of their
115 parents identifiers. ie. efs mount targets (parent efs), route53 resource
116 records (parent hosted zone), ecs services (ecs cluster).
117 """
119 parent_key = 'c7n:parent-id'
121 def __init__(self, session_factory, manager, capture_parent_id=False):
122 self.session_factory = session_factory
123 self.manager = manager
124 self.capture_parent_id = capture_parent_id
126 def filter(self, resource_manager, parent_ids=None, **params):
127 """Query a set of resources."""
128 m = self.resolve(resource_manager.resource_type)
129 if resource_manager.get_client:
130 client = resource_manager.get_client()
131 else:
132 client = local_session(self.session_factory).client(m.service)
134 enum_op, path, extra_args = m.enum_spec
135 if extra_args:
136 params.update(extra_args)
138 parent_type, parent_key, annotate_parent = m.parent_spec
139 parents = self.manager.get_resource_manager(parent_type)
140 if not parent_ids:
141 parent_ids = []
142 for p in parents.resources(augment=False):
143 if isinstance(p, str):
144 parent_ids.append(p)
145 else:
146 parent_ids.append(p[parents.resource_type.id])
148 # Bail out with no parent ids...
149 existing_param = parent_key in params
150 if not existing_param and len(parent_ids) == 0:
151 return []
153 # Handle a query with parent id
154 if existing_param:
155 return self._invoke_client_enum(client, enum_op, params, path)
157 # Have to query separately for each parent's children.
158 results = []
159 for parent_id in parent_ids:
160 merged_params = self.get_parent_parameters(params, parent_id, parent_key)
161 subset = self._invoke_client_enum(
162 client, enum_op, merged_params, path, retry=self.manager.retry)
163 if annotate_parent:
164 for r in subset:
165 r[self.parent_key] = parent_id
166 if subset:
167 if self.capture_parent_id:
168 results.extend([(parent_id, s) for s in subset])
169 else:
170 results.extend(subset)
171 return results
173 def get_parent_parameters(self, params, parent_id, parent_key):
174 return dict(params, **{parent_key: parent_id})
177class QueryMeta(type):
179 def __new__(cls, name, parents, attrs):
180 if 'resource_type' not in attrs:
181 return super(QueryMeta, cls).__new__(cls, name, parents, attrs)
183 if 'filter_registry' not in attrs:
184 attrs['filter_registry'] = FilterRegistry(
185 '%s.filters' % name.lower())
186 if 'action_registry' not in attrs:
187 attrs['action_registry'] = ActionRegistry(
188 '%s.actions' % name.lower())
190 if attrs['resource_type']:
191 m = ResourceQuery.resolve(attrs['resource_type'])
192 # Generic cloud watch metrics support
193 if m.dimension:
194 attrs['filter_registry'].register('metrics', MetricsFilter)
195 # EC2 Service boilerplate ...
196 if m.service == 'ec2':
197 # Generic ec2 resource tag support
198 if getattr(m, 'taggable', True):
199 register_ec2_tags(
200 attrs['filter_registry'], attrs['action_registry'])
201 if getattr(m, 'universal_taggable', False):
202 compatibility = isinstance(m.universal_taggable, bool) and True or False
203 register_universal_tags(
204 attrs['filter_registry'], attrs['action_registry'],
205 compatibility=compatibility)
207 return super(QueryMeta, cls).__new__(cls, name, parents, attrs)
210def _napi(op_name):
211 return op_name.title().replace('_', '')
214sources = PluginRegistry('sources')
217@sources.register('describe')
218class DescribeSource:
220 resource_query_factory = ResourceQuery
222 def __init__(self, manager):
223 self.manager = manager
224 self.query = self.get_query()
226 def get_resources(self, ids, cache=True):
227 return self.query.get(self.manager, ids)
229 def resources(self, query):
230 return self.query.filter(self.manager, **query)
232 def get_query(self):
233 return self.resource_query_factory(self.manager.session_factory)
235 def get_query_params(self, query_params):
236 return query_params
238 def get_permissions(self):
239 m = self.manager.get_model()
240 prefix = m.permission_prefix or m.service
241 if m.permissions_enum:
242 perms = list(m.permissions_enum)
243 else:
244 perms = ['%s:%s' % (prefix, _napi(m.enum_spec[0]))]
245 if m.universal_taggable is not False:
246 perms.append("tag:GetResources")
247 if m.permissions_augment:
248 perms.extend(m.permissions_augment)
250 if getattr(m, 'detail_spec', None):
251 perms.append("%s:%s" % (prefix, _napi(m.detail_spec[0])))
252 if getattr(m, 'batch_detail_spec', None):
253 perms.append("%s:%s" % (prefix, _napi(m.batch_detail_spec[0])))
254 return perms
256 def augment(self, resources):
257 model = self.manager.get_model()
258 if getattr(model, 'detail_spec', None):
259 detail_spec = getattr(model, 'detail_spec', None)
260 _augment = _scalar_augment
261 elif getattr(model, 'batch_detail_spec', None):
262 detail_spec = getattr(model, 'batch_detail_spec', None)
263 _augment = _batch_augment
264 else:
265 return resources
266 if self.manager.get_client:
267 client = self.manager.get_client()
268 else:
269 client = local_session(self.manager.session_factory).client(
270 model.service, region_name=self.manager.config.region)
271 _augment = functools.partial(
272 _augment, self.manager, model, detail_spec, client)
273 with self.manager.executor_factory(
274 max_workers=self.manager.max_workers) as w:
275 results = list(w.map(
276 _augment, chunks(resources, self.manager.chunk_size)))
277 return list(itertools.chain(*results))
280class DescribeWithResourceTags(DescribeSource):
282 def augment(self, resources):
283 return universal_augment(self.manager, super().augment(resources))
286@sources.register('describe-child')
287class ChildDescribeSource(DescribeSource):
289 resource_query_factory = ChildResourceQuery
291 def get_query(self, capture_parent_id=False):
292 return self.resource_query_factory(
293 self.manager.session_factory, self.manager, capture_parent_id=capture_parent_id)
296@sources.register('config')
297class ConfigSource:
299 retry = staticmethod(get_retry(('ThrottlingException',)))
301 def __init__(self, manager):
302 self.manager = manager
303 self.titleCase = self.manager.resource_type.id[0].isupper()
305 def get_permissions(self):
306 return ["config:GetResourceConfigHistory",
307 "config:ListDiscoveredResources"]
309 def get_resources(self, ids, cache=True):
310 client = local_session(self.manager.session_factory).client('config')
311 results = []
312 m = self.manager.get_model()
313 for i in ids:
314 revisions = self.retry(
315 client.get_resource_config_history,
316 resourceId=i,
317 resourceType=m.config_type,
318 limit=1).get('configurationItems')
319 if not revisions:
320 continue
321 results.append(self.load_resource(revisions[0]))
322 return list(filter(None, results))
324 def get_query_params(self, query):
325 """Parse config select expression from policy and parameter.
327 On policy config supports a full statement being given, or
328 a clause that will be added to the where expression.
330 If no query is specified, a default query is utilized.
332 A valid query should at minimum select fields
333 for configuration, supplementaryConfiguration and
334 must have resourceType qualifier.
335 """
336 if query and not isinstance(query, dict):
337 raise PolicyExecutionError("invalid config source query %s" % (query,))
339 if query is None and 'query' in self.manager.data:
340 _q = [q for q in self.manager.data['query'] if 'expr' in q]
341 if _q:
342 query = _q.pop()
344 if query is None and 'query' in self.manager.data:
345 _c = [q['clause'] for q in self.manager.data['query'] if 'clause' in q]
346 if _c:
347 _c = _c.pop()
348 elif query:
349 return query
350 else:
351 _c = None
353 s = ("select resourceId, configuration, supplementaryConfiguration "
354 "where resourceType = '{}'").format(self.manager.resource_type.config_type)
356 if _c:
357 s += "AND {}".format(_c)
359 return {'expr': s}
361 def load_resource(self, item):
362 item_config = self._load_item_config(item)
363 resource = camelResource(
364 item_config, implicitDate=True, implicitTitle=self.titleCase)
365 self._load_resource_tags(resource, item)
366 return resource
368 def _load_item_config(self, item):
369 if isinstance(item['configuration'], str):
370 item_config = json.loads(item['configuration'])
371 else:
372 item_config = item['configuration']
373 return item_config
375 def _load_resource_tags(self, resource, item):
376 # normalized tag loading across the many variants of config's inconsistencies.
377 if 'Tags' in resource:
378 return
379 elif item.get('tags'):
380 resource['Tags'] = [
381 {u'Key': k, u'Value': v} for k, v in item['tags'].items()]
382 elif item['supplementaryConfiguration'].get('Tags'):
383 stags = item['supplementaryConfiguration']['Tags']
384 if isinstance(stags, str):
385 stags = json.loads(stags)
386 if isinstance(stags, list):
387 resource['Tags'] = [
388 {u'Key': t.get('key', t.get('tagKey')),
389 u'Value': t.get('value', t.get('tagValue'))}
390 for t in stags
391 ]
392 elif isinstance(stags, dict):
393 resource['Tags'] = [{u'Key': k, u'Value': v} for k, v in stags.items()]
395 def get_listed_resources(self, client):
396 # fallback for when config decides to arbitrarily break select
397 # resource for a given resource type.
398 paginator = client.get_paginator('list_discovered_resources')
399 paginator.PAGE_ITERATOR_CLS = RetryPageIterator
400 pages = paginator.paginate(
401 resourceType=self.manager.get_model().config_type)
402 results = []
404 with self.manager.executor_factory(max_workers=2) as w:
405 ridents = pages.build_full_result()
406 resource_ids = [
407 r['resourceId'] for r in ridents.get('resourceIdentifiers', ())]
408 self.manager.log.debug(
409 "querying %d %s resources",
410 len(resource_ids),
411 self.manager.__class__.__name__.lower())
413 for resource_set in chunks(resource_ids, 50):
414 futures = []
415 futures.append(w.submit(self.get_resources, resource_set))
416 for f in as_completed(futures):
417 if f.exception():
418 self.manager.log.error(
419 "Exception getting resources from config \n %s" % (
420 f.exception()))
421 results.extend(f.result())
422 return results
424 def resources(self, query=None):
425 client = local_session(self.manager.session_factory).client('config')
426 query = self.get_query_params(query)
427 pager = Paginator(
428 client.select_resource_config,
429 {'input_token': 'NextToken', 'output_token': 'NextToken',
430 'result_key': 'Results'},
431 client.meta.service_model.operation_model('SelectResourceConfig'))
432 pager.PAGE_ITERATOR_CLS = RetryPageIterator
434 results = []
435 for page in pager.paginate(Expression=query['expr']):
436 results.extend([
437 self.load_resource(json.loads(r)) for r in page['Results']])
439 # Config arbitrarily breaks which resource types its supports for query/select
440 # on any given day, if we don't have a user defined query, then fallback
441 # to iteration mode.
442 if not results and query == self.get_query_params({}):
443 results = self.get_listed_resources(client)
444 return results
446 def augment(self, resources):
447 return resources
450class QueryResourceManager(ResourceManager, metaclass=QueryMeta):
452 resource_type = ""
454 # TODO Check if we can move to describe source
455 max_workers = 3
456 chunk_size = 20
458 _generate_arn = None
460 retry = staticmethod(
461 get_retry((
462 'TooManyRequestsException',
463 'ThrottlingException',
464 'RequestLimitExceeded',
465 'Throttled',
466 'ThrottledException',
467 'Throttling',
468 'Client.RequestLimitExceeded')))
470 source_mapping = sources
472 def __init__(self, ctx, data):
473 super(QueryResourceManager, self).__init__(ctx, data)
474 self.source = self.get_source(self.source_type)
476 @property
477 def source_type(self):
478 return self.data.get('source', 'describe')
480 def get_source(self, source_type):
481 if source_type in self.source_mapping:
482 return self.source_mapping.get(source_type)(self)
483 if source_type in sources:
484 return sources[source_type](self)
485 raise KeyError("Invalid Source %s" % source_type)
487 @classmethod
488 def has_arn(cls):
489 if cls.resource_type.arn is not None:
490 return bool(cls.resource_type.arn)
491 elif getattr(cls.resource_type, 'arn_type', None) is not None:
492 return True
493 elif cls.__dict__.get('get_arns'):
494 return True
495 return False
497 @classmethod
498 def get_model(cls):
499 return ResourceQuery.resolve(cls.resource_type)
501 @classmethod
502 def match_ids(cls, ids):
503 """return ids that match this resource type's id format."""
504 id_prefix = getattr(cls.get_model(), 'id_prefix', None)
505 if id_prefix is not None:
506 return [i for i in ids if i.startswith(id_prefix)]
507 return ids
509 def get_permissions(self):
510 perms = self.source.get_permissions()
511 if getattr(self, 'permissions', None):
512 perms.extend(self.permissions)
513 return perms
515 def get_cache_key(self, query):
516 return {
517 'account': self.account_id,
518 'region': self.config.region,
519 'resource': str(self.__class__.__name__),
520 'source': self.source_type,
521 'q': query
522 }
524 def resources(self, query=None, augment=True) -> List[dict]:
525 query = self.source.get_query_params(query)
526 cache_key = self.get_cache_key(query)
527 resources = None
529 with self._cache:
530 resources = self._cache.get(cache_key)
531 if resources is not None:
532 self.log.debug("Using cached %s: %d" % (
533 "%s.%s" % (self.__class__.__module__, self.__class__.__name__),
534 len(resources)))
536 if resources is None:
537 if query is None:
538 query = {}
539 with self.ctx.tracer.subsegment('resource-fetch'):
540 resources = self.source.resources(query)
541 if augment:
542 with self.ctx.tracer.subsegment('resource-augment'):
543 resources = self.augment(resources)
544 # Don't pollute cache with unaugmented resources.
545 self._cache.save(cache_key, resources)
547 resource_count = len(resources)
548 with self.ctx.tracer.subsegment('filter'):
549 resources = self.filter_resources(resources)
551 # Check if we're out of a policies execution limits.
552 if self.data == self.ctx.policy.data:
553 self.check_resource_limit(len(resources), resource_count)
554 return resources
556 def check_resource_limit(self, selection_count, population_count):
557 """Check if policy's execution affects more resources then its limit.
559 Ideally this would be at a higher level but we've hidden
560 filtering behind the resource manager facade for default usage.
561 """
562 p = self.ctx.policy
563 max_resource_limits = MaxResourceLimit(p, selection_count, population_count)
564 return max_resource_limits.check_resource_limits()
566 def _get_cached_resources(self, ids):
567 key = self.get_cache_key(None)
568 with self._cache:
569 resources = self._cache.get(key)
570 if resources is not None:
571 self.log.debug("Using cached results for get_resources")
572 m = self.get_model()
573 id_set = set(ids)
574 return [r for r in resources if r[m.id] in id_set]
575 return None
577 def get_resources(self, ids, cache=True, augment=True):
578 if not ids:
579 return []
580 if cache:
581 resources = self._get_cached_resources(ids)
582 if resources is not None:
583 return resources
584 try:
585 resources = self.source.get_resources(ids)
586 if augment:
587 resources = self.augment(resources)
588 return resources
589 except ClientError as e:
590 self.log.warning("event ids not resolved: %s error:%s" % (ids, e))
591 return []
593 def augment(self, resources):
594 """subclasses may want to augment resources with additional information.
596 ie. we want tags by default (rds, elb), and policy, location, acl for
597 s3 buckets.
598 """
599 return self.source.augment(resources)
601 @property
602 def account_id(self):
603 """ Return the current account ID.
605 This should now be passed in using the --account-id flag, but for a
606 period of time we will support the old behavior of inferring this from
607 IAM.
608 """
609 return self.config.account_id
611 @property
612 def region(self):
613 """ Return the current region.
614 """
615 return self.config.region
617 def get_arns(self, resources):
618 arns = []
620 m = self.get_model()
621 arn_key = getattr(m, 'arn', None)
622 if arn_key is False:
623 raise ValueError("%s do not have arns" % self.type)
625 for r in resources:
626 if arn_key:
627 arns.append(get_path(arn_key, r))
628 else:
629 _id = get_path(m.id, r)
631 if 'arn' in _id[:3]:
632 arns.append(_id)
633 else:
634 arns.append(self.generate_arn(_id))
636 return arns
638 @property
639 def generate_arn(self):
640 """ Generates generic arn if ID is not already arn format.
641 """
642 if self._generate_arn is None:
643 self._generate_arn = functools.partial(
644 generate_arn,
645 self.resource_type.arn_service or self.resource_type.service,
646 region=not self.resource_type.global_resource and self.config.region or "",
647 account_id=self.account_id,
648 resource_type=self.resource_type.arn_type,
649 separator=self.resource_type.arn_separator)
650 return self._generate_arn
653class MaxResourceLimit:
655 C7N_MAXRES_OP = os.environ.get("C7N_MAXRES_OP", 'or')
657 def __init__(self, policy, selection_count, population_count):
658 self.p = policy
659 self.op = MaxResourceLimit.C7N_MAXRES_OP
660 self.selection_count = selection_count
661 self.population_count = population_count
662 self.amount = None
663 self.percentage_amount = None
664 self.percent = None
665 self._parse_policy()
667 def _parse_policy(self,):
668 if isinstance(self.p.max_resources, dict):
669 self.op = self.p.max_resources.get("op", MaxResourceLimit.C7N_MAXRES_OP).lower()
670 self.percent = self.p.max_resources.get("percent")
671 self.amount = self.p.max_resources.get("amount")
673 if isinstance(self.p.max_resources, int):
674 self.amount = self.p.max_resources
676 if isinstance(self.p.max_resources_percent, (int, float)):
677 self.percent = self.p.max_resources_percent
679 if self.percent:
680 self.percentage_amount = self.population_count * (self.percent / 100.0)
682 def check_resource_limits(self):
683 if self.percentage_amount and self.amount:
684 if (self.selection_count > self.amount and
685 self.selection_count > self.percentage_amount and self.op == "and"):
686 raise ResourceLimitExceeded(
687 ("policy:%s exceeded resource-limit:{limit} and percentage-limit:%s%% "
688 "found:{selection_count} total:{population_count}")
689 % (self.p.name, self.percent), "max-resource and max-percent",
690 self.amount, self.selection_count, self.population_count)
692 if self.amount:
693 if self.selection_count > self.amount and self.op != "and":
694 raise ResourceLimitExceeded(
695 ("policy:%s exceeded resource-limit:{limit} "
696 "found:{selection_count} total: {population_count}") % self.p.name,
697 "max-resource", self.amount, self.selection_count, self.population_count)
699 if self.percentage_amount:
700 if self.selection_count > self.percentage_amount and self.op != "and":
701 raise ResourceLimitExceeded(
702 ("policy:%s exceeded resource-limit:{limit}%% "
703 "found:{selection_count} total:{population_count}") % self.p.name,
704 "max-percent", self.percent, self.selection_count, self.population_count)
707class ChildResourceManager(QueryResourceManager):
709 child_source = 'describe-child'
711 @property
712 def source_type(self):
713 source = self.data.get('source', self.child_source)
714 if source == 'describe':
715 source = self.child_source
716 return source
718 def get_parent_manager(self):
719 return self.get_resource_manager(self.resource_type.parent_spec[0])
722def _batch_augment(manager, model, detail_spec, client, resource_set):
723 detail_op, param_name, param_key, detail_path, detail_args = detail_spec
724 op = getattr(client, detail_op)
725 if manager.retry:
726 args = (op,)
727 op = manager.retry
728 else:
729 args = ()
730 kw = {param_name: [param_key and r[param_key] or r for r in resource_set]}
731 if detail_args:
732 kw.update(detail_args)
733 response = op(*args, **kw)
734 return response[detail_path]
737def _scalar_augment(manager, model, detail_spec, client, resource_set):
738 detail_op, param_name, param_key, detail_path = detail_spec
739 op = getattr(client, detail_op)
740 if manager.retry:
741 args = (op,)
742 op = manager.retry
743 else:
744 args = ()
745 results = []
746 for r in resource_set:
747 kw = {param_name: param_key and r[param_key] or r}
748 response = op(*args, **kw)
749 if detail_path:
750 response = response[detail_path]
751 else:
752 response.pop('ResponseMetadata')
753 if param_key is None:
754 response[model.id] = r
755 r = response
756 else:
757 r.update(response)
758 results.append(r)
759 return results
762class RetryPageIterator(PageIterator):
764 retry = staticmethod(QueryResourceManager.retry)
766 def _make_request(self, current_kwargs):
767 return self.retry(self._method, **current_kwargs)
770class TypeMeta(type):
772 def __repr__(cls):
773 if cls.config_type:
774 identifier = cls.config_type
775 elif cls.cfn_type:
776 identifier = cls.cfn_type
777 elif cls.arn_type and cls.service:
778 identifier = "AWS::%s::%s" % (cls.service.title(), cls.arn_type.title())
779 elif cls.enum_spec and cls.service:
780 identifier = "AWS::%s::%s" % (cls.service.title(), cls.enum_spec[1])
781 elif cls.service:
782 identifier = "AWS::%s::%s" % (cls.service.title(), cls.id)
783 else:
784 identifier = cls.__name__
785 return "<TypeInfo %s>" % identifier
788class TypeInfo(metaclass=TypeMeta):
790 """
791 Resource Type Metadata
794 **Required**
796 :param id: For resource types that use QueryResourceManager this field
797 names the field in the enum_spec response that contains the identifier to use
798 in calls to other API's of this service.
799 Therefore, this "id" field might be the "arn" field for some API's but
800 in other API's it's a name or other identifier value.
802 :param name: Defines the name of a field in the resource that contains the "name" of
803 the resource for report purposes.
804 This name value appears in the "report" command output.
805 By default, the id field is automatically included in the report
806 and if name and id fields are the same field name then it's only shown once.
807 example: custodian report --format csv -s . my-policy.yml
810 :param service: Which aws service (per sdk) has the api for this resource.
811 See the "client" info for each service in the boto documentation.
812 https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/index.html #noqa
814 :param enum_spec: Defines the boto3 call used to find at least basic
815 details on all resource instances of the relevant type. The data per
816 resource can be further enriched by a supplying a detail_spec function.
818 enum_spec is also used when we've received an event in which
819 case the results from enum_spec are filtered to include only
820 those identified by the event. If the enum function API allows a filter param to be
821 specified then the filtering can be done on the server
822 side.
824 For instance, ASG uses "describe_auto_scaling_groups"
825 as the enum function and "AutoScalingGroupNames" as a filter
826 param to that function so the API returns only relevant resources.
827 However, it seems that most Cloud Custodian integrations
828 do not use this approach. App mesh list_meshes for instance doesn't
829 support filtering ...
831 https://boto3.amazonaws.com/v1/documentation/reference/services/appmesh/client/list_meshes.html
833 However, if the enum op doesn't support filtering then the enum op must return all
834 instances of the resource and cloud custodian will perform client side filtering.
836 Params to the enum_spec:
837 - enum_op - the aws api operation
838 - path - JMESPATH path to the field in the response that is the collection of result objects
839 - extra_args - optional eg {'maxResults': 100}
841 **Permissions - Optional**
843 :param permission_prefix: Permission string prefix if not service
844 :param permissions_enum: Permissions for resource enumeration/get.
845 Normally we autogen but in some cases we need to specify statically
846 :param permissions_augment: Permissions for resource augment
848 **Arn handling / generation metadata - Optional**
850 :param arn: Defines a field in the resource definition that contains the ARN value, when
851 the resource has an ARM..
853 This value is accessed used by the 'get_arns(..)' fn on the super-class
854 QueryResourceManager. This value must be a simple field name and cannot be a path.
856 If this value is not defined then 'get_arns' contains fallback logic.
857 - First fallback logic is to look at what's defined in the 'id' field of the resource.
858 If the value of the "id" field starts with "arn:" then that value is used as the arn.
859 - Otherwise, an attempt at generating (guessing!) the ARN by assembling it from
860 various fields and runtime values based on a recipe defined in 'generate_arn()' on
861 the super-class QueryResourceManager.
863 If you aren't going to define the "arn" field and can't rely on the "id" to be an
864 ARN then you might get lucky that "generate_arn" works for your resource type.
865 However, failing that then you should override "get_arns" function entirely and
866 implement your own logic.
868 Testing: Whatever approach you use (above) you REALLY SHOULD (!!!) include a unit
869 test that verifies that "get_arns" yields the right shape of ARNs for your resources.
870 This test should be implemented as an additional assertion within the unit tests
871 you'll be already planning to write.
873 :param arn_type: Type, used for arn construction. also required for universal tag augment
874 Only required when you are NOT providing the ARN value directly via the "arn" cfg field.
875 When arn is not provided then QueryResourceManager.generate_arn uses the arn_type value,
876 plus other fields, to construct an ARN; basically, a best guess but not 100% reliable.
877 If generate_arn() isn't good enough for your needs then you should override the
878 QueryResourceManager.get_arn() function and do it yourself.
880 :param arn_separator: How arn type is separated from rest of arn
881 :param arn_service: For services that need custom labeling for arns
883 **Resource retrieval - Optional**
885 :param filter_name: When fetching a single resource via enum_spec this is technically optional,
886 but effectively required for serverless event policies else we have to enumerate the
887 population
888 :param filter_type: filter_type, scalar or list
889 :param detail_spec: Used to enrich the resource descriptions returned by enum_spec.
890 In many cases the enum_spec function is one of the
891 describe style functions that return a fullish spec that
892 is sufficient for the user policy. However, in other cases
893 the enum_spec is a list style function then the
894 response to then enum call will be lacking in detail and
895 might even just be a list of id's. In these cases it is generally
896 necessary to define a "detail_spec" function that may be called for each id returned
897 by the enum_spec which can be used to enrich the values provided by the enum_spec.
899 Params to the detail_spec:
900 - detail_op - the boto api call name
901 - param_name - name of the identifier argument in the boto api call
902 - param_key - name of field in enum_spec response tha that will be pushed into
903 the identifier argument of the boto api call.
904 - detail_path - path to extract from the boto response and merge into the resource model.
905 if not provided then whole response is merged into the results
907 :param batch_detail_spec: Used when the api supports getting resource details enmasse
909 **Misc - Optional**
911 :param default_report_fields: Used for reporting, array of fields
912 :param date: Latest date associated to resource, generally references either create date or
913 modified date. If this field is defined then it will appear in report output
914 such as you would get from ....
915 example: custodian report --format csv -s . my-policy.yml
917 :param dimension: Defines that resource has cloud watch metrics and the resource id can be
918 passed as this value. Further customizations of dimensions require subclass metrics filter
920 :param cfn_type: AWS Cloudformation type.
921 See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html
923 :param config_type: AWS Config Service resource type name.
924 See https://docs.aws.amazon.com/config/latest/developerguide/resource-config-reference.html
925 Typically cfn_type and config_type will have the sane value,
926 but there are some exceptions, so check.
927 The constants defined will be verified by the PolicyMetaLint tests during the build.
930 :param config_id: Resource attribute that maps to the resourceId field in AWS Config. Intended
931 for resources which use one ID attribute for service API calls and a different one for
932 AWS Config (example: IAM resources).
934 :param universal_taggable: Determined whether resource group tagging will be used to
935 augment the resource model, in which case we'll automatically register tag actions/filters.
936 Note:
937 - values of False will disable tag filters/actions,
938 - values of True will register legacy tag filters/actions,
939 - values of object() will just register current standard tag/filters/actions.
941 :param global_resource: Denotes if this resource exists across all regions (iam, cloudfront,
942 r53)
943 :param metrics_namespace: Generally we utilize a service to namespace mapping in the metrics
944 filter. However, some resources have a type specific namespace (ig. ebs)
945 :param id_prefix: Specific to ec2 service resources used to disambiguate a resource by its id
947 """
949 # Required
950 id = None
951 name = None
952 service = None
953 enum_spec = None
955 # Permissions
956 permission_prefix = None
957 permissions_enum = None
958 permissions_augment = None
960 # Arn handling / generation metadata
961 arn = None
962 arn_type = None
963 arn_separator = "/"
964 arn_service = None
966 # Resource retrieval
967 filter_name = None
968 filter_type = None
969 detail_spec = None
970 batch_detail_spec = None
972 # Misc
973 default_report_fields = ()
974 date = None
975 dimension = None
976 cfn_type = None
977 config_type = None
978 config_id = None
979 universal_taggable = False
980 global_resource = False
981 metrics_namespace = None
982 id_prefix = None