Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/c7n_gcp/filters/metrics.py: 23%

1# Copyright The Cloud Custodian Authors. 

2# SPDX-License-Identifier: Apache-2.0 

3""" 

4Monitoring Metrics suppport for resources 

5""" 

6from datetime import datetime, timedelta 

7 

8from c7n.filters.core import Filter, OPERATORS, FilterValidationError 

9from c7n.utils import local_session, type_schema, jmespath_search 

10 

11from c7n_gcp.provider import resources as gcp_resources 

12 

13REDUCERS = [ 

14 'REDUCE_NONE', 

15 'REDUCE_MEAN', 

16 'REDUCE_MIN', 

17 'REDUCE_MAX', 

18 'REDUCE_MEAN', 

19 'REDUCE_SUM', 

20 'REDUCE_STDDEV', 

21 'REDUCE_COUNT', 

22 'REDUCE_COUNT_TRUE', 

23 'REDUCE_COUNT_FALSE', 

24 'REDUCE_FRACTION_TRUE', 

25 'REDUCE_PERCENTILE_99', 

26 'REDUCE_PERCENTILE_95', 

27 'REDUCE_PERCENTILE_50', 

28 'REDUCE_PERCENTILE_05'] 

29 

30ALIGNERS = [ 

31 'ALIGN_NONE', 

32 'ALIGN_DELTA', 

33 'ALIGN_RATE', 

34 'ALIGN_INTERPOLATE', 

35 'ALIGN_MIN', 

36 'ALIGN_MAX', 

37 'ALIGN_MEAN', 

38 'ALIGN_COUNT', 

39 'ALIGN_SUM', 

40 'REDUCE_COUNT_FALSE', 

41 'ALIGN_STDDEV', 

42 'ALIGN_COUNT_TRUE', 

43 'ALIGN_COUNT_FALSE', 

44 'ALIGN_FRACTION_TRUE', 

45 'ALIGN_PERCENTILE_99', 

46 'ALIGN_PERCENTILE_95', 

47 'ALIGN_PERCENTILE_50', 

48 'ALIGN_PERCENTILE_05', 

49 'ALIGN_PERCENT_CHANG'] 

50 

51BATCH_SIZE = 10000 

52 

53 

54class GCPMetricsFilter(Filter): 

55 """Supports metrics filters on resources. 

56 

57 All resources that have cloud watch metrics are supported. 

58 

59 Docs on cloud watch metrics 

60 

61 - Google Supported Metrics 

62 https://cloud.google.com/monitoring/api/metrics_gcp 

63 

64 - Custom Metrics 

65 https://cloud.google.com/monitoring/api/v3/metric-model#intro-custom-metrics 

66 

67 .. code-block:: yaml 

68 

69 - name: firewall-hit-count 

70 resource: gcp.firewall 

71 filters: 

72 - type: metrics 

73 name: firewallinsights.googleapis.com/subnet/firewall_hit_count 

74 aligner: ALIGN_COUNT 

75 days: 14 

76 value: 1 

77 op: greater-than 

78 """ 

79 

80 schema = type_schema( 

81 'metrics', 

82 **{'name': {'type': 'string'}, 

83 'metric-key': {'type': 'string'}, 

84 'group-by-fields': {'type': 'array', 'items': {'type': 'string'}}, 

85 'days': {'type': 'number'}, 

86 'op': {'type': 'string', 'enum': list(OPERATORS.keys())}, 

87 'reducer': {'type': 'string', 'enum': REDUCERS}, 

88 'aligner': {'type': 'string', 'enum': ALIGNERS}, 

89 'value': {'type': 'number'}, 

90 'filter': {'type': 'string'}, 

91 'missing-value': {'type': 'number'}, 

92 'required': ('value', 'name', 'op')}) 

93 permissions = ("monitoring.timeSeries.list",) 

94 

95 def validate(self): 

96 if not self.data.get('metric-key') and \ 

97 not hasattr(self.manager.resource_type, 'metric_key'): 

98 raise FilterValidationError("metric-key not defined for resource %s," 

99 "so must be provided in the policy" % (self.manager.type)) 

100 return self 

101 

102 def process(self, resources, event=None): 

103 days = self.data.get('days', 14) 

104 duration = timedelta(days) 

105 

106 self.metric = self.data['name'] 

107 self.metric_key = self.data.get('metric-key') or self.manager.resource_type.metric_key 

108 self.aligner = self.data.get('aligner', 'ALIGN_NONE') 

109 self.reducer = self.data.get('reducer', 'REDUCE_NONE') 

110 self.group_by_fields = self.data.get('group-by-fields', []) 

111 self.missing_value = self.data.get('missing-value') 

112 self.end = datetime.utcnow().replace(microsecond=0) 

113 self.start = self.end - duration 

114 self.period = str((self.end - self.start).total_seconds()) + 's' 

115 self.resource_metric_dict = {} 

116 self.op = OPERATORS[self.data.get('op', 'less-than')] 

117 self.value = self.data['value'] 

118 self.filter = self.data.get('filter', '') 

119 self.c7n_metric_key = "%s.%s.%s" % (self.metric, self.aligner, self.reducer) 

120 

121 session = local_session(self.manager.session_factory) 

122 client = session.client("monitoring", "v3", "projects.timeSeries") 

123 project = session.get_default_project() 

124 

125 time_series_data = [] 

126 for batched_filter in self.get_batched_query_filter(resources): 

127 query_params = { 

128 'filter': batched_filter, 

129 'interval_startTime': self.start.isoformat() + 'Z', 

130 'interval_endTime': self.end.isoformat() + 'Z', 

131 'aggregation_alignmentPeriod': self.period, 

132 "aggregation_perSeriesAligner": self.aligner, 

133 "aggregation_crossSeriesReducer": self.reducer, 

134 "aggregation_groupByFields": self.group_by_fields, 

135 'view': 'FULL' 

136 } 

137 metric_list = client.execute_query('list', 

138 {'name': 'projects/' + project, **query_params}) 

139 time_series_data.extend(metric_list.get('timeSeries', [])) 

140 

141 if not time_series_data: 

142 self.log.info("No metrics found for {}".format(self.c7n_metric_key)) 

143 return [] 

144 

145 self.split_by_resource(time_series_data) 

146 matched = [r for r in resources if self.process_resource(r)] 

147 

148 return matched 

149 

150 def batch_resources(self, resources): 

151 if not resources: 

152 return [] 

153 

154 batched_resources = [] 

155 

156 resource_filter = [] 

157 batch_size = len(self.filter) 

158 for r in resources: 

159 resource_name = self.manager.resource_type.get_metric_resource_name(r) 

160 resource_filter_item = '{} = "{}"'.format(self.metric_key, resource_name) 

161 resource_filter.append(resource_filter_item) 

162 resource_filter.append(' OR ') 

163 batch_size += len(resource_filter_item) + 4 

164 if batch_size >= BATCH_SIZE: 

165 resource_filter.pop() 

166 batched_resources.append(resource_filter) 

167 resource_filter = [] 

168 batch_size = len(self.filter) 

169 

170 resource_filter.pop() 

171 batched_resources.append(resource_filter) 

172 return batched_resources 

173 

174 def get_batched_query_filter(self, resources): 

175 batched_filters = [] 

176 metric_filter_type = 'metric.type = "{}" AND ( '.format(self.metric) 

177 user_filter = '' 

178 if self.filter: 

179 user_filter = " AND " + self.filter 

180 

181 for batch in self.batch_resources(resources): 

182 batched_filters.append(''.join([ 

183 metric_filter_type, 

184 ''.join(batch), 

185 ' ) ', 

186 user_filter 

187 ])) 

188 return batched_filters 

189 

190 def split_by_resource(self, metric_list): 

191 for m in metric_list: 

192 resource_name = jmespath_search(self.metric_key, m) 

193 self.resource_metric_dict[resource_name] = m 

194 

195 def process_resource(self, resource): 

196 resource_metric = resource.setdefault('c7n.metrics', {}) 

197 resource_name = self.manager.resource_type.get_metric_resource_name(resource) 

198 metric = self.resource_metric_dict.get(resource_name) 

199 if not metric and not self.missing_value: 

200 return False 

201 if not metric: 

202 metric_value = self.missing_value 

203 else: 

204 metric_value = float(list(metric["points"][0]["value"].values())[0]) 

205 

206 resource_metric[self.c7n_metric_key] = metric 

207 

208 matched = self.op(metric_value, self.value) 

209 return matched 

210 

211 @classmethod 

212 def register_resources(klass, registry, resource_class): 

213 if resource_class.filter_registry: 

214 resource_class.filter_registry.register('metrics', klass) 

215 

216 

217gcp_resources.subscribe(GCPMetricsFilter.register_resources)