/proc/self/cwd/source/extensions/common/aws/utility.cc
Line | Count | Source (jump to first uncovered line) |
1 | | #include "source/extensions/common/aws/utility.h" |
2 | | |
3 | | #include "envoy/upstream/cluster_manager.h" |
4 | | |
5 | | #include "source/common/common/empty_string.h" |
6 | | #include "source/common/common/fmt.h" |
7 | | #include "source/common/common/utility.h" |
8 | | #include "source/common/protobuf/message_validator_impl.h" |
9 | | #include "source/common/protobuf/utility.h" |
10 | | |
11 | | #include "absl/strings/match.h" |
12 | | #include "absl/strings/str_join.h" |
13 | | #include "absl/strings/str_split.h" |
14 | | #include "curl/curl.h" |
15 | | #include "fmt/printf.h" |
16 | | |
17 | | namespace Envoy { |
18 | | namespace Extensions { |
19 | | namespace Common { |
20 | | namespace Aws { |
21 | | |
22 | | constexpr absl::string_view PATH_SPLITTER = "/"; |
23 | | constexpr absl::string_view QUERY_PARAM_SEPERATOR = "="; |
24 | | constexpr absl::string_view QUERY_SEPERATOR = "&"; |
25 | | constexpr absl::string_view QUERY_SPLITTER = "?"; |
26 | | constexpr absl::string_view RESERVED_CHARS = "-._~"; |
27 | | constexpr absl::string_view S3_SERVICE_NAME = "s3"; |
28 | | constexpr absl::string_view URI_ENCODE = "%{:02X}"; |
29 | | constexpr absl::string_view URI_DOUBLE_ENCODE = "%25{:02X}"; |
30 | | |
31 | | std::map<std::string, std::string> |
32 | | Utility::canonicalizeHeaders(const Http::RequestHeaderMap& headers, |
33 | 0 | const std::vector<Matchers::StringMatcherPtr>& excluded_headers) { |
34 | 0 | std::map<std::string, std::string> out; |
35 | 0 | headers.iterate( |
36 | 0 | [&out, &excluded_headers](const Http::HeaderEntry& entry) -> Http::HeaderMap::Iterate { |
37 | | // Skip empty headers |
38 | 0 | if (entry.key().empty() || entry.value().empty()) { |
39 | 0 | return Http::HeaderMap::Iterate::Continue; |
40 | 0 | } |
41 | | // Pseudo-headers should not be canonicalized |
42 | 0 | if (!entry.key().getStringView().empty() && entry.key().getStringView()[0] == ':') { |
43 | 0 | return Http::HeaderMap::Iterate::Continue; |
44 | 0 | } |
45 | 0 | const auto key = entry.key().getStringView(); |
46 | 0 | if (std::any_of(excluded_headers.begin(), excluded_headers.end(), |
47 | 0 | [&key](const Matchers::StringMatcherPtr& matcher) { |
48 | 0 | return matcher->match(key); |
49 | 0 | })) { |
50 | 0 | return Http::HeaderMap::Iterate::Continue; |
51 | 0 | } |
52 | | |
53 | 0 | std::string value(entry.value().getStringView()); |
54 | | // Remove leading, trailing, and deduplicate repeated ascii spaces |
55 | 0 | absl::RemoveExtraAsciiWhitespace(&value); |
56 | 0 | const auto iter = out.find(std::string(entry.key().getStringView())); |
57 | | // If the entry already exists, append the new value to the end |
58 | 0 | if (iter != out.end()) { |
59 | 0 | iter->second += fmt::format(",{}", value); |
60 | 0 | } else { |
61 | 0 | out.emplace(std::string(entry.key().getStringView()), value); |
62 | 0 | } |
63 | 0 | return Http::HeaderMap::Iterate::Continue; |
64 | 0 | }); |
65 | | // The AWS SDK has a quirk where it removes "default ports" (80, 443) from the host headers |
66 | | // Additionally, we canonicalize the :authority header as "host" |
67 | | // TODO(suniltheta): This may need to be tweaked to canonicalize :authority for HTTP/2 requests |
68 | 0 | const absl::string_view authority_header = headers.getHostValue(); |
69 | 0 | if (!authority_header.empty()) { |
70 | 0 | const auto parts = StringUtil::splitToken(authority_header, ":"); |
71 | 0 | if (parts.size() > 1 && (parts[1] == "80" || parts[1] == "443")) { |
72 | | // Has default port, so use only the host part |
73 | 0 | out.emplace(Http::Headers::get().HostLegacy.get(), std::string(parts[0])); |
74 | 0 | } else { |
75 | 0 | out.emplace(Http::Headers::get().HostLegacy.get(), std::string(authority_header)); |
76 | 0 | } |
77 | 0 | } |
78 | 0 | return out; |
79 | 0 | } |
80 | | |
81 | | std::string Utility::createCanonicalRequest( |
82 | | absl::string_view service_name, absl::string_view method, absl::string_view path, |
83 | 0 | const std::map<std::string, std::string>& canonical_headers, absl::string_view content_hash) { |
84 | 0 | std::vector<absl::string_view> parts; |
85 | 0 | parts.emplace_back(method); |
86 | | // don't include the query part of the path |
87 | 0 | const auto path_part = StringUtil::cropRight(path, QUERY_SPLITTER); |
88 | 0 | const auto canonicalized_path = path_part.empty() |
89 | 0 | ? std::string{PATH_SPLITTER} |
90 | 0 | : canonicalizePathString(path_part, service_name); |
91 | 0 | parts.emplace_back(canonicalized_path); |
92 | 0 | const auto query_part = StringUtil::cropLeft(path, QUERY_SPLITTER); |
93 | | // if query_part == path_part, then there is no query |
94 | 0 | const auto canonicalized_query = |
95 | 0 | query_part == path_part ? EMPTY_STRING : Utility::canonicalizeQueryString(query_part); |
96 | 0 | parts.emplace_back(absl::string_view(canonicalized_query)); |
97 | 0 | std::vector<std::string> formatted_headers; |
98 | 0 | formatted_headers.reserve(canonical_headers.size()); |
99 | 0 | for (const auto& header : canonical_headers) { |
100 | 0 | formatted_headers.emplace_back(fmt::format("{}:{}", header.first, header.second)); |
101 | 0 | parts.emplace_back(formatted_headers.back()); |
102 | 0 | } |
103 | | // need an extra blank space after the canonical headers |
104 | 0 | parts.emplace_back(EMPTY_STRING); |
105 | 0 | const auto signed_headers = Utility::joinCanonicalHeaderNames(canonical_headers); |
106 | 0 | parts.emplace_back(signed_headers); |
107 | 0 | parts.emplace_back(content_hash); |
108 | 0 | return absl::StrJoin(parts, "\n"); |
109 | 0 | } |
110 | | |
111 | | /** |
112 | | * Normalizes the path string based on AWS requirements. |
113 | | * See step 2 in https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html |
114 | | */ |
115 | | std::string Utility::canonicalizePathString(absl::string_view path_string, |
116 | 0 | absl::string_view service_name) { |
117 | | // If service is S3, do not normalize but only encode the path |
118 | 0 | if (absl::EqualsIgnoreCase(service_name, S3_SERVICE_NAME)) { |
119 | 0 | return encodePathSegment(path_string, service_name); |
120 | 0 | } |
121 | | // If service is not S3, normalize and encode the path |
122 | 0 | const auto path_segments = StringUtil::splitToken(path_string, std::string{PATH_SPLITTER}); |
123 | 0 | std::vector<std::string> path_list; |
124 | 0 | path_list.reserve(path_segments.size()); |
125 | 0 | for (const auto& path_segment : path_segments) { |
126 | 0 | if (path_segment.empty()) { |
127 | 0 | continue; |
128 | 0 | } |
129 | 0 | path_list.emplace_back(encodePathSegment(path_segment, service_name)); |
130 | 0 | } |
131 | 0 | auto canonical_path_string = |
132 | 0 | fmt::format("{}{}", PATH_SPLITTER, absl::StrJoin(path_list, PATH_SPLITTER)); |
133 | | // Handle corner case when path ends with '/' |
134 | 0 | if (absl::EndsWith(path_string, PATH_SPLITTER) && canonical_path_string.size() > 1) { |
135 | 0 | canonical_path_string.push_back(PATH_SPLITTER[0]); |
136 | 0 | } |
137 | 0 | return canonical_path_string; |
138 | 0 | } |
139 | | |
140 | 0 | bool isReservedChar(const char c) { |
141 | 0 | return std::isalnum(c) || RESERVED_CHARS.find(c) != std::string::npos; |
142 | 0 | } |
143 | | |
144 | 0 | void encodeS3Path(std::string& encoded, const char& c) { |
145 | | // Do not encode '/' for S3 |
146 | 0 | if (c == PATH_SPLITTER[0]) { |
147 | 0 | encoded.push_back(c); |
148 | 0 | } else { |
149 | 0 | absl::StrAppend(&encoded, fmt::format(URI_ENCODE, c)); |
150 | 0 | } |
151 | 0 | } |
152 | | |
153 | 0 | std::string Utility::encodePathSegment(absl::string_view decoded, absl::string_view service_name) { |
154 | 0 | std::string encoded; |
155 | 0 | for (char c : decoded) { |
156 | 0 | if (isReservedChar(c)) { |
157 | | // Escape unreserved chars from RFC 3986 |
158 | 0 | encoded.push_back(c); |
159 | 0 | } else if (absl::EqualsIgnoreCase(service_name, S3_SERVICE_NAME)) { |
160 | 0 | encodeS3Path(encoded, c); |
161 | 0 | } else { |
162 | | // TODO: @aws, There is some inconsistency between AWS services if this should be double |
163 | | // encoded or not. We need to parameterize this and expose this in the config. Ref: |
164 | | // https://github.com/aws/aws-sdk-cpp/blob/main/aws-cpp-sdk-core/source/auth/AWSAuthSigner.cpp#L79-L93 |
165 | 0 | absl::StrAppend(&encoded, fmt::format(URI_ENCODE, c)); |
166 | 0 | } |
167 | 0 | } |
168 | 0 | return encoded; |
169 | 0 | } |
170 | | |
171 | | /** |
172 | | * Normalizes the query string based on AWS requirements. |
173 | | * See step 3 in https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html |
174 | | */ |
175 | 0 | std::string Utility::canonicalizeQueryString(absl::string_view query_string) { |
176 | | // Sort query string based on param name and append "=" if value is missing |
177 | 0 | const auto query_fragments = StringUtil::splitToken(query_string, QUERY_SEPERATOR); |
178 | 0 | std::vector<std::pair<std::string, std::string>> query_list; |
179 | 0 | for (const auto& query_fragment : query_fragments) { |
180 | | // Only split at the first "=" and encode the rest |
181 | 0 | const std::vector<std::string> query = |
182 | 0 | absl::StrSplit(query_fragment, absl::MaxSplits(QUERY_PARAM_SEPERATOR, 1)); |
183 | 0 | if (!query.empty()) { |
184 | 0 | const absl::string_view param = query[0]; |
185 | 0 | const absl::string_view value = query.size() > 1 ? query[1] : EMPTY_STRING; |
186 | 0 | query_list.emplace_back(std::make_pair(param, value)); |
187 | 0 | } |
188 | 0 | } |
189 | | // Sort query params by name and value |
190 | 0 | std::sort(query_list.begin(), query_list.end()); |
191 | | // Encode query params name and value separately |
192 | 0 | for (auto& query : query_list) { |
193 | 0 | query = std::make_pair(Utility::encodeQueryParam(query.first), |
194 | 0 | Utility::encodeQueryParam(query.second)); |
195 | 0 | } |
196 | 0 | return absl::StrJoin(query_list, QUERY_SEPERATOR, absl::PairFormatter(QUERY_PARAM_SEPERATOR)); |
197 | 0 | } |
198 | | |
199 | 0 | std::string Utility::encodeQueryParam(absl::string_view decoded) { |
200 | 0 | std::string encoded; |
201 | 0 | for (char c : decoded) { |
202 | 0 | if (isReservedChar(c) || c == '%') { |
203 | | // Escape unreserved chars from RFC 3986 |
204 | 0 | encoded.push_back(c); |
205 | 0 | } else if (c == '+') { |
206 | | // Encode '+' as space |
207 | 0 | absl::StrAppend(&encoded, "%20"); |
208 | 0 | } else if (c == QUERY_PARAM_SEPERATOR[0]) { |
209 | | // Double encode '=' |
210 | 0 | absl::StrAppend(&encoded, fmt::format(URI_DOUBLE_ENCODE, c)); |
211 | 0 | } else { |
212 | 0 | absl::StrAppend(&encoded, fmt::format(URI_ENCODE, c)); |
213 | 0 | } |
214 | 0 | } |
215 | 0 | return encoded; |
216 | 0 | } |
217 | | |
218 | | std::string |
219 | 0 | Utility::joinCanonicalHeaderNames(const std::map<std::string, std::string>& canonical_headers) { |
220 | 0 | return absl::StrJoin(canonical_headers, ";", [](auto* out, const auto& pair) { |
221 | 0 | return absl::StrAppend(out, pair.first); |
222 | 0 | }); |
223 | 0 | } |
224 | | |
225 | 0 | static size_t curlCallback(char* ptr, size_t, size_t nmemb, void* data) { |
226 | 0 | auto buf = static_cast<std::string*>(data); |
227 | 0 | buf->append(ptr, nmemb); |
228 | 0 | return nmemb; |
229 | 0 | } |
230 | | |
231 | 0 | absl::optional<std::string> Utility::fetchMetadata(Http::RequestMessage& message) { |
232 | 0 | static const size_t MAX_RETRIES = 4; |
233 | 0 | static const std::chrono::milliseconds RETRY_DELAY{1000}; |
234 | 0 | static const std::chrono::seconds TIMEOUT{5}; |
235 | |
|
236 | 0 | CURL* const curl = curl_easy_init(); |
237 | 0 | if (!curl) { |
238 | 0 | return absl::nullopt; |
239 | 0 | }; |
240 | |
|
241 | 0 | const auto host = message.headers().getHostValue(); |
242 | 0 | const auto path = message.headers().getPathValue(); |
243 | 0 | const auto method = message.headers().getMethodValue(); |
244 | |
|
245 | 0 | const std::string url = fmt::format("http://{}{}", host, path); |
246 | 0 | curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); |
247 | 0 | curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT.count()); |
248 | 0 | curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L); |
249 | 0 | curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); |
250 | |
|
251 | 0 | std::string buffer; |
252 | 0 | curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer); |
253 | 0 | curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlCallback); |
254 | |
|
255 | 0 | struct curl_slist* headers = nullptr; |
256 | 0 | message.headers().iterate([&headers](const Http::HeaderEntry& entry) -> Http::HeaderMap::Iterate { |
257 | | // Skip pseudo-headers |
258 | 0 | if (!entry.key().getStringView().empty() && entry.key().getStringView()[0] == ':') { |
259 | 0 | return Http::HeaderMap::Iterate::Continue; |
260 | 0 | } |
261 | 0 | const std::string header = |
262 | 0 | fmt::format("{}: {}", entry.key().getStringView(), entry.value().getStringView()); |
263 | 0 | headers = curl_slist_append(headers, header.c_str()); |
264 | 0 | return Http::HeaderMap::Iterate::Continue; |
265 | 0 | }); |
266 | | |
267 | | // This function only support doing PUT(UPLOAD) other than GET(_default_) operation. |
268 | 0 | if (Http::Headers::get().MethodValues.Put == method) { |
269 | | // https://curl.se/libcurl/c/CURLOPT_PUT.html is deprecated |
270 | | // so using https://curl.se/libcurl/c/CURLOPT_UPLOAD.html. |
271 | 0 | curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); |
272 | | // To call PUT on HTTP 1.0 we must specify a value for the upload size |
273 | | // since some old EC2's metadata service will be serving on HTTP 1.0. |
274 | | // https://curl.se/libcurl/c/CURLOPT_INFILESIZE.html |
275 | 0 | curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0); |
276 | | // Disabling `Expect: 100-continue` header to get a response |
277 | | // in the first attempt as the put size is zero. |
278 | | // https://everything.curl.dev/http/post/expect100 |
279 | 0 | headers = curl_slist_append(headers, "Expect:"); |
280 | 0 | } |
281 | |
|
282 | 0 | if (headers != nullptr) { |
283 | 0 | curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); |
284 | 0 | } |
285 | |
|
286 | 0 | for (size_t retry = 0; retry < MAX_RETRIES; retry++) { |
287 | 0 | const CURLcode res = curl_easy_perform(curl); |
288 | 0 | if (res == CURLE_OK) { |
289 | 0 | break; |
290 | 0 | } |
291 | 0 | ENVOY_LOG_MISC(debug, "Could not fetch AWS metadata: {}", curl_easy_strerror(res)); |
292 | 0 | buffer.clear(); |
293 | 0 | std::this_thread::sleep_for(RETRY_DELAY); |
294 | 0 | } |
295 | |
|
296 | 0 | curl_easy_cleanup(curl); |
297 | 0 | curl_slist_free_all(headers); |
298 | |
|
299 | 0 | return buffer.empty() ? absl::nullopt : absl::optional<std::string>(buffer); |
300 | 0 | } |
301 | | |
302 | | bool Utility::addInternalClusterStatic( |
303 | | Upstream::ClusterManager& cm, absl::string_view cluster_name, |
304 | 21 | const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, absl::string_view uri) { |
305 | | // Check if local cluster exists with that name. |
306 | 21 | if (cm.getThreadLocalCluster(cluster_name) == nullptr) { |
307 | | // Make sure we run this on main thread. |
308 | 21 | TRY_ASSERT_MAIN_THREAD { |
309 | 21 | envoy::config::cluster::v3::Cluster cluster; |
310 | 21 | absl::string_view host_port; |
311 | 21 | absl::string_view path; |
312 | 21 | Http::Utility::extractHostPathFromUri(uri, host_port, path); |
313 | 21 | const auto host_attributes = Http::Utility::parseAuthority(host_port); |
314 | 21 | const auto host = host_attributes.host_; |
315 | 21 | const auto port = host_attributes.port_ ? host_attributes.port_.value() : 80; |
316 | | |
317 | 21 | cluster.set_name(cluster_name); |
318 | 21 | cluster.set_type(cluster_type); |
319 | 21 | cluster.mutable_connect_timeout()->set_seconds(5); |
320 | 21 | cluster.mutable_load_assignment()->set_cluster_name(cluster_name); |
321 | 21 | auto* endpoint = cluster.mutable_load_assignment() |
322 | 21 | ->add_endpoints() |
323 | 21 | ->add_lb_endpoints() |
324 | 21 | ->mutable_endpoint(); |
325 | 21 | auto* addr = endpoint->mutable_address(); |
326 | 21 | addr->mutable_socket_address()->set_address(host); |
327 | 21 | addr->mutable_socket_address()->set_port_value(port); |
328 | 21 | cluster.set_lb_policy(envoy::config::cluster::v3::Cluster::ROUND_ROBIN); |
329 | 21 | envoy::extensions::upstreams::http::v3::HttpProtocolOptions protocol_options; |
330 | 21 | auto* http_protocol_options = |
331 | 21 | protocol_options.mutable_explicit_http_config()->mutable_http_protocol_options(); |
332 | 21 | http_protocol_options->set_accept_http_10(true); |
333 | 21 | (*cluster.mutable_typed_extension_protocol_options()) |
334 | 21 | ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"] |
335 | 21 | .PackFrom(protocol_options); |
336 | | |
337 | | // TODO(suniltheta): use random number generator here for cluster version. |
338 | 21 | cm.addOrUpdateCluster(cluster, "12345"); |
339 | 21 | ENVOY_LOG_MISC(info, |
340 | 21 | "Added a {} internal cluster [name: {}, address:{}:{}] to fetch aws " |
341 | 21 | "credentials", |
342 | 21 | cluster_type, cluster_name, host, port); |
343 | 21 | } |
344 | 21 | END_TRY |
345 | 21 | CATCH(const EnvoyException& e, { |
346 | 21 | ENVOY_LOG_MISC(error, "Failed to add internal cluster {}: {}", cluster_name, e.what()); |
347 | 21 | return false; |
348 | 21 | }); |
349 | 21 | } |
350 | 21 | return true; |
351 | 21 | } |
352 | | |
353 | | } // namespace Aws |
354 | | } // namespace Common |
355 | | } // namespace Extensions |
356 | | } // namespace Envoy |