Coverage Report

Created: 2023-11-12 09:30

/proc/self/cwd/source/extensions/common/aws/utility.cc
Line
Count
Source (jump to first uncovered line)
1
#include "source/extensions/common/aws/utility.h"
2
3
#include "envoy/upstream/cluster_manager.h"
4
5
#include "source/common/common/empty_string.h"
6
#include "source/common/common/fmt.h"
7
#include "source/common/common/utility.h"
8
#include "source/common/protobuf/message_validator_impl.h"
9
#include "source/common/protobuf/utility.h"
10
11
#include "absl/strings/match.h"
12
#include "absl/strings/str_join.h"
13
#include "absl/strings/str_split.h"
14
#include "curl/curl.h"
15
#include "fmt/printf.h"
16
17
namespace Envoy {
18
namespace Extensions {
19
namespace Common {
20
namespace Aws {
21
22
constexpr absl::string_view PATH_SPLITTER = "/";
23
constexpr absl::string_view QUERY_PARAM_SEPERATOR = "=";
24
constexpr absl::string_view QUERY_SEPERATOR = "&";
25
constexpr absl::string_view QUERY_SPLITTER = "?";
26
constexpr absl::string_view RESERVED_CHARS = "-._~";
27
constexpr absl::string_view S3_SERVICE_NAME = "s3";
28
constexpr absl::string_view URI_ENCODE = "%{:02X}";
29
constexpr absl::string_view URI_DOUBLE_ENCODE = "%25{:02X}";
30
31
std::map<std::string, std::string>
32
Utility::canonicalizeHeaders(const Http::RequestHeaderMap& headers,
33
0
                             const std::vector<Matchers::StringMatcherPtr>& excluded_headers) {
34
0
  std::map<std::string, std::string> out;
35
0
  headers.iterate(
36
0
      [&out, &excluded_headers](const Http::HeaderEntry& entry) -> Http::HeaderMap::Iterate {
37
        // Skip empty headers
38
0
        if (entry.key().empty() || entry.value().empty()) {
39
0
          return Http::HeaderMap::Iterate::Continue;
40
0
        }
41
        // Pseudo-headers should not be canonicalized
42
0
        if (!entry.key().getStringView().empty() && entry.key().getStringView()[0] == ':') {
43
0
          return Http::HeaderMap::Iterate::Continue;
44
0
        }
45
0
        const auto key = entry.key().getStringView();
46
0
        if (std::any_of(excluded_headers.begin(), excluded_headers.end(),
47
0
                        [&key](const Matchers::StringMatcherPtr& matcher) {
48
0
                          return matcher->match(key);
49
0
                        })) {
50
0
          return Http::HeaderMap::Iterate::Continue;
51
0
        }
52
53
0
        std::string value(entry.value().getStringView());
54
        // Remove leading, trailing, and deduplicate repeated ascii spaces
55
0
        absl::RemoveExtraAsciiWhitespace(&value);
56
0
        const auto iter = out.find(std::string(entry.key().getStringView()));
57
        // If the entry already exists, append the new value to the end
58
0
        if (iter != out.end()) {
59
0
          iter->second += fmt::format(",{}", value);
60
0
        } else {
61
0
          out.emplace(std::string(entry.key().getStringView()), value);
62
0
        }
63
0
        return Http::HeaderMap::Iterate::Continue;
64
0
      });
65
  // The AWS SDK has a quirk where it removes "default ports" (80, 443) from the host headers
66
  // Additionally, we canonicalize the :authority header as "host"
67
  // TODO(suniltheta): This may need to be tweaked to canonicalize :authority for HTTP/2 requests
68
0
  const absl::string_view authority_header = headers.getHostValue();
69
0
  if (!authority_header.empty()) {
70
0
    const auto parts = StringUtil::splitToken(authority_header, ":");
71
0
    if (parts.size() > 1 && (parts[1] == "80" || parts[1] == "443")) {
72
      // Has default port, so use only the host part
73
0
      out.emplace(Http::Headers::get().HostLegacy.get(), std::string(parts[0]));
74
0
    } else {
75
0
      out.emplace(Http::Headers::get().HostLegacy.get(), std::string(authority_header));
76
0
    }
77
0
  }
78
0
  return out;
79
0
}
80
81
std::string Utility::createCanonicalRequest(
82
    absl::string_view service_name, absl::string_view method, absl::string_view path,
83
0
    const std::map<std::string, std::string>& canonical_headers, absl::string_view content_hash) {
84
0
  std::vector<absl::string_view> parts;
85
0
  parts.emplace_back(method);
86
  // don't include the query part of the path
87
0
  const auto path_part = StringUtil::cropRight(path, QUERY_SPLITTER);
88
0
  const auto canonicalized_path = path_part.empty()
89
0
                                      ? std::string{PATH_SPLITTER}
90
0
                                      : canonicalizePathString(path_part, service_name);
91
0
  parts.emplace_back(canonicalized_path);
92
0
  const auto query_part = StringUtil::cropLeft(path, QUERY_SPLITTER);
93
  // if query_part == path_part, then there is no query
94
0
  const auto canonicalized_query =
95
0
      query_part == path_part ? EMPTY_STRING : Utility::canonicalizeQueryString(query_part);
96
0
  parts.emplace_back(absl::string_view(canonicalized_query));
97
0
  std::vector<std::string> formatted_headers;
98
0
  formatted_headers.reserve(canonical_headers.size());
99
0
  for (const auto& header : canonical_headers) {
100
0
    formatted_headers.emplace_back(fmt::format("{}:{}", header.first, header.second));
101
0
    parts.emplace_back(formatted_headers.back());
102
0
  }
103
  // need an extra blank space after the canonical headers
104
0
  parts.emplace_back(EMPTY_STRING);
105
0
  const auto signed_headers = Utility::joinCanonicalHeaderNames(canonical_headers);
106
0
  parts.emplace_back(signed_headers);
107
0
  parts.emplace_back(content_hash);
108
0
  return absl::StrJoin(parts, "\n");
109
0
}
110
111
/**
112
 * Normalizes the path string based on AWS requirements.
113
 * See step 2 in https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
114
 */
115
std::string Utility::canonicalizePathString(absl::string_view path_string,
116
0
                                            absl::string_view service_name) {
117
  // If service is S3, do not normalize but only encode the path
118
0
  if (absl::EqualsIgnoreCase(service_name, S3_SERVICE_NAME)) {
119
0
    return encodePathSegment(path_string, service_name);
120
0
  }
121
  // If service is not S3, normalize and encode the path
122
0
  const auto path_segments = StringUtil::splitToken(path_string, std::string{PATH_SPLITTER});
123
0
  std::vector<std::string> path_list;
124
0
  path_list.reserve(path_segments.size());
125
0
  for (const auto& path_segment : path_segments) {
126
0
    if (path_segment.empty()) {
127
0
      continue;
128
0
    }
129
0
    path_list.emplace_back(encodePathSegment(path_segment, service_name));
130
0
  }
131
0
  auto canonical_path_string =
132
0
      fmt::format("{}{}", PATH_SPLITTER, absl::StrJoin(path_list, PATH_SPLITTER));
133
  // Handle corner case when path ends with '/'
134
0
  if (absl::EndsWith(path_string, PATH_SPLITTER) && canonical_path_string.size() > 1) {
135
0
    canonical_path_string.push_back(PATH_SPLITTER[0]);
136
0
  }
137
0
  return canonical_path_string;
138
0
}
139
140
0
bool isReservedChar(const char c) {
141
0
  return std::isalnum(c) || RESERVED_CHARS.find(c) != std::string::npos;
142
0
}
143
144
0
void encodeS3Path(std::string& encoded, const char& c) {
145
  // Do not encode '/' for S3
146
0
  if (c == PATH_SPLITTER[0]) {
147
0
    encoded.push_back(c);
148
0
  } else {
149
0
    absl::StrAppend(&encoded, fmt::format(URI_ENCODE, c));
150
0
  }
151
0
}
152
153
0
std::string Utility::encodePathSegment(absl::string_view decoded, absl::string_view service_name) {
154
0
  std::string encoded;
155
0
  for (char c : decoded) {
156
0
    if (isReservedChar(c)) {
157
      // Escape unreserved chars from RFC 3986
158
0
      encoded.push_back(c);
159
0
    } else if (absl::EqualsIgnoreCase(service_name, S3_SERVICE_NAME)) {
160
0
      encodeS3Path(encoded, c);
161
0
    } else {
162
      // TODO: @aws, There is some inconsistency between AWS services if this should be double
163
      // encoded or not. We need to parameterize this and expose this in the config. Ref:
164
      // https://github.com/aws/aws-sdk-cpp/blob/main/aws-cpp-sdk-core/source/auth/AWSAuthSigner.cpp#L79-L93
165
0
      absl::StrAppend(&encoded, fmt::format(URI_ENCODE, c));
166
0
    }
167
0
  }
168
0
  return encoded;
169
0
}
170
171
/**
172
 * Normalizes the query string based on AWS requirements.
173
 * See step 3 in https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
174
 */
175
0
std::string Utility::canonicalizeQueryString(absl::string_view query_string) {
176
  // Sort query string based on param name and append "=" if value is missing
177
0
  const auto query_fragments = StringUtil::splitToken(query_string, QUERY_SEPERATOR);
178
0
  std::vector<std::pair<std::string, std::string>> query_list;
179
0
  for (const auto& query_fragment : query_fragments) {
180
    // Only split at the first "=" and encode the rest
181
0
    const std::vector<std::string> query =
182
0
        absl::StrSplit(query_fragment, absl::MaxSplits(QUERY_PARAM_SEPERATOR, 1));
183
0
    if (!query.empty()) {
184
0
      const absl::string_view param = query[0];
185
0
      const absl::string_view value = query.size() > 1 ? query[1] : EMPTY_STRING;
186
0
      query_list.emplace_back(std::make_pair(param, value));
187
0
    }
188
0
  }
189
  // Sort query params by name and value
190
0
  std::sort(query_list.begin(), query_list.end());
191
  // Encode query params name and value separately
192
0
  for (auto& query : query_list) {
193
0
    query = std::make_pair(Utility::encodeQueryParam(query.first),
194
0
                           Utility::encodeQueryParam(query.second));
195
0
  }
196
0
  return absl::StrJoin(query_list, QUERY_SEPERATOR, absl::PairFormatter(QUERY_PARAM_SEPERATOR));
197
0
}
198
199
0
std::string Utility::encodeQueryParam(absl::string_view decoded) {
200
0
  std::string encoded;
201
0
  for (char c : decoded) {
202
0
    if (isReservedChar(c) || c == '%') {
203
      // Escape unreserved chars from RFC 3986
204
0
      encoded.push_back(c);
205
0
    } else if (c == '+') {
206
      // Encode '+' as space
207
0
      absl::StrAppend(&encoded, "%20");
208
0
    } else if (c == QUERY_PARAM_SEPERATOR[0]) {
209
      // Double encode '='
210
0
      absl::StrAppend(&encoded, fmt::format(URI_DOUBLE_ENCODE, c));
211
0
    } else {
212
0
      absl::StrAppend(&encoded, fmt::format(URI_ENCODE, c));
213
0
    }
214
0
  }
215
0
  return encoded;
216
0
}
217
218
std::string
219
0
Utility::joinCanonicalHeaderNames(const std::map<std::string, std::string>& canonical_headers) {
220
0
  return absl::StrJoin(canonical_headers, ";", [](auto* out, const auto& pair) {
221
0
    return absl::StrAppend(out, pair.first);
222
0
  });
223
0
}
224
225
0
static size_t curlCallback(char* ptr, size_t, size_t nmemb, void* data) {
226
0
  auto buf = static_cast<std::string*>(data);
227
0
  buf->append(ptr, nmemb);
228
0
  return nmemb;
229
0
}
230
231
0
absl::optional<std::string> Utility::fetchMetadata(Http::RequestMessage& message) {
232
0
  static const size_t MAX_RETRIES = 4;
233
0
  static const std::chrono::milliseconds RETRY_DELAY{1000};
234
0
  static const std::chrono::seconds TIMEOUT{5};
235
236
0
  CURL* const curl = curl_easy_init();
237
0
  if (!curl) {
238
0
    return absl::nullopt;
239
0
  };
240
241
0
  const auto host = message.headers().getHostValue();
242
0
  const auto path = message.headers().getPathValue();
243
0
  const auto method = message.headers().getMethodValue();
244
245
0
  const std::string url = fmt::format("http://{}{}", host, path);
246
0
  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
247
0
  curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT.count());
248
0
  curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
249
0
  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
250
251
0
  std::string buffer;
252
0
  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &buffer);
253
0
  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlCallback);
254
255
0
  struct curl_slist* headers = nullptr;
256
0
  message.headers().iterate([&headers](const Http::HeaderEntry& entry) -> Http::HeaderMap::Iterate {
257
    // Skip pseudo-headers
258
0
    if (!entry.key().getStringView().empty() && entry.key().getStringView()[0] == ':') {
259
0
      return Http::HeaderMap::Iterate::Continue;
260
0
    }
261
0
    const std::string header =
262
0
        fmt::format("{}: {}", entry.key().getStringView(), entry.value().getStringView());
263
0
    headers = curl_slist_append(headers, header.c_str());
264
0
    return Http::HeaderMap::Iterate::Continue;
265
0
  });
266
267
  // This function only support doing PUT(UPLOAD) other than GET(_default_) operation.
268
0
  if (Http::Headers::get().MethodValues.Put == method) {
269
    // https://curl.se/libcurl/c/CURLOPT_PUT.html is deprecated
270
    // so using https://curl.se/libcurl/c/CURLOPT_UPLOAD.html.
271
0
    curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);
272
    // To call PUT on HTTP 1.0 we must specify a value for the upload size
273
    // since some old EC2's metadata service will be serving on HTTP 1.0.
274
    // https://curl.se/libcurl/c/CURLOPT_INFILESIZE.html
275
0
    curl_easy_setopt(curl, CURLOPT_INFILESIZE, 0);
276
    // Disabling `Expect: 100-continue` header to get a response
277
    // in the first attempt as the put size is zero.
278
    // https://everything.curl.dev/http/post/expect100
279
0
    headers = curl_slist_append(headers, "Expect:");
280
0
  }
281
282
0
  if (headers != nullptr) {
283
0
    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
284
0
  }
285
286
0
  for (size_t retry = 0; retry < MAX_RETRIES; retry++) {
287
0
    const CURLcode res = curl_easy_perform(curl);
288
0
    if (res == CURLE_OK) {
289
0
      break;
290
0
    }
291
0
    ENVOY_LOG_MISC(debug, "Could not fetch AWS metadata: {}", curl_easy_strerror(res));
292
0
    buffer.clear();
293
0
    std::this_thread::sleep_for(RETRY_DELAY);
294
0
  }
295
296
0
  curl_easy_cleanup(curl);
297
0
  curl_slist_free_all(headers);
298
299
0
  return buffer.empty() ? absl::nullopt : absl::optional<std::string>(buffer);
300
0
}
301
302
bool Utility::addInternalClusterStatic(
303
    Upstream::ClusterManager& cm, absl::string_view cluster_name,
304
21
    const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, absl::string_view uri) {
305
  // Check if local cluster exists with that name.
306
21
  if (cm.getThreadLocalCluster(cluster_name) == nullptr) {
307
    // Make sure we run this on main thread.
308
21
    TRY_ASSERT_MAIN_THREAD {
309
21
      envoy::config::cluster::v3::Cluster cluster;
310
21
      absl::string_view host_port;
311
21
      absl::string_view path;
312
21
      Http::Utility::extractHostPathFromUri(uri, host_port, path);
313
21
      const auto host_attributes = Http::Utility::parseAuthority(host_port);
314
21
      const auto host = host_attributes.host_;
315
21
      const auto port = host_attributes.port_ ? host_attributes.port_.value() : 80;
316
317
21
      cluster.set_name(cluster_name);
318
21
      cluster.set_type(cluster_type);
319
21
      cluster.mutable_connect_timeout()->set_seconds(5);
320
21
      cluster.mutable_load_assignment()->set_cluster_name(cluster_name);
321
21
      auto* endpoint = cluster.mutable_load_assignment()
322
21
                           ->add_endpoints()
323
21
                           ->add_lb_endpoints()
324
21
                           ->mutable_endpoint();
325
21
      auto* addr = endpoint->mutable_address();
326
21
      addr->mutable_socket_address()->set_address(host);
327
21
      addr->mutable_socket_address()->set_port_value(port);
328
21
      cluster.set_lb_policy(envoy::config::cluster::v3::Cluster::ROUND_ROBIN);
329
21
      envoy::extensions::upstreams::http::v3::HttpProtocolOptions protocol_options;
330
21
      auto* http_protocol_options =
331
21
          protocol_options.mutable_explicit_http_config()->mutable_http_protocol_options();
332
21
      http_protocol_options->set_accept_http_10(true);
333
21
      (*cluster.mutable_typed_extension_protocol_options())
334
21
          ["envoy.extensions.upstreams.http.v3.HttpProtocolOptions"]
335
21
              .PackFrom(protocol_options);
336
337
      // TODO(suniltheta): use random number generator here for cluster version.
338
21
      cm.addOrUpdateCluster(cluster, "12345");
339
21
      ENVOY_LOG_MISC(info,
340
21
                     "Added a {} internal cluster [name: {}, address:{}:{}] to fetch aws "
341
21
                     "credentials",
342
21
                     cluster_type, cluster_name, host, port);
343
21
    }
344
21
    END_TRY
345
21
    CATCH(const EnvoyException& e, {
346
21
      ENVOY_LOG_MISC(error, "Failed to add internal cluster {}: {}", cluster_name, e.what());
347
21
      return false;
348
21
    });
349
21
  }
350
21
  return true;
351
21
}
352
353
} // namespace Aws
354
} // namespace Common
355
} // namespace Extensions
356
} // namespace Envoy