Line data Source code
1 : #include "source/extensions/transport_sockets/tls/ssl_socket.h"
2 :
3 : #include "envoy/stats/scope.h"
4 :
5 : #include "source/common/common/assert.h"
6 : #include "source/common/common/empty_string.h"
7 : #include "source/common/common/hex.h"
8 : #include "source/common/http/headers.h"
9 : #include "source/common/runtime/runtime_features.h"
10 : #include "source/extensions/transport_sockets/tls/io_handle_bio.h"
11 : #include "source/extensions/transport_sockets/tls/ssl_handshaker.h"
12 : #include "source/extensions/transport_sockets/tls/utility.h"
13 :
14 : #include "absl/strings/str_replace.h"
15 : #include "openssl/err.h"
16 : #include "openssl/x509v3.h"
17 :
18 : using Envoy::Network::PostIoAction;
19 :
20 : namespace Envoy {
21 : namespace Extensions {
22 : namespace TransportSockets {
23 : namespace Tls {
24 :
25 : namespace {
26 :
27 : constexpr absl::string_view NotReadyReason{"TLS error: Secret is not supplied by SDS"};
28 :
29 : // This SslSocket will be used when SSL secret is not fetched from SDS server.
30 : class NotReadySslSocket : public Network::TransportSocket {
31 : public:
32 : // Network::TransportSocket
33 0 : void setTransportSocketCallbacks(Network::TransportSocketCallbacks&) override {}
34 0 : std::string protocol() const override { return EMPTY_STRING; }
35 0 : absl::string_view failureReason() const override { return NotReadyReason; }
36 0 : bool canFlushClose() override { return true; }
37 0 : void closeSocket(Network::ConnectionEvent) override {}
38 0 : Network::IoResult doRead(Buffer::Instance&) override { return {PostIoAction::Close, 0, false}; }
39 0 : Network::IoResult doWrite(Buffer::Instance&, bool) override {
40 0 : return {PostIoAction::Close, 0, false};
41 0 : }
42 0 : void onConnected() override {}
43 0 : Ssl::ConnectionInfoConstSharedPtr ssl() const override { return nullptr; }
44 0 : bool startSecureTransport() override { return false; }
45 0 : void configureInitialCongestionWindow(uint64_t, std::chrono::microseconds) override {}
46 : };
47 :
48 : } // namespace
49 :
50 : SslSocket::SslSocket(Envoy::Ssl::ContextSharedPtr ctx, InitialState state,
51 : const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
52 : Ssl::HandshakerFactoryCb handshaker_factory_cb)
53 : : transport_socket_options_(transport_socket_options),
54 : ctx_(std::dynamic_pointer_cast<ContextImpl>(ctx)),
55 : info_(std::dynamic_pointer_cast<SslHandshakerImpl>(handshaker_factory_cb(
56 0 : ctx_->newSsl(transport_socket_options_), ctx_->sslExtendedSocketInfoIndex(), this))) {
57 0 : if (state == InitialState::Client) {
58 0 : SSL_set_connect_state(rawSsl());
59 0 : } else {
60 0 : ASSERT(state == InitialState::Server);
61 0 : SSL_set_accept_state(rawSsl());
62 0 : }
63 0 : }
64 :
65 0 : void SslSocket::setTransportSocketCallbacks(Network::TransportSocketCallbacks& callbacks) {
66 0 : ASSERT(!callbacks_);
67 0 : callbacks_ = &callbacks;
68 :
69 : // Associate this SSL connection with all the certificates (with their potentially different
70 : // private key methods).
71 0 : for (auto const& provider : ctx_->getPrivateKeyMethodProviders()) {
72 0 : provider->registerPrivateKeyMethod(rawSsl(), *this, callbacks_->connection().dispatcher());
73 0 : }
74 :
75 : // Use custom BIO that reads from/writes to IoHandle
76 0 : BIO* bio = BIO_new_io_handle(&callbacks_->ioHandle());
77 0 : SSL_set_bio(rawSsl(), bio, bio);
78 0 : SSL_set_ex_data(rawSsl(), ContextImpl::sslSocketIndex(), static_cast<void*>(callbacks_));
79 0 : }
80 :
81 0 : SslSocket::ReadResult SslSocket::sslReadIntoSlice(Buffer::RawSlice& slice) {
82 0 : ReadResult result;
83 0 : uint8_t* mem = static_cast<uint8_t*>(slice.mem_);
84 0 : size_t remaining = slice.len_;
85 0 : while (remaining > 0) {
86 0 : int rc = SSL_read(rawSsl(), mem, remaining);
87 0 : ENVOY_CONN_LOG(trace, "ssl read returns: {}", callbacks_->connection(), rc);
88 0 : if (rc > 0) {
89 0 : ASSERT(static_cast<size_t>(rc) <= remaining);
90 0 : mem += rc;
91 0 : remaining -= rc;
92 0 : result.bytes_read_ += rc;
93 0 : } else {
94 0 : result.error_ = absl::make_optional<int>(rc);
95 0 : break;
96 0 : }
97 0 : }
98 :
99 0 : return result;
100 0 : }
101 :
102 0 : Network::IoResult SslSocket::doRead(Buffer::Instance& read_buffer) {
103 0 : if (info_->state() != Ssl::SocketState::HandshakeComplete &&
104 0 : info_->state() != Ssl::SocketState::ShutdownSent) {
105 0 : PostIoAction action = doHandshake();
106 0 : if (action == PostIoAction::Close || info_->state() != Ssl::SocketState::HandshakeComplete) {
107 : // end_stream is false because either a hard error occurred (action == Close) or
108 : // the handshake isn't complete, so a half-close cannot occur yet.
109 0 : return {action, 0, false};
110 0 : }
111 0 : }
112 :
113 0 : bool keep_reading = true;
114 0 : bool end_stream = false;
115 0 : PostIoAction action = PostIoAction::KeepOpen;
116 0 : uint64_t bytes_read = 0;
117 0 : while (keep_reading) {
118 0 : uint64_t bytes_read_this_iteration = 0;
119 0 : Buffer::Reservation reservation = read_buffer.reserveForRead();
120 0 : for (uint64_t i = 0; i < reservation.numSlices(); i++) {
121 0 : auto result = sslReadIntoSlice(reservation.slices()[i]);
122 0 : bytes_read_this_iteration += result.bytes_read_;
123 0 : if (result.error_.has_value()) {
124 0 : keep_reading = false;
125 0 : int err = SSL_get_error(rawSsl(), result.error_.value());
126 0 : ENVOY_CONN_LOG(trace, "ssl error occurred while read: {}", callbacks_->connection(),
127 0 : Utility::getErrorDescription(err));
128 0 : switch (err) {
129 0 : case SSL_ERROR_WANT_READ:
130 0 : break;
131 0 : case SSL_ERROR_ZERO_RETURN:
132 : // Graceful shutdown using close_notify TLS alert.
133 0 : end_stream = true;
134 0 : break;
135 0 : case SSL_ERROR_SYSCALL:
136 0 : if (result.error_.value() == 0) {
137 : // Non-graceful shutdown by closing the underlying socket.
138 0 : end_stream = true;
139 0 : break;
140 0 : }
141 0 : FALLTHRU;
142 0 : case SSL_ERROR_WANT_WRITE:
143 : // Renegotiation has started. We don't handle renegotiation so just fall through.
144 0 : default:
145 0 : drainErrorQueue();
146 0 : action = PostIoAction::Close;
147 0 : break;
148 0 : }
149 :
150 0 : break;
151 0 : }
152 0 : }
153 :
154 0 : reservation.commit(bytes_read_this_iteration);
155 0 : if (bytes_read_this_iteration > 0 && callbacks_->shouldDrainReadBuffer()) {
156 0 : callbacks_->setTransportSocketIsReadable();
157 0 : keep_reading = false;
158 0 : }
159 :
160 0 : bytes_read += bytes_read_this_iteration;
161 0 : }
162 :
163 0 : ENVOY_CONN_LOG(trace, "ssl read {} bytes", callbacks_->connection(), bytes_read);
164 :
165 0 : return {action, bytes_read, end_stream};
166 0 : }
167 :
168 0 : void SslSocket::onPrivateKeyMethodComplete() { resumeHandshake(); }
169 :
170 0 : void SslSocket::resumeHandshake() {
171 0 : ASSERT(callbacks_ != nullptr && callbacks_->connection().dispatcher().isThreadSafe());
172 0 : ASSERT(info_->state() == Ssl::SocketState::HandshakeInProgress);
173 :
174 : // Resume handshake.
175 0 : PostIoAction action = doHandshake();
176 0 : if (action == PostIoAction::Close) {
177 0 : ENVOY_CONN_LOG(debug, "async handshake completion error", callbacks_->connection());
178 0 : callbacks_->connection().close(Network::ConnectionCloseType::FlushWrite,
179 0 : "failed_resuming_async_handshake");
180 0 : }
181 0 : }
182 :
183 0 : Network::Connection& SslSocket::connection() const { return callbacks_->connection(); }
184 :
185 0 : void SslSocket::onSuccess(SSL* ssl) {
186 0 : ctx_->logHandshake(ssl);
187 0 : if (callbacks_->connection().streamInfo().upstreamInfo()) {
188 0 : callbacks_->connection()
189 0 : .streamInfo()
190 0 : .upstreamInfo()
191 0 : ->upstreamTiming()
192 0 : .onUpstreamHandshakeComplete(callbacks_->connection().dispatcher().timeSource());
193 0 : } else {
194 0 : callbacks_->connection().streamInfo().downstreamTiming().onDownstreamHandshakeComplete(
195 0 : callbacks_->connection().dispatcher().timeSource());
196 0 : }
197 0 : callbacks_->raiseEvent(Network::ConnectionEvent::Connected);
198 0 : }
199 :
200 0 : void SslSocket::onFailure() { drainErrorQueue(); }
201 :
202 0 : PostIoAction SslSocket::doHandshake() { return info_->doHandshake(); }
203 :
204 0 : void SslSocket::drainErrorQueue() {
205 0 : bool saw_error = false;
206 0 : bool saw_counted_error = false;
207 0 : bool new_ssl_failure_format = Runtime::runtimeFeatureEnabled(
208 0 : "envoy.reloadable_features.ssl_transport_failure_reason_format");
209 0 : while (uint64_t err = ERR_get_error()) {
210 0 : if (ERR_GET_LIB(err) == ERR_LIB_SSL) {
211 0 : if (ERR_GET_REASON(err) == SSL_R_PEER_DID_NOT_RETURN_A_CERTIFICATE) {
212 0 : ctx_->stats().fail_verify_no_cert_.inc();
213 0 : saw_counted_error = true;
214 0 : } else if (ERR_GET_REASON(err) == SSL_R_CERTIFICATE_VERIFY_FAILED) {
215 0 : saw_counted_error = true;
216 0 : }
217 0 : } else if (ERR_GET_LIB(err) == ERR_LIB_SYS) {
218 : // Any syscall errors that result in connection closure are already tracked in other
219 : // connection related stats. We will still retain the specific syscall failure for
220 : // transport failure reasons.
221 0 : saw_counted_error = true;
222 0 : }
223 0 : saw_error = true;
224 :
225 0 : if (failure_reason_.empty()) {
226 0 : failure_reason_ = new_ssl_failure_format ? "TLS_error:" : "TLS error:";
227 0 : }
228 :
229 0 : absl::StrAppend(&failure_reason_, new_ssl_failure_format ? "|" : " ", err, ":",
230 0 : absl::NullSafeStringView(ERR_lib_error_string(err)), ":",
231 0 : absl::NullSafeStringView(ERR_func_error_string(err)), ":",
232 0 : absl::NullSafeStringView(ERR_reason_error_string(err)));
233 0 : }
234 :
235 0 : if (!failure_reason_.empty()) {
236 0 : if (new_ssl_failure_format) {
237 0 : absl::StrAppend(&failure_reason_, ":TLS_error_end");
238 0 : }
239 0 : ENVOY_CONN_LOG(debug, "remote address:{},{}", callbacks_->connection(),
240 0 : callbacks_->connection().connectionInfoProvider().remoteAddress()->asString(),
241 0 : failure_reason_);
242 0 : }
243 :
244 0 : if (saw_error && !saw_counted_error) {
245 0 : ctx_->stats().connection_error_.inc();
246 0 : }
247 0 : }
248 :
249 0 : Network::IoResult SslSocket::doWrite(Buffer::Instance& write_buffer, bool end_stream) {
250 0 : ASSERT(info_->state() != Ssl::SocketState::ShutdownSent || write_buffer.length() == 0);
251 0 : if (info_->state() != Ssl::SocketState::HandshakeComplete &&
252 0 : info_->state() != Ssl::SocketState::ShutdownSent) {
253 0 : PostIoAction action = doHandshake();
254 0 : if (action == PostIoAction::Close || info_->state() != Ssl::SocketState::HandshakeComplete) {
255 0 : return {action, 0, false};
256 0 : }
257 0 : }
258 :
259 0 : uint64_t bytes_to_write;
260 0 : if (bytes_to_retry_) {
261 0 : bytes_to_write = bytes_to_retry_;
262 0 : bytes_to_retry_ = 0;
263 0 : } else {
264 0 : bytes_to_write = std::min(write_buffer.length(), static_cast<uint64_t>(16384));
265 0 : }
266 :
267 0 : uint64_t total_bytes_written = 0;
268 0 : while (bytes_to_write > 0) {
269 : // TODO(mattklein123): As it relates to our fairness efforts, we might want to limit the number
270 : // of iterations of this loop, either by pure iterations, bytes written, etc.
271 :
272 : // SSL_write() requires that if a previous call returns SSL_ERROR_WANT_WRITE, we need to call
273 : // it again with the same parameters. This is done by tracking last write size, but not write
274 : // data, since linearize() will return the same undrained data anyway.
275 0 : ASSERT(bytes_to_write <= write_buffer.length());
276 0 : int rc = SSL_write(rawSsl(), write_buffer.linearize(bytes_to_write), bytes_to_write);
277 0 : ENVOY_CONN_LOG(trace, "ssl write returns: {}", callbacks_->connection(), rc);
278 0 : if (rc > 0) {
279 0 : ASSERT(rc == static_cast<int>(bytes_to_write));
280 0 : total_bytes_written += rc;
281 0 : write_buffer.drain(rc);
282 0 : bytes_to_write = std::min(write_buffer.length(), static_cast<uint64_t>(16384));
283 0 : } else {
284 0 : int err = SSL_get_error(rawSsl(), rc);
285 0 : ENVOY_CONN_LOG(trace, "ssl error occurred while write: {}", callbacks_->connection(),
286 0 : Utility::getErrorDescription(err));
287 0 : switch (err) {
288 0 : case SSL_ERROR_WANT_WRITE:
289 0 : bytes_to_retry_ = bytes_to_write;
290 0 : break;
291 0 : case SSL_ERROR_WANT_READ:
292 : // Renegotiation has started. We don't handle renegotiation so just fall through.
293 0 : default:
294 0 : drainErrorQueue();
295 0 : return {PostIoAction::Close, total_bytes_written, false};
296 0 : }
297 :
298 0 : break;
299 0 : }
300 0 : }
301 :
302 0 : if (write_buffer.length() == 0 && end_stream) {
303 0 : shutdownSsl();
304 0 : }
305 :
306 0 : return {PostIoAction::KeepOpen, total_bytes_written, false};
307 0 : }
308 :
309 0 : void SslSocket::onConnected() { ASSERT(info_->state() == Ssl::SocketState::PreHandshake); }
310 :
311 0 : Ssl::ConnectionInfoConstSharedPtr SslSocket::ssl() const { return info_; }
312 :
313 0 : void SslSocket::shutdownSsl() {
314 0 : ASSERT(info_->state() != Ssl::SocketState::PreHandshake);
315 0 : if (info_->state() != Ssl::SocketState::ShutdownSent &&
316 0 : callbacks_->connection().state() != Network::Connection::State::Closed) {
317 0 : int rc = SSL_shutdown(rawSsl());
318 0 : if constexpr (Event::PlatformDefaultTriggerType == Event::FileTriggerType::EmulatedEdge) {
319 : // Windows operate under `EmulatedEdge`. These are level events that are artificially
320 : // made to behave like edge events. And if the rc is 0 then in that case we want read
321 : // activation resumption. This code is protected with an `constexpr` if, to minimize the tax
322 : // on POSIX systems that operate in Edge events.
323 0 : if (rc == 0) {
324 : // See https://www.openssl.org/docs/manmaster/man3/SSL_shutdown.html
325 : // if return value is 0, Call SSL_read() to do a bidirectional shutdown.
326 0 : callbacks_->setTransportSocketIsReadable();
327 0 : }
328 0 : }
329 0 : ENVOY_CONN_LOG(debug, "SSL shutdown: rc={}", callbacks_->connection(), rc);
330 0 : drainErrorQueue();
331 0 : info_->setState(Ssl::SocketState::ShutdownSent);
332 0 : }
333 0 : }
334 :
335 0 : void SslSocket::shutdownBasic() {
336 0 : if (info_->state() != Ssl::SocketState::ShutdownSent &&
337 0 : callbacks_->connection().state() != Network::Connection::State::Closed) {
338 0 : callbacks_->ioHandle().shutdown(ENVOY_SHUT_WR);
339 0 : drainErrorQueue();
340 0 : info_->setState(Ssl::SocketState::ShutdownSent);
341 0 : }
342 0 : }
343 :
344 0 : void SslSocket::closeSocket(Network::ConnectionEvent) {
345 : // Unregister the SSL connection object from private key method providers.
346 0 : for (auto const& provider : ctx_->getPrivateKeyMethodProviders()) {
347 0 : provider->unregisterPrivateKeyMethod(rawSsl());
348 0 : }
349 :
350 : // Attempt to send a shutdown before closing the socket. It's possible this won't go out if
351 : // there is no room on the socket. We can extend the state machine to handle this at some point
352 : // if needed.
353 0 : if (info_->state() == Ssl::SocketState::HandshakeInProgress ||
354 0 : info_->state() == Ssl::SocketState::HandshakeComplete) {
355 0 : shutdownSsl();
356 0 : } else {
357 : // We're not in a state to do the full SSL shutdown so perform a basic shutdown to flush any
358 : // outstanding alerts
359 0 : shutdownBasic();
360 0 : }
361 0 : }
362 :
363 0 : std::string SslSocket::protocol() const { return ssl()->alpn(); }
364 :
365 0 : absl::string_view SslSocket::failureReason() const { return failure_reason_; }
366 :
367 0 : void SslSocket::onAsynchronousCertValidationComplete() {
368 0 : ENVOY_CONN_LOG(debug, "Async cert validation completed", callbacks_->connection());
369 0 : if (info_->state() == Ssl::SocketState::HandshakeInProgress) {
370 0 : resumeHandshake();
371 0 : }
372 0 : }
373 :
374 : namespace {
375 0 : SslSocketFactoryStats generateStats(const std::string& prefix, Stats::Scope& store) {
376 0 : return {
377 0 : ALL_SSL_SOCKET_FACTORY_STATS(POOL_COUNTER_PREFIX(store, prefix + "_ssl_socket_factory."))};
378 0 : }
379 : } // namespace
380 :
381 : ClientSslSocketFactory::ClientSslSocketFactory(Envoy::Ssl::ClientContextConfigPtr config,
382 : Envoy::Ssl::ContextManager& manager,
383 : Stats::Scope& stats_scope)
384 : : manager_(manager), stats_scope_(stats_scope), stats_(generateStats("client", stats_scope)),
385 : config_(std::move(config)),
386 0 : ssl_ctx_(manager_.createSslClientContext(stats_scope_, *config_)) {
387 0 : config_->setSecretUpdateCallback([this]() { onAddOrUpdateSecret(); });
388 0 : }
389 :
390 0 : ClientSslSocketFactory::~ClientSslSocketFactory() { manager_.removeContext(ssl_ctx_); }
391 :
392 : Network::TransportSocketPtr ClientSslSocketFactory::createTransportSocket(
393 : Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
394 0 : Upstream::HostDescriptionConstSharedPtr) const {
395 : // onAddOrUpdateSecret() could be invoked in the middle of checking the existence of ssl_ctx and
396 : // creating SslSocket using ssl_ctx. Capture ssl_ctx_ into a local variable so that we check and
397 : // use the same ssl_ctx to create SslSocket.
398 0 : Envoy::Ssl::ClientContextSharedPtr ssl_ctx;
399 0 : {
400 0 : absl::ReaderMutexLock l(&ssl_ctx_mu_);
401 0 : ssl_ctx = ssl_ctx_;
402 0 : }
403 0 : if (ssl_ctx) {
404 0 : return std::make_unique<SslSocket>(std::move(ssl_ctx), InitialState::Client,
405 0 : transport_socket_options, config_->createHandshaker());
406 0 : } else {
407 0 : ENVOY_LOG(debug, "Create NotReadySslSocket");
408 0 : stats_.upstream_context_secrets_not_ready_.inc();
409 0 : return std::make_unique<NotReadySslSocket>();
410 0 : }
411 0 : }
412 :
413 0 : bool ClientSslSocketFactory::implementsSecureTransport() const { return true; }
414 :
415 0 : void ClientSslSocketFactory::onAddOrUpdateSecret() {
416 0 : ENVOY_LOG(debug, "Secret is updated.");
417 0 : auto ctx = manager_.createSslClientContext(stats_scope_, *config_);
418 0 : {
419 0 : absl::WriterMutexLock l(&ssl_ctx_mu_);
420 0 : std::swap(ctx, ssl_ctx_);
421 0 : }
422 0 : manager_.removeContext(ctx);
423 0 : stats_.ssl_context_update_by_sds_.inc();
424 0 : }
425 :
426 : ServerSslSocketFactory::ServerSslSocketFactory(Envoy::Ssl::ServerContextConfigPtr config,
427 : Envoy::Ssl::ContextManager& manager,
428 : Stats::Scope& stats_scope,
429 : const std::vector<std::string>& server_names)
430 : : manager_(manager), stats_scope_(stats_scope), stats_(generateStats("server", stats_scope)),
431 : config_(std::move(config)), server_names_(server_names),
432 0 : ssl_ctx_(manager_.createSslServerContext(stats_scope_, *config_, server_names_)) {
433 0 : config_->setSecretUpdateCallback([this]() { onAddOrUpdateSecret(); });
434 0 : }
435 :
436 0 : ServerSslSocketFactory::~ServerSslSocketFactory() { manager_.removeContext(ssl_ctx_); }
437 :
438 0 : Envoy::Ssl::ClientContextSharedPtr ClientSslSocketFactory::sslCtx() {
439 0 : absl::ReaderMutexLock l(&ssl_ctx_mu_);
440 0 : return ssl_ctx_;
441 0 : }
442 :
443 0 : Network::TransportSocketPtr ServerSslSocketFactory::createDownstreamTransportSocket() const {
444 : // onAddOrUpdateSecret() could be invoked in the middle of checking the existence of ssl_ctx and
445 : // creating SslSocket using ssl_ctx. Capture ssl_ctx_ into a local variable so that we check and
446 : // use the same ssl_ctx to create SslSocket.
447 0 : Envoy::Ssl::ServerContextSharedPtr ssl_ctx;
448 0 : {
449 0 : absl::ReaderMutexLock l(&ssl_ctx_mu_);
450 0 : ssl_ctx = ssl_ctx_;
451 0 : }
452 0 : if (ssl_ctx) {
453 0 : return std::make_unique<SslSocket>(std::move(ssl_ctx), InitialState::Server, nullptr,
454 0 : config_->createHandshaker());
455 0 : } else {
456 0 : ENVOY_LOG(debug, "Create NotReadySslSocket");
457 0 : stats_.downstream_context_secrets_not_ready_.inc();
458 0 : return std::make_unique<NotReadySslSocket>();
459 0 : }
460 0 : }
461 :
462 0 : bool ServerSslSocketFactory::implementsSecureTransport() const { return true; }
463 :
464 0 : void ServerSslSocketFactory::onAddOrUpdateSecret() {
465 0 : ENVOY_LOG(debug, "Secret is updated.");
466 0 : auto ctx = manager_.createSslServerContext(stats_scope_, *config_, server_names_);
467 0 : {
468 0 : absl::WriterMutexLock l(&ssl_ctx_mu_);
469 0 : std::swap(ctx, ssl_ctx_);
470 0 : }
471 0 : manager_.removeContext(ctx);
472 :
473 0 : stats_.ssl_context_update_by_sds_.inc();
474 0 : }
475 :
476 : } // namespace Tls
477 : } // namespace TransportSockets
478 : } // namespace Extensions
479 : } // namespace Envoy
|
