1
#pragma once
2

            
3
#include <cstdint>
4
#include <memory>
5

            
6
#include "envoy/common/optref.h"
7
#include "envoy/http/persistent_quic_info.h"
8
#include "envoy/server/overload/overload_manager.h"
9
#include "envoy/upstream/upstream.h"
10

            
11
#include "source/common/http/codec_client.h"
12
#include "source/common/http/conn_pool_base.h"
13

            
14
#ifdef ENVOY_ENABLE_QUIC
15
#include "source/common/quic/client_connection_factory_impl.h"
16
#include "source/common/quic/envoy_quic_network_observer_registry_factory.h"
17
#include "source/common/quic/envoy_quic_utils.h"
18
#include "source/common/quic/quic_transport_socket_factory.h"
19

            
20
#include "quiche/quic/core/deterministic_connection_id_generator.h"
21

            
22
#else
23
#error "http3 conn pool should not be built with QUIC disabled"
24
#endif
25

            
26
namespace Envoy {
27
namespace Http {
28
namespace Http3 {
29

            
30
class ActiveClient : public MultiplexedActiveClientBase {
31
public:
32
  ActiveClient(Envoy::Http::HttpConnPoolImplBase& parent,
33
               Upstream::Host::CreateConnectionData& data);
34

            
35
943
  ~ActiveClient() override {
36
943
    if (async_connect_callback_ != nullptr && async_connect_callback_->enabled()) {
37
1
      async_connect_callback_->cancel();
38
1
    }
39
943
  }
40
  // Http::ConnectionCallbacks
41
  void onMaxStreamsChanged(uint32_t num_streams) override;
42

            
43
1450
  void updateQuicheCapacity() {
44
1450
    ASSERT(quiche_capacity_ != 0);
45
1450
    has_created_stream_ = true;
46
    // Each time a quic stream is allocated the quic capacity needs to get
47
    // decremented. See comments by quiche_capacity_.
48
1450
    updateCapacity(quiche_capacity_ - 1);
49
1450
  }
50

            
51
1
  RequestEncoder& newStreamEncoder(ResponseDecoder& response_decoder) override {
52
1
    updateQuicheCapacity();
53
1
    return MultiplexedActiveClientBase::newStreamEncoder(response_decoder);
54
1
  }
55

            
56
1449
  RequestEncoder& newStreamEncoder(ResponseDecoderHandlePtr response_decoder_handle) override {
57
1449
    updateQuicheCapacity();
58
1449
    return MultiplexedActiveClientBase::newStreamEncoder(std::move(response_decoder_handle));
59
1449
  }
60

            
61
15858
  uint32_t effectiveConcurrentStreamLimit() const override {
62
15858
    return std::min<int64_t>(MultiplexedActiveClientBase::effectiveConcurrentStreamLimit(),
63
15858
                             quiche_capacity_);
64
15858
  }
65

            
66
  // Overload the default capacity calculations to return the quic capacity
67
  // (modified by any stream limits in Envoy config)
68
15858
  int64_t currentUnusedCapacity() const override {
69
15858
    return std::min<int64_t>(quiche_capacity_, effectiveConcurrentStreamLimit());
70
15858
  }
71

            
72
  // Overridden to return true as long as the client is doing handshake even when it is ready for
73
  // early data streams.
74
5405
  bool hasHandshakeCompleted() const override { return has_handshake_completed_; }
75

            
76
  // Overridden to include ReadyForEarlyData state.
77
1853
  bool readyForStream() const override {
78
1853
    return state() == State::Ready || state() == State::ReadyForEarlyData;
79
1853
  }
80

            
81
3296
  void updateCapacity(uint64_t new_quiche_capacity) {
82
    // Each time we update the capacity make sure to reflect the update in the
83
    // connection pool.
84
    //
85
    // Due to interplay between the max number of concurrent streams Envoy will
86
    // allow and the max number of streams per connection this is not as simple
87
    // as just updating based on the delta between quiche_capacity_ and
88
    // new_quiche_capacity, so we use the delta between the actual calculated
89
    // capacity before and after the update.
90
3296
    uint64_t old_capacity = currentUnusedCapacity();
91
3296
    quiche_capacity_ = new_quiche_capacity;
92
3296
    uint64_t new_capacity = currentUnusedCapacity();
93

            
94
3296
    if (new_capacity < old_capacity) {
95
1441
      parent_.decrConnectingAndConnectedStreamCapacity(old_capacity - new_capacity, *this);
96
2202
    } else if (old_capacity < new_capacity) {
97
37
      parent_.incrConnectingAndConnectedStreamCapacity(new_capacity - old_capacity, *this);
98
37
    }
99
3296
  }
100

            
101
35
  bool hasCreatedStream() const { return has_created_stream_; }
102

            
103
protected:
104
49
  bool supportsEarlyData() const override { return true; }
105

            
106
private:
107
  // Unlike HTTP/2 and HTTP/1, rather than having a cap on the number of active
108
  // streams, QUIC has a fixed number of streams available which is updated via
109
  // the MAX_STREAMS frame.
110
  //
111
  // As such each time we create a new stream for QUIC, the capacity goes down
112
  // by one, but unlike the other two codecs it is _not_ restored on stream
113
  // closure.
114
  //
115
  // We track the QUIC capacity here, and overload currentUnusedCapacity so the
116
  // connection pool can accurately keep track of when it is safe to create new
117
  // streams.
118
  //
119
  // Though HTTP/3 should arguably start out with 0 stream capacity until the
120
  // initial handshake is complete and MAX_STREAMS frame has been received,
121
  // assume optimistically it will get ~100 streams, so that the connection pool
122
  // won't fetch a connection for each incoming stream but will assume that the
123
  // first connection will likely be able to serve 100.
124
  // This number will be updated to the correct value before the connection is
125
  // deemed connected, at which point further connections will be established if
126
  // necessary.
127
  uint64_t quiche_capacity_ = 100;
128
  // Used to schedule a deferred connect() call. Because HTTP/3 codec client can
129
  // do 0-RTT during connect(), deferring it to avoid handling network events during CodecClient
130
  // construction.
131
  Event::SchedulableCallbackPtr async_connect_callback_;
132
  // True if newStream() is ever called.
133
  bool has_created_stream_{false};
134
};
135

            
136
// An interface to propagate H3 handshake result.
137
// TODO(danzh) add an API to propagate 0-RTT handshake failure.
138
class PoolConnectResultCallback {
139
public:
140
99
  virtual ~PoolConnectResultCallback() = default;
141

            
142
  // Called when the mandatory handshake is complete. This is when a HTTP/3 connection is regarded
143
  // as connected and is able to send requests.
144
  virtual void onHandshakeComplete() PURE;
145
  // Called upon connection close event from a client who hasn't finish handshake but already sent
146
  // early data.
147
  // TODO(danzh) actually call it from h3 pool.
148
  virtual void onZeroRttHandshakeFailed() PURE;
149
};
150

            
151
// Http3 subclass of FixedHttpConnPoolImpl which exists to store quic data.
152
class Http3ConnPoolImpl : public FixedHttpConnPoolImpl {
153
public:
154
  Http3ConnPoolImpl(Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
155
                    Event::Dispatcher& dispatcher,
156
                    const Network::ConnectionSocket::OptionsSharedPtr& options,
157
                    const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
158
                    Random::RandomGenerator& random_generator,
159
                    Upstream::ClusterConnectivityState& state, CreateClientFn client_fn,
160
                    CreateCodecFn codec_fn, std::vector<Http::Protocol> protocol,
161
                    OptRef<PoolConnectResultCallback> connect_callback,
162
                    Http::PersistentQuicInfo& quic_info,
163
                    OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry,
164
                    Server::OverloadManager& overload_manager, bool attempt_happy_eyeballs = false);
165

            
166
  ~Http3ConnPoolImpl() override;
167
  ConnectionPool::Cancellable* newStream(Http::ResponseDecoder& response_decoder,
168
                                         ConnectionPool::Callbacks& callbacks,
169
                                         const Instance::StreamOptions& options) override;
170

            
171
13
  void drainConnections(Envoy::ConnectionPool::DrainBehavior drain_behavior) override {
172
13
    if (drain_behavior ==
173
13
            Envoy::ConnectionPool::DrainBehavior::DrainExistingNonMigratableConnections &&
174
13
        quic_info_.migration_config_.migrate_session_on_network_change) {
175
      // If connection migration is enabled, don't drain existing connections.
176
      // Each connection will observe network change signals and decide whether
177
      // to migrate or drain.
178
1
      return;
179
1
    }
180
12
    FixedHttpConnPoolImpl::drainConnections(drain_behavior);
181
12
  }
182

            
183
  // For HTTP/3 the base connection pool does not track stream capacity, rather
184
  // the HTTP3 active client does.
185
2900
  bool trackStreamCapacity() override { return false; }
186

            
187
  std::unique_ptr<Network::ClientConnection>
188
  createClientConnection(Quic::QuicStatNames& quic_stat_names,
189
                         OptRef<Http::HttpServerPropertiesCache> rtt_cache, Stats::Scope& scope);
190

            
191
protected:
192
  void onConnected(Envoy::ConnectionPool::ActiveClient&) override;
193
  void onConnectFailed(Envoy::ConnectionPool::ActiveClient&) override;
194

            
195
private:
196
  friend class Http3ConnPoolImplPeer;
197

            
198
  // Latches Quic helpers shared across the cluster
199
  Quic::PersistentQuicInfoImpl& quic_info_;
200
  // server-id can change over the lifetime of Envoy but will be consistent for a
201
  // given connection pool.
202
  quic::QuicServerId server_id_;
203
  // If not nullopt, called when the handshake state changes.
204
  OptRef<PoolConnectResultCallback> connect_callback_;
205

            
206
  quic::DeterministicConnectionIdGenerator connection_id_generator_{
207
      quic::kQuicDefaultConnectionIdLength};
208

            
209
  // Make a best effort attempt to find an address family other than the initial
210
  // address. This fails over to using the primary address if the second address
211
  // in the list isn't of a different address family.
212
  bool attempt_happy_eyeballs_;
213
  OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry_;
214
};
215

            
216
std::unique_ptr<Http3ConnPoolImpl>
217
allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_generator,
218
                 Upstream::HostConstSharedPtr host, Upstream::ResourcePriority priority,
219
                 const Network::ConnectionSocket::OptionsSharedPtr& options,
220
                 const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
221
                 Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names,
222
                 OptRef<Http::HttpServerPropertiesCache> rtt_cache, Stats::Scope& scope,
223
                 OptRef<PoolConnectResultCallback> connect_callback,
224
                 Http::PersistentQuicInfo& quic_info,
225
                 OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry,
226
                 Server::OverloadManager& overload_manager, bool attempt_happy_eyeballs = false);
227

            
228
} // namespace Http3
229
} // namespace Http
230
} // namespace Envoy