1
#pragma once
2

            
3
#include <chrono>
4
#include <functional>
5
#include <memory>
6
#include <string>
7

            
8
#include "envoy/access_log/access_log.h"
9
#include "envoy/api/api.h"
10
#include "envoy/common/random_generator.h"
11
#include "envoy/config/bootstrap/v3/bootstrap.pb.h"
12
#include "envoy/config/cluster/v3/cluster.pb.h"
13
#include "envoy/config/core/v3/address.pb.h"
14
#include "envoy/config/core/v3/config_source.pb.h"
15
#include "envoy/config/core/v3/protocol.pb.h"
16
#include "envoy/config/eds_resources_cache.h"
17
#include "envoy/config/grpc_mux.h"
18
#include "envoy/config/subscription_factory.h"
19
#include "envoy/grpc/async_client_manager.h"
20
#include "envoy/http/conn_pool.h"
21
#include "envoy/http/persistent_quic_info.h"
22
#include "envoy/local_info/local_info.h"
23
#include "envoy/runtime/runtime.h"
24
#include "envoy/secret/secret_manager.h"
25
#include "envoy/server/admin.h"
26
#include "envoy/server/options.h"
27
#include "envoy/singleton/manager.h"
28
#include "envoy/ssl/context_manager.h"
29
#include "envoy/stats/store.h"
30
#include "envoy/tcp/conn_pool.h"
31
#include "envoy/thread_local/thread_local.h"
32
#include "envoy/upstream/health_checker.h"
33
#include "envoy/upstream/load_balancer.h"
34
#include "envoy/upstream/thread_local_cluster.h"
35
#include "envoy/upstream/upstream.h"
36

            
37
#include "absl/container/flat_hash_set.h"
38
#include "absl/container/node_hash_map.h"
39

            
40
namespace Envoy {
41

            
42
namespace Quic {
43

            
44
class EnvoyQuicNetworkObserverRegistryFactory;
45
class EnvoyQuicNetworkObserverRegistry;
46

            
47
} // namespace Quic
48

            
49
namespace Config {
50
// TODO(adisuissa): This forward declaration is needed because OD-CDS code is
51
// part of the Envoy::Upstream namespace but should be eventually moved to the
52
// Envoy::Config namespace (next to the XdsManager).
53
class XdsManager;
54
} // namespace Config
55

            
56
namespace Upstream {
57

            
58
/**
59
 * ClusterUpdateCallbacks provide a way to expose Cluster lifecycle events in the
60
 * ClusterManager.
61
 */
62
using ThreadLocalClusterCommand = std::function<ThreadLocalCluster&()>;
63
class ClusterUpdateCallbacks {
64
public:
65
22481
  virtual ~ClusterUpdateCallbacks() = default;
66

            
67
  /**
68
   * onClusterAddOrUpdate is called when a new cluster is added or an existing cluster
69
   * is updated in the ClusterManager.
70
   * @param cluster_name the name of the changed cluster.
71
   * @param get_cluster is a callable that will provide the ThreadLocalCluster that represents the
72
   * updated cluster. It should be used within the call or discarded.
73
   */
74
  virtual void onClusterAddOrUpdate(absl::string_view cluster_name,
75
                                    ThreadLocalClusterCommand& get_cluster) PURE;
76
  /**
77
   * onClusterRemoval is called when a cluster is removed; the argument is the cluster name.
78
   * @param cluster_name is the name of the removed cluster.
79
   */
80
  virtual void onClusterRemoval(const std::string& cluster_name) PURE;
81
};
82

            
83
/**
84
 * ClusterUpdateCallbacksHandle is a RAII wrapper for a ClusterUpdateCallbacks. Deleting
85
 * the ClusterUpdateCallbacksHandle will remove the callbacks from ClusterManager in O(1).
86
 */
87
class ClusterUpdateCallbacksHandle {
88
public:
89
22169
  virtual ~ClusterUpdateCallbacksHandle() = default;
90
};
91

            
92
using ClusterUpdateCallbacksHandlePtr = std::unique_ptr<ClusterUpdateCallbacksHandle>;
93

            
94
/**
95
 * Status enum for the result of an attempted cluster discovery.
96
 */
97
enum class ClusterDiscoveryStatus {
98
  /**
99
   * The discovery process timed out. This means that we haven't yet received any reply from
100
   * on-demand CDS about it.
101
   */
102
  Timeout,
103
  /**
104
   * The discovery process has concluded and on-demand CDS has no such cluster.
105
   */
106
  Missing,
107
  /**
108
   * Cluster found and currently available through ClusterManager.
109
   */
110
  Available,
111
};
112

            
113
/**
114
 * ClusterDiscoveryCallback is a callback called at the end of the on-demand cluster discovery
115
 * process. The status of the discovery is sent as a parameter.
116
 */
117
using ClusterDiscoveryCallback = std::function<void(ClusterDiscoveryStatus)>;
118
using ClusterDiscoveryCallbackPtr = std::unique_ptr<ClusterDiscoveryCallback>;
119

            
120
/**
121
 * ClusterDiscoveryCallbackHandle is a RAII wrapper for a ClusterDiscoveryCallback. Deleting the
122
 * ClusterDiscoveryCallbackHandle will remove the callbacks from ClusterManager.
123
 */
124
class ClusterDiscoveryCallbackHandle {
125
public:
126
472
  virtual ~ClusterDiscoveryCallbackHandle() = default;
127
};
128

            
129
using ClusterDiscoveryCallbackHandlePtr = std::unique_ptr<ClusterDiscoveryCallbackHandle>;
130

            
131
/**
132
 * A handle to an on-demand CDS.
133
 */
134
class OdCdsApiHandle {
135
public:
136
205
  virtual ~OdCdsApiHandle() = default;
137

            
138
  /**
139
   * Request an on-demand discovery of a cluster with a passed name. This ODCDS may be used to
140
   * perform the discovery process in the main thread if there is no discovery going on for this
141
   * cluster. When the requested cluster is added and warmed up, the passed callback will be invoked
142
   * in the same thread that invoked this function.
143
   *
144
   * The returned handle can be destroyed to prevent the callback from being invoked. Note that the
145
   * handle can only be destroyed in the same thread that invoked the function. Destroying the
146
   * handle might not stop the discovery process, though. As soon as the callback is invoked,
147
   * destroying the handle does nothing. It is a responsibility of the caller to make sure that the
148
   * objects captured in the callback outlive the callback.
149
   *
150
   * This function is thread-safe.
151
   *
152
   * @param name is the name of the cluster to be discovered.
153
   * @param callback will be called when the discovery is finished.
154
   * @param timeout describes how long the operation may take before failing.
155
   * @return the discovery process handle.
156
   */
157
  virtual ClusterDiscoveryCallbackHandlePtr
158
  requestOnDemandClusterDiscovery(absl::string_view name, ClusterDiscoveryCallbackPtr callback,
159
                                  std::chrono::milliseconds timeout) PURE;
160
};
161

            
162
using OdCdsApiHandlePtr = std::unique_ptr<OdCdsApiHandle>;
163

            
164
class ClusterManagerFactory;
165

            
166
// These are per-cluster per-thread, so not "global" stats.
167
struct ClusterConnectivityState {
168
21969
  ~ClusterConnectivityState() {
169
21969
    ASSERT(pending_streams_ == 0);
170
21969
    ASSERT(active_streams_ == 0);
171
21969
    ASSERT(connecting_and_connected_stream_capacity_ == 0);
172
21969
  }
173

            
174
160720
  template <class T> void checkAndDecrement(T& value, uint32_t delta) {
175
160720
    ASSERT(std::numeric_limits<T>::min() + delta <= value);
176
160720
    value -= delta;
177
160720
  }
178

            
179
156013
  template <class T> void checkAndIncrement(T& value, uint32_t delta) {
180
156013
    ASSERT(std::numeric_limits<T>::max() - delta >= value);
181
156013
    value += delta;
182
156013
  }
183

            
184
31837
  void incrPendingStreams(uint32_t delta) { checkAndIncrement(pending_streams_, delta); }
185
31884
  void decrPendingStreams(uint32_t delta) { checkAndDecrement(pending_streams_, delta); }
186
75389
  void incrConnectingAndConnectedStreamCapacity(uint32_t delta) {
187
75389
    checkAndIncrement(connecting_and_connected_stream_capacity_, delta);
188
75389
  }
189
80049
  void decrConnectingAndConnectedStreamCapacity(uint32_t delta) {
190
80049
    checkAndDecrement(connecting_and_connected_stream_capacity_, delta);
191
80049
  }
192
48787
  void incrActiveStreams(uint32_t delta) { checkAndIncrement(active_streams_, delta); }
193
48787
  void decrActiveStreams(uint32_t delta) { checkAndDecrement(active_streams_, delta); }
194

            
195
  // Tracks the number of pending streams for this ClusterManager.
196
  uint32_t pending_streams_{};
197
  // Tracks the number of active streams for this ClusterManager.
198
  uint32_t active_streams_{};
199
  // Tracks the available stream capacity if all connecting connections were connected.
200
  //
201
  // For example, if an H2 connection is started with concurrent stream limit of 100, this
202
  // goes up by 100. If the connection is established and 2 streams are in use, it
203
  // would be reduced to 98 (as 2 of the 100 are not available).
204
  //
205
  // Note that if more HTTP/2 streams have been established than are allowed by
206
  // a late-received SETTINGS frame, this MAY BE NEGATIVE.
207
  // Note this tracks the sum of multiple 32 bit stream capacities so must remain 64 bit.
208
  int64_t connecting_and_connected_stream_capacity_{};
209
};
210

            
211
/**
212
 * An interface for on-demand CDS. Defined to allow mocking.
213
 */
214
class OdCdsApi {
215
public:
216
197
  virtual ~OdCdsApi() = default;
217

            
218
  // Subscribe to a cluster with a given name. It's meant to eventually send a discovery request
219
  // with the cluster name to the management server.
220
  virtual void updateOnDemand(std::string cluster_name) PURE;
221
};
222

            
223
using OdCdsApiSharedPtr = std::shared_ptr<OdCdsApi>;
224

            
225
/**
226
 * An interface used by OdCdsApiImpl for sending notifications about the missing cluster that was
227
 * requested.
228
 */
229
class MissingClusterNotifier {
230
public:
231
11047
  virtual ~MissingClusterNotifier() = default;
232

            
233
  virtual void notifyMissingCluster(absl::string_view name) PURE;
234
};
235

            
236
/**
237
 * Manages connection pools and load balancing for upstream clusters. The cluster manager is
238
 * persistent and shared among multiple ongoing requests/connections.
239
 * Cluster manager is initialized in two phases. In the first phase which begins at the construction
240
 * all primary clusters (i.e. with endpoint assignments provisioned statically in bootstrap,
241
 * discovered through DNS or file based CDS) are initialized. This phase may complete synchronously
242
 * with cluster manager construction iff all clusters are STATIC and without health checks
243
 * configured. At the completion of the first phase cluster manager invokes callback set through the
244
 * `setPrimaryClustersInitializedCb` method.
245
 * After the first phase has completed the server instance initializes services (i.e. RTDS) needed
246
 * to successfully deploy the rest of dynamic configuration.
247
 * In the second phase all secondary clusters (with endpoint assignments provisioned by xDS servers)
248
 * are initialized and then the rest of the configuration provisioned through xDS.
249
 */
250
class ClusterManager {
251
public:
252
  using PrimaryClustersReadyCallback = std::function<void()>;
253
  using InitializationCompleteCallback = std::function<void()>;
254

            
255
53701
  virtual ~ClusterManager() = default;
256

            
257
  // API to initialize the ClusterManagerImpl instance based on the given Bootstrap config.
258
  //
259
  // This method *must* be called prior to invoking any other methods on the class and *must* only
260
  // be called once. This method should be called immediately after ClusterManagerImpl construction
261
  // and from the same thread in which the ClusterManagerImpl was constructed.
262
  //
263
  // The initialization is separated from the constructor because lots of work, including ADS
264
  // initialization, is done in this method. If the contents of this method are invoked during
265
  // construction, a derived class cannot override any of the virtual methods and have them invoked
266
  // instead, since the base class's methods are used when in a base class constructor.
267
  virtual absl::Status initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) PURE;
268

            
269
  // API to return whether the ClusterManagerImpl instance is initialized.
270
  virtual bool initialized() PURE;
271

            
272
  /**
273
   * Add or update a cluster via API. The semantics of this API are:
274
   * 1) The hash of the config is used to determine if an already existing cluster has changed.
275
   *    Nothing is done if the hash matches the previously running configuration.
276
   * 2) Statically defined clusters (those present when Envoy starts) can not be updated via API.
277
   *
278
   * @param cluster supplies the cluster configuration.
279
   * @param version_info supplies the xDS version of the cluster.
280
   * @param avoid_cds_removal If set to true, the cluster will be ignored from removal during CDS
281
   *                       update. It can be overridden by setting `remove_ignored` to true while
282
   *                       calling removeCluster(). This is useful for clusters whose lifecycle
283
   *                       is managed with custom implementation, e.g., DFP clusters.
284
   * @return true if the action results in an add/update of a cluster, an error
285
   * status if the config is invalid.
286
   */
287
  virtual absl::StatusOr<bool>
288
  addOrUpdateCluster(const envoy::config::cluster::v3::Cluster& cluster,
289
                     const std::string& version_info, const bool avoid_cds_removal = false) PURE;
290

            
291
  /**
292
   * Set a callback that will be invoked when all primary clusters have been initialized.
293
   */
294
  virtual void setPrimaryClustersInitializedCb(PrimaryClustersReadyCallback callback) PURE;
295

            
296
  /**
297
   * Set a callback that will be invoked when all owned clusters have been initialized.
298
   */
299
  virtual void setInitializedCb(InitializationCompleteCallback callback) PURE;
300

            
301
  /**
302
   * Start initialization of secondary clusters and then dynamically configured clusters.
303
   * The "initialized callback" set in the method above is invoked when secondary and
304
   * dynamically provisioned clusters have finished initializing.
305
   */
306
  virtual absl::Status
307
  initializeSecondaryClusters(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) PURE;
308

            
309
  using ClusterInfoMap = absl::flat_hash_map<std::string, std::reference_wrapper<const Cluster>>;
310
  struct ClusterInfoMaps {
311
962
    bool hasCluster(absl::string_view cluster) const {
312
962
      return active_clusters_.find(cluster) != active_clusters_.end() ||
313
962
             warming_clusters_.find(cluster) != warming_clusters_.end();
314
962
    }
315

            
316
90
    ClusterConstOptRef getCluster(absl::string_view cluster) const {
317
90
      auto active_cluster = active_clusters_.find(cluster);
318
90
      if (active_cluster != active_clusters_.cend()) {
319
84
        return active_cluster->second;
320
84
      }
321
6
      auto warming_cluster = warming_clusters_.find(cluster);
322
6
      if (warming_cluster != warming_clusters_.cend()) {
323
        return warming_cluster->second;
324
      }
325
6
      return absl::nullopt;
326
6
    }
327

            
328
    ClusterInfoMap active_clusters_;
329
    ClusterInfoMap warming_clusters_;
330

            
331
    // Number of clusters that were dynamically added via API (xDS). This will be
332
    // less than or equal to the number of `active_clusters_` and `warming_clusters_`.
333
    uint32_t added_via_api_clusters_num_{0};
334
  };
335

            
336
  /**
337
   * @return ClusterInfoMap all current clusters including active and warming.
338
   *
339
   * NOTE: This method is only thread safe on the main thread. It should not be called elsewhere.
340
   */
341
  virtual ClusterInfoMaps clusters() const PURE;
342

            
343
  /**
344
   * Receives a cluster name and returns an active cluster (if found).
345
   * @param cluster_name the name of the cluster.
346
   * @return OptRef<const Cluster> A reference to the cluster if found, and nullopt otherwise.
347
   *
348
   * NOTE: This method is only thread safe on the main thread. It should not be called elsewhere.
349
   */
350
  virtual OptRef<const Cluster> getActiveCluster(const std::string& cluster_name) const PURE;
351

            
352
  /**
353
   * Receives a cluster name and returns an active or warming cluster (if found).
354
   * @param cluster_name the name of the cluster.
355
   * @return OptRef<const Cluster> A reference to the cluster if found, and nullopt otherwise.
356
   *
357
   * NOTE: This method is only thread safe on the main thread. It should not be called elsewhere.
358
   */
359
  virtual OptRef<const Cluster>
360
  getActiveOrWarmingCluster(const std::string& cluster_name) const PURE;
361

            
362
  /**
363
   * Returns true iff the given cluster name is known in the cluster-manager
364
   * (either as active or as warming).
365
   * @param cluster_name the name of the cluster.
366
   * @return bool true if the cluster name is known, and false otherwise.
367
   *
368
   * NOTE: This method is only thread safe on the main thread. It should not be called elsewhere.
369
   */
370
  virtual bool hasCluster(const std::string& cluster_name) const PURE;
371

            
372
  /**
373
   * Returns true iff there's an active cluster in the cluster-manager.
374
   * @return bool true if there is an active cluster, and false otherwise.
375
   *
376
   * NOTE: This method is only thread safe on the main thread. It should not be called elsewhere.
377
   */
378
  virtual bool hasActiveClusters() const PURE;
379

            
380
  using ClusterSet = absl::flat_hash_set<std::string>;
381

            
382
  /**
383
   * @return const ClusterSet& providing the cluster names that are eligible as
384
   *         xDS API config sources. These must be static (i.e. in the
385
   *         bootstrap) and non-EDS.
386
   */
387
  virtual const ClusterSet& primaryClusters() PURE;
388

            
389
  /**
390
   * @return ThreadLocalCluster* the thread local cluster with the given name or nullptr if it
391
   * does not exist. This is thread safe.
392
   *
393
   * NOTE: The pointer returned by this function is ONLY safe to use in the context of the owning
394
   * call (or if the caller knows that the cluster is fully static and will never be deleted). In
395
   * the case of dynamic clusters, subsequent event loop iterations may invalidate this pointer.
396
   * If information about the cluster needs to be kept, use the ThreadLocalCluster::info() method to
397
   * obtain cluster information that is safe to store.
398
   *
399
   * NOTE: This method may return nullptr even if the cluster exists (if it hasn't been warmed yet,
400
   * propagated to workers, etc.). Use clusters() for general configuration checking on the main
401
   * thread.
402
   */
403
  virtual ThreadLocalCluster* getThreadLocalCluster(absl::string_view cluster) PURE;
404

            
405
  /**
406
   * Remove a cluster via API. Only clusters added via addOrUpdateCluster() can
407
   * be removed in this manner. Statically defined clusters present when Envoy starts cannot be
408
   * removed.
409
   * Cluster created using `addOrUpdateCluster()` with `avoid_cds_removal` set to true.
410
   * can be removed by setting `remove_ignored` to true while removeCluster().
411
   * @return true if the action results in the removal of a cluster.
412
   */
413
  virtual bool removeCluster(const std::string& cluster, const bool remove_ignored = false) PURE;
414

            
415
  /**
416
   * Shutdown the cluster manager prior to destroying connection pools and other thread local data.
417
   */
418
  virtual void shutdown() PURE;
419

            
420
  /**
421
   * @return whether the shutdown method has been called.
422
   */
423
  virtual bool isShutdown() PURE;
424

            
425
  /**
426
   * @return cluster manager wide bind configuration for new upstream connections.
427
   */
428
  virtual const absl::optional<envoy::config::core::v3::BindConfig>& bindConfig() const PURE;
429

            
430
  /**
431
   * Returns a shared_ptr to the singleton xDS-over-gRPC provider for upstream control plane muxing
432
   * of xDS. This is treated somewhat as a special case in ClusterManager, since it does not relate
433
   * logically to the management of clusters but instead is required early in ClusterManager/server
434
   * initialization and in various sites that need ClusterManager for xDS API interfacing.
435
   *
436
   * @return GrpcMux& ADS API provider referencee.
437
   */
438
  virtual Config::GrpcMuxSharedPtr adsMux() PURE;
439

            
440
  /**
441
   * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager.
442
   */
443
  virtual Grpc::AsyncClientManager& grpcAsyncClientManager() PURE;
444

            
445
  /**
446
   * Return the local cluster name, if it was configured.
447
   *
448
   * @return absl::optional<std::string> the local cluster name, or empty if no local cluster was
449
   * configured.
450
   */
451
  virtual const absl::optional<std::string>& localClusterName() const PURE;
452

            
453
  /**
454
   * This method allows to register callbacks for cluster lifecycle events in the ClusterManager.
455
   * The callbacks will be registered in a thread local slot and the callbacks will be executed
456
   * on the thread that registered them.
457
   * To be executed on all threads, Callbacks need to be registered on all threads.
458
   *
459
   * @param callbacks are the ClusterUpdateCallbacks to add or remove to the cluster manager.
460
   * @return ClusterUpdateCallbacksHandlePtr a RAII that needs to be deleted to
461
   * unregister the callback.
462
   */
463
  virtual ClusterUpdateCallbacksHandlePtr
464
  addThreadLocalClusterUpdateCallbacks(ClusterUpdateCallbacks& callbacks) PURE;
465

            
466
  /**
467
   * Obtain the subscription factory for the cluster manager. Since subscriptions may have an
468
   * upstream component, the factory is a facet of the cluster manager.
469
   *
470
   * @return Config::SubscriptionFactory& the subscription factory.
471
   */
472
  virtual Config::SubscriptionFactory& subscriptionFactory() PURE;
473

            
474
  /**
475
   * Returns a struct with all the Stats::StatName objects needed by
476
   * Clusters. This helps factor out some relatively heavy name
477
   * construction which occur when there is a large CDS update during operation,
478
   * relative to recreating all stats from strings on-the-fly.
479
   *
480
   * @return the stat names.
481
   */
482
  virtual const ClusterTrafficStatNames& clusterStatNames() const PURE;
483
  virtual const ClusterConfigUpdateStatNames& clusterConfigUpdateStatNames() const PURE;
484
  virtual const ClusterLbStatNames& clusterLbStatNames() const PURE;
485
  virtual const ClusterEndpointStatNames& clusterEndpointStatNames() const PURE;
486
  virtual const ClusterLoadReportStatNames& clusterLoadReportStatNames() const PURE;
487
  virtual const ClusterCircuitBreakersStatNames& clusterCircuitBreakersStatNames() const PURE;
488
  virtual const ClusterRequestResponseSizeStatNames&
489
  clusterRequestResponseSizeStatNames() const PURE;
490
  virtual const ClusterTimeoutBudgetStatNames& clusterTimeoutBudgetStatNames() const PURE;
491

            
492
  /**
493
   * Predicate function used in drainConnections().
494
   * @param host supplies the host that is about to be drained.
495
   * @return true if the host should be drained, and false otherwise.
496
   *
497
   * IMPORTANT: This predicate must be completely self contained and thread safe. It will be posted
498
   * to all worker threads and run concurrently.
499
   */
500
  using DrainConnectionsHostPredicate = std::function<bool(const Host&)>;
501

            
502
  /**
503
   * Drain all connection pool connections owned by this cluster.
504
   * @param cluster, the cluster to drain.
505
   * @param predicate supplies the optional drain connections host predicate. If not supplied, all
506
   *                  hosts are drained.
507
   */
508
  virtual void drainConnections(const std::string& cluster,
509
                                DrainConnectionsHostPredicate predicate) PURE;
510

            
511
  /**
512
   * Drain all connection pool connections owned by all clusters in the cluster manager.
513
   * @param predicate supplies the optional drain connections host predicate. If not supplied, all
514
   *                  hosts are drained.
515
   */
516
  virtual void drainConnections(DrainConnectionsHostPredicate predicate,
517
                                ConnectionPool::DrainBehavior drain_behavior) PURE;
518

            
519
  /**
520
   * Check if the cluster is active and statically configured, and if not, return an error
521
   * @param cluster, the cluster to check.
522
   */
523
  virtual absl::Status checkActiveStaticCluster(const std::string& cluster) PURE;
524

            
525
  /**
526
   * Allocates an on-demand CDS API provider from configuration proto or locator.
527
   *
528
   * @param odcds_config is a configuration proto. Used when odcds_resources_locator is a nullopt.
529
   * @param odcds_resources_locator is a locator for ODCDS. Used over odcds_config if not a nullopt.
530
   * @param validation_visitor
531
   * @return OdCdsApiHandlePtr the ODCDS handle.
532
   */
533
  // TODO(adisuissa): once the xDS-TP config-sources are fully supported, the
534
  // `odcds_config` parameter should become optional, and the comment above
535
  // should be updated.
536
  using OdCdsCreationFunction = std::function<absl::StatusOr<std::shared_ptr<OdCdsApi>>(
537
      const envoy::config::core::v3::ConfigSource& odcds_config,
538
      OptRef<xds::core::v3::ResourceLocator> odcds_resources_locator,
539
      Config::XdsManager& xds_manager, ClusterManager& cm, MissingClusterNotifier& notifier,
540
      Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor,
541
      Server::Configuration::ServerFactoryContext& server_factory_context)>;
542

            
543
  virtual absl::StatusOr<OdCdsApiHandlePtr>
544
  allocateOdCdsApi(OdCdsCreationFunction creation_function,
545
                   const envoy::config::core::v3::ConfigSource& odcds_config,
546
                   OptRef<xds::core::v3::ResourceLocator> odcds_resources_locator,
547
                   ProtobufMessage::ValidationVisitor& validation_visitor) PURE;
548

            
549
  /**
550
   * @param common_lb_config The config field to be stored
551
   * @return shared_ptr to the CommonLbConfig
552
   */
553
  virtual std::shared_ptr<const envoy::config::cluster::v3::Cluster::CommonLbConfig>
554
  getCommonLbConfigPtr(
555
      const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_lb_config) PURE;
556

            
557
  /**
558
   * Returns an EdsResourcesCache that is unique for the cluster manager.
559
   */
560
  virtual Config::EdsResourcesCacheOptRef edsResourcesCache() PURE;
561

            
562
  /**
563
   * Create a QUIC network observer registry for each worker thread using the given factory.
564
   * @param factory used to create a registry object.
565
   */
566
  virtual void createNetworkObserverRegistries(
567
      Envoy::Quic::EnvoyQuicNetworkObserverRegistryFactory& factory) PURE;
568
};
569

            
570
using ClusterManagerPtr = std::unique_ptr<ClusterManager>;
571

            
572
/**
573
 * Abstract interface for a CDS API provider.
574
 */
575
class CdsApi {
576
public:
577
734
  virtual ~CdsApi() = default;
578

            
579
  /**
580
   * Start the first fetch of CDS data.
581
   */
582
  virtual void initialize() PURE;
583

            
584
  /**
585
   * Set a callback that will be called when the CDS API has done an initial load from the remote
586
   * server. If the initial load fails, the callback will also be called.
587
   */
588
  virtual void setInitializedCb(std::function<void()> callback) PURE;
589

            
590
  /**
591
   * @return std::string last accepted version from fetch.
592
   */
593
  virtual const std::string versionInfo() const PURE;
594
};
595

            
596
using CdsApiPtr = std::unique_ptr<CdsApi>;
597

            
598
/**
599
 * Factory for objects needed during cluster manager operation.
600
 */
601
class ClusterManagerFactory {
602
public:
603
11041
  virtual ~ClusterManagerFactory() = default;
604

            
605
  /**
606
   * Allocate a cluster manager from configuration proto.
607
   * The cluster manager initialize() method needs to be called right after this method.
608
   * Please check https://github.com/envoyproxy/envoy/issues/33218 for details.
609
   */
610
  virtual absl::StatusOr<ClusterManagerPtr>
611
  clusterManagerFromProto(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) PURE;
612

            
613
  /**
614
   * Allocate an HTTP connection pool for the host. Pools are separated by 'priority',
615
   * 'protocol', and 'options->hashKey()', if any.
616
   * @param network_observer_registry if not null all the QUIC connections created by this pool
617
   * should register to it for network events.
618
   */
619
  virtual Http::ConnectionPool::InstancePtr
620
  allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
621
                   ResourcePriority priority, std::vector<Http::Protocol>& protocol,
622
                   const absl::optional<envoy::config::core::v3::AlternateProtocolsCacheOptions>&
623
                       alternate_protocol_options,
624
                   const Network::ConnectionSocket::OptionsSharedPtr& options,
625
                   const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
626
                   TimeSource& time_source, ClusterConnectivityState& state,
627
                   Http::PersistentQuicInfoPtr& quic_info,
628
                   OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry) PURE;
629

            
630
  /**
631
   * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and
632
   * 'options->hashKey()', if any.
633
   */
634
  virtual Tcp::ConnectionPool::InstancePtr
635
  allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
636
                      ResourcePriority priority,
637
                      const Network::ConnectionSocket::OptionsSharedPtr& options,
638
                      Network::TransportSocketOptionsConstSharedPtr transport_socket_options,
639
                      ClusterConnectivityState& state,
640
                      absl::optional<std::chrono::milliseconds> tcp_pool_idle_timeout) PURE;
641

            
642
  /**
643
   * Allocate a cluster from configuration proto.
644
   */
645
  virtual absl::StatusOr<std::pair<ClusterSharedPtr, ThreadAwareLoadBalancerPtr>>
646
  clusterFromProto(const envoy::config::cluster::v3::Cluster& cluster,
647
                   Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) PURE;
648

            
649
  /**
650
   * Create a CDS API provider from configuration proto.
651
   */
652
  virtual absl::StatusOr<CdsApiPtr>
653
  createCds(const envoy::config::core::v3::ConfigSource& cds_config,
654
            const xds::core::v3::ResourceLocator* cds_resources_locator, ClusterManager& cm,
655
            bool support_multi_ads_sources) PURE;
656
};
657

            
658
/**
659
 * Factory for creating ClusterInfo
660
 */
661
class ClusterInfoFactory {
662
public:
663
86
  virtual ~ClusterInfoFactory() = default;
664

            
665
  /**
666
   * Parameters for createClusterInfo().
667
   */
668
  struct CreateClusterInfoParams {
669
    Server::Configuration::ServerFactoryContext& server_context_;
670
    const envoy::config::cluster::v3::Cluster& cluster_;
671
    const envoy::config::core::v3::BindConfig& bind_config_;
672
    Stats::Store& stats_;
673
    Ssl::ContextManager& ssl_context_manager_;
674
    const bool added_via_api_;
675
    ThreadLocal::SlotAllocator& tls_;
676
  };
677

            
678
  /**
679
   * This method returns a Upstream::ClusterInfoConstSharedPtr given construction parameters.
680
   */
681
  virtual Upstream::ClusterInfoConstSharedPtr
682
  createClusterInfo(const CreateClusterInfoParams& params) PURE;
683
};
684

            
685
} // namespace Upstream
686
} // namespace Envoy