1
#pragma once
2

            
3
#include "envoy/config/config_provider.h"
4
#include "envoy/config/config_provider_manager.h"
5
#include "envoy/init/manager.h"
6
#include "envoy/server/admin.h"
7
#include "envoy/server/config_tracker.h"
8
#include "envoy/singleton/instance.h"
9
#include "envoy/thread_local/thread_local.h"
10

            
11
#include "source/common/common/thread.h"
12
#include "source/common/common/utility.h"
13
#include "source/common/config/utility.h"
14
#include "source/common/init/manager_impl.h"
15
#include "source/common/init/target_impl.h"
16
#include "source/common/init/watcher_impl.h"
17
#include "source/common/protobuf/protobuf.h"
18

            
19
namespace Envoy {
20
namespace Config {
21

            
22
// This file provides a set of base classes, (ImmutableConfigProviderBase,
23
// MutableConfigProviderCommonBase, ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase,
24
// ConfigSubscriptionInstance, DeltaConfigSubscriptionInstance), conforming to the
25
// ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a framework for
26
// implementing statically defined (i.e., immutable) and dynamic (mutable via subscriptions)
27
// configuration for Envoy.
28
//
29
// The mutability property applies to the ConfigProvider itself and _not_ the underlying config
30
// proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto
31
// updates via xDS subscriptions, resulting in new ConfigProvider::Config objects being instantiated
32
// with the corresponding change in behavior corresponding to updated config. ConfigProvider::Config
33
// objects must be latched/associated with the appropriate objects in the connection and request
34
// processing pipeline, such that configuration stays consistent for the lifetime of the connection
35
// and/or stream/request (if required by the configuration being processed).
36
//
37
// Dynamic configuration is distributed via xDS APIs (see
38
// https://github.com/envoyproxy/data-plane-api/blob/main/xds_protocol.rst). The framework exposed
39
// by these classes simplifies creation of client xDS implementations following a shared ownership
40
// model, where according to the config source specification, a config subscription, config protos
41
// received over the subscription and the subsequent config "implementation" (i.e., data structures
42
// and associated business logic) are shared across ConfigProvider objects and Envoy worker threads.
43
//
44
// This approach enables linear memory scalability based primarily on the size of the configuration
45
// set.
46
//
47
// A blueprint to follow for implementing mutable or immutable config providers is as follows:
48
//
49
// For both:
50
//   1) Create a class derived from ConfigProviderManagerImplBase and implement the required
51
//   interface.
52
//      When implementing createXdsConfigProvider(), it is expected that getSubscription<T>() will
53
//      be called to fetch either an existing ConfigSubscriptionCommonBase if the config
54
//      source configuration matches, or a newly instantiated subscription otherwise.
55
//
56
// For immutable providers:
57
//   1) Create a class derived from ImmutableConfigProviderBase and implement the required
58
//   interface.
59
//
60
// For mutable (xDS) providers:
61
//   1) According to the API type, create a class derived from MutableConfigProviderCommonBase and
62
//   implement the required interface.
63
//   2) According to the API type, create a class derived from
64
//   ConfigSubscriptionInstance or DeltaConfigSubscriptionInstance; this is the entity responsible
65
//   for owning and managing the Envoy::Config::Subscription<ConfigProto> that provides the
66
//   underlying config subscription, and the Config implementation shared by associated providers.
67
//     a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a
68
//     ConfigSubscriptionInstance child):
69
//     - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the
70
//     underlying subscription, the corresponding ConfigSubscriptionInstance functions
71
//     must be called as well.
72
//     - On a successful config update, checkAndApplyConfigUpdate() should be called to instantiate
73
//     the new config implementation and propagate it to the shared config providers and all worker
74
//     threads.
75
//       - On a successful return from checkAndApplyConfigUpdate(), the config proto must be latched
76
//       into this class and returned via the getConfigProto() override.
77
//    b) For a ConfigProvider::ApiType::Delta subscription instance (i.e., a
78
//    DeltaConfigSubscriptionInstance child):
79
//    - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the
80
//    underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called
81
//    as well.
82
//    - On a successful config update, applyConfigUpdate() should be called to propagate the
83
//    config updates to all bound config providers and worker threads.
84

            
85
class ConfigProviderManagerImplBase;
86

            
87
/**
88
 * Specifies the type of config associated with a ConfigProvider.
89
 */
90
enum class ConfigProviderInstanceType {
91
  // Configuration defined as a static resource in the bootstrap config.
92
  Static,
93
  // Configuration defined inline in a resource that may be specified statically or obtained via
94
  // xDS.
95
  Inline,
96
  // Configuration obtained from an xDS subscription.
97
  Xds
98
};
99

            
100
/**
101
 * ConfigProvider implementation for immutable configuration.
102
 *
103
 * TODO(AndresGuedez): support sharing of config protos and config impls, as is
104
 * done with the MutableConfigProviderCommonBase.
105
 *
106
 * This class can not be instantiated directly; instead, it provides the foundation for
107
 * immutable config provider implementations which derive from it.
108
 */
109
class ImmutableConfigProviderBase : public ConfigProvider {
110
public:
111
  ~ImmutableConfigProviderBase() override;
112

            
113
  // Envoy::Config::ConfigProvider
114
6
  SystemTime lastUpdated() const override { return last_updated_; }
115
  ApiType apiType() const override { return api_type_; }
116

            
117
38
  ConfigProviderInstanceType instanceType() const { return instance_type_; }
118

            
119
protected:
120
  ImmutableConfigProviderBase(Server::Configuration::ServerFactoryContext& factory_context,
121
                              ConfigProviderManagerImplBase& config_provider_manager,
122
                              ConfigProviderInstanceType instance_type, ApiType api_type);
123

            
124
private:
125
  SystemTime last_updated_;
126
  ConfigProviderManagerImplBase& config_provider_manager_;
127
  ConfigProviderInstanceType instance_type_;
128
  ApiType api_type_;
129
};
130

            
131
class MutableConfigProviderCommonBase;
132

            
133
/**
134
 * Provides common DS API subscription functionality required by the ConfigProvider::ApiType.
135
 *
136
 * This class can not be instantiated directly; instead, it provides the foundation for
137
 * config subscription implementations which derive from it.
138
 *
139
 * A subscription is intended to be co-owned by config providers with the same config source, it's
140
 * designed to be created/destructed on admin thread only.
141
 *
142
 */
143
class ConfigSubscriptionCommonBase : protected Logger::Loggable<Logger::Id::config> {
144
public:
145
  // Callback for updating a Config implementation held in each worker thread, the callback is
146
  // called in applyConfigUpdate() with the current version Config, and is expected to return the
147
  // new version Config.
148
  using ConfigUpdateCb =
149
      std::function<ConfigProvider::ConfigConstSharedPtr(ConfigProvider::ConfigConstSharedPtr)>;
150

            
151
  struct LastConfigInfo {
152
    absl::optional<uint64_t> last_config_hash_;
153
    std::string last_config_version_;
154
  };
155

            
156
  virtual ~ConfigSubscriptionCommonBase();
157

            
158
  /**
159
   * Starts the subscription corresponding to a config source.
160
   * A derived class must own the configuration proto specific Envoy::Config::Subscription to be
161
   * started.
162
   */
163
  virtual void start() PURE;
164

            
165
12
  const SystemTime& lastUpdated() const { return last_updated_; }
166

            
167
23
  const absl::optional<LastConfigInfo>& configInfo() const { return config_info_; }
168

            
169
258
  ConfigProvider::ConfigConstSharedPtr getConfig() { return tls_->config_; }
170

            
171
  /**
172
   * Must be called by derived classes when the onConfigUpdate() callback associated with the
173
   * underlying subscription is issued.
174
   */
175
146
  absl::Status onConfigUpdate() {
176
146
    setLastUpdated();
177
146
    local_init_target_.ready();
178
146
    return absl::OkStatus();
179
146
  }
180

            
181
  /**
182
   * Must be called by derived classes when the onConfigUpdateFailed() callback associated with the
183
   * underlying subscription is issued.
184
   */
185
8
  void onConfigUpdateFailed() {
186
8
    setLastUpdated();
187
8
    local_init_target_.ready();
188
8
  }
189

            
190
protected:
191
  struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject {
192
    explicit ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config)
193
191
        : config_(std::move(initial_config)) {}
194

            
195
    ConfigProvider::ConfigConstSharedPtr config_;
196
  };
197

            
198
  ConfigSubscriptionCommonBase(const std::string& name, const uint64_t manager_identifier,
199
                               ConfigProviderManagerImplBase& config_provider_manager,
200
                               Server::Configuration::ServerFactoryContext& factory_context);
201

            
202
  /**
203
   * Propagates a config update to worker threads.
204
   *
205
   * @param update_fn the callback to run on each thread, it takes the previous version Config and
206
   * returns a updated/new version Config.
207
   */
208
  void applyConfigUpdate(const ConfigUpdateCb& update_fn);
209

            
210
154
  void setLastUpdated() { last_updated_ = time_source_.systemTime(); }
211
246
  Init::Manager& localInitManager() { return local_init_manager_; }
212
138
  void setLastConfigInfo(absl::optional<LastConfigInfo>&& config_info) {
213
138
    config_info_ = std::move(config_info);
214
138
  }
215

            
216
  const std::string name_;
217
  absl::optional<LastConfigInfo> config_info_;
218
  // This slot holds a Config implementation in each thread, which is intended to be shared between
219
  // config providers from the same config source.
220
  ThreadLocal::TypedSlot<ThreadLocalConfig> tls_;
221

            
222
private:
223
  // Local init target which signals first RPC interaction with management server.
224
  Init::TargetImpl local_init_target_;
225
  // Target added to factory context's initManager.
226
  Init::TargetImpl parent_init_target_;
227
  // Watcher that marks parent_init_target_ ready when the local init manager is ready.
228
  Init::WatcherImpl local_init_watcher_;
229
  // Local manager that tracks the subscription initialization, it is also used for sub-resource
230
  // initialization if the sub-resource is not initialized.
231
  Init::ManagerImpl local_init_manager_;
232

            
233
  const uint64_t manager_identifier_;
234
  ConfigProviderManagerImplBase& config_provider_manager_;
235
  TimeSource& time_source_;
236
  SystemTime last_updated_;
237

            
238
  // ConfigSubscriptionCommonBase, MutableConfigProviderCommonBase and
239
  // ConfigProviderManagerImplBase are tightly coupled with the current shared ownership model; use
240
  // friend classes to explicitly denote the binding between them.
241
  //
242
  // TODO(AndresGuedez): Investigate whether a shared ownership model avoiding the <shared_ptr>s and
243
  // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit
244
  // reference counting would be more maintainable.
245
  friend class ConfigProviderManagerImplBase;
246
};
247

            
248
using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr<ConfigSubscriptionCommonBase>;
249

            
250
/**
251
 * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs.
252
 * A single Config instance is shared across all providers and all workers associated with this
253
 * subscription.
254
 */
255
class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase {
256
public:
257
  ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier,
258
                             ConfigProviderManagerImplBase& config_provider_manager,
259
                             Server::Configuration::ServerFactoryContext& factory_context)
260
5
      : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager,
261
5
                                     factory_context) {}
262

            
263
  /**
264
   * Must be called by the derived class' constructor.
265
   * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated
266
   * with the underlying subscription, shared across all providers and workers.
267
   */
268
4
  void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) {
269
4
    tls_.set([initial_config](Event::Dispatcher&) {
270
4
      return std::make_shared<ThreadLocalConfig>(initial_config);
271
4
    });
272
4
  }
273

            
274
  /**
275
   * Determines whether a configuration proto is a new update, and if so, propagates it to all
276
   * config providers associated with this subscription.
277
   * @param config_proto supplies the newly received config proto.
278
   * @param config_name supplies the name associated with the config.
279
   * @param version_info supplies the version associated with the config.
280
   * @return bool false when the config proto has no delta from the previous config, true
281
   * otherwise.
282
   */
283
  bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto,
284
                                 const std::string& config_name, const std::string& version_info);
285

            
286
protected:
287
  /**
288
   * Called when a new config proto is received via an xDS subscription.
289
   * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config
290
   * implementation that will be propagated to all mutable config providers sharing the
291
   * subscription.
292
   * Note that this function is called _once_ across all shared config providers per xDS
293
   * subscription config update.
294
   * @param config_proto supplies the configuration proto.
295
   * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers.
296
   */
297
  virtual ConfigProvider::ConfigConstSharedPtr
298
  onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE;
299
};
300

            
301
/**
302
 * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs.
303
 */
304
class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase {
305
protected:
306
  using ConfigSubscriptionCommonBase::ConfigSubscriptionCommonBase;
307

            
308
  /**
309
   * Must be called by the derived class' constructor.
310
   * @param init_cb supplies an initial Envoy::Config::ConfigProvider::Config associated with the
311
   * underlying subscription for each worker thread.
312
   */
313
107
  void initialize(const std::function<ConfigProvider::ConfigConstSharedPtr()>& init_cb) {
314
107
    tls_.set(
315
187
        [init_cb](Event::Dispatcher&) { return std::make_shared<ThreadLocalConfig>(init_cb()); });
316
107
  }
317
};
318

            
319
/**
320
 * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config
321
 * providers.
322
 *
323
 * This class can not be instantiated directly; instead, it provides the foundation for
324
 * dynamic config provider implementations which derive from it.
325
 */
326
class MutableConfigProviderCommonBase : public ConfigProvider {
327
public:
328
  // Envoy::Config::ConfigProvider
329
2
  SystemTime lastUpdated() const override { return subscription_->lastUpdated(); }
330
  ApiType apiType() const override { return api_type_; }
331

            
332
protected:
333
  MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription,
334
                                  ApiType api_type)
335
122
      : subscription_(subscription), api_type_(api_type) {}
336

            
337
  // Envoy::Config::ConfigProvider
338
254
  ConfigConstSharedPtr getConfig() const override { return subscription_->getConfig(); }
339

            
340
  ConfigSubscriptionCommonBaseSharedPtr subscription_;
341

            
342
private:
343
  ApiType api_type_;
344
};
345

            
346
/**
347
 * Provides generic functionality required by all config provider managers, such as managing
348
 * shared lifetime of subscriptions and dynamic config providers, along with determining which
349
 * subscriptions should be associated with newly instantiated providers.
350
 *
351
 * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase
352
 * and ConfigSubscriptionCommonBase call the corresponding {bind,unbind}* functions exposed
353
 * by this class.
354
 *
355
 * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects
356
 * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care
357
 * must be taken with regards to destruction of these objects, since it must also happen on the
358
 * main thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they
359
 * were created.
360
 *
361
 * This class can not be instantiated directly; instead, it provides the foundation for
362
 * dynamic config provider implementations which derive from it.
363
 */
364
class ConfigProviderManagerImplBase : public ConfigProviderManager {
365
public:
366
  /**
367
   * This is invoked by the /config_dump admin handler.
368
   * @return ProtobufTypes::MessagePtr the config dump proto corresponding to the associated
369
   *                                   config providers.
370
   */
371
  virtual ProtobufTypes::MessagePtr
372
  dumpConfigs(const Matchers::StringMatcher& name_matcher) const PURE;
373

            
374
protected:
375
  // Ordered set for deterministic config dump output.
376
  using ConfigProviderSet = std::set<ConfigProvider*>;
377
  using ConfigProviderMap = absl::node_hash_map<ConfigProviderInstanceType,
378
                                                std::unique_ptr<ConfigProviderSet>, EnumClassHash>;
379
  using ConfigSubscriptionMap =
380
      absl::node_hash_map<uint64_t, std::weak_ptr<ConfigSubscriptionCommonBase>>;
381

            
382
  ConfigProviderManagerImplBase(OptRef<Server::Admin> admin, const std::string& config_name);
383

            
384
35
  const ConfigSubscriptionMap& configSubscriptions() const { return config_subscriptions_; }
385

            
386
  /**
387
   * Returns the set of bound ImmutableConfigProviderBase-derived providers of a given type.
388
   * @param type supplies the type of config providers to return.
389
   * @return const ConfigProviderSet* the set of config providers corresponding to the type.
390
   */
391
  const ConfigProviderSet& immutableConfigProviders(ConfigProviderInstanceType type) const;
392

            
393
  /**
394
   * Returns the subscription associated with the config_source_proto; if none exists, a new one
395
   * is allocated according to the subscription_factory_fn.
396
   * @param config_source_proto supplies the proto specifying the config subscription parameters.
397
   * @param init_manager supplies the init manager.
398
   * @param subscription_factory_fn supplies a function to be called when a new subscription needs
399
   *                                to be allocated.
400
   * @return std::shared_ptr<T> an existing (if a match is found) or newly allocated subscription.
401
   */
402
  template <typename T>
403
  std::shared_ptr<T>
404
  getSubscription(const Protobuf::Message& config_source_proto, Init::Manager& init_manager,
405
                  const std::function<ConfigSubscriptionCommonBaseSharedPtr(
406
123
                      const uint64_t, ConfigProviderManagerImplBase&)>& subscription_factory_fn) {
407
123
    static_assert(std::is_base_of<ConfigSubscriptionCommonBase, T>::value,
408
123
                  "T must be a subclass of ConfigSubscriptionCommonBase");
409

            
410
123
    ConfigSubscriptionCommonBaseSharedPtr subscription;
411
123
    const uint64_t manager_identifier = MessageUtil::hash(config_source_proto);
412

            
413
123
    auto it = config_subscriptions_.find(manager_identifier);
414
123
    if (it == config_subscriptions_.end()) {
415
      // std::make_shared does not work for classes with private constructors. There are ways
416
      // around it. However, since this is not a performance critical path we err on the side
417
      // of simplicity.
418
112
      subscription = subscription_factory_fn(manager_identifier, *this);
419
112
      init_manager.add(subscription->parent_init_target_);
420

            
421
112
      bindSubscription(manager_identifier, subscription);
422
112
    } else {
423
      // Because the ConfigProviderManagerImplBase's weak_ptrs only get cleaned up
424
      // in the ConfigSubscriptionCommonBase destructor, and the single threaded nature
425
      // of this code, locking the weak_ptr will not fail.
426
11
      subscription = it->second.lock();
427
11
    }
428
123
    ASSERT(subscription);
429

            
430
123
    return std::static_pointer_cast<T>(subscription);
431
123
  }
432

            
433
private:
434
  void bindSubscription(const uint64_t manager_identifier,
435
111
                        ConfigSubscriptionCommonBaseSharedPtr& subscription) {
436
111
    config_subscriptions_.insert({manager_identifier, subscription});
437
111
  }
438

            
439
111
  void unbindSubscription(const uint64_t manager_identifier) {
440
111
    config_subscriptions_.erase(manager_identifier);
441
111
  }
442

            
443
  void bindImmutableConfigProvider(ImmutableConfigProviderBase* provider);
444
  void unbindImmutableConfigProvider(ImmutableConfigProviderBase* provider);
445

            
446
  // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map
447
  // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists
448
  // Then the lifetime management stuff is centralized and opaque.
449
  ConfigSubscriptionMap config_subscriptions_;
450
  ConfigProviderMap immutable_config_providers_map_;
451

            
452
  Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_;
453

            
454
  // See comment for friend classes in the ConfigSubscriptionCommonBase for more details on
455
  // the use of friends.
456
  friend class ConfigSubscriptionCommonBase;
457
  friend class ImmutableConfigProviderBase;
458
};
459

            
460
} // namespace Config
461
} // namespace Envoy