Line data Source code
1 : #pragma once
2 :
3 : #include "envoy/config/config_provider.h"
4 : #include "envoy/config/config_provider_manager.h"
5 : #include "envoy/init/manager.h"
6 : #include "envoy/server/admin.h"
7 : #include "envoy/server/config_tracker.h"
8 : #include "envoy/singleton/instance.h"
9 : #include "envoy/thread_local/thread_local.h"
10 :
11 : #include "source/common/common/thread.h"
12 : #include "source/common/common/utility.h"
13 : #include "source/common/config/utility.h"
14 : #include "source/common/init/manager_impl.h"
15 : #include "source/common/init/target_impl.h"
16 : #include "source/common/init/watcher_impl.h"
17 : #include "source/common/protobuf/protobuf.h"
18 :
19 : namespace Envoy {
20 : namespace Config {
21 :
22 : // This file provides a set of base classes, (ImmutableConfigProviderBase,
23 : // MutableConfigProviderCommonBase, ConfigProviderManagerImplBase, ConfigSubscriptionCommonBase,
24 : // ConfigSubscriptionInstance, DeltaConfigSubscriptionInstance), conforming to the
25 : // ConfigProvider/ConfigProviderManager interfaces, which in tandem provide a framework for
26 : // implementing statically defined (i.e., immutable) and dynamic (mutable via subscriptions)
27 : // configuration for Envoy.
28 : //
29 : // The mutability property applies to the ConfigProvider itself and _not_ the underlying config
30 : // proto, which is always immutable. MutableConfigProviderCommonBase objects receive config proto
31 : // updates via xDS subscriptions, resulting in new ConfigProvider::Config objects being instantiated
32 : // with the corresponding change in behavior corresponding to updated config. ConfigProvider::Config
33 : // objects must be latched/associated with the appropriate objects in the connection and request
34 : // processing pipeline, such that configuration stays consistent for the lifetime of the connection
35 : // and/or stream/request (if required by the configuration being processed).
36 : //
37 : // Dynamic configuration is distributed via xDS APIs (see
38 : // https://github.com/envoyproxy/data-plane-api/blob/main/xds_protocol.rst). The framework exposed
39 : // by these classes simplifies creation of client xDS implementations following a shared ownership
40 : // model, where according to the config source specification, a config subscription, config protos
41 : // received over the subscription and the subsequent config "implementation" (i.e., data structures
42 : // and associated business logic) are shared across ConfigProvider objects and Envoy worker threads.
43 : //
44 : // This approach enables linear memory scalability based primarily on the size of the configuration
45 : // set.
46 : //
47 : // A blueprint to follow for implementing mutable or immutable config providers is as follows:
48 : //
49 : // For both:
50 : // 1) Create a class derived from ConfigProviderManagerImplBase and implement the required
51 : // interface.
52 : // When implementing createXdsConfigProvider(), it is expected that getSubscription<T>() will
53 : // be called to fetch either an existing ConfigSubscriptionCommonBase if the config
54 : // source configuration matches, or a newly instantiated subscription otherwise.
55 : //
56 : // For immutable providers:
57 : // 1) Create a class derived from ImmutableConfigProviderBase and implement the required
58 : // interface.
59 : //
60 : // For mutable (xDS) providers:
61 : // 1) According to the API type, create a class derived from MutableConfigProviderCommonBase and
62 : // implement the required interface.
63 : // 2) According to the API type, create a class derived from
64 : // ConfigSubscriptionInstance or DeltaConfigSubscriptionInstance; this is the entity responsible
65 : // for owning and managing the Envoy::Config::Subscription<ConfigProto> that provides the
66 : // underlying config subscription, and the Config implementation shared by associated providers.
67 : // a) For a ConfigProvider::ApiType::Full subscription instance (i.e., a
68 : // ConfigSubscriptionInstance child):
69 : // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the
70 : // underlying subscription, the corresponding ConfigSubscriptionInstance functions
71 : // must be called as well.
72 : // - On a successful config update, checkAndApplyConfigUpdate() should be called to instantiate
73 : // the new config implementation and propagate it to the shared config providers and all worker
74 : // threads.
75 : // - On a successful return from checkAndApplyConfigUpdate(), the config proto must be latched
76 : // into this class and returned via the getConfigProto() override.
77 : // b) For a ConfigProvider::ApiType::Delta subscription instance (i.e., a
78 : // DeltaConfigSubscriptionInstance child):
79 : // - When subscription callbacks (onConfigUpdate, onConfigUpdateFailed) are issued by the
80 : // underlying subscription, the corresponding ConfigSubscriptionInstance functions must be called
81 : // as well.
82 : // - On a successful config update, applyConfigUpdate() should be called to propagate the
83 : // config updates to all bound config providers and worker threads.
84 :
85 : class ConfigProviderManagerImplBase;
86 :
87 : /**
88 : * Specifies the type of config associated with a ConfigProvider.
89 : */
90 : enum class ConfigProviderInstanceType {
91 : // Configuration defined as a static resource in the bootstrap config.
92 : Static,
93 : // Configuration defined inline in a resource that may be specified statically or obtained via
94 : // xDS.
95 : Inline,
96 : // Configuration obtained from an xDS subscription.
97 : Xds
98 : };
99 :
100 : /**
101 : * ConfigProvider implementation for immutable configuration.
102 : *
103 : * TODO(AndresGuedez): support sharing of config protos and config impls, as is
104 : * done with the MutableConfigProviderCommonBase.
105 : *
106 : * This class can not be instantiated directly; instead, it provides the foundation for
107 : * immutable config provider implementations which derive from it.
108 : */
109 : class ImmutableConfigProviderBase : public ConfigProvider {
110 : public:
111 : ~ImmutableConfigProviderBase() override;
112 :
113 : // Envoy::Config::ConfigProvider
114 0 : SystemTime lastUpdated() const override { return last_updated_; }
115 0 : ApiType apiType() const override { return api_type_; }
116 :
117 0 : ConfigProviderInstanceType instanceType() const { return instance_type_; }
118 :
119 : protected:
120 : ImmutableConfigProviderBase(Server::Configuration::ServerFactoryContext& factory_context,
121 : ConfigProviderManagerImplBase& config_provider_manager,
122 : ConfigProviderInstanceType instance_type, ApiType api_type);
123 :
124 : private:
125 : SystemTime last_updated_;
126 : ConfigProviderManagerImplBase& config_provider_manager_;
127 : ConfigProviderInstanceType instance_type_;
128 : ApiType api_type_;
129 : };
130 :
131 : class MutableConfigProviderCommonBase;
132 :
133 : /**
134 : * Provides common DS API subscription functionality required by the ConfigProvider::ApiType.
135 : *
136 : * This class can not be instantiated directly; instead, it provides the foundation for
137 : * config subscription implementations which derive from it.
138 : *
139 : * A subscription is intended to be co-owned by config providers with the same config source, it's
140 : * designed to be created/destructed on admin thread only.
141 : *
142 : */
143 : class ConfigSubscriptionCommonBase : protected Logger::Loggable<Logger::Id::config> {
144 : public:
145 : // Callback for updating a Config implementation held in each worker thread, the callback is
146 : // called in applyConfigUpdate() with the current version Config, and is expected to return the
147 : // new version Config.
148 : using ConfigUpdateCb =
149 : std::function<ConfigProvider::ConfigConstSharedPtr(ConfigProvider::ConfigConstSharedPtr)>;
150 :
151 : struct LastConfigInfo {
152 : absl::optional<uint64_t> last_config_hash_;
153 : std::string last_config_version_;
154 : };
155 :
156 : virtual ~ConfigSubscriptionCommonBase();
157 :
158 : /**
159 : * Starts the subscription corresponding to a config source.
160 : * A derived class must own the configuration proto specific Envoy::Config::Subscription to be
161 : * started.
162 : */
163 : virtual void start() PURE;
164 :
165 0 : const SystemTime& lastUpdated() const { return last_updated_; }
166 :
167 0 : const absl::optional<LastConfigInfo>& configInfo() const { return config_info_; }
168 :
169 0 : ConfigProvider::ConfigConstSharedPtr getConfig() { return tls_->config_; }
170 :
171 : /**
172 : * Must be called by derived classes when the onConfigUpdate() callback associated with the
173 : * underlying subscription is issued.
174 : */
175 0 : absl::Status onConfigUpdate() {
176 0 : setLastUpdated();
177 0 : local_init_target_.ready();
178 0 : return absl::OkStatus();
179 0 : }
180 :
181 : /**
182 : * Must be called by derived classes when the onConfigUpdateFailed() callback associated with the
183 : * underlying subscription is issued.
184 : */
185 0 : void onConfigUpdateFailed() {
186 0 : setLastUpdated();
187 0 : local_init_target_.ready();
188 0 : }
189 :
190 : protected:
191 : struct ThreadLocalConfig : public ThreadLocal::ThreadLocalObject {
192 : explicit ThreadLocalConfig(ConfigProvider::ConfigConstSharedPtr initial_config)
193 0 : : config_(std::move(initial_config)) {}
194 :
195 : ConfigProvider::ConfigConstSharedPtr config_;
196 : };
197 :
198 : ConfigSubscriptionCommonBase(const std::string& name, const uint64_t manager_identifier,
199 : ConfigProviderManagerImplBase& config_provider_manager,
200 : Server::Configuration::ServerFactoryContext& factory_context)
201 : : name_(name), tls_(factory_context.threadLocal()),
202 : local_init_target_(
203 : fmt::format("ConfigSubscriptionCommonBase local init target '{}'", name_),
204 0 : [this]() { start(); }),
205 : parent_init_target_(fmt::format("ConfigSubscriptionCommonBase init target '{}'", name_),
206 0 : [this]() { local_init_manager_.initialize(local_init_watcher_); }),
207 : local_init_watcher_(fmt::format("ConfigSubscriptionCommonBase local watcher '{}'", name_),
208 0 : [this]() { parent_init_target_.ready(); }),
209 : local_init_manager_(
210 : fmt::format("ConfigSubscriptionCommonBase local init manager '{}'", name_)),
211 : manager_identifier_(manager_identifier), config_provider_manager_(config_provider_manager),
212 : time_source_(factory_context.timeSource()),
213 0 : last_updated_(factory_context.timeSource().systemTime()) {
214 0 : Envoy::Config::Utility::checkLocalInfo(name, factory_context.localInfo());
215 0 : local_init_manager_.add(local_init_target_);
216 0 : }
217 :
218 : /**
219 : * Propagates a config update to worker threads.
220 : *
221 : * @param update_fn the callback to run on each thread, it takes the previous version Config and
222 : * returns a updated/new version Config.
223 : */
224 : void applyConfigUpdate(const ConfigUpdateCb& update_fn);
225 :
226 0 : void setLastUpdated() { last_updated_ = time_source_.systemTime(); }
227 0 : Init::Manager& localInitManager() { return local_init_manager_; }
228 0 : void setLastConfigInfo(absl::optional<LastConfigInfo>&& config_info) {
229 0 : config_info_ = std::move(config_info);
230 0 : }
231 :
232 : const std::string name_;
233 : absl::optional<LastConfigInfo> config_info_;
234 : // This slot holds a Config implementation in each thread, which is intended to be shared between
235 : // config providers from the same config source.
236 : ThreadLocal::TypedSlot<ThreadLocalConfig> tls_;
237 :
238 : private:
239 : // Local init target which signals first RPC interaction with management server.
240 : Init::TargetImpl local_init_target_;
241 : // Target added to factory context's initManager.
242 : Init::TargetImpl parent_init_target_;
243 : // Watcher that marks parent_init_target_ ready when the local init manager is ready.
244 : Init::WatcherImpl local_init_watcher_;
245 : // Local manager that tracks the subscription initialization, it is also used for sub-resource
246 : // initialization if the sub-resource is not initialized.
247 : Init::ManagerImpl local_init_manager_;
248 :
249 : const uint64_t manager_identifier_;
250 : ConfigProviderManagerImplBase& config_provider_manager_;
251 : TimeSource& time_source_;
252 : SystemTime last_updated_;
253 :
254 : // ConfigSubscriptionCommonBase, MutableConfigProviderCommonBase and
255 : // ConfigProviderManagerImplBase are tightly coupled with the current shared ownership model; use
256 : // friend classes to explicitly denote the binding between them.
257 : //
258 : // TODO(AndresGuedez): Investigate whether a shared ownership model avoiding the <shared_ptr>s and
259 : // instead centralizing lifetime management in the ConfigProviderManagerImplBase with explicit
260 : // reference counting would be more maintainable.
261 : friend class ConfigProviderManagerImplBase;
262 : };
263 :
264 : using ConfigSubscriptionCommonBaseSharedPtr = std::shared_ptr<ConfigSubscriptionCommonBase>;
265 :
266 : /**
267 : * Provides common subscription functionality required by ConfigProvider::ApiType::Full DS APIs.
268 : * A single Config instance is shared across all providers and all workers associated with this
269 : * subscription.
270 : */
271 : class ConfigSubscriptionInstance : public ConfigSubscriptionCommonBase {
272 : public:
273 : ConfigSubscriptionInstance(const std::string& name, const uint64_t manager_identifier,
274 : ConfigProviderManagerImplBase& config_provider_manager,
275 : Server::Configuration::ServerFactoryContext& factory_context)
276 : : ConfigSubscriptionCommonBase(name, manager_identifier, config_provider_manager,
277 0 : factory_context) {}
278 :
279 : /**
280 : * Must be called by the derived class' constructor.
281 : * @param initial_config supplies an initial Envoy::Config::ConfigProvider::Config associated
282 : * with the underlying subscription, shared across all providers and workers.
283 : */
284 0 : void initialize(const ConfigProvider::ConfigConstSharedPtr& initial_config) {
285 0 : tls_.set([initial_config](Event::Dispatcher&) {
286 0 : return std::make_shared<ThreadLocalConfig>(initial_config);
287 0 : });
288 0 : }
289 :
290 : /**
291 : * Determines whether a configuration proto is a new update, and if so, propagates it to all
292 : * config providers associated with this subscription.
293 : * @param config_proto supplies the newly received config proto.
294 : * @param config_name supplies the name associated with the config.
295 : * @param version_info supplies the version associated with the config.
296 : * @return bool false when the config proto has no delta from the previous config, true
297 : * otherwise.
298 : */
299 : bool checkAndApplyConfigUpdate(const Protobuf::Message& config_proto,
300 : const std::string& config_name, const std::string& version_info);
301 :
302 : protected:
303 : /**
304 : * Called when a new config proto is received via an xDS subscription.
305 : * On successful validation of the config, must return a shared_ptr to a ConfigProvider::Config
306 : * implementation that will be propagated to all mutable config providers sharing the
307 : * subscription.
308 : * Note that this function is called _once_ across all shared config providers per xDS
309 : * subscription config update.
310 : * @param config_proto supplies the configuration proto.
311 : * @return ConfigConstSharedPtr the ConfigProvider::Config to share with other providers.
312 : */
313 : virtual ConfigProvider::ConfigConstSharedPtr
314 : onConfigProtoUpdate(const Protobuf::Message& config_proto) PURE;
315 : };
316 :
317 : /**
318 : * Provides common subscription functionality required by ConfigProvider::ApiType::Delta DS APIs.
319 : */
320 : class DeltaConfigSubscriptionInstance : public ConfigSubscriptionCommonBase {
321 : protected:
322 : using ConfigSubscriptionCommonBase::ConfigSubscriptionCommonBase;
323 :
324 : /**
325 : * Must be called by the derived class' constructor.
326 : * @param init_cb supplies an initial Envoy::Config::ConfigProvider::Config associated with the
327 : * underlying subscription for each worker thread.
328 : */
329 0 : void initialize(const std::function<ConfigProvider::ConfigConstSharedPtr()>& init_cb) {
330 0 : tls_.set(
331 0 : [init_cb](Event::Dispatcher&) { return std::make_shared<ThreadLocalConfig>(init_cb()); });
332 0 : }
333 : };
334 :
335 : /**
336 : * Provides generic functionality required by the ConfigProvider::ApiType specific dynamic config
337 : * providers.
338 : *
339 : * This class can not be instantiated directly; instead, it provides the foundation for
340 : * dynamic config provider implementations which derive from it.
341 : */
342 : class MutableConfigProviderCommonBase : public ConfigProvider {
343 : public:
344 : // Envoy::Config::ConfigProvider
345 0 : SystemTime lastUpdated() const override { return subscription_->lastUpdated(); }
346 0 : ApiType apiType() const override { return api_type_; }
347 :
348 : protected:
349 : MutableConfigProviderCommonBase(ConfigSubscriptionCommonBaseSharedPtr&& subscription,
350 : ApiType api_type)
351 0 : : subscription_(subscription), api_type_(api_type) {}
352 :
353 : // Envoy::Config::ConfigProvider
354 0 : ConfigConstSharedPtr getConfig() const override { return subscription_->getConfig(); }
355 :
356 : ConfigSubscriptionCommonBaseSharedPtr subscription_;
357 :
358 : private:
359 : ApiType api_type_;
360 : };
361 :
362 : /**
363 : * Provides generic functionality required by all config provider managers, such as managing
364 : * shared lifetime of subscriptions and dynamic config providers, along with determining which
365 : * subscriptions should be associated with newly instantiated providers.
366 : *
367 : * The implementation of this class is not thread safe. Note that ImmutableConfigProviderBase
368 : * and ConfigSubscriptionCommonBase call the corresponding {bind,unbind}* functions exposed
369 : * by this class.
370 : *
371 : * All config processing is done on the main thread, so instantiation of *ConfigProvider* objects
372 : * via createStaticConfigProvider() and createXdsConfigProvider() is naturally thread safe. Care
373 : * must be taken with regards to destruction of these objects, since it must also happen on the
374 : * main thread _prior_ to destruction of the ConfigProviderManagerImplBase object from which they
375 : * were created.
376 : *
377 : * This class can not be instantiated directly; instead, it provides the foundation for
378 : * dynamic config provider implementations which derive from it.
379 : */
380 : class ConfigProviderManagerImplBase : public ConfigProviderManager, public Singleton::Instance {
381 : public:
382 : /**
383 : * This is invoked by the /config_dump admin handler.
384 : * @return ProtobufTypes::MessagePtr the config dump proto corresponding to the associated
385 : * config providers.
386 : */
387 : virtual ProtobufTypes::MessagePtr
388 : dumpConfigs(const Matchers::StringMatcher& name_matcher) const PURE;
389 :
390 : protected:
391 : // Ordered set for deterministic config dump output.
392 : using ConfigProviderSet = std::set<ConfigProvider*>;
393 : using ConfigProviderMap = absl::node_hash_map<ConfigProviderInstanceType,
394 : std::unique_ptr<ConfigProviderSet>, EnumClassHash>;
395 : using ConfigSubscriptionMap =
396 : absl::node_hash_map<uint64_t, std::weak_ptr<ConfigSubscriptionCommonBase>>;
397 :
398 : ConfigProviderManagerImplBase(OptRef<Server::Admin> admin, const std::string& config_name);
399 :
400 70 : const ConfigSubscriptionMap& configSubscriptions() const { return config_subscriptions_; }
401 :
402 : /**
403 : * Returns the set of bound ImmutableConfigProviderBase-derived providers of a given type.
404 : * @param type supplies the type of config providers to return.
405 : * @return const ConfigProviderSet* the set of config providers corresponding to the type.
406 : */
407 : const ConfigProviderSet& immutableConfigProviders(ConfigProviderInstanceType type) const;
408 :
409 : /**
410 : * Returns the subscription associated with the config_source_proto; if none exists, a new one
411 : * is allocated according to the subscription_factory_fn.
412 : * @param config_source_proto supplies the proto specifying the config subscription parameters.
413 : * @param init_manager supplies the init manager.
414 : * @param subscription_factory_fn supplies a function to be called when a new subscription needs
415 : * to be allocated.
416 : * @return std::shared_ptr<T> an existing (if a match is found) or newly allocated subscription.
417 : */
418 : template <typename T>
419 : std::shared_ptr<T>
420 : getSubscription(const Protobuf::Message& config_source_proto, Init::Manager& init_manager,
421 : const std::function<ConfigSubscriptionCommonBaseSharedPtr(
422 0 : const uint64_t, ConfigProviderManagerImplBase&)>& subscription_factory_fn) {
423 0 : static_assert(std::is_base_of<ConfigSubscriptionCommonBase, T>::value,
424 0 : "T must be a subclass of ConfigSubscriptionCommonBase");
425 :
426 0 : ConfigSubscriptionCommonBaseSharedPtr subscription;
427 0 : const uint64_t manager_identifier = MessageUtil::hash(config_source_proto);
428 :
429 0 : auto it = config_subscriptions_.find(manager_identifier);
430 0 : if (it == config_subscriptions_.end()) {
431 : // std::make_shared does not work for classes with private constructors. There are ways
432 : // around it. However, since this is not a performance critical path we err on the side
433 : // of simplicity.
434 0 : subscription = subscription_factory_fn(manager_identifier, *this);
435 0 : init_manager.add(subscription->parent_init_target_);
436 :
437 0 : bindSubscription(manager_identifier, subscription);
438 0 : } else {
439 : // Because the ConfigProviderManagerImplBase's weak_ptrs only get cleaned up
440 : // in the ConfigSubscriptionCommonBase destructor, and the single threaded nature
441 : // of this code, locking the weak_ptr will not fail.
442 0 : subscription = it->second.lock();
443 0 : }
444 0 : ASSERT(subscription);
445 :
446 0 : return std::static_pointer_cast<T>(subscription);
447 0 : }
448 :
449 : private:
450 : void bindSubscription(const uint64_t manager_identifier,
451 0 : ConfigSubscriptionCommonBaseSharedPtr& subscription) {
452 0 : config_subscriptions_.insert({manager_identifier, subscription});
453 0 : }
454 :
455 0 : void unbindSubscription(const uint64_t manager_identifier) {
456 0 : config_subscriptions_.erase(manager_identifier);
457 0 : }
458 :
459 : void bindImmutableConfigProvider(ImmutableConfigProviderBase* provider);
460 : void unbindImmutableConfigProvider(ImmutableConfigProviderBase* provider);
461 :
462 : // TODO(jsedgwick) These two members are prime candidates for the owned-entry list/map
463 : // as in ConfigTracker. I.e. the ProviderImpls would have an EntryOwner for these lists
464 : // Then the lifetime management stuff is centralized and opaque.
465 : ConfigSubscriptionMap config_subscriptions_;
466 : ConfigProviderMap immutable_config_providers_map_;
467 :
468 : Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_;
469 :
470 : // See comment for friend classes in the ConfigSubscriptionCommonBase for more details on
471 : // the use of friends.
472 : friend class ConfigSubscriptionCommonBase;
473 : friend class ImmutableConfigProviderBase;
474 : };
475 :
476 : } // namespace Config
477 : } // namespace Envoy
|
