1
#include "source/common/runtime/runtime_impl.h"
2

            
3
#include <cstdint>
4
#include <string>
5

            
6
#include "envoy/config/bootstrap/v3/bootstrap.pb.h"
7
#include "envoy/config/core/v3/config_source.pb.h"
8
#include "envoy/event/dispatcher.h"
9
#include "envoy/service/discovery/v3/discovery.pb.h"
10
#include "envoy/thread_local/thread_local.h"
11
#include "envoy/type/v3/percent.pb.h"
12
#include "envoy/type/v3/percent.pb.validate.h"
13

            
14
#include "source/common/common/assert.h"
15
#include "source/common/common/fmt.h"
16
#include "source/common/common/utility.h"
17
#include "source/common/config/api_version.h"
18
#include "source/common/filesystem/directory.h"
19
#include "source/common/grpc/common.h"
20
#include "source/common/http/utility.h"
21
#include "source/common/protobuf/message_validator_impl.h"
22
#include "source/common/protobuf/utility.h"
23
#include "source/common/runtime/runtime_features.h"
24

            
25
#include "absl/container/node_hash_map.h"
26
#include "absl/container/node_hash_set.h"
27
#include "absl/flags/flag.h"
28
#include "absl/strings/match.h"
29
#include "absl/strings/numbers.h"
30
#include "re2/re2.h"
31

            
32
#ifdef ENVOY_ENABLE_QUIC
33
#include "quiche/common/platform/api/quiche_flags.h"
34
#include "quiche_platform_impl/quiche_flags_impl.h"
35
#endif
36

            
37
namespace Envoy {
38
namespace Runtime {
39

            
40
namespace {
41

            
42
1506
void countDeprecatedFeatureUseInternal(const RuntimeStats& stats) {
43
1506
  stats.deprecated_feature_use_.inc();
44
  // Similar to the above, but a gauge that isn't imported during a hot restart.
45
1506
  stats.deprecated_feature_seen_since_process_start_.inc();
46
1506
}
47

            
48
17704
void refreshReloadableFlags(const Snapshot::EntryMap& flag_map) {
49
38179
  for (const auto& it : flag_map) {
50
34666
    if (it.second.bool_value_.has_value() && isRuntimeFeature(it.first)) {
51
33453
      maybeSetRuntimeGuard(it.first, it.second.bool_value_.value());
52
33453
    }
53
34666
  }
54
17704
#ifdef ENVOY_ENABLE_QUIC
55
17704
  absl::flat_hash_map<std::string, bool> quiche_flags_override;
56
38179
  for (const auto& it : flag_map) {
57
34666
    if (absl::StartsWith(it.first, quiche::EnvoyQuicheReloadableFlagPrefix) &&
58
34666
        it.second.bool_value_.has_value()) {
59
5
      quiche_flags_override[it.first.substr(quiche::EnvoyFeaturePrefix.length())] =
60
5
          it.second.bool_value_.value();
61
5
    }
62
34666
  }
63

            
64
17704
  quiche::FlagRegistry::getInstance().updateReloadableFlags(quiche_flags_override);
65

            
66
  // Because this is a QUICHE protocol flag, this behavior can't be flipped with the above
67
  // code, so it needs its own runtime flag and code to set it.
68
17704
  SetQuicheFlag(quic_always_support_server_preferred_address,
69
17704
                Runtime::runtimeFeatureEnabled(
70
17704
                    "envoy.reloadable_features.quic_send_server_preferred_address_to_all_clients"));
71

            
72
17704
#endif
73
  // Make sure ints are parsed after the flag allowing deprecated ints is parsed.
74
38179
  for (const auto& it : flag_map) {
75
34666
    if (it.second.uint_value_.has_value()) {
76
340
      maybeSetDeprecatedInts(it.first, it.second.uint_value_.value());
77
340
    }
78
34666
  }
79
17704
  markRuntimeInitialized();
80
17704
}
81

            
82
} // namespace
83

            
84
199
bool SnapshotImpl::deprecatedFeatureEnabled(absl::string_view key, bool default_value) const {
85
  // A deprecated feature is enabled if at least one of the following conditions holds:
86
  // 1. A boolean runtime entry <key> doesn't exist, and default_value is true.
87
  // 2. A boolean runtime entry <key> exists, with a value of "true".
88
  // 3. A boolean runtime entry "envoy.features.enable_all_deprecated_features" with a value of
89
  //    "true" exists, and there isn't a boolean runtime entry <key> with a value of "false".
90

            
91
199
  if (!getBoolean(key,
92
199
                  getBoolean("envoy.features.enable_all_deprecated_features", default_value))) {
93
12
    return false;
94
12
  }
95

            
96
  // The feature is allowed. It is assumed this check is called when the feature
97
  // is about to be used, so increment the feature use stat.
98
187
  countDeprecatedFeatureUseInternal(stats_);
99

            
100
#ifdef ENVOY_DISABLE_DEPRECATED_FEATURES
101
  return false;
102
#endif
103

            
104
187
  return true;
105
199
}
106

            
107
2
bool SnapshotImpl::runtimeFeatureEnabled(absl::string_view key) const {
108
  // If the value is not explicitly set as a runtime boolean, the default value is based on
109
  // the underlying value.
110
2
  return getBoolean(key, Runtime::runtimeFeatureEnabled(key));
111
2
}
112

            
113
bool SnapshotImpl::featureEnabled(absl::string_view key, uint64_t default_value,
114
5
                                  uint64_t random_value, uint64_t num_buckets) const {
115
5
  return random_value % num_buckets < std::min(getInteger(key, default_value), num_buckets);
116
5
}
117

            
118
64038
bool SnapshotImpl::featureEnabled(absl::string_view key, uint64_t default_value) const {
119
  // Avoid PRNG if we know we don't need it.
120
64038
  uint64_t cutoff = std::min(getInteger(key, default_value), static_cast<uint64_t>(100));
121
64038
  if (cutoff == 0) {
122
56915
    return false;
123
56935
  } else if (cutoff == 100) {
124
7120
    return true;
125
7122
  } else {
126
3
    return generator_.random() % 100 < cutoff;
127
3
  }
128
64038
}
129

            
130
bool SnapshotImpl::featureEnabled(absl::string_view key, uint64_t default_value,
131
2
                                  uint64_t random_value) const {
132
2
  return featureEnabled(key, default_value, random_value, 100);
133
2
}
134

            
135
23510
Snapshot::ConstStringOptRef SnapshotImpl::get(absl::string_view key) const {
136
23510
  ASSERT(!isRuntimeFeature(key)); // Make sure runtime guarding is only used for getBoolean
137
23510
  auto entry = key.empty() ? values_.end() : values_.find(key);
138
23510
  if (entry == values_.end()) {
139
23401
    return absl::nullopt;
140
23475
  } else {
141
109
    return entry->second.raw_string_value_;
142
109
  }
143
23510
}
144

            
145
bool SnapshotImpl::featureEnabled(absl::string_view key,
146
10511
                                  const envoy::type::v3::FractionalPercent& default_value) const {
147
10511
  return featureEnabled(key, default_value, generator_.random());
148
10511
}
149

            
150
bool SnapshotImpl::featureEnabled(absl::string_view key,
151
                                  const envoy::type::v3::FractionalPercent& default_value,
152
1010686
                                  uint64_t random_value) const {
153
1010686
  const auto& entry = key.empty() ? values_.end() : values_.find(key);
154
1010686
  envoy::type::v3::FractionalPercent percent;
155
1010686
  if (entry != values_.end() && entry->second.fractional_percent_value_.has_value()) {
156
29
    percent = entry->second.fractional_percent_value_.value();
157
1010657
  } else if (entry != values_.end() && entry->second.uint_value_.has_value()) {
158
    // Check for > 100 because the runtime value is assumed to be specified as
159
    // an integer, and it also ensures that truncating the uint64_t runtime
160
    // value into a uint32_t percent numerator later is safe
161
15
    if (entry->second.uint_value_.value() > 100) {
162
5
      return true;
163
5
    }
164

            
165
    // The runtime value was specified as an integer rather than a fractional
166
    // percent proto. To preserve legacy semantics, we treat it as a percentage
167
    // (i.e. denominator of 100).
168
10
    percent.set_numerator(entry->second.uint_value_.value());
169
10
    percent.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED);
170
1010644
  } else {
171
1010642
    percent = default_value;
172
1010642
  }
173

            
174
  // When numerator > denominator condition is always evaluates to TRUE
175
  // It becomes hard to debug why configuration does not work in case of wrong numerator.
176
  // Log debug message that numerator is invalid.
177
1010681
  uint64_t denominator_value =
178
1010681
      ProtobufPercentHelper::fractionalPercentDenominatorToInt(percent.denominator());
179
1010681
  if (percent.numerator() > denominator_value) {
180
1
    ENVOY_LOG(debug,
181
1
              "WARNING runtime key '{}': numerator ({}) > denominator ({}), condition always "
182
1
              "evaluates to true",
183
1
              key, percent.numerator(), denominator_value);
184
1
    return true;
185
1
  }
186

            
187
1010680
  return ProtobufPercentHelper::evaluateFractionalPercent(percent, random_value);
188
1010681
}
189

            
190
2657266
uint64_t SnapshotImpl::getInteger(absl::string_view key, uint64_t default_value) const {
191
2657266
  ASSERT(!isRuntimeFeature(key));
192
2657266
  const auto& entry = key.empty() ? values_.end() : values_.find(key);
193
2657266
  if (entry == values_.end() || !entry->second.uint_value_) {
194
2615690
    return default_value;
195
2615701
  } else {
196
41576
    return entry->second.uint_value_.value();
197
41576
  }
198
2657266
}
199

            
200
6317
double SnapshotImpl::getDouble(absl::string_view key, double default_value) const {
201
6317
  ASSERT(!isRuntimeFeature(key)); // Make sure runtime guarding is only used for getBoolean
202
6317
  const auto& entry = key.empty() ? values_.end() : values_.find(key);
203
6317
  if (entry == values_.end() || !entry->second.double_value_) {
204
6306
    return default_value;
205
6311
  } else {
206
11
    return entry->second.double_value_.value();
207
11
  }
208
6317
}
209

            
210
264607
bool SnapshotImpl::getBoolean(absl::string_view key, bool default_value) const {
211
264607
  const auto& entry = key.empty() ? values_.end() : values_.find(key);
212
264607
  if (entry == values_.end() || !entry->second.bool_value_.has_value()) {
213
264493
    return default_value;
214
264494
  } else {
215
114
    return entry->second.bool_value_.value();
216
114
  }
217
264607
}
218

            
219
135
const std::vector<Snapshot::OverrideLayerConstPtr>& SnapshotImpl::getLayers() const {
220
135
  return layers_;
221
135
}
222

            
223
17704
const Snapshot::EntryMap& SnapshotImpl::values() const { return values_; }
224

            
225
SnapshotImpl::SnapshotImpl(Random::RandomGenerator& generator, RuntimeStats& stats,
226
                           std::vector<OverrideLayerConstPtr>&& layers)
227
17704
    : layers_{std::move(layers)}, generator_{generator}, stats_{stats} {
228
28490
  for (const auto& layer : layers_) {
229
41601
    for (const auto& kv : layer->values()) {
230
34734
      values_.erase(kv.first);
231
34734
      values_.emplace(kv.first, kv.second);
232
34734
    }
233
28330
  }
234
17704
  stats.num_keys_.set(values_.size());
235
17704
}
236

            
237
87
void parseFractionValue(SnapshotImpl::Entry& entry, const Protobuf::Struct& value) {
238
87
  envoy::type::v3::FractionalPercent percent;
239
87
  static_assert(envoy::type::v3::FractionalPercent::MILLION ==
240
87
                envoy::type::v3::FractionalPercent::DenominatorType_MAX);
241
87
  percent.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED);
242
126
  for (const auto& f : value.fields()) {
243
126
    if (f.first == "numerator") {
244
57
      if (f.second.has_number_value()) {
245
57
        percent.set_numerator(f.second.number_value());
246
57
      }
247
85
    } else if (f.first == "denominator" && f.second.has_string_value()) {
248
68
      if (f.second.string_value() == "HUNDRED") {
249
34
        percent.set_denominator(envoy::type::v3::FractionalPercent::HUNDRED);
250
34
      } else if (f.second.string_value() == "TEN_THOUSAND") {
251
1
        percent.set_denominator(envoy::type::v3::FractionalPercent::TEN_THOUSAND);
252
33
      } else if (f.second.string_value() == "MILLION") {
253
1
        percent.set_denominator(envoy::type::v3::FractionalPercent::MILLION);
254
32
      } else {
255
32
        return;
256
32
      }
257
68
    } else {
258
1
      return;
259
1
    }
260
126
  }
261

            
262
54
  entry.fractional_percent_value_ = percent;
263
54
}
264

            
265
398
void setNumberValue(Envoy::Runtime::Snapshot::Entry& entry, double value) {
266
398
  entry.double_value_ = value;
267
398
  if (value < std::numeric_limits<int>::max() && value == static_cast<int>(value)) {
268
238
    entry.bool_value_ = value != 0;
269
238
  }
270
398
  if (entry.double_value_ >= 0 && entry.double_value_ <= std::numeric_limits<uint64_t>::max()) {
271
    // Valid uint values will always be parseable as doubles, so we assign the value to both the
272
    // uint and double fields. In cases where the value is something like "3.1", we will floor the
273
    // number by casting it to a uint and assigning the uint value.
274
336
    entry.uint_value_ = entry.double_value_;
275
336
  }
276
398
}
277

            
278
// Handle corner cases in parsing: negatives and decimals aren't always parsed as doubles.
279
790
bool parseEntryDoubleValue(Envoy::Runtime::Snapshot::Entry& entry) {
280
790
  double converted_double;
281
790
  if (absl::SimpleAtod(entry.raw_string_value_, &converted_double)) {
282
224
    setNumberValue(entry, converted_double);
283
224
    return true;
284
224
  }
285
566
  return false;
286
790
}
287

            
288
void SnapshotImpl::addEntry(Snapshot::EntryMap& values, const std::string& key,
289
33806
                            const Protobuf::Value& value, absl::string_view raw_string) {
290
33806
  values.emplace(key, SnapshotImpl::createEntry(value, raw_string));
291
33806
}
292

            
293
SnapshotImpl::Entry SnapshotImpl::createEntry(const Protobuf::Value& value,
294
33807
                                              absl::string_view raw_string) {
295
33807
  Entry entry;
296
33807
  entry.raw_string_value_ = value.string_value();
297
33807
  if (!raw_string.empty()) {
298
4168
    entry.raw_string_value_ = raw_string;
299
4168
  }
300
33807
  switch (value.kind_case()) {
301
174
  case Protobuf::Value::kNumberValue:
302
174
    setNumberValue(entry, value.number_value());
303
174
    if (entry.raw_string_value_.empty()) {
304
33
      entry.raw_string_value_ = absl::StrCat(value.number_value());
305
33
    }
306
174
    break;
307
32724
  case Protobuf::Value::kBoolValue:
308
32724
    entry.bool_value_ = value.bool_value();
309
32724
    if (entry.raw_string_value_.empty()) {
310
      // Convert boolean to "true"/"false"
311
29278
      entry.raw_string_value_ = value.bool_value() ? "true" : "false";
312
29278
    }
313
32724
    break;
314
87
  case Protobuf::Value::kStructValue:
315
87
    if (entry.raw_string_value_.empty()) {
316
25
      entry.raw_string_value_ = value.struct_value().DebugString();
317
25
    }
318
87
    parseFractionValue(entry, value.struct_value());
319
87
    break;
320
790
  case Protobuf::Value::kStringValue:
321
790
    parseEntryDoubleValue(entry);
322
790
    break;
323
32
  default:
324
32
    break;
325
33807
  }
326

            
327
33807
  return entry;
328
33807
}
329

            
330
3427
absl::Status AdminLayer::mergeValues(const absl::node_hash_map<std::string, std::string>& values) {
331
3427
#ifdef ENVOY_ENABLE_YAML
332
3433
  for (const auto& kv : values) {
333
3433
    values_.erase(kv.first);
334
3433
    if (!kv.second.empty()) {
335
3421
      SnapshotImpl::addEntry(values_, kv.first, ValueUtil::loadFromYaml(kv.second), kv.second);
336
3421
    }
337
3433
  }
338
3427
  stats_.admin_overrides_active_.set(values_.empty() ? 0 : 1);
339
3427
  return absl::OkStatus();
340
#else
341
  UNREFERENCED_PARAMETER(values);
342
  return absl::InvalidArgumentError("Runtime admin reload requires YAML support");
343
#endif
344
3427
}
345

            
346
DiskLayer::DiskLayer(absl::string_view name, const std::string& path, Api::Api& api,
347
                     absl::Status& creation_status)
348
68
    : OverrideLayerImpl{name} {
349
68
  creation_status = walkDirectory(path, "", 1, api);
350
68
}
351

            
352
absl::Status DiskLayer::walkDirectory(const std::string& path, const std::string& prefix,
353
162
                                      uint32_t depth, Api::Api& api) {
354
  // Maximum recursion depth for walkDirectory().
355
162
  static constexpr uint32_t MaxWalkDepth = 16;
356

            
357
162
  ENVOY_LOG(debug, "walking directory: {}", path);
358
162
  if (depth > MaxWalkDepth) {
359
2
    return absl::InvalidArgumentError(absl::StrCat("Walk recursion depth exceeded ", MaxWalkDepth));
360
2
  }
361
  // Check if this is an obviously bad path.
362
160
  if (api.fileSystem().illegalPath(path)) {
363
1
    return absl::InvalidArgumentError(absl::StrCat("Invalid path: ", path));
364
1
  }
365

            
366
159
  Filesystem::Directory directory(path);
367
159
  Filesystem::DirectoryIteratorImpl it = directory.begin();
368
159
  RETURN_IF_NOT_OK_REF(it.status());
369
1253
  for (; it != directory.end(); ++it) {
370
1126
    RETURN_IF_NOT_OK_REF(it.status());
371
1126
    Filesystem::DirectoryEntry entry = *it;
372
1126
    std::string full_path = path + "/" + entry.name_;
373
1126
    std::string full_prefix;
374
1126
    if (prefix.empty()) {
375
910
      full_prefix = entry.name_;
376
910
    } else {
377
216
      full_prefix = prefix + "." + entry.name_;
378
216
    }
379

            
380
1126
    if (entry.type_ == Filesystem::FileType::Directory && entry.name_ != "." &&
381
1126
        entry.name_ != "..") {
382
94
      absl::Status status = walkDirectory(full_path, full_prefix, depth + 1, api);
383
94
      RETURN_IF_NOT_OK(status);
384
1032
    } else if (entry.type_ == Filesystem::FileType::Regular) {
385
      // Suck the file into a string. This is not very efficient but it should be good enough
386
      // for small files. Also, as noted elsewhere, none of this is non-blocking which could
387
      // theoretically lead to issues.
388
778
      ENVOY_LOG(debug, "reading file: {}", full_path);
389
778
      std::string value;
390

            
391
      // Read the file and remove any comments. A comment is a line starting with a '#' character.
392
      // Comments are useful for placeholder files with no value.
393
778
      auto file_or_error = api.fileSystem().fileReadToEnd(full_path);
394
778
      RETURN_IF_NOT_OK_REF(file_or_error.status());
395
778
      const std::string text_file{file_or_error.value()};
396

            
397
778
      const auto lines = StringUtil::splitToken(text_file, "\n");
398
1057
      for (const auto& line : lines) {
399
1057
        if (!line.empty() && line.front() == '#') {
400
186
          continue;
401
186
        }
402
871
        if (line == lines.back()) {
403
747
          const absl::string_view trimmed = StringUtil::rtrim(line);
404
747
          value.append(trimmed.data(), trimmed.size());
405
747
        } else {
406
124
          value.append(std::string{line} + "\n");
407
124
        }
408
871
      }
409
      // Separate erase/insert calls required due to the value type being constant; this prevents
410
      // the use of the [] operator. Can leverage insert_or_assign in C++17 in the future.
411
778
      values_.erase(full_prefix);
412
778
#ifdef ENVOY_ENABLE_YAML
413
778
      SnapshotImpl::addEntry(values_, full_prefix, ValueUtil::loadFromYaml(value), value);
414
#else
415
      IS_ENVOY_BUG("Runtime admin reload requires YAML support");
416
      UNREFERENCED_PARAMETER(value);
417
      return absl::OkStatus();
418
#endif
419
778
    }
420
1126
  }
421
127
  RETURN_IF_NOT_OK_REF(it.status());
422
127
  return absl::OkStatus();
423
127
}
424

            
425
ProtoLayer::ProtoLayer(absl::string_view name, const Protobuf::Struct& proto,
426
                       absl::Status& creation_status)
427
10777
    : OverrideLayerImpl{name} {
428
10777
  creation_status = absl::OkStatus();
429
30464
  for (const auto& f : proto.fields()) {
430
29609
    creation_status = walkProtoValue(f.second, f.first);
431
29609
    if (!creation_status.ok()) {
432
2
      return;
433
2
    }
434
29609
  }
435
10777
}
436

            
437
29620
absl::Status ProtoLayer::walkProtoValue(const Protobuf::Value& v, const std::string& prefix) {
438
29620
  switch (v.kind_case()) {
439
  case Protobuf::Value::KIND_NOT_SET:
440
1
  case Protobuf::Value::kListValue:
441
2
  case Protobuf::Value::kNullValue:
442
2
    return absl::InvalidArgumentError(absl::StrCat("Invalid runtime entry value for ", prefix));
443
    break;
444
271
  case Protobuf::Value::kStringValue:
445
271
    SnapshotImpl::addEntry(values_, prefix, v, "");
446
271
    break;
447
33
  case Protobuf::Value::kNumberValue:
448
29311
  case Protobuf::Value::kBoolValue:
449
29311
    if (hasRuntimePrefix(prefix) && !isRuntimeFeature(prefix) && !isLegacyRuntimeFeature(prefix)) {
450
2
      IS_ENVOY_BUG(absl::StrCat(
451
2
          "Using a removed guard ", prefix,
452
2
          ". In future version of Envoy this will be treated as invalid configuration"));
453
2
    }
454
29311
    SnapshotImpl::addEntry(values_, prefix, v, "");
455
29311
    break;
456
36
  case Protobuf::Value::kStructValue: {
457
36
    const Protobuf::Struct& s = v.struct_value();
458
36
    if (s.fields().empty() || s.fields().find("numerator") != s.fields().end() ||
459
36
        s.fields().find("denominator") != s.fields().end()) {
460
25
      SnapshotImpl::addEntry(values_, prefix, v, "");
461
25
      break;
462
25
    }
463
11
    for (const auto& f : s.fields()) {
464
11
      absl::Status status = walkProtoValue(f.second, prefix + "." + f.first);
465
11
      RETURN_IF_NOT_OK(status);
466
11
    }
467
11
    break;
468
11
  }
469
29620
  }
470
29618
  return absl::OkStatus();
471
29620
}
472

            
473
LoaderImpl::LoaderImpl(ThreadLocal::SlotAllocator& tls,
474
                       const envoy::config::bootstrap::v3::LayeredRuntime& config,
475
                       const LocalInfo::LocalInfo& local_info, Stats::Store& store,
476
                       Random::RandomGenerator& generator, Api::Api& api)
477
14231
    : generator_(generator), stats_(generateStats(store)), tls_(tls.allocateSlot()),
478
14231
      config_(config), service_cluster_(local_info.clusterName()), api_(api),
479
14231
      init_watcher_("RTDS", [this]() { onRtdsReady(); }), store_(store) {}
480

            
481
absl::StatusOr<std::unique_ptr<LoaderImpl>>
482
LoaderImpl::create(Event::Dispatcher& dispatcher, ThreadLocal::SlotAllocator& tls,
483
                   const envoy::config::bootstrap::v3::LayeredRuntime& config,
484
                   const LocalInfo::LocalInfo& local_info, Stats::Store& store,
485
                   Random::RandomGenerator& generator,
486
14231
                   ProtobufMessage::ValidationVisitor& validation_visitor, Api::Api& api) {
487
14231
  auto loader =
488
14231
      std::unique_ptr<LoaderImpl>(new LoaderImpl(tls, config, local_info, store, generator, api));
489
14231
  auto result = loader->initLayers(dispatcher, validation_visitor);
490
14231
  RETURN_IF_NOT_OK(result);
491
14226
  return loader;
492
14231
}
493

            
494
absl::Status LoaderImpl::initLayers(Event::Dispatcher& dispatcher,
495
14231
                                    ProtobufMessage::ValidationVisitor& validation_visitor) {
496
14231
  absl::Status creation_status;
497
14231
  absl::node_hash_set<std::string> layer_names;
498
24875
  for (const auto& layer : config_.layers()) {
499
24716
    auto ret = layer_names.insert(layer.name());
500
24716
    if (!ret.second) {
501
1
      return absl::InvalidArgumentError(absl::StrCat("Duplicate layer name: ", layer.name()));
502
1
    }
503
24715
    switch (layer.layer_specifier_case()) {
504
10611
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kStaticLayer:
505
      // Nothing needs to be done here.
506
10611
      break;
507
14033
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kAdminLayer:
508
14033
      if (admin_layer_ != nullptr) {
509
1
        return absl::InvalidArgumentError(
510
1
            "Too many admin layers specified in LayeredRuntime, at most one may be specified");
511
1
      }
512
14032
      admin_layer_ = std::make_unique<AdminLayer>(layer.name(), stats_);
513
14032
      break;
514
29
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kDiskLayer:
515
29
      if (watcher_ == nullptr) {
516
15
        watcher_ = dispatcher.createFilesystemWatcher();
517
15
      }
518
29
      creation_status = watcher_->addWatch(layer.disk_layer().symlink_root(),
519
29
                                           Filesystem::Watcher::Events::MovedTo,
520
29
                                           [this](uint32_t) { return loadNewSnapshot(); });
521
29
      RETURN_IF_NOT_OK(creation_status);
522
28
      break;
523
42
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kRtdsLayer:
524
42
      subscriptions_.emplace_back(std::make_unique<RtdsSubscription>(*this, layer.rtds_layer(),
525
42
                                                                     store_, validation_visitor));
526
42
      init_manager_.add(subscriptions_.back()->init_target_);
527
42
      break;
528
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::LAYER_SPECIFIER_NOT_SET:
529
      return absl::InvalidArgumentError("layer specifier not set");
530
24715
    }
531
24715
  }
532

            
533
14228
  return loadNewSnapshot();
534
14231
}
535

            
536
10720
absl::Status LoaderImpl::initialize(Upstream::ClusterManager& cm) {
537
10720
  cm_ = &cm;
538

            
539
10721
  for (const auto& s : subscriptions_) {
540
41
    RETURN_IF_NOT_OK(s->createSubscription());
541
40
  }
542
10719
  return absl::OkStatus();
543
10720
}
544

            
545
10663
void LoaderImpl::startRtdsSubscriptions(ReadyCallback on_done) {
546
10663
  on_rtds_initialized_ = on_done;
547
10663
  init_manager_.initialize(init_watcher_);
548
10663
}
549

            
550
10662
void LoaderImpl::onRtdsReady() {
551
10662
  ENVOY_LOG(info, "RTDS has finished initialization");
552
10662
  on_rtds_initialized_();
553
10662
}
554

            
555
RtdsSubscription::RtdsSubscription(
556
    LoaderImpl& parent, const envoy::config::bootstrap::v3::RuntimeLayer::RtdsLayer& rtds_layer,
557
    Stats::Store& store, ProtobufMessage::ValidationVisitor& validation_visitor)
558
42
    : Envoy::Config::SubscriptionBase<envoy::service::runtime::v3::Runtime>(validation_visitor,
559
42
                                                                            "name"),
560
42
      parent_(parent), config_source_(rtds_layer.rtds_config()), store_(store),
561
42
      stats_scope_(store_.createScope("runtime")), resource_name_(rtds_layer.name()),
562
42
      init_target_("RTDS " + resource_name_, [this]() { start(); }) {}
563

            
564
41
absl::Status RtdsSubscription::createSubscription() {
565
41
  const auto resource_name = getResourceName();
566
41
  auto subscription_or_error = parent_.cm_->subscriptionFactory().subscriptionFromConfigSource(
567
41
      config_source_, Grpc::Common::typeUrl(resource_name), *stats_scope_, *this, resource_decoder_,
568
41
      {});
569
41
  RETURN_IF_NOT_OK(subscription_or_error.status());
570
40
  subscription_ = std::move(*subscription_or_error);
571
40
  return absl::OkStatus();
572
41
}
573

            
574
absl::Status
575
RtdsSubscription::onConfigUpdate(const std::vector<Config::DecodedResourceRef>& resources,
576
52
                                 const std::string&) {
577
52
  absl::Status valid = validateUpdateSize(resources.size(), 0);
578
52
  if (!valid.ok()) {
579
2
    return valid;
580
2
  }
581
50
  const auto& runtime =
582
50
      dynamic_cast<const envoy::service::runtime::v3::Runtime&>(resources[0].get().resource());
583
50
  if (runtime.name() != resource_name_) {
584
1
    return absl::InvalidArgumentError(
585
1
        fmt::format("Unexpected RTDS runtime (expecting {}): {}", resource_name_, runtime.name()));
586
1
  }
587
49
  ENVOY_LOG(debug, "Reloading RTDS snapshot for onConfigUpdate");
588
49
  proto_.CopyFrom(runtime.layer());
589
49
  RETURN_IF_NOT_OK(parent_.loadNewSnapshot());
590
49
  init_target_.ready();
591
49
  return absl::OkStatus();
592
49
}
593

            
594
absl::Status
595
RtdsSubscription::onConfigUpdate(const std::vector<Config::DecodedResourceRef>& added_resources,
596
                                 const Protobuf::RepeatedPtrField<std::string>& removed_resources,
597
18
                                 const std::string&) {
598
18
  absl::Status valid = validateUpdateSize(added_resources.size(), removed_resources.size());
599
18
  if (!valid.ok()) {
600
    return valid;
601
  }
602

            
603
  // This is a singleton subscription, so we can only have the subscribed resource added or removed,
604
  // but not both.
605
18
  if (!added_resources.empty()) {
606
16
    return onConfigUpdate(added_resources, added_resources[0].get().version());
607
16
  } else {
608
2
    return onConfigRemoved(removed_resources);
609
2
  }
610
18
}
611

            
612
void RtdsSubscription::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason,
613
1
                                            const EnvoyException*) {
614
1
  ASSERT(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure != reason);
615
  // We need to allow server startup to continue, even if we have a bad
616
  // config.
617
1
  init_target_.ready();
618
1
}
619

            
620
39
void RtdsSubscription::start() { subscription_->start({resource_name_}); }
621

            
622
absl::Status RtdsSubscription::validateUpdateSize(uint32_t added_resources_num,
623
70
                                                  uint32_t removed_resources_num) {
624
70
  if (added_resources_num + removed_resources_num != 1) {
625
2
    init_target_.ready();
626
2
    return absl::InvalidArgumentError(
627
2
        fmt::format("Unexpected RTDS resource length, number of added resources "
628
2
                    "{}, number of removed resources {}",
629
2
                    added_resources_num, removed_resources_num));
630
2
  }
631
68
  return absl::OkStatus();
632
70
}
633

            
634
absl::Status RtdsSubscription::onConfigRemoved(
635
2
    const Protobuf::RepeatedPtrField<std::string>& removed_resources) {
636
2
  if (removed_resources[0] != resource_name_) {
637
1
    return absl::InvalidArgumentError(
638
1
        fmt::format("Unexpected removal of unknown RTDS runtime layer {}, expected {}",
639
1
                    removed_resources[0], resource_name_));
640
1
  }
641
1
  ENVOY_LOG(debug, "Clear RTDS snapshot for onConfigUpdate");
642
1
  proto_.Clear();
643
1
  RETURN_IF_NOT_OK(parent_.loadNewSnapshot());
644
1
  init_target_.ready();
645
1
  return absl::OkStatus();
646
1
}
647

            
648
17706
absl::Status LoaderImpl::loadNewSnapshot() {
649
17706
  auto snapshot_or_error = createNewSnapshot();
650
17706
  RETURN_IF_NOT_OK_REF(snapshot_or_error.status());
651
17704
  std::shared_ptr<SnapshotImpl> ptr = std::move(snapshot_or_error.value());
652
28375
  tls_->set([ptr](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr {
653
28375
    return std::static_pointer_cast<ThreadLocal::ThreadLocalObject>(ptr);
654
28375
  });
655

            
656
17704
  refreshReloadableFlags(ptr->values());
657

            
658
17704
  {
659
17704
    absl::MutexLock lock(snapshot_mutex_);
660
17704
    thread_safe_snapshot_ = ptr;
661
17704
  }
662
17704
  return absl::OkStatus();
663
17706
}
664

            
665
3938817
const Snapshot& LoaderImpl::snapshot() const {
666
3938817
  ASSERT(tls_->currentThreadRegistered(),
667
3938817
         "snapshot can only be called from a worker thread or after the main thread is registered");
668
3938817
  return tls_->getTyped<Snapshot>();
669
3938817
}
670

            
671
23005
SnapshotConstSharedPtr LoaderImpl::threadsafeSnapshot() {
672
23005
  if (tls_->currentThreadRegistered()) {
673
22998
    return std::dynamic_pointer_cast<const Snapshot>(tls_->get());
674
22998
  }
675

            
676
7
  {
677
7
    absl::ReaderMutexLock lock(snapshot_mutex_);
678
7
    return thread_safe_snapshot_;
679
23005
  }
680
23005
}
681

            
682
3428
absl::Status LoaderImpl::mergeValues(const absl::node_hash_map<std::string, std::string>& values) {
683
3428
  if (admin_layer_ == nullptr) {
684
1
    return absl::InvalidArgumentError("No admin layer specified");
685
1
  }
686
3427
  RETURN_IF_NOT_OK(admin_layer_->mergeValues(values));
687
3427
  return loadNewSnapshot();
688
3427
}
689

            
690
Stats::Scope& LoaderImpl::getRootScope() { return *store_.rootScope(); }
691

            
692
1319
void LoaderImpl::countDeprecatedFeatureUse() const { countDeprecatedFeatureUseInternal(stats_); }
693

            
694
14231
RuntimeStats LoaderImpl::generateStats(Stats::Store& store) {
695
14231
  std::string prefix = "runtime.";
696
14231
  RuntimeStats stats{
697
14231
      ALL_RUNTIME_STATS(POOL_COUNTER_PREFIX(store, prefix), POOL_GAUGE_PREFIX(store, prefix))};
698
14231
  return stats;
699
14231
}
700

            
701
17706
absl::StatusOr<SnapshotImplPtr> LoaderImpl::createNewSnapshot() {
702
17706
  std::vector<Snapshot::OverrideLayerConstPtr> layers;
703
17706
  uint32_t disk_layers = 0;
704
17706
  uint32_t error_layers = 0;
705
17706
  uint32_t rtds_layer = 0;
706
17706
  absl::Status creation_status;
707
28497
  for (const auto& layer : config_.layers()) {
708
28337
    switch (layer.layer_specifier_case()) {
709
10683
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kStaticLayer:
710
10683
      layers.emplace_back(
711
10683
          std::make_unique<const ProtoLayer>(layer.name(), layer.static_layer(), creation_status));
712
10683
      RETURN_IF_NOT_OK(creation_status);
713
10681
      break;
714
70
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kDiskLayer: {
715
70
      std::string path =
716
70
          layer.disk_layer().symlink_root() + "/" + layer.disk_layer().subdirectory();
717
70
      if (layer.disk_layer().append_service_cluster()) {
718
35
        absl::StrAppend(&path, "/", service_cluster_);
719
35
      }
720
70
      if (api_.fileSystem().directoryExists(path)) {
721
66
        std::unique_ptr<DiskLayer> disk_layer;
722
66
        std::string error;
723
66
        TRY_ASSERT_MAIN_THREAD {
724
66
          absl::Status creation_status;
725
66
          disk_layer = std::make_unique<DiskLayer>(layer.name(), path, api_, creation_status);
726
66
          if (!creation_status.ok()) {
727
1
            error = creation_status.message();
728
1
          }
729
66
          END_TRY
730
66
        }
731
66
        CATCH(EnvoyException & e, { error = e.what(); });
732
66
        if (error.empty()) {
733
65
          layers.emplace_back(std::move(disk_layer));
734
65
          ++disk_layers;
735
65
        } else {
736
          // TODO(htuch): Consider latching here, rather than ignoring the
737
          // layer. This would be consistent with filesystem RTDS.
738
1
          ++error_layers;
739
1
          ENVOY_LOG(debug, "error loading runtime values for layer {} from disk: {}",
740
1
                    layer.DebugString(), error);
741
1
        }
742
66
      }
743
70
      break;
744
70
    }
745
17510
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kAdminLayer:
746
17490
      layers.push_back(std::make_unique<AdminLayer>(*admin_layer_));
747
17490
      break;
748
94
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::kRtdsLayer: {
749
94
      auto* subscription = subscriptions_[rtds_layer++].get();
750
94
      layers.emplace_back(
751
94
          std::make_unique<const ProtoLayer>(layer.name(), subscription->proto_, creation_status));
752
94
      RETURN_IF_NOT_OK(creation_status);
753
94
      break;
754
94
    }
755
    case envoy::config::bootstrap::v3::RuntimeLayer::LayerSpecifierCase::LAYER_SPECIFIER_NOT_SET:
756
      PANIC_DUE_TO_PROTO_UNSET;
757
28337
    }
758
28337
  }
759
17704
  stats_.num_layers_.set(layers.size());
760
17704
  if (error_layers == 0) {
761
17703
    stats_.load_success_.inc();
762
17703
  } else {
763
1
    stats_.load_error_.inc();
764
1
  }
765
17704
  if (disk_layers > 1) {
766
32
    stats_.override_dir_exists_.inc();
767
17672
  } else {
768
17672
    stats_.override_dir_not_exists_.inc();
769
17672
  }
770
17704
  return std::make_unique<SnapshotImpl>(generator_, stats_, std::move(layers));
771
17706
}
772

            
773
} // namespace Runtime
774
} // namespace Envoy