Line data    Source code

       1             : #include "source/server/admin/clusters_handler.h"
       2             : 
       3             : #include "envoy/admin/v3/clusters.pb.h"
       4             : 
       5             : #include "source/common/http/headers.h"
       6             : #include "source/common/http/utility.h"
       7             : #include "source/common/network/utility.h"
       8             : #include "source/common/upstream/host_utility.h"
       9             : #include "source/server/admin/utils.h"
      10             : 
      11             : namespace Envoy {
      12             : namespace Server {
      13             : 
      14             : namespace {
      15             : 
      16             : void addCircuitBreakerSettingsAsText(const std::string& cluster_name,
      17             :                                      const std::string& priority_str,
      18             :                                      Upstream::ResourceManager& resource_manager,
      19           0 :                                      Buffer::Instance& response) {
      20           0 :   response.add(fmt::format("{}::{}_priority::max_connections::{}\n", cluster_name, priority_str,
      21           0 :                            resource_manager.connections().max()));
      22           0 :   response.add(fmt::format("{}::{}_priority::max_pending_requests::{}\n", cluster_name,
      23           0 :                            priority_str, resource_manager.pendingRequests().max()));
      24           0 :   response.add(fmt::format("{}::{}_priority::max_requests::{}\n", cluster_name, priority_str,
      25           0 :                            resource_manager.requests().max()));
      26           0 :   response.add(fmt::format("{}::{}_priority::max_retries::{}\n", cluster_name, priority_str,
      27           0 :                            resource_manager.retries().max()));
      28           0 : }
      29             : 
      30             : void addCircuitBreakerSettingsAsJson(const envoy::config::core::v3::RoutingPriority& priority,
      31             :                                      Upstream::ResourceManager& resource_manager,
      32           0 :                                      envoy::admin::v3::ClusterStatus& cluster_status) {
      33           0 :   auto& thresholds = *cluster_status.mutable_circuit_breakers()->add_thresholds();
      34           0 :   thresholds.set_priority(priority);
      35           0 :   thresholds.mutable_max_connections()->set_value(resource_manager.connections().max());
      36           0 :   thresholds.mutable_max_pending_requests()->set_value(resource_manager.pendingRequests().max());
      37           0 :   thresholds.mutable_max_requests()->set_value(resource_manager.requests().max());
      38           0 :   thresholds.mutable_max_retries()->set_value(resource_manager.retries().max());
      39           0 : }
      40             : 
      41             : } // namespace
      42             : 
      43         134 : ClustersHandler::ClustersHandler(Server::Instance& server) : HandlerContextBase(server) {}
      44             : 
      45             : Http::Code ClustersHandler::handlerClusters(Http::ResponseHeaderMap& response_headers,
      46           0 :                                             Buffer::Instance& response, AdminStream& admin_stream) {
      47           0 :   const auto format_value = Utility::formatParam(admin_stream.queryParams());
      48             : 
      49           0 :   if (format_value.has_value() && format_value.value() == "json") {
      50           0 :     writeClustersAsJson(response);
      51           0 :     response_headers.setReferenceContentType(Http::Headers::get().ContentTypeValues.Json);
      52           0 :   } else {
      53           0 :     writeClustersAsText(response);
      54           0 :   }
      55             : 
      56           0 :   return Http::Code::OK;
      57           0 : }
      58             : 
      59             : // Helper method that ensures that we've setting flags based on all the health flag values on the
      60             : // host.
      61             : void setHealthFlag(Upstream::Host::HealthFlag flag, const Upstream::Host& host,
      62           0 :                    envoy::admin::v3::HostHealthStatus& health_status) {
      63           0 :   switch (flag) {
      64           0 :   case Upstream::Host::HealthFlag::FAILED_ACTIVE_HC:
      65           0 :     health_status.set_failed_active_health_check(
      66           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::FAILED_ACTIVE_HC));
      67           0 :     break;
      68           0 :   case Upstream::Host::HealthFlag::FAILED_OUTLIER_CHECK:
      69           0 :     health_status.set_failed_outlier_check(
      70           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::FAILED_OUTLIER_CHECK));
      71           0 :     break;
      72           0 :   case Upstream::Host::HealthFlag::FAILED_EDS_HEALTH:
      73           0 :   case Upstream::Host::HealthFlag::DEGRADED_EDS_HEALTH:
      74           0 :     if (host.healthFlagGet(Upstream::Host::HealthFlag::FAILED_EDS_HEALTH)) {
      75           0 :       health_status.set_eds_health_status(envoy::config::core::v3::UNHEALTHY);
      76           0 :     } else if (host.healthFlagGet(Upstream::Host::HealthFlag::DEGRADED_EDS_HEALTH)) {
      77           0 :       health_status.set_eds_health_status(envoy::config::core::v3::DEGRADED);
      78           0 :     } else {
      79           0 :       health_status.set_eds_health_status(envoy::config::core::v3::HEALTHY);
      80           0 :     }
      81           0 :     break;
      82           0 :   case Upstream::Host::HealthFlag::DEGRADED_ACTIVE_HC:
      83           0 :     health_status.set_failed_active_degraded_check(
      84           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::DEGRADED_ACTIVE_HC));
      85           0 :     break;
      86           0 :   case Upstream::Host::HealthFlag::PENDING_DYNAMIC_REMOVAL:
      87           0 :     health_status.set_pending_dynamic_removal(
      88           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::PENDING_DYNAMIC_REMOVAL));
      89           0 :     break;
      90           0 :   case Upstream::Host::HealthFlag::PENDING_ACTIVE_HC:
      91           0 :     health_status.set_pending_active_hc(
      92           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::PENDING_ACTIVE_HC));
      93           0 :     break;
      94           0 :   case Upstream::Host::HealthFlag::EXCLUDED_VIA_IMMEDIATE_HC_FAIL:
      95           0 :     health_status.set_excluded_via_immediate_hc_fail(
      96           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::EXCLUDED_VIA_IMMEDIATE_HC_FAIL));
      97           0 :     break;
      98           0 :   case Upstream::Host::HealthFlag::ACTIVE_HC_TIMEOUT:
      99           0 :     health_status.set_active_hc_timeout(
     100           0 :         host.healthFlagGet(Upstream::Host::HealthFlag::ACTIVE_HC_TIMEOUT));
     101           0 :     break;
     102           0 :   }
     103           0 : }
     104             : 
     105             : // TODO(efimki): Add support of text readouts stats.
     106           0 : void ClustersHandler::writeClustersAsJson(Buffer::Instance& response) {
     107           0 :   envoy::admin::v3::Clusters clusters;
     108             :   // TODO(mattklein123): Add ability to see warming clusters in admin output.
     109           0 :   auto all_clusters = server_.clusterManager().clusters();
     110           0 :   for (const auto& [name, cluster_ref] : all_clusters.active_clusters_) {
     111           0 :     UNREFERENCED_PARAMETER(name);
     112           0 :     const Upstream::Cluster& cluster = cluster_ref.get();
     113           0 :     Upstream::ClusterInfoConstSharedPtr cluster_info = cluster.info();
     114             : 
     115           0 :     envoy::admin::v3::ClusterStatus& cluster_status = *clusters.add_cluster_statuses();
     116           0 :     cluster_status.set_name(cluster_info->name());
     117           0 :     cluster_status.set_observability_name(cluster_info->observabilityName());
     118           0 :     if (const auto& name = cluster_info->edsServiceName(); !name.empty()) {
     119           0 :       cluster_status.set_eds_service_name(name);
     120           0 :     }
     121           0 :     addCircuitBreakerSettingsAsJson(
     122           0 :         envoy::config::core::v3::RoutingPriority::DEFAULT,
     123           0 :         cluster.info()->resourceManager(Upstream::ResourcePriority::Default), cluster_status);
     124           0 :     addCircuitBreakerSettingsAsJson(
     125           0 :         envoy::config::core::v3::RoutingPriority::HIGH,
     126           0 :         cluster.info()->resourceManager(Upstream::ResourcePriority::High), cluster_status);
     127             : 
     128           0 :     const Upstream::Outlier::Detector* outlier_detector = cluster.outlierDetector();
     129           0 :     if (outlier_detector != nullptr &&
     130           0 :         outlier_detector->successRateEjectionThreshold(
     131           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin) > 0.0) {
     132           0 :       cluster_status.mutable_success_rate_ejection_threshold()->set_value(
     133           0 :           outlier_detector->successRateEjectionThreshold(
     134           0 :               Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin));
     135           0 :     }
     136           0 :     if (outlier_detector != nullptr &&
     137           0 :         outlier_detector->successRateEjectionThreshold(
     138           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin) > 0.0) {
     139           0 :       cluster_status.mutable_local_origin_success_rate_ejection_threshold()->set_value(
     140           0 :           outlier_detector->successRateEjectionThreshold(
     141           0 :               Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin));
     142           0 :     }
     143             : 
     144           0 :     cluster_status.set_added_via_api(cluster_info->addedViaApi());
     145             : 
     146           0 :     for (auto& host_set : cluster.prioritySet().hostSetsPerPriority()) {
     147           0 :       for (auto& host : host_set->hosts()) {
     148           0 :         envoy::admin::v3::HostStatus& host_status = *cluster_status.add_host_statuses();
     149           0 :         Network::Utility::addressToProtobufAddress(*host->address(),
     150           0 :                                                    *host_status.mutable_address());
     151           0 :         host_status.set_hostname(host->hostname());
     152           0 :         host_status.mutable_locality()->MergeFrom(host->locality());
     153             : 
     154           0 :         for (const auto& [counter_name, counter] : host->counters()) {
     155           0 :           auto& metric = *host_status.add_stats();
     156           0 :           metric.set_name(std::string(counter_name));
     157           0 :           metric.set_value(counter.get().value());
     158           0 :           metric.set_type(envoy::admin::v3::SimpleMetric::COUNTER);
     159           0 :         }
     160             : 
     161           0 :         for (const auto& [gauge_name, gauge] : host->gauges()) {
     162           0 :           auto& metric = *host_status.add_stats();
     163           0 :           metric.set_name(std::string(gauge_name));
     164           0 :           metric.set_value(gauge.get().value());
     165           0 :           metric.set_type(envoy::admin::v3::SimpleMetric::GAUGE);
     166           0 :         }
     167             : 
     168           0 :         envoy::admin::v3::HostHealthStatus& health_status = *host_status.mutable_health_status();
     169             : 
     170             : // Invokes setHealthFlag for each health flag.
     171           0 : #define SET_HEALTH_FLAG(name, notused)                                                             \
     172           0 :   setHealthFlag(Upstream::Host::HealthFlag::name, *host, health_status);
     173           0 :         HEALTH_FLAG_ENUM_VALUES(SET_HEALTH_FLAG)
     174           0 : #undef SET_HEALTH_FLAG
     175             : 
     176           0 :         double success_rate = host->outlierDetector().successRate(
     177           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin);
     178           0 :         if (success_rate >= 0.0) {
     179           0 :           host_status.mutable_success_rate()->set_value(success_rate);
     180           0 :         }
     181             : 
     182           0 :         host_status.set_weight(host->weight());
     183             : 
     184           0 :         host_status.set_priority(host->priority());
     185           0 :         success_rate = host->outlierDetector().successRate(
     186           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin);
     187           0 :         if (success_rate >= 0.0) {
     188           0 :           host_status.mutable_local_origin_success_rate()->set_value(success_rate);
     189           0 :         }
     190           0 :       }
     191           0 :     }
     192           0 :   }
     193           0 :   response.add(MessageUtil::getJsonStringFromMessageOrError(clusters, true)); // pretty-print
     194           0 : }
     195             : 
     196             : // TODO(efimki): Add support of text readouts stats.
     197           0 : void ClustersHandler::writeClustersAsText(Buffer::Instance& response) {
     198             :   // TODO(mattklein123): Add ability to see warming clusters in admin output.
     199           0 :   auto all_clusters = server_.clusterManager().clusters();
     200           0 :   for (const auto& [name, cluster_ref] : all_clusters.active_clusters_) {
     201           0 :     UNREFERENCED_PARAMETER(name);
     202           0 :     const Upstream::Cluster& cluster = cluster_ref.get();
     203           0 :     const std::string& cluster_name = cluster.info()->name();
     204           0 :     response.add(fmt::format("{}::observability_name::{}\n", cluster_name,
     205           0 :                              cluster.info()->observabilityName()));
     206           0 :     addOutlierInfo(cluster_name, cluster.outlierDetector(), response);
     207             : 
     208           0 :     addCircuitBreakerSettingsAsText(
     209           0 :         cluster_name, "default",
     210           0 :         cluster.info()->resourceManager(Upstream::ResourcePriority::Default), response);
     211           0 :     addCircuitBreakerSettingsAsText(
     212           0 :         cluster_name, "high", cluster.info()->resourceManager(Upstream::ResourcePriority::High),
     213           0 :         response);
     214             : 
     215           0 :     response.add(
     216           0 :         fmt::format("{}::added_via_api::{}\n", cluster_name, cluster.info()->addedViaApi()));
     217           0 :     if (const auto& name = cluster.info()->edsServiceName(); !name.empty()) {
     218           0 :       response.add(fmt::format("{}::eds_service_name::{}\n", cluster_name, name));
     219           0 :     }
     220           0 :     for (auto& host_set : cluster.prioritySet().hostSetsPerPriority()) {
     221           0 :       for (auto& host : host_set->hosts()) {
     222           0 :         const std::string& host_address = host->address()->asString();
     223           0 :         std::map<absl::string_view, uint64_t> all_stats;
     224           0 :         for (const auto& [counter_name, counter] : host->counters()) {
     225           0 :           all_stats[counter_name] = counter.get().value();
     226           0 :         }
     227             : 
     228           0 :         for (const auto& [gauge_name, gauge] : host->gauges()) {
     229           0 :           all_stats[gauge_name] = gauge.get().value();
     230           0 :         }
     231             : 
     232           0 :         for (const auto& [stat_name, stat] : all_stats) {
     233           0 :           response.add(
     234           0 :               fmt::format("{}::{}::{}::{}\n", cluster_name, host_address, stat_name, stat));
     235           0 :         }
     236             : 
     237           0 :         response.add(
     238           0 :             fmt::format("{}::{}::hostname::{}\n", cluster_name, host_address, host->hostname()));
     239           0 :         response.add(fmt::format("{}::{}::health_flags::{}\n", cluster_name, host_address,
     240           0 :                                  Upstream::HostUtility::healthFlagsToString(*host)));
     241           0 :         response.add(
     242           0 :             fmt::format("{}::{}::weight::{}\n", cluster_name, host_address, host->weight()));
     243           0 :         response.add(fmt::format("{}::{}::region::{}\n", cluster_name, host_address,
     244           0 :                                  host->locality().region()));
     245           0 :         response.add(
     246           0 :             fmt::format("{}::{}::zone::{}\n", cluster_name, host_address, host->locality().zone()));
     247           0 :         response.add(fmt::format("{}::{}::sub_zone::{}\n", cluster_name, host_address,
     248           0 :                                  host->locality().sub_zone()));
     249           0 :         response.add(
     250           0 :             fmt::format("{}::{}::canary::{}\n", cluster_name, host_address, host->canary()));
     251           0 :         response.add(
     252           0 :             fmt::format("{}::{}::priority::{}\n", cluster_name, host_address, host->priority()));
     253           0 :         response.add(fmt::format(
     254           0 :             "{}::{}::success_rate::{}\n", cluster_name, host_address,
     255           0 :             host->outlierDetector().successRate(
     256           0 :                 Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)));
     257           0 :         response.add(fmt::format(
     258           0 :             "{}::{}::local_origin_success_rate::{}\n", cluster_name, host_address,
     259           0 :             host->outlierDetector().successRate(
     260           0 :                 Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)));
     261           0 :       }
     262           0 :     }
     263           0 :   }
     264           0 : }
     265             : 
     266             : void ClustersHandler::addOutlierInfo(const std::string& cluster_name,
     267             :                                      const Upstream::Outlier::Detector* outlier_detector,
     268           0 :                                      Buffer::Instance& response) {
     269           0 :   if (outlier_detector) {
     270           0 :     response.add(fmt::format(
     271           0 :         "{}::outlier::success_rate_average::{:g}\n", cluster_name,
     272           0 :         outlier_detector->successRateAverage(
     273           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)));
     274           0 :     response.add(fmt::format(
     275           0 :         "{}::outlier::success_rate_ejection_threshold::{:g}\n", cluster_name,
     276           0 :         outlier_detector->successRateEjectionThreshold(
     277           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::ExternalOrigin)));
     278           0 :     response.add(fmt::format(
     279           0 :         "{}::outlier::local_origin_success_rate_average::{:g}\n", cluster_name,
     280           0 :         outlier_detector->successRateAverage(
     281           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)));
     282           0 :     response.add(fmt::format(
     283           0 :         "{}::outlier::local_origin_success_rate_ejection_threshold::{:g}\n", cluster_name,
     284           0 :         outlier_detector->successRateEjectionThreshold(
     285           0 :             Upstream::Outlier::DetectorHostMonitor::SuccessRateMonitorType::LocalOrigin)));
     286           0 :   }
     287           0 : }
     288             : 
     289             : } // namespace Server
     290             : } // namespace Envoy