use polkadot_node_core_pvf_common::prepare::MemoryStats;
use polkadot_node_metrics::metrics::{self, prometheus};
#[derive(Default, Clone)]
pub struct Metrics(Option<MetricsInner>);
impl Metrics {
pub(crate) fn prepare_worker(&'_ self) -> WorkerRelatedMetrics<'_> {
WorkerRelatedMetrics { metrics: self, flavor: WorkerFlavor::Prepare }
}
pub(crate) fn execute_worker(&'_ self) -> WorkerRelatedMetrics<'_> {
WorkerRelatedMetrics { metrics: self, flavor: WorkerFlavor::Execute }
}
pub(crate) fn prepare_enqueued(&self) {
if let Some(metrics) = &self.0 {
metrics.prepare_enqueued.inc();
}
}
pub(crate) fn prepare_concluded(&self) {
if let Some(metrics) = &self.0 {
metrics.prepare_concluded.inc();
}
}
pub(crate) fn execute_enqueued(&self) {
if let Some(metrics) = &self.0 {
metrics.execute_enqueued.inc();
}
}
pub(crate) fn execute_finished(&self) {
if let Some(metrics) = &self.0 {
metrics.execute_finished.inc();
}
}
pub(crate) fn time_preparation(
&self,
) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.preparation_time.start_timer())
}
pub(crate) fn time_execution(&self) -> Option<metrics::prometheus::prometheus::HistogramTimer> {
self.0.as_ref().map(|metrics| metrics.execution_time.start_timer())
}
#[allow(unused_variables)]
pub(crate) fn observe_preparation_memory_metrics(&self, memory_stats: MemoryStats) {
if let Some(metrics) = &self.0 {
#[cfg(target_os = "linux")]
if let Some(max_rss) = memory_stats.max_rss {
metrics.preparation_max_rss.observe(max_rss as f64);
}
#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
if let Some(tracker_stats) = memory_stats.memory_tracker_stats {
let max_resident_kb = (tracker_stats.resident / 1024) as f64;
let max_allocated_kb = (tracker_stats.allocated / 1024) as f64;
metrics.preparation_max_resident.observe(max_resident_kb);
metrics.preparation_max_allocated.observe(max_allocated_kb);
}
}
}
}
#[derive(Clone)]
struct MetricsInner {
worker_spawning: prometheus::CounterVec<prometheus::U64>,
worker_spawned: prometheus::CounterVec<prometheus::U64>,
worker_retired: prometheus::CounterVec<prometheus::U64>,
prepare_enqueued: prometheus::Counter<prometheus::U64>,
prepare_concluded: prometheus::Counter<prometheus::U64>,
execute_enqueued: prometheus::Counter<prometheus::U64>,
execute_finished: prometheus::Counter<prometheus::U64>,
preparation_time: prometheus::Histogram,
execution_time: prometheus::Histogram,
#[cfg(target_os = "linux")]
preparation_max_rss: prometheus::Histogram,
#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
preparation_max_allocated: prometheus::Histogram,
#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
preparation_max_resident: prometheus::Histogram,
}
impl metrics::Metrics for Metrics {
fn try_register(registry: &prometheus::Registry) -> Result<Self, prometheus::PrometheusError> {
let inner = MetricsInner {
worker_spawning: prometheus::register(
prometheus::CounterVec::new(
prometheus::Opts::new(
"polkadot_pvf_worker_spawning",
"The total number of workers began to spawn",
),
&["flavor"],
)?,
registry,
)?,
worker_spawned: prometheus::register(
prometheus::CounterVec::new(
prometheus::Opts::new(
"polkadot_pvf_worker_spawned",
"The total number of workers spawned successfully",
),
&["flavor"],
)?,
registry,
)?,
worker_retired: prometheus::register(
prometheus::CounterVec::new(
prometheus::Opts::new(
"polkadot_pvf_worker_retired",
"The total number of workers retired, either killed by the host or died on duty",
),
&["flavor"],
)?,
registry,
)?,
prepare_enqueued: prometheus::register(
prometheus::Counter::new(
"polkadot_pvf_prepare_enqueued",
"The total number of jobs enqueued into the preparation pipeline"
)?,
registry,
)?,
prepare_concluded: prometheus::register(
prometheus::Counter::new(
"polkadot_pvf_prepare_concluded",
"The total number of jobs concluded in the preparation pipeline"
)?,
registry,
)?,
execute_enqueued: prometheus::register(
prometheus::Counter::new(
"polkadot_pvf_execute_enqueued",
"The total number of jobs enqueued into the execution pipeline"
)?,
registry,
)?,
execute_finished: prometheus::register(
prometheus::Counter::new(
"polkadot_pvf_execute_finished",
"The total number of jobs done in the execution pipeline"
)?,
registry,
)?,
preparation_time: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"polkadot_pvf_preparation_time",
"Time spent in preparing PVF artifacts in seconds",
)
.buckets(vec![
0.1,
0.5,
1.0,
2.0,
3.0,
10.0,
20.0,
30.0,
60.0,
120.0,
240.0,
360.0,
480.0,
]),
)?,
registry,
)?,
execution_time: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"polkadot_pvf_execution_time",
"Time spent in executing PVFs",
).buckets(vec![
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1.0,
2.0,
3.0,
4.0,
5.0,
6.0,
8.0,
10.0,
12.0,
]),
)?,
registry,
)?,
#[cfg(target_os = "linux")]
preparation_max_rss: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"polkadot_pvf_preparation_max_rss",
"ru_maxrss (maximum resident set size) observed for preparation (in kilobytes)",
).buckets(
prometheus::exponential_buckets(8192.0, 2.0, 10)
.expect("arguments are always valid; qed"),
),
)?,
registry,
)?,
#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
preparation_max_resident: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"polkadot_pvf_preparation_max_resident",
"max resident memory observed for preparation (in kilobytes)",
).buckets(
prometheus::exponential_buckets(8192.0, 2.0, 10)
.expect("arguments are always valid; qed"),
),
)?,
registry,
)?,
#[cfg(any(target_os = "linux", feature = "jemalloc-allocator"))]
preparation_max_allocated: prometheus::register(
prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new(
"polkadot_pvf_preparation_max_allocated",
"max allocated memory observed for preparation (in kilobytes)",
).buckets(
prometheus::exponential_buckets(8192.0, 2.0, 10)
.expect("arguments are always valid; qed"),
),
)?,
registry,
)?,
};
Ok(Metrics(Some(inner)))
}
}
enum WorkerFlavor {
Prepare,
Execute,
}
impl WorkerFlavor {
fn as_label(&self) -> &'static str {
match *self {
WorkerFlavor::Prepare => "prepare",
WorkerFlavor::Execute => "execute",
}
}
}
pub(crate) struct WorkerRelatedMetrics<'a> {
metrics: &'a Metrics,
flavor: WorkerFlavor,
}
impl<'a> WorkerRelatedMetrics<'a> {
pub(crate) fn on_begin_spawn(&self) {
if let Some(metrics) = &self.metrics.0 {
metrics.worker_spawning.with_label_values(&[self.flavor.as_label()]).inc();
}
}
pub(crate) fn on_spawned(&self) {
if let Some(metrics) = &self.metrics.0 {
metrics.worker_spawned.with_label_values(&[self.flavor.as_label()]).inc();
}
}
pub(crate) fn on_retired(&self) {
if let Some(metrics) = &self.metrics.0 {
metrics.worker_retired.with_label_values(&[self.flavor.as_label()]).inc();
}
}
}