diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 491077b4476..eb2a5473ebc 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -15,7 +15,7 @@ ethrex-crypto.workspace = true ethrex-storage.workspace = true ethrex-trie.workspace = true ethrex-vm.workspace = true -ethrex-metrics = { path = "./metrics", default-features = false } +ethrex-metrics = { path = "./metrics" } serde = { workspace = true, optional = true } serde_json = { workspace = true, optional = true } diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index bda3b98f735..970e2d4f442 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -2,7 +2,8 @@ use axum::{Router, routing::get}; use crate::{ MetricsApiError, blocks::METRICS_BLOCKS, gather_default_metrics, node::METRICS_NODE, - p2p::METRICS_P2P, process::METRICS_PROCESS, transactions::METRICS_TX, + p2p::METRICS_P2P, process::METRICS_PROCESS, snapsync::METRICS_SNAPSYNC, + transactions::METRICS_TX, }; pub async fn start_prometheus_metrics_api( @@ -54,6 +55,12 @@ pub(crate) async fn get_metrics() -> String { Err(_) => tracing::error!("Failed to gather METRICS_P2P"), }; + ret_string.push('\n'); + match METRICS_SNAPSYNC.gather_metrics() { + Ok(s) => ret_string.push_str(&s), + Err(_) => tracing::error!("Failed to gather METRICS_SNAPSYNC"), + }; + ret_string.push('\n'); if let Some(node_metrics) = METRICS_NODE.get() { match node_metrics.gather_metrics() { diff --git a/crates/blockchain/metrics/mod.rs b/crates/blockchain/metrics/mod.rs index 47f188a09ca..b0ba23cd523 100644 --- a/crates/blockchain/metrics/mod.rs +++ b/crates/blockchain/metrics/mod.rs @@ -4,6 +4,8 @@ pub mod api; pub mod blocks; #[cfg(feature = "api")] pub mod l2; +#[cfg(any(feature = "api", feature = "metrics"))] +pub mod snapsync; #[cfg(feature = "api")] pub mod node; #[cfg(any(feature = "api", feature = "metrics"))] diff --git a/crates/blockchain/metrics/snapsync.rs b/crates/blockchain/metrics/snapsync.rs new file mode 100644 index 00000000000..10f20aa5556 --- /dev/null +++ b/crates/blockchain/metrics/snapsync.rs @@ -0,0 +1,239 @@ +use prometheus::{Encoder, Gauge, IntGauge, Registry, TextEncoder}; +use std::sync::LazyLock; + +use crate::MetricsError; + +pub static METRICS_SNAPSYNC: LazyLock = LazyLock::new(MetricsSnapSync::default); + +#[derive(Debug, Clone)] +pub struct MetricsSnapSync { + // Phase + stage: IntGauge, + target_block: IntGauge, + + // Headers + headers_downloaded: IntGauge, + headers_total: IntGauge, + headers_per_second: Gauge, + headers_stage_start_timestamp: Gauge, + + // Accounts + accounts_downloaded: IntGauge, + accounts_inserted: IntGauge, + accounts_per_second: Gauge, + accounts_stage_start_timestamp: Gauge, + + // Storage + storage_downloaded: IntGauge, + storage_inserted: IntGauge, + storage_per_second: Gauge, + storage_stage_start_timestamp: Gauge, + + // Healing + state_leaves_healed: IntGauge, + storage_leaves_healed: IntGauge, + healing_per_second: Gauge, + healing_stage_start_timestamp: Gauge, + + // Bytecodes + bytecodes_downloaded: IntGauge, + bytecodes_total: IntGauge, + bytecodes_per_second: Gauge, + bytecodes_stage_start_timestamp: Gauge, +} + +impl Default for MetricsSnapSync { + fn default() -> Self { + Self::new() + } +} + +impl MetricsSnapSync { + pub fn new() -> Self { + MetricsSnapSync { + stage: IntGauge::new( + "snapsync_stage", + "Current snap sync stage: 0=none, 1=healing_storage, 2=healing_state, 3=bytecodes, 4=account_ranges, 5=storage_ranges, 6=headers, 7=inserting_storage, 8=inserting_accounts, 9=inserting_accounts_nodb", + ).expect("Failed to create snapsync_stage"), + target_block: IntGauge::new( + "snapsync_target_block", + "Snap sync pivot block number", + ).expect("Failed to create snapsync_target_block"), + + // Headers + headers_downloaded: IntGauge::new( + "snapsync_headers_downloaded", + "Headers downloaded so far", + ).expect("Failed to create snapsync_headers_downloaded"), + headers_total: IntGauge::new( + "snapsync_headers_total", + "Total headers to download", + ).expect("Failed to create snapsync_headers_total"), + headers_per_second: Gauge::new( + "snapsync_headers_per_second", + "Header download rate (5m avg via internal calc)", + ).expect("Failed to create snapsync_headers_per_second"), + headers_stage_start_timestamp: Gauge::new( + "snapsync_headers_stage_start_timestamp", + "Unix timestamp when header download began", + ).expect("Failed to create snapsync_headers_stage_start_timestamp"), + + // Accounts + accounts_downloaded: IntGauge::new( + "snapsync_accounts_downloaded", + "Account ranges downloaded so far", + ).expect("Failed to create snapsync_accounts_downloaded"), + accounts_inserted: IntGauge::new( + "snapsync_accounts_inserted", + "Account ranges inserted to DB so far", + ).expect("Failed to create snapsync_accounts_inserted"), + accounts_per_second: Gauge::new( + "snapsync_accounts_per_second", + "Account download/insert rate", + ).expect("Failed to create snapsync_accounts_per_second"), + accounts_stage_start_timestamp: Gauge::new( + "snapsync_accounts_stage_start_timestamp", + "Unix timestamp when account download began", + ).expect("Failed to create snapsync_accounts_stage_start_timestamp"), + + // Storage + storage_downloaded: IntGauge::new( + "snapsync_storage_downloaded", + "Storage leaves downloaded", + ).expect("Failed to create snapsync_storage_downloaded"), + storage_inserted: IntGauge::new( + "snapsync_storage_inserted", + "Storage leaves inserted to DB", + ).expect("Failed to create snapsync_storage_inserted"), + storage_per_second: Gauge::new( + "snapsync_storage_per_second", + "Storage download/insert rate", + ).expect("Failed to create snapsync_storage_per_second"), + storage_stage_start_timestamp: Gauge::new( + "snapsync_storage_stage_start_timestamp", + "Unix timestamp when storage download began", + ).expect("Failed to create snapsync_storage_stage_start_timestamp"), + + // Healing + state_leaves_healed: IntGauge::new( + "snapsync_state_leaves_healed", + "State trie leaves healed", + ).expect("Failed to create snapsync_state_leaves_healed"), + storage_leaves_healed: IntGauge::new( + "snapsync_storage_leaves_healed", + "Storage trie leaves healed", + ).expect("Failed to create snapsync_storage_leaves_healed"), + healing_per_second: Gauge::new( + "snapsync_healing_per_second", + "Healing rate (leaves/sec)", + ).expect("Failed to create snapsync_healing_per_second"), + healing_stage_start_timestamp: Gauge::new( + "snapsync_healing_stage_start_timestamp", + "Unix timestamp when healing began", + ).expect("Failed to create snapsync_healing_stage_start_timestamp"), + + // Bytecodes + bytecodes_downloaded: IntGauge::new( + "snapsync_bytecodes_downloaded", + "Bytecodes downloaded so far", + ).expect("Failed to create snapsync_bytecodes_downloaded"), + bytecodes_total: IntGauge::new( + "snapsync_bytecodes_total", + "Total bytecodes to download", + ).expect("Failed to create snapsync_bytecodes_total"), + bytecodes_per_second: Gauge::new( + "snapsync_bytecodes_per_second", + "Bytecode download rate", + ).expect("Failed to create snapsync_bytecodes_per_second"), + bytecodes_stage_start_timestamp: Gauge::new( + "snapsync_bytecodes_stage_start_timestamp", + "Unix timestamp when bytecode download began", + ).expect("Failed to create snapsync_bytecodes_stage_start_timestamp"), + } + } + + // Phase setters + pub fn set_stage(&self, stage: i64) { self.stage.set(stage); } + pub fn set_target_block(&self, block: u64) { self.target_block.set(block.cast_signed()); } + + // Headers + pub fn set_headers_downloaded(&self, n: u64) { self.headers_downloaded.set(n.cast_signed()); } + pub fn set_headers_total(&self, n: u64) { self.headers_total.set(n.cast_signed()); } + pub fn set_headers_per_second(&self, r: f64) { self.headers_per_second.set(r); } + pub fn set_headers_stage_start_now(&self) { self.headers_stage_start_timestamp.set(now_secs()); } + + // Accounts + pub fn set_accounts_downloaded(&self, n: u64) { self.accounts_downloaded.set(n.cast_signed()); } + pub fn set_accounts_inserted(&self, n: u64) { self.accounts_inserted.set(n.cast_signed()); } + pub fn set_accounts_per_second(&self, r: f64) { self.accounts_per_second.set(r); } + pub fn set_accounts_stage_start_now(&self) { self.accounts_stage_start_timestamp.set(now_secs()); } + + // Storage + pub fn set_storage_downloaded(&self, n: u64) { self.storage_downloaded.set(n as i64); } + pub fn set_storage_inserted(&self, n: u64) { self.storage_inserted.set(n as i64); } + pub fn set_storage_per_second(&self, r: f64) { self.storage_per_second.set(r); } + pub fn set_storage_stage_start_now(&self) { self.storage_stage_start_timestamp.set(now_secs()); } + + // Healing + pub fn set_state_leaves_healed(&self, n: u64) { self.state_leaves_healed.set(n.cast_signed()); } + pub fn set_storage_leaves_healed(&self, n: u64) { self.storage_leaves_healed.set(n.cast_signed()); } + pub fn set_healing_per_second(&self, r: f64) { self.healing_per_second.set(r); } + pub fn get_healing_stage_start_timestamp(&self) -> f64 { self.healing_stage_start_timestamp.get() } + pub fn set_healing_stage_start_now(&self) { self.healing_stage_start_timestamp.set(now_secs()); } + + // Bytecodes + pub fn set_bytecodes_downloaded(&self, n: u64) { self.bytecodes_downloaded.set(n.cast_signed()); } + pub fn get_bytecodes_total(&self) -> i64 { self.bytecodes_total.get() } + pub fn set_bytecodes_total(&self, n: u64) { self.bytecodes_total.set(n.cast_signed()); } + pub fn set_bytecodes_per_second(&self, r: f64) { self.bytecodes_per_second.set(r); } + pub fn set_bytecodes_stage_start_now(&self) { self.bytecodes_stage_start_timestamp.set(now_secs()); } + + pub fn gather_metrics(&self) -> Result { + let r = Registry::new(); + + let metrics: Vec> = vec![ + Box::new(self.stage.clone()), + Box::new(self.target_block.clone()), + Box::new(self.headers_downloaded.clone()), + Box::new(self.headers_total.clone()), + Box::new(self.headers_per_second.clone()), + Box::new(self.headers_stage_start_timestamp.clone()), + Box::new(self.accounts_downloaded.clone()), + Box::new(self.accounts_inserted.clone()), + Box::new(self.accounts_per_second.clone()), + Box::new(self.accounts_stage_start_timestamp.clone()), + Box::new(self.storage_downloaded.clone()), + Box::new(self.storage_inserted.clone()), + Box::new(self.storage_per_second.clone()), + Box::new(self.storage_stage_start_timestamp.clone()), + Box::new(self.state_leaves_healed.clone()), + Box::new(self.storage_leaves_healed.clone()), + Box::new(self.healing_per_second.clone()), + Box::new(self.healing_stage_start_timestamp.clone()), + Box::new(self.bytecodes_downloaded.clone()), + Box::new(self.bytecodes_total.clone()), + Box::new(self.bytecodes_per_second.clone()), + Box::new(self.bytecodes_stage_start_timestamp.clone()), + ]; + + for metric in metrics { + r.register(metric) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + } + + let encoder = TextEncoder::new(); + let metric_families = r.gather(); + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + Ok(String::from_utf8(buffer)?) + } +} + +fn now_secs() -> f64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64() +} diff --git a/crates/networking/p2p/network.rs b/crates/networking/p2p/network.rs index 3fc2bac185f..0b76135bf28 100644 --- a/crates/networking/p2p/network.rs +++ b/crates/networking/p2p/network.rs @@ -390,6 +390,14 @@ pub async fn periodically_show_peer_stats_during_syncing( phase_elapsed_str, &phase_metrics(previous_step, &phase_start).await, ); + + // Emit final metrics for completed phase + #[cfg(feature = "metrics")] + push_snapsync_prometheus_metrics( + previous_step, + phase_start_time.elapsed(), + &phase_start, + ); } // Start new phase @@ -416,6 +424,10 @@ pub async fn periodically_show_peer_stats_during_syncing( ) .await; + // Push snap sync metrics to Prometheus + #[cfg(feature = "metrics")] + push_snapsync_prometheus_metrics(current_step, phase_elapsed, &prev_interval); + // Update previous interval counters for next rate calculation prev_interval = PhaseCounters::capture_current(); @@ -684,6 +696,100 @@ async fn log_phase_progress( } } +#[cfg(feature = "metrics")] +fn push_snapsync_prometheus_metrics( + step: CurrentStepValue, + phase_elapsed: Duration, + prev_interval: &PhaseCounters, +) { + use ethrex_metrics::snapsync::METRICS_SNAPSYNC; + + let step_num: u8 = step.into(); + METRICS_SNAPSYNC.set_stage(step_num as i64); + METRICS_SNAPSYNC.set_target_block(METRICS.sync_head_block.load(Ordering::Relaxed)); + + let interval_secs = PROGRESS_INTERVAL_SECS.max(1) as f64; + + match step { + CurrentStepValue::DownloadingHeaders => { + let total = METRICS.sync_head_block.load(Ordering::Relaxed); + let downloaded = u64::min(METRICS.downloaded_headers.get(), total); + let interval = downloaded.saturating_sub(prev_interval.headers); + METRICS_SNAPSYNC.set_headers_downloaded(downloaded); + METRICS_SNAPSYNC.set_headers_total(total); + METRICS_SNAPSYNC.set_headers_per_second(interval as f64 / interval_secs); + if phase_elapsed.as_secs() < 2 { + METRICS_SNAPSYNC.set_headers_stage_start_now(); + } + } + CurrentStepValue::RequestingAccountRanges => { + let downloaded = METRICS.downloaded_account_tries.load(Ordering::Relaxed); + let interval = downloaded.saturating_sub(prev_interval.accounts); + METRICS_SNAPSYNC.set_accounts_downloaded(downloaded); + METRICS_SNAPSYNC.set_accounts_per_second(interval as f64 / interval_secs); + if phase_elapsed.as_secs() < 2 { + METRICS_SNAPSYNC.set_accounts_stage_start_now(); + } + } + CurrentStepValue::InsertingAccountRanges | CurrentStepValue::InsertingAccountRangesNoDb => { + let total = METRICS.downloaded_account_tries.load(Ordering::Relaxed); + let inserted = METRICS.account_tries_inserted.load(Ordering::Relaxed); + let interval = inserted.saturating_sub(prev_interval.accounts_inserted); + METRICS_SNAPSYNC.set_accounts_downloaded(total); + METRICS_SNAPSYNC.set_accounts_inserted(inserted); + METRICS_SNAPSYNC.set_accounts_per_second(interval as f64 / interval_secs); + } + CurrentStepValue::RequestingStorageRanges => { + let downloaded = METRICS.storage_leaves_downloaded.get(); + let interval = downloaded.saturating_sub(prev_interval.storage); + METRICS_SNAPSYNC.set_storage_downloaded(downloaded); + METRICS_SNAPSYNC.set_storage_per_second(interval as f64 / interval_secs); + if phase_elapsed.as_secs() < 2 { + METRICS_SNAPSYNC.set_storage_stage_start_now(); + } + } + CurrentStepValue::InsertingStorageRanges => { + let inserted = METRICS.storage_leaves_inserted.get(); + let interval = inserted.saturating_sub(prev_interval.storage_inserted); + METRICS_SNAPSYNC.set_storage_inserted(inserted); + METRICS_SNAPSYNC.set_storage_per_second(interval as f64 / interval_secs); + } + CurrentStepValue::HealingState => { + let healed = METRICS.global_state_trie_leafs_healed.load(Ordering::Relaxed); + let interval = healed.saturating_sub(prev_interval.healed_accounts); + METRICS_SNAPSYNC.set_state_leaves_healed(healed); + METRICS_SNAPSYNC.set_healing_per_second(interval as f64 / interval_secs); + if METRICS_SNAPSYNC.get_healing_stage_start_timestamp() == 0.0 { + METRICS_SNAPSYNC.set_healing_stage_start_now(); + } + } + CurrentStepValue::HealingStorage => { + let healed = METRICS.global_storage_tries_leafs_healed.load(Ordering::Relaxed); + let interval = healed.saturating_sub(prev_interval.healed_storage); + METRICS_SNAPSYNC.set_storage_leaves_healed(healed); + METRICS_SNAPSYNC.set_healing_per_second(interval as f64 / interval_secs); + if METRICS_SNAPSYNC.get_healing_stage_start_timestamp() == 0.0 { + METRICS_SNAPSYNC.set_healing_stage_start_now(); + } + } + CurrentStepValue::RequestingBytecodes => { + let total = METRICS.bytecodes_to_download.load(Ordering::Relaxed); + let downloaded = METRICS.downloaded_bytecodes.load(Ordering::Relaxed); + let interval = downloaded.saturating_sub(prev_interval.bytecodes); + METRICS_SNAPSYNC.set_bytecodes_downloaded(downloaded); + // Lock bytecodes_total when phase starts to prevent ETA bouncing + if METRICS_SNAPSYNC.get_bytecodes_total() == 0 { + METRICS_SNAPSYNC.set_bytecodes_total(total); + } + METRICS_SNAPSYNC.set_bytecodes_per_second(interval as f64 / interval_secs); + if phase_elapsed.as_secs() < 2 { + METRICS_SNAPSYNC.set_bytecodes_stage_start_now(); + } + } + CurrentStepValue::None => {} + } +} + fn progress_bar(percentage: f64, width: usize) -> String { let clamped_percentage = percentage.clamp(0.0, 100.0); let filled = ((clamped_percentage / 100.0) * width as f64) as usize; diff --git a/metrics/provisioning/grafana/dashboards/common_dashboards/snapsync_dashboard.json b/metrics/provisioning/grafana/dashboards/common_dashboards/snapsync_dashboard.json new file mode 100644 index 00000000000..71b8df10bdf --- /dev/null +++ b/metrics/provisioning/grafana/dashboards/common_dashboards/snapsync_dashboard.json @@ -0,0 +1,332 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "graphTooltip": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "title": "Sync Overview", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { + "0": { "text": "None" }, "1": { "text": "Healing Storage" }, "2": { "text": "Healing State" }, + "3": { "text": "Requesting Bytecodes" }, "4": { "text": "Requesting Accounts" }, + "5": { "text": "Requesting Storage" }, "6": { "text": "Downloading Headers" }, + "7": { "text": "Inserting Storage" }, "8": { "text": "Inserting Accounts" }, + "9": { "text": "Inserting Accounts (no DB)" } + }, "type": "value" } + ], + "thresholds": { "steps": [ + { "color": "green", "value": null }, { "color": "yellow", "value": 1 }, + { "color": "orange", "value": 3 }, { "color": "blue", "value": 6 } + ]}, + "color": { "mode": "thresholds" } + } + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "textMode": "value", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "snapsync_stage{instance=~\"$instance\"}", "legendFormat": "Stage" }], + "title": "Sync Stage", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0 } }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "snapsync_target_block{instance=~\"$instance\"}", "legendFormat": "Target Block" }], + "title": "Pivot Block", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0 } }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "ethrex_p2p_peer_count{instance=~\"$instance\"}", "legendFormat": "Peers" }], + "title": "Peers", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none", "decimals": 2 } }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "gigagas{instance=~\"$instance\"}", "legendFormat": "Ggas/s" }], + "title": "Throughput (Ggas/s)", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "title": "Header Download", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "percent", "decimals": 1, "thresholds": { "steps": [{ "color": "yellow", "value": null }, { "color": "green", "value": 95 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 6 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_headers_downloaded{instance=~\"$instance\"} / clamp_min(snapsync_headers_total{instance=~\"$instance\"}, 1) * 100", "legendFormat": "Headers" }], + "title": "Header Progress", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "unit": "none", "decimals": 0, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 500 }, { "color": "green", "value": 1500 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 6 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_headers_per_second{instance=~\"$instance\"}", "legendFormat": "hdr/s" }], + "title": "Headers per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 6 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "(snapsync_headers_total{instance=~\"$instance\"} - snapsync_headers_downloaded{instance=~\"$instance\"}) / clamp_min(snapsync_headers_per_second{instance=~\"$instance\"}, 0.001) and snapsync_headers_per_second{instance=~\"$instance\"} > 0", "legendFormat": "ETA" }], + "title": "Header ETA", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 6 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "time() - snapsync_headers_stage_start_timestamp{instance=~\"$instance\"} and snapsync_headers_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" }], + "title": "Header Elapsed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0, "unit": "none" } }, + "gridPos": { "h": 2, "w": 6, "x": 0, "y": 11 }, + "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "horizontal" }, + "targets": [ + { "expr": "snapsync_headers_downloaded{instance=~\"$instance\"}", "legendFormat": "Downloaded" }, + { "expr": "snapsync_headers_total{instance=~\"$instance\"}", "legendFormat": "Total" } + ], + "title": "", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "title": "Account Ranges", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "percent", "decimals": 1, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 33 }, { "color": "green", "value": 66 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 14 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_accounts_inserted{instance=~\"$instance\"} / clamp_min(snapsync_accounts_downloaded{instance=~\"$instance\"}, 1) * 100", "legendFormat": "Accounts" }], + "title": "Account Progress", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "unit": "none", "decimals": 0, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 100 }, { "color": "green", "value": 500 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 14 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_accounts_per_second{instance=~\"$instance\"}", "legendFormat": "acc/s" }], + "title": "Accounts per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 14 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "(snapsync_accounts_downloaded{instance=~\"$instance\"} - snapsync_accounts_inserted{instance=~\"$instance\"}) / clamp_min(snapsync_accounts_per_second{instance=~\"$instance\"}, 0.001) and snapsync_accounts_per_second{instance=~\"$instance\"} > 0", "legendFormat": "ETA" }], + "title": "Account ETA", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 14 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "time() - snapsync_accounts_stage_start_timestamp{instance=~\"$instance\"} and snapsync_accounts_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" }], + "title": "Account Elapsed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0, "unit": "none" } }, + "gridPos": { "h": 2, "w": 6, "x": 0, "y": 19 }, + "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "horizontal" }, + "targets": [ + { "expr": "snapsync_accounts_inserted{instance=~\"$instance\"}", "legendFormat": "Inserted" }, + { "expr": "snapsync_accounts_downloaded{instance=~\"$instance\"}", "legendFormat": "Downloaded" } + ], + "title": "", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "title": "Storage Ranges", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 22 }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "snapsync_storage_downloaded{instance=~\"$instance\"}", "legendFormat": "Downloaded" }, + { "expr": "snapsync_storage_inserted{instance=~\"$instance\"}", "legendFormat": "Inserted" } + ], + "title": "Storage Leaves", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "unit": "none", "decimals": 0, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 500 }, { "color": "green", "value": 2000 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 22 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_storage_per_second{instance=~\"$instance\"}", "legendFormat": "slots/s" }], + "title": "Storage Leaves per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 22 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "time() - snapsync_storage_stage_start_timestamp{instance=~\"$instance\"} and snapsync_storage_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" }], + "title": "Storage Elapsed", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "title": "Healing", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 28 }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "snapsync_state_leaves_healed{instance=~\"$instance\"}", "legendFormat": "State Healed" }, + { "expr": "snapsync_storage_leaves_healed{instance=~\"$instance\"}", "legendFormat": "Storage Healed" } + ], + "title": "Leaves Healed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "unit": "none", "decimals": 0, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 100 }, { "color": "green", "value": 500 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 28 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_healing_per_second{instance=~\"$instance\"}", "legendFormat": "leaves/s" }], + "title": "Healing per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 12, "x": 12, "y": 28 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "time() - snapsync_healing_stage_start_timestamp{instance=~\"$instance\"} and snapsync_healing_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" }], + "title": "Healing Elapsed", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 33 }, + "title": "Bytecodes", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "max": 100, "unit": "percent", "decimals": 1, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 33 }, { "color": "green", "value": 66 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 34 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_bytecodes_downloaded{instance=~\"$instance\"} / clamp_min(snapsync_bytecodes_total{instance=~\"$instance\"}, 1) * 100", "legendFormat": "Bytecodes" }], + "title": "Bytecode Progress", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "min": 0, "unit": "none", "decimals": 0, "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 50 }, { "color": "green", "value": 200 }] }, "color": { "mode": "thresholds" } } }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 34 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "snapsync_bytecodes_per_second{instance=~\"$instance\"}", "legendFormat": "codes/s" }], + "title": "Bytecodes per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 34 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "(snapsync_bytecodes_total{instance=~\"$instance\"} - snapsync_bytecodes_downloaded{instance=~\"$instance\"}) / clamp_min(snapsync_bytecodes_per_second{instance=~\"$instance\"}, 0.001) and snapsync_bytecodes_per_second{instance=~\"$instance\"} > 0", "legendFormat": "ETA" }], + "title": "Bytecode ETA", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 34 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "time() - snapsync_bytecodes_stage_start_timestamp{instance=~\"$instance\"} and snapsync_bytecodes_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" }], + "title": "Bytecode Elapsed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0, "unit": "none" } }, + "gridPos": { "h": 2, "w": 6, "x": 0, "y": 39 }, + "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "horizontal" }, + "targets": [ + { "expr": "snapsync_bytecodes_downloaded{instance=~\"$instance\"}", "legendFormat": "Downloaded" }, + { "expr": "snapsync_bytecodes_total{instance=~\"$instance\"}", "legendFormat": "Total" } + ], + "title": "", + "type": "stat" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["ethrex", "snapsync"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "localhost:3701", "value": "localhost:3701" }, + "datasource": { "type": "prometheus" }, + "definition": "label_values(snapsync_stage, instance)", + "name": "instance", + "query": "label_values(snapsync_stage, instance)", + "refresh": 2, + "type": "query" + }, + { + "hide": 2, + "name": "DS_PROMETHEUS", + "query": "prometheus", + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "ethrex Snap Sync", + "uid": "ethrex-snapsync" +}