From 94214853b5e33b774bbc462d19ab632432639ef7 Mon Sep 17 00:00:00 2001 From: KtorZ Date: Wed, 5 Aug 2026 22:46:29 +0200 Subject: [PATCH] fix: cleanup metrics and observability setup and communication with the TUI No-changelog Signed-off-by: KtorZ --- .../workflows/nightly-tool-integrations.yml | 2 +- .../template-ledger-epoch-snapshots.yml | 1 - CHANGELOG.md | 1 + Cargo.lock | 3 +- Cargo.toml | 2 + .../amaru-kernel/src/cardano/era_history.rs | 60 +- crates/amaru-kernel/src/lib.rs | 3 +- crates/amaru-kernel/src/utils.rs | 1 + crates/amaru-kernel/src/utils/process.rs | 111 +++ .../src/store/columns/proposals.rs | 4 +- crates/amaru-metrics/src/consensus.rs | 10 +- crates/amaru-metrics/src/ledger.rs | 18 +- crates/amaru-metrics/src/lib.rs | 47 +- crates/amaru-metrics/src/mempool.rs | 4 + crates/amaru-metrics/src/metrics.rs | 52 +- crates/amaru-metrics/src/protocol.rs | 12 + crates/amaru-metrics/src/system.rs | 103 ++- crates/amaru-node/Cargo.toml | 2 +- crates/amaru-node/src/stages/build_node.rs | 12 +- crates/amaru-node/src/tests/setup.rs | 3 +- .../macros/src/define_schemas.rs | 11 +- .../amaru-observability/macros/src/traces.rs | 2 +- .../tests/schema_access.rs | 2 + crates/amaru-protocols/src/metrics_effects.rs | 4 +- crates/amaru-stores/src/lib.rs | 9 +- .../src/rocksdb/consensus/tests.rs | 1 + crates/amaru-tui/AGENTS.md | 18 +- crates/amaru-tui/Cargo.toml | 1 - crates/amaru-tui/README.md | 41 +- crates/amaru-tui/src/capture.rs | 42 +- crates/amaru-tui/src/config.rs | 23 +- crates/amaru-tui/src/events.rs | 6 +- crates/amaru-tui/src/events/field_value.rs | 61 ++ crates/amaru-tui/src/events/host_sample.rs | 41 - crates/amaru-tui/src/events/message.rs | 3 +- .../amaru-tui/src/events/telemetry_record.rs | 3 +- crates/amaru-tui/src/host_metrics.rs | 249 ------ crates/amaru-tui/src/lib.rs | 2 - crates/amaru-tui/src/metrics.rs | 49 -- crates/amaru-tui/src/model.rs | 713 +++++++----------- .../exponential_moving_average.rs} | 25 +- crates/amaru-tui/src/model/interaction.rs | 7 +- crates/amaru-tui/src/model/log_buffer.rs | 111 +++ crates/amaru-tui/src/model/metrics_update.rs | 136 +--- crates/amaru-tui/src/model/peer_state.rs | 38 +- crates/amaru-tui/src/model/queries.rs | 54 +- crates/amaru-tui/src/model/rate_counter.rs | 52 ++ crates/amaru-tui/src/model/telemetry_event.rs | 6 + .../amaru-tui/src/model/telemetry_update.rs | 26 +- crates/amaru-tui/src/model/tip_state.rs | 2 + crates/amaru-tui/src/session.rs | 77 +- crates/amaru-tui/src/startup.rs | 68 +- crates/amaru-tui/src/ui/components/logs.rs | 17 +- crates/amaru-tui/src/ui/screens/amaru.rs | 33 +- crates/amaru-tui/src/ui/screens/config.rs | 12 +- crates/amaru/src/bin/amaru/cmd/node/run.rs | 96 +-- crates/amaru/src/bin/amaru/main.rs | 41 +- crates/amaru/src/bin/amaru/pid.rs | 4 +- crates/amaru/src/lifecycle.rs | 25 +- crates/amaru/src/metrics.rs | 79 +- crates/amaru/src/observability.rs | 182 +++-- .../030-embedded-terminal-observability-ui.md | 40 +- monitoring/Makefile | 1 - monitoring/README.md | 14 +- monitoring/docker-compose.yml | 1 - monitoring/otlp-collector.yml | 2 - 66 files changed, 1395 insertions(+), 1486 deletions(-) create mode 100644 crates/amaru-kernel/src/utils/process.rs delete mode 100644 crates/amaru-tui/src/events/host_sample.rs delete mode 100644 crates/amaru-tui/src/host_metrics.rs delete mode 100644 crates/amaru-tui/src/metrics.rs rename crates/amaru-tui/src/{events/telemetry_kind.rs => model/exponential_moving_average.rs} (51%) create mode 100644 crates/amaru-tui/src/model/log_buffer.rs create mode 100644 crates/amaru-tui/src/model/rate_counter.rs diff --git a/.github/workflows/nightly-tool-integrations.yml b/.github/workflows/nightly-tool-integrations.yml index c8311b323d..2d098dd93e 100644 --- a/.github/workflows/nightly-tool-integrations.yml +++ b/.github/workflows/nightly-tool-integrations.yml @@ -101,7 +101,7 @@ jobs: AMARU_LISTEN_ADDRESS: 127.0.0.1:3000 AMARU_PEER_ADDRESS: preprod-node.play.dev.cardano.org:3001 AMARU_WITH_OPEN_TELEMETRY: ${{ matrix.tool.metrics || 'false' }} - OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318" + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4317" TOOL_TEST_TIMEOUT: ${{ matrix.tool.timeout || 120 }} strategy: fail-fast: false diff --git a/.github/workflows/template-ledger-epoch-snapshots.yml b/.github/workflows/template-ledger-epoch-snapshots.yml index 832fa66dfd..1aa476a23b 100644 --- a/.github/workflows/template-ledger-epoch-snapshots.yml +++ b/.github/workflows/template-ledger-epoch-snapshots.yml @@ -57,7 +57,6 @@ jobs: TRACE_COMPARE_LOG: trace-compare.log AMARU_WITH_OPEN_TELEMETRY: "true" OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4317" - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: "http://localhost:4318/v1/metrics" OTEL_METRIC_EXPORT_INTERVAL: 1000 RUST_BACKTRACE: 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index d14107b97a..26cef2011a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ Other guiding principles: - **amaru-ledger**: keep only slim stake summaries in runtime memory, and rebuild the full account-heavy stake distribution from snapshots when computing rewards. - **amaru-ledger**: compute rewards and stake distributions asynchronously to prevent blocking the main roll forward loop from times to times. - **amaru**: bootstrap snapshots now are retrieved directly from R2 (no embedded manifests) and compressed with zstandard. ([#1012][]) +- **amaru**: metrics are now (also) exported through gRPC on `:4317` by default instead of `:4318` over HTTP. ### Removed diff --git a/Cargo.lock b/Cargo.lock index 9b0289c063..99365bf328 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -443,6 +443,7 @@ dependencies = [ "amaru-kernel", "amaru-ledger", "amaru-mempool", + "amaru-metrics", "amaru-network", "amaru-observability", "amaru-ouroboros", @@ -459,7 +460,6 @@ dependencies = [ "hex", "minicbor", "num", - "opentelemetry", "parking_lot", "quote", "reqwest", @@ -683,7 +683,6 @@ dependencies = [ "crossterm 0.29.0", "ratatui", "signal-hook", - "sysinfo", "tracing", "tracing-subscriber", ] diff --git a/Cargo.toml b/Cargo.toml index eee3ba423d..966f630a24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,7 @@ once_cell = "1.21.4" opentelemetry = "0.32.0" opentelemetry-appender-tracing = "0.32.0" opentelemetry-otlp = { version = "0.32.0", features = [ "grpc-tonic", "http-proto", "logs", "reqwest-blocking-client", ] } +opentelemetry-proto = { version = "0.32.0", default-features = false, features = [ "gen-tonic", "trace", "logs", "metrics" ] } opentelemetry-semantic-conventions = { version = "0.32.1", features = ["semconv_experimental"] } opentelemetry_sdk = { version = "0.32.1", features = ["logs"] } ouroboros = "0.18.5" @@ -111,6 +112,7 @@ thiserror = "2.0.19" tokio = { version = "1.53.0", features = ["sync"] } tokio-util = "0.7.18" toml = "1.1.3" +tonic = "0.14.6" tracing = { version = "0.1.40", features = ["valuable"] } tracing-opentelemetry = "0.33.0" tracing-subscriber = { version = "0.3.23", features = [ "env-filter", "std", "json", ] } diff --git a/crates/amaru-kernel/src/cardano/era_history.rs b/crates/amaru-kernel/src/cardano/era_history.rs index 196bfcfb89..8b22231c88 100644 --- a/crates/amaru-kernel/src/cardano/era_history.rs +++ b/crates/amaru-kernel/src/cardano/era_history.rs @@ -377,32 +377,6 @@ pub enum EraHistoryFileError { JsonParseError(#[from] serde_json::Error), } -/// Load an `EraHistory` from a JSON file. -/// -/// # Arguments -/// -/// * `path` - Path to the JSON file containing era history data -/// -/// # Returns -/// -/// Returns a Result containing the `EraHistory` if successful, or an `EraHistoryFileError` if the file -/// cannot be read or parsed. -/// -/// # Example -/// -/// ```no_run -/// use amaru_kernel::load_era_history_from_file; -/// use std::path::Path; -/// -/// let era_history = load_era_history_from_file(Path::new("era_history.json")).unwrap(); -/// ``` -pub fn load_era_history_from_file(path: &Path) -> Result { - let file = File::open(path).map_err(EraHistoryFileError::FileOpenError)?; - let reader = BufReader::new(file); - - serde_json::from_reader(reader).map_err(EraHistoryFileError::JsonParseError) -} - impl cbor::Encode for EraHistory { fn encode( &self, @@ -451,6 +425,32 @@ pub struct EpochEraBounds { // horizon is the end of the epoch containing the end of the current era's safe zone relative to // the current tip. Returns number of milliseconds elapsed since the system start time. impl EraHistory { + /// Load an `EraHistory` from a JSON file. + /// + /// # Arguments + /// + /// * `path` - Path to the JSON file containing era history data + /// + /// # Returns + /// + /// Returns a Result containing the `EraHistory` if successful, or an `EraHistoryFileError` if the file + /// cannot be read or parsed. + /// + /// # Example + /// + /// ```no_run + /// use amaru_kernel::EraHistory; + /// use std::path::Path; + /// + /// let era_history = EraHistory::load(Path::new("era_history.json")).unwrap(); + /// ``` + pub fn load(path: &Path) -> Result { + let file = File::open(path).map_err(EraHistoryFileError::FileOpenError)?; + let reader = BufReader::new(file); + + serde_json::from_reader(reader).map_err(EraHistoryFileError::JsonParseError) + } + pub fn new(eras: &[EraSummary], stability_window: Slot) -> EraHistory { #[expect(clippy::panic)] if eras.is_empty() { @@ -751,7 +751,7 @@ mod tests { use super::*; use crate::{ Epoch, MAINNET_ERA_HISTORY, PREPROD_ERA_HISTORY, PREVIEW_ERA_HISTORY, Slot, any_era_params, any_network_name, - from_cbor_no_leftovers_with, load_era_history_from_file, to_cbor, + from_cbor_no_leftovers_with, to_cbor, }; prop_compose! { @@ -1268,7 +1268,7 @@ mod tests { file.write_all(json_data.as_bytes()).expect("Failed to write JSON data to file"); let loaded_era_history = - load_era_history_from_file(temp_file_path.as_path()).expect("Failed to load EraHistory from file"); + EraHistory::load(temp_file_path.as_path()).expect("Failed to load EraHistory from file"); assert_eq!(*original_era_history, loaded_era_history, "Era histories don't match"); @@ -1289,7 +1289,7 @@ mod tests { fn test_era_history_file_open_error() { let non_existent_path = Path::new("non_existent_file.json"); - let result = load_era_history_from_file(non_existent_path); + let result = EraHistory::load(non_existent_path); match result { Err(EraHistoryFileError::FileOpenError(_)) => { @@ -1310,7 +1310,7 @@ mod tests { file.write_all(invalid_json.as_bytes()).expect("Failed to write invalid JSON data to file"); - let result = load_era_history_from_file(temp_file_path.as_path()); + let result = EraHistory::load(temp_file_path.as_path()); match result { Err(EraHistoryFileError::JsonParseError(_)) => { diff --git a/crates/amaru-kernel/src/lib.rs b/crates/amaru-kernel/src/lib.rs index db0e7a86c6..0a4a41ad06 100644 --- a/crates/amaru-kernel/src/lib.rs +++ b/crates/amaru-kernel/src/lib.rs @@ -96,8 +96,7 @@ pub use cardano::{ epoch::Epoch, era_bound::EraBound, era_history::{ - EraHistory, EraHistoryError, EraHistoryFileError, MAINNET_ERA_HISTORY, PREPROD_ERA_HISTORY, - PREVIEW_ERA_HISTORY, load_era_history_from_file, + EraHistory, EraHistoryError, EraHistoryFileError, MAINNET_ERA_HISTORY, PREPROD_ERA_HISTORY, PREVIEW_ERA_HISTORY, }, era_name::{EraName, EraNameError}, era_params::EraParams, diff --git a/crates/amaru-kernel/src/utils.rs b/crates/amaru-kernel/src/utils.rs index bb11321006..be438167b1 100644 --- a/crates/amaru-kernel/src/utils.rs +++ b/crates/amaru-kernel/src/utils.rs @@ -18,6 +18,7 @@ pub mod duration; #[cfg(all(any(test, feature = "test-utils"), not(target_family = "wasm"), not(target_arch = "riscv32")))] pub mod memory; pub mod path; +pub mod process; pub mod serde; pub mod string; #[cfg(any(test, feature = "test-utils"))] diff --git a/crates/amaru-kernel/src/utils/process.rs b/crates/amaru-kernel/src/utils/process.rs new file mode 100644 index 0000000000..becf639fae --- /dev/null +++ b/crates/amaru-kernel/src/utils/process.rs @@ -0,0 +1,111 @@ +// Copyright 2026 PRAGMA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![cfg(unix)] + +use std::process::Command; + +pub fn sample_process_memory(pid: u32) -> Option { + let output = if cfg!(target_os = "macos") { + Command::new("top").args(["-l", "1", "-pid", &pid.to_string(), "-stats", "pid,mem"]).output().ok()? + } else { + Command::new("top").args(["-b", "-n", "1", "-p", &pid.to_string()]).output().ok()? + }; + + if !output.status.success() { + return None; + } + + parse_top_mem(&String::from_utf8_lossy(&output.stdout), pid) +} + +fn parse_top_mem(output: &str, pid: u32) -> Option { + output.lines().rev().find_map(|line| { + let mut fields = line.split_whitespace(); + if fields.next()?.parse::().ok()? != pid { + return None; + } + + if cfg!(target_os = "linux") { + for _ in 0..4 { + fields.next()?; + } + } + + let multiplier = if cfg!(target_os = "linux") { 1024 } else { 1 }; + + parse_value_with_unit(fields.next()?, multiplier) + }) +} + +fn parse_value_with_unit(value: &str, plain_multiplier: u64) -> Option { + let value = value.trim_end_matches('+'); + let suffix = value.chars().last()?; + + let multiplier = match suffix { + 'K' | 'k' => 1_024f64, + 'M' | 'm' => 1_024f64 * 1_024f64, + 'G' | 'g' => 1_024f64 * 1_024f64 * 1_024f64, + 'T' | 't' => 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64, + 'P' | 'p' => 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64, + '0'..='9' => { + let amount = value.parse::().ok()?; + return Some(amount.saturating_mul(plain_multiplier)); + } + _ => return None, + }; + + let amount = value[..value.len() - 1].parse::().ok()?; + + Some((amount * multiplier).round() as u64) +} + +#[cfg(all(test, unix))] +mod tests { + use test_case::test_case; + + use super::{parse_top_mem, parse_value_with_unit}; + + #[test_case("1201K", 1 => Some(1_229_824))] + #[test_case("1.5M", 1 => Some(1_572_864))] + #[test_case("2.0G", 1 => Some(2_147_483_648))] + #[test_case("42", 1 => Some(42))] + #[test_case("42", 1024 => Some(43_008))] + #[test_case("1.5g", 1 => Some(1_610_612_736))] + #[test_case("150M+", 1 => Some(157_286_400))] + fn parses_top_memory_suffixes(value: &str, plain_multiplier: u64) -> Option { + parse_value_with_unit(value, plain_multiplier) + } + + #[test] + fn parse_top_process_for_memory() { + let output = if cfg!(target_os = "linux") { + [ + "top - 12:00:00 up 1 day, 1 user, load average: 0.00, 0.00, 0.00", + "Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie", + "%Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st ", + "MiB Mem : 1024.0 total, 256.0 free, 512.0 used, 256.0 buff/cache", + "", + " PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND", + " 73194 user 20 0 1234567 654321 12345 S 0.0 0.1 0:00.01 amaru", + "", + ] + .join("\n") + } else { + ["Processes: 1 total", "PID MEM", "73194 654321K", ""].join("\n") + }; + + assert_eq!(parse_top_mem(&output, 73_194), Some(670_024_704)); + } +} diff --git a/crates/amaru-ledger/src/store/columns/proposals.rs b/crates/amaru-ledger/src/store/columns/proposals.rs index 3fe2be1df6..0ec4367e7b 100644 --- a/crates/amaru-ledger/src/store/columns/proposals.rs +++ b/crates/amaru-ledger/src/store/columns/proposals.rs @@ -52,13 +52,13 @@ impl<'a, C> cbor::decode::Decode<'a, C> for Row { #[cfg(any(test, feature = "test-utils"))] pub mod tests { - use amaru_kernel::{any_proposal, any_proposal_pointer, prop_cbor_roundtrip}; + use amaru_kernel::{any_proposal, any_proposal_pointer}; use proptest::{prelude::*, prop_compose}; use super::*; #[cfg(not(target_os = "windows"))] - prop_cbor_roundtrip!(prop_cbor_roundtrip_row, Row, any_row(u64::MAX)); + amaru_kernel::prop_cbor_roundtrip!(prop_cbor_roundtrip_row, Row, any_row(u64::MAX)); prop_compose! { pub fn any_row(max_slot: u64)( diff --git a/crates/amaru-metrics/src/consensus.rs b/crates/amaru-metrics/src/consensus.rs index 1e9a3093ea..c27e25edc0 100644 --- a/crates/amaru-metrics/src/consensus.rs +++ b/crates/amaru-metrics/src/consensus.rs @@ -17,6 +17,8 @@ use std::sync::OnceLock; #[cfg(not(target_arch = "wasm32"))] use opentelemetry::KeyValue; +#[cfg(not(target_arch = "wasm32"))] +use opentelemetry::metrics::Meter as OpenTelemetryMeter; #[cfg(not(target_arch = "wasm32"))] use crate::{Counter, Histogram}; @@ -62,6 +64,10 @@ impl MetricRecorder for ConsensusMetrics { static FORK_SWITCH_DURATION: OnceLock> = OnceLock::new(); static FORK_SWITCH_TOTAL: OnceLock> = OnceLock::new(); + let Some(meter) = meter.get() else { + return; + }; + match self { ConsensusMetrics::HeaderLifecycle { outcome, @@ -133,7 +139,7 @@ impl MetricRecorder for ConsensusMetrics { /// Record a duration to its histogram, if present, without touching any counter. #[cfg(not(target_arch = "wasm32"))] fn record_optional_duration( - meter: &Meter, + meter: &OpenTelemetryMeter, duration: &'static OnceLock>, duration_name: &'static str, duration_description: &'static str, @@ -153,7 +159,7 @@ fn record_optional_duration( #[cfg(not(target_arch = "wasm32"))] #[allow(clippy::too_many_arguments)] fn record_duration( - meter: &Meter, + meter: &OpenTelemetryMeter, duration: &'static OnceLock>, total: &'static OnceLock>, duration_name: &'static str, diff --git a/crates/amaru-metrics/src/ledger.rs b/crates/amaru-metrics/src/ledger.rs index fe42968824..f7b4f81698 100644 --- a/crates/amaru-metrics/src/ledger.rs +++ b/crates/amaru-metrics/src/ledger.rs @@ -63,6 +63,17 @@ impl MetricRecorder for LedgerMetrics { #[cfg(not(target_arch = "wasm32"))] impl MetricRecorder for LedgerMetrics { fn record_to_meter(&self, meter: &Meter) { + crate::protocol::TipBlockMetrics { + hash: self.block_header_hash.clone(), + parent_hash: self.parent_block_header_hash.clone(), + issuer_verification_key_hash: self.issuer_verification_key_hash.clone(), + } + .record_to_meter(meter); + + let Some(meter) = meter.get() else { + return; + }; + static BLOCK_HEIGHT: OnceLock> = OnceLock::new(); static SLOT_NUM: OnceLock> = OnceLock::new(); static SLOT_IN_EPOCH: OnceLock> = OnceLock::new(); @@ -128,13 +139,6 @@ impl MetricRecorder for LedgerMetrics { density.record(self.density, &[]); current_kes_period.record(self.current_kes_period, &[]); remaining_kes_periods.record(self.remaining_kes_periods, &[]); - - crate::protocol::TipBlockMetrics { - hash: self.block_header_hash.clone(), - parent_hash: self.parent_block_header_hash.clone(), - issuer_verification_key_hash: self.issuer_verification_key_hash.clone(), - } - .record_to_meter(meter); } } diff --git a/crates/amaru-metrics/src/lib.rs b/crates/amaru-metrics/src/lib.rs index 06886c17eb..45d3cfdef2 100644 --- a/crates/amaru-metrics/src/lib.rs +++ b/crates/amaru-metrics/src/lib.rs @@ -12,14 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{ - collections::BTreeMap, - sync::{ - Arc, LazyLock, Mutex, MutexGuard, - atomic::{AtomicU32, Ordering}, - }, -}; - pub use crate::{ consensus::ConsensusMetrics, ledger::LedgerMetrics, @@ -53,6 +45,7 @@ pub trait MetricRecorder { impl MetricRecorder for MetricsEvent { fn record_to_meter(&self, meter: &Meter) { + meter.notify_local_observer_if_any(self); match self { MetricsEvent::LedgerMetrics(ledger_metrics) => ledger_metrics.record_to_meter(meter), MetricsEvent::MempoolMetrics(mempool_metrics) => mempool_metrics.record_to_meter(meter), @@ -62,41 +55,3 @@ impl MetricRecorder for MetricsEvent { } } } - -#[derive(Debug)] -pub struct Subscription { - id: u32, -} - -static NEXT_SUBSCRIBER_ID: AtomicU32 = AtomicU32::new(1); -type Subscriber = dyn Fn(&MetricsEvent) + Send + Sync; - -static SUBSCRIBERS: LazyLock>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); - -fn subscribers() -> MutexGuard<'static, BTreeMap>> { - SUBSCRIBERS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) -} - -pub fn subscribe(subscriber: Arc) -> Subscription { - let id = NEXT_SUBSCRIBER_ID.fetch_add(1, Ordering::Relaxed); - subscribers().insert(id, subscriber); - Subscription { id } -} - -pub fn has_subscribers() -> bool { - !subscribers().is_empty() -} - -pub fn notify_subscribers(event: &MetricsEvent) { - let subscribers = subscribers().values().cloned().collect::>(); - - for subscriber in subscribers { - subscriber(event); - } -} - -impl Drop for Subscription { - fn drop(&mut self) { - subscribers().remove(&self.id); - } -} diff --git a/crates/amaru-metrics/src/mempool.rs b/crates/amaru-metrics/src/mempool.rs index 1b3ab30e59..9618d65101 100644 --- a/crates/amaru-metrics/src/mempool.rs +++ b/crates/amaru-metrics/src/mempool.rs @@ -104,6 +104,10 @@ impl MetricRecorder for MempoolMetrics { #[cfg(not(target_arch = "wasm32"))] impl MetricRecorder for MempoolMetrics { fn record_to_meter(&self, meter: &Meter) { + let Some(meter) = meter.get() else { + return; + }; + static SIZE_BYTES: OnceLock> = OnceLock::new(); static TX_COUNT: OnceLock> = OnceLock::new(); static TXS_PROCESSED: OnceLock> = OnceLock::new(); diff --git a/crates/amaru-metrics/src/metrics.rs b/crates/amaru-metrics/src/metrics.rs index 11be784667..9d8a96aded 100644 --- a/crates/amaru-metrics/src/metrics.rs +++ b/crates/amaru-metrics/src/metrics.rs @@ -13,15 +13,57 @@ // limitations under the License. #[cfg(not(target_arch = "wasm32"))] -pub use opentelemetry::metrics::{Counter, Gauge, Histogram, Meter}; +pub use native::*; + +#[cfg(not(target_arch = "wasm32"))] +mod native { + use opentelemetry::metrics::Meter as OpenTelemetryMeter; + pub use opentelemetry::metrics::{Counter, Gauge, Histogram}; + + use crate::MetricsEvent; + + #[derive(Default)] + pub struct Meter { + open_telemetry_meter: Option, + local_observer: Option>, + } + + impl From for Meter { + fn from(open_telemetry_meter: OpenTelemetryMeter) -> Self { + Self { open_telemetry_meter: Some(open_telemetry_meter), local_observer: None } + } + } + + impl Meter { + pub fn get(&self) -> Option<&OpenTelemetryMeter> { + self.open_telemetry_meter.as_ref() + } + + pub fn set_local_observer(&mut self, local_observer: Box) { + self.local_observer = Some(local_observer); + } + + pub fn notify_local_observer_if_any(&self, event: &MetricsEvent) { + if let Some(notify) = self.local_observer.as_ref() { + notify(event); + } + } + } +} + +#[cfg(target_arch = "wasm32")] +pub use wasm::*; #[cfg(target_arch = "wasm32")] mod wasm { - pub type Meter = (); + use crate::MetricsEvent; + + pub struct Meter; pub type Gauge = (); pub type Counter = (); pub type Histogram = (); -} -#[cfg(target_arch = "wasm32")] -pub use wasm::*; + impl Meter { + pub fn notify_local_observer_if_any(&self, _event: &MetricsEvent) {} + } +} diff --git a/crates/amaru-metrics/src/protocol.rs b/crates/amaru-metrics/src/protocol.rs index 2abe0839bf..887c9e2d78 100644 --- a/crates/amaru-metrics/src/protocol.rs +++ b/crates/amaru-metrics/src/protocol.rs @@ -40,6 +40,10 @@ fn update_observable_gauge( ) where T: Clone + Send + 'static, { + let Some(meter) = meter.get() else { + return; + }; + let state_ref = state.get_or_init(|| { let shared: Arc>> = Arc::new(Mutex::new(None)); let shared_cb = shared.clone(); @@ -118,6 +122,10 @@ impl MetricRecorder for ConnectionManagerMetrics { static OUTBOUND_CONNECTIONS: OnceLock> = OnceLock::new(); static UNIDIRECTIONAL_CONNECTIONS: OnceLock> = OnceLock::new(); + let Some(meter) = meter.get() else { + return; + }; + let inbound_connections = INBOUND_CONNECTIONS.get_or_init(|| { meter .u64_gauge("cardano_node_metrics_connectionManager_inboundConns_int") @@ -156,6 +164,10 @@ impl MetricRecorder for ServedBlockCountMetrics { fn record_to_meter(&self, meter: &Meter) { static SERVED_BLOCK_COUNT: OnceLock> = OnceLock::new(); + let Some(meter) = meter.get() else { + return; + }; + let served_block_count = SERVED_BLOCK_COUNT.get_or_init(|| { meter .u64_counter("cardano_node_metrics_served_block_count_int") diff --git a/crates/amaru-metrics/src/system.rs b/crates/amaru-metrics/src/system.rs index 72b8ffe942..c922810871 100644 --- a/crates/amaru-metrics/src/system.rs +++ b/crates/amaru-metrics/src/system.rs @@ -19,17 +19,36 @@ use std::sync::OnceLock; use crate::Gauge; use crate::{Meter, MetricRecorder, MetricsEvent}; -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub const PROCESS_RUNTIME: &str = "process_runtime"; +pub const PROCESS_CPU_LIVE: &str = "process_cpu_live"; +pub const PROCESS_DISK_TOTAL_READ: &str = "process_disk_total_read"; +pub const PROCESS_DISK_TOTAL_WRITE: &str = "process_disk_total_write"; +pub const PROCESS_DISK_LIVE_READ: &str = "process_disk_live_read"; +pub const PROCESS_DISK_LIVE_WRITE: &str = "process_disk_live_write"; +pub const PROCESS_MEMORY_FOOTPRINT: &str = "process_memory_footprint"; +pub const PROCESS_MEMORY_LIVE_RESIDENT: &str = "process_memory_live_resident"; +pub const PROCESS_MEMORY_AVAILABLE_VIRTUAL: &str = "process_memory_available_virtual"; +pub const PROCESS_OPEN_FILES: &str = "process_open_files"; +pub const HOST_MEMORY_USED: &str = "host_memory_used"; +pub const HOST_MEMORY_TOTAL: &str = "host_memory_total"; +pub const HOST_DISK_LIVE_READ: &str = "host_disk_live_read"; +pub const HOST_DISK_LIVE_WRITE: &str = "host_disk_live_write"; + +#[derive(Debug, Clone, Default, PartialEq, serde::Serialize, serde::Deserialize)] pub struct SystemMetrics { pub runtime_seconds: u64, pub cpu_percent: f64, pub process_memory_bytes: u64, - pub rss_bytes: u64, - pub virtual_bytes: u64, + pub process_memory_live_resident: u64, + pub process_memory_available_virtual: u64, + pub memory_used_bytes: u64, + pub memory_total_bytes: u64, pub disk_read_bytes: u64, pub disk_write_bytes: u64, pub disk_live_read_bytes: u64, pub disk_live_write_bytes: u64, + pub host_live_read_bytes: u64, + pub host_live_write_bytes: u64, pub open_files: u64, } @@ -46,81 +65,115 @@ impl MetricRecorder for SystemMetrics { static DISK_TOTAL_WRITE_BYTES: OnceLock> = OnceLock::new(); static DISK_LIVE_READ_BYTES: OnceLock> = OnceLock::new(); static DISK_LIVE_WRITE_BYTES: OnceLock> = OnceLock::new(); + static HOST_DISK_LIVE_READ_BYTES: OnceLock> = OnceLock::new(); + static HOST_DISK_LIVE_WRITE_BYTES: OnceLock> = OnceLock::new(); static CPU_PERCENT: OnceLock> = OnceLock::new(); - static PROCESS_MEMORY_BYTES: OnceLock> = OnceLock::new(); + static FOOTPRINT_BYTES: OnceLock> = OnceLock::new(); static RSS_BYTES: OnceLock> = OnceLock::new(); static VIRTUAL_BYTES: OnceLock> = OnceLock::new(); + static HOST_MEMORY_USED_BYTES: OnceLock> = OnceLock::new(); + static HOST_MEMORY_TOTAL_BYTES: OnceLock> = OnceLock::new(); static OPEN_FILES: OnceLock> = OnceLock::new(); + let Some(meter) = meter.get() else { + return; + }; + let runtime_seconds = RUNTIME_SECONDS.get_or_init(|| { meter - .u64_gauge("process_runtime") + .u64_gauge(PROCESS_RUNTIME) .with_description("How much time the process has been running (in seconds)") .with_unit("seconds") .build() }); let disk_total_read_bytes = DISK_TOTAL_READ_BYTES.get_or_init(|| { meter - .u64_gauge("process_disk_total_read") + .u64_gauge(PROCESS_DISK_TOTAL_READ) .with_description("Total number of read bytes (in bytes).") .with_unit("bytes") .build() }); let disk_total_write_bytes = DISK_TOTAL_WRITE_BYTES.get_or_init(|| { meter - .u64_gauge("process_disk_total_write") + .u64_gauge(PROCESS_DISK_TOTAL_WRITE) .with_description("Total number of written bytes (in bytes).") .with_unit("bytes") .build() }); let cpu_percent = CPU_PERCENT.get_or_init(|| { - meter - .f64_gauge("process_cpu_live") - .with_description("Current CPU utilization (in %)") - .with_unit("%") - .build() + meter.f64_gauge(PROCESS_CPU_LIVE).with_description("Current CPU utilization (in %)").with_unit("%").build() }); let disk_live_read_bytes = DISK_LIVE_READ_BYTES.get_or_init(|| { meter - .u64_gauge("process_disk_live_read") + .u64_gauge(PROCESS_DISK_LIVE_READ) .with_description("Number of read bytes since the last refresh (in bytes)") .with_unit("bytes") .build() }); let disk_live_write_bytes = DISK_LIVE_WRITE_BYTES.get_or_init(|| { meter - .u64_gauge("process_disk_live_write") + .u64_gauge(PROCESS_DISK_LIVE_WRITE) .with_description("Number of written bytes since the last refresh (in bytes)") .with_unit("bytes") .build() }); - let process_memory_bytes = PROCESS_MEMORY_BYTES.get_or_init(|| { + let host_disk_live_read_bytes = HOST_DISK_LIVE_READ_BYTES.get_or_init(|| { meter - .u64_gauge("amaru_metrics_process_memory_footprint_bytes") - .with_description("Current process memory footprint (in bytes)") + .u64_gauge(HOST_DISK_LIVE_READ) + .with_description("Number of read bytes observed across the host since the last refresh (in bytes)") + .with_unit("bytes") + .build() + }); + let host_disk_live_write_bytes = HOST_DISK_LIVE_WRITE_BYTES.get_or_init(|| { + meter + .u64_gauge(HOST_DISK_LIVE_WRITE) + .with_description("Number of written bytes observed across the host since the last refresh (in bytes)") .with_unit("bytes") .build() }); let rss_bytes = RSS_BYTES.get_or_init(|| { meter - .u64_gauge("process_memory_live_resident") + .u64_gauge(PROCESS_MEMORY_LIVE_RESIDENT) .with_description( "The amount of memory that the process allocated and which is currently mapped in physical RAM (in bytes).", ) .with_unit("bytes") .build() }); + let footprint_bytes = FOOTPRINT_BYTES.get_or_init(|| { + meter + .u64_gauge(PROCESS_MEMORY_FOOTPRINT) + .with_description( + "Process memory footprint as reported by the local host tooling when available, falling back to the resident set size otherwise (in bytes).", + ) + .with_unit("bytes") + .build() + }); let virtual_bytes = VIRTUAL_BYTES.get_or_init(|| { meter - .u64_gauge("process_memory_available_virtual") + .u64_gauge(PROCESS_MEMORY_AVAILABLE_VIRTUAL) .with_description( "The amount of memory that the process can access, whether it is currently mapped in physical RAM or not (in bytes).", ) .with_unit("bytes") .build() }); + let host_memory_used_bytes = HOST_MEMORY_USED_BYTES.get_or_init(|| { + meter + .u64_gauge(HOST_MEMORY_USED) + .with_description("Amount of RAM currently used on the host (in bytes)") + .with_unit("bytes") + .build() + }); + let host_memory_total_bytes = HOST_MEMORY_TOTAL_BYTES.get_or_init(|| { + meter + .u64_gauge(HOST_MEMORY_TOTAL) + .with_description("Total amount of RAM available on the host (in bytes)") + .with_unit("bytes") + .build() + }); let open_files = OPEN_FILES.get_or_init(|| { - meter.u64_gauge("process_open_files").with_description("Total number of file descriptors.").build() + meter.u64_gauge(PROCESS_OPEN_FILES).with_description("Total number of file descriptors.").build() }); runtime_seconds.record(self.runtime_seconds, &[]); @@ -128,10 +181,14 @@ impl MetricRecorder for SystemMetrics { disk_total_write_bytes.record(self.disk_write_bytes, &[]); disk_live_read_bytes.record(self.disk_live_read_bytes, &[]); disk_live_write_bytes.record(self.disk_live_write_bytes, &[]); + host_disk_live_read_bytes.record(self.host_live_read_bytes, &[]); + host_disk_live_write_bytes.record(self.host_live_write_bytes, &[]); cpu_percent.record(self.cpu_percent, &[]); - process_memory_bytes.record(self.process_memory_bytes, &[]); - rss_bytes.record(self.rss_bytes, &[]); - virtual_bytes.record(self.virtual_bytes, &[]); + footprint_bytes.record(self.process_memory_bytes, &[]); + rss_bytes.record(self.process_memory_live_resident, &[]); + virtual_bytes.record(self.process_memory_available_virtual, &[]); + host_memory_used_bytes.record(self.memory_used_bytes, &[]); + host_memory_total_bytes.record(self.memory_total_bytes, &[]); open_files.record(self.open_files, &[]); } } diff --git a/crates/amaru-node/Cargo.toml b/crates/amaru-node/Cargo.toml index 1d746e5a3f..a33a21cc19 100644 --- a/crates/amaru-node/Cargo.toml +++ b/crates/amaru-node/Cargo.toml @@ -35,7 +35,6 @@ futures-util.workspace = true hex.workspace = true minicbor.workspace = true num.workspace = true -opentelemetry.workspace = true parking_lot.workspace = true reqwest.workspace = true serde.workspace = true @@ -57,6 +56,7 @@ amaru-consensus.workspace = true amaru-kernel = { workspace = true, features = ["tracing", "clap"] } amaru-ledger.workspace = true amaru-mempool.workspace = true +amaru-metrics.workspace = true amaru-network.workspace = true amaru-observability.workspace = true amaru-ouroboros.workspace = true diff --git a/crates/amaru-node/src/stages/build_node.rs b/crates/amaru-node/src/stages/build_node.rs index 00737b5b4c..00bb521f88 100644 --- a/crates/amaru-node/src/stages/build_node.rs +++ b/crates/amaru-node/src/stages/build_node.rs @@ -28,6 +28,7 @@ use amaru_ledger::{ state::State, }; use amaru_mempool::{InMemoryMempool, MempoolConfig}; +use amaru_metrics::Meter; use amaru_network::connection::TokioConnections; use amaru_ouroboros::{ChainStore, ConnectionsResource, MempoolMsg, PoolSummaries, ResourceMempool}; use amaru_plutus::arena_pool::ArenaPool; @@ -42,7 +43,6 @@ use amaru_pure_stage::{ }; use amaru_stores::rocksdb::{RocksDB, RocksDBHistoricalStores, consensus::RocksDBStore}; use anyhow::anyhow; -use opentelemetry::metrics::Meter; use parking_lot::Mutex; use tokio::runtime::Handle; @@ -55,7 +55,7 @@ use crate::{ }; /// Build a node given the provided configuration and run it using Tokio. -pub fn build_and_run_node(config: Config, meter: Option) -> anyhow::Result { +pub fn build_and_run_node(config: Config, meter: Arc) -> anyhow::Result { let trace_buffer = TraceBuffer::new_shared(config.trace_buffer_min_entries, config.trace_buffer_max_size); let mut stage_builder = TokioBuilder::default() .with_trace_buffer(trace_buffer) @@ -115,7 +115,7 @@ impl NodeRunning { pub fn build_node( config: &Config, global_parameters: &GlobalParameters, - meter: Option, + meter: Arc, stage_builder: &mut impl StageGraph, ) -> anyhow::Result { // NOTE: Open the chain store first so incompatible DB versions fail before the slower ledger open. @@ -217,7 +217,7 @@ fn register_resources( block_validator: Arc>, consensus_parameters: Arc, era_history: EraHistory, - meter: Option, + meter: Arc, mempool_config: MempoolConfig, ) { stage_graph.resources().put::(chain_store); @@ -233,9 +233,7 @@ fn register_resources( stage_graph.resources().put::(consensus_parameters); stage_graph.resources().put::(era_history); - if let Some(meter) = meter { - stage_graph.resources().put::(Arc::new(meter)); - }; + stage_graph.resources().put::(meter); stage_graph.resources().put::(Arc::new( amaru_consensus::performance::Performance::new(), diff --git a/crates/amaru-node/src/tests/setup.rs b/crates/amaru-node/src/tests/setup.rs index 145e9f2a88..0aab652bf1 100644 --- a/crates/amaru-node/src/tests/setup.rs +++ b/crates/amaru-node/src/tests/setup.rs @@ -23,6 +23,7 @@ use amaru_consensus::{ stages::test_utils::start_in_era, }; use amaru_kernel::{BlockHeight, ConsensusParameters, IsHeader, NetworkName, NonEmptyVec, Tip, Transaction}; +use amaru_metrics::Meter; use amaru_ouroboros::{ BaseReadChainStore, ConnectionsResource, DiagnosticChainStore, MockBlockValidator, MockCanValidateTxs, Nonces, PoolSummaries, ResourceMempool, has_stake_pools::MockHasStakePools, @@ -92,7 +93,7 @@ pub fn create_node(node_config: &NodeTestConfig, stage_graph: &mut impl StageGra // in order to simulate what happens when new tips are added and trigger a move of the best // chain anchor. global_parameters.consensus_security_param = node_config.chain_length as u64; - let node_stages = build_node(&config, &global_parameters, None, stage_graph) + let node_stages = build_node(&config, &global_parameters, Arc::new(Meter::default()), stage_graph) .map_err(|e| anyhow!("Cannot build node.\nThe node config is\n{:?}\n\nThe error is {e:?}", node_config))?; // The actions stage allows us to send NewTip messages to the manager so that chainsync diff --git a/crates/amaru-observability/macros/src/define_schemas.rs b/crates/amaru-observability/macros/src/define_schemas.rs index c90a817400..0e2169371b 100644 --- a/crates/amaru-observability/macros/src/define_schemas.rs +++ b/crates/amaru-observability/macros/src/define_schemas.rs @@ -1586,6 +1586,14 @@ fn generate_schema_item(schema: &Schema, config: &GenerationConfig) -> proc_macr let field_count = schema.required_fields.len() + schema.optional_fields.len(); let is_public = schema.public; let record_fields = record_fields_trait_path(); + let field_constants = schema.required_fields.iter().chain(schema.optional_fields.iter()).map(|field| { + let field_const_ident = make_ident(&format!("FIELD_{}", field.name.to_uppercase())); + let field_name = field.name.as_str(); + + quote! { + pub const #field_const_ident: &str = #field_name; + } + }); let accessors = if config.export_macros { let required_accessors = schema.required_fields.iter().map(|field| { @@ -1640,9 +1648,10 @@ fn generate_schema_item(schema: &Schema, config: &GenerationConfig) -> proc_macr pub const TARGET: &str = #schema_target; pub const PATH: &str = #schema_path; pub const VALIDATION: &str = #validation_string; + #(#field_constants)* #[doc(hidden)] - pub const FIELD_COUNT: usize = #field_count; + pub const SCHEMA_FIELD_COUNT: usize = #field_count; #[doc(hidden)] pub const PUBLIC: bool = #is_public; diff --git a/crates/amaru-observability/macros/src/traces.rs b/crates/amaru-observability/macros/src/traces.rs index 4653da4675..7832bd618d 100644 --- a/crates/amaru-observability/macros/src/traces.rs +++ b/crates/amaru-observability/macros/src/traces.rs @@ -789,7 +789,7 @@ pub fn expand_trace_span(input: TokenStream) -> TokenStream { let categories = meta.categories(); let record_macro_ident = make_ident(&make_record_macro_name(&categories, &meta.schema_name)); let assign_macro_ident = make_ident(&make_assign_macro_name(&categories, &meta.schema_name)); - let field_count_path = build_schema_associated_const_path(&meta, &args.schema_path, "FIELD_COUNT"); + let field_count_path = build_schema_associated_const_path(&meta, &args.schema_path, "SCHEMA_FIELD_COUNT"); let public_const_path = build_schema_associated_const_path(&meta, &args.schema_path, "PUBLIC"); let private_emit_guard = private_emit_guard_tokens(); diff --git a/crates/amaru-observability/tests/schema_access.rs b/crates/amaru-observability/tests/schema_access.rs index 88b191ec5c..5682c000e1 100644 --- a/crates/amaru-observability/tests/schema_access.rs +++ b/crates/amaru-observability/tests/schema_access.rs @@ -56,6 +56,8 @@ fn generated_schema_identity_and_accessors_are_typed() { assert_eq!(amaru::protocols::peer_selection::peer::CONNECTED::TARGET, "amaru::protocols"); assert_eq!(amaru::protocols::peer_selection::peer::CONNECTED::NAME, "peer_selection.peer.connected"); + assert_eq!(amaru::protocols::peer_selection::peer::CONNECTED::FIELD_PEER, "peer"); + assert_eq!(amaru::protocols::peer_selection::peer::CONNECTED::FIELD_DIRECTION, "direction"); assert!(amaru::protocols::peer_selection::peer::CONNECTED::matches( "amaru::protocols", "peer_selection.peer.connected" diff --git a/crates/amaru-protocols/src/metrics_effects.rs b/crates/amaru-protocols/src/metrics_effects.rs index 36da65571b..92b9b6200b 100644 --- a/crates/amaru-protocols/src/metrics_effects.rs +++ b/crates/amaru-protocols/src/metrics_effects.rs @@ -14,7 +14,7 @@ use std::sync::Arc; -use amaru_metrics::{Meter, MetricRecorder, MetricsEvent, notify_subscribers}; +use amaru_metrics::{Meter, MetricRecorder, MetricsEvent}; use amaru_pure_stage::{BoxFuture, Effects, ExternalEffect, ExternalEffectAPI, Resources, SendData}; /// Metrics operations available to a stage. This allows a stage to record a MetricsEvent that @@ -67,8 +67,6 @@ impl ExternalEffect for RecordMetricsEffect { if let Ok(meter) = resources.get::() { self.event.record_to_meter(&meter); } - notify_subscribers(&self.event); - // Metric export is optional, but subscribers still receive the typed event stream. }) } } diff --git a/crates/amaru-stores/src/lib.rs b/crates/amaru-stores/src/lib.rs index 855d089a64..65d1880aae 100644 --- a/crates/amaru-stores/src/lib.rs +++ b/crates/amaru-stores/src/lib.rs @@ -19,19 +19,22 @@ pub mod rocksdb; pub mod tests { use std::collections::BTreeMap; + #[cfg(not(target_os = "windows"))] + use amaru_kernel::any_proposal_id; use amaru_kernel::{ Anchor, DRepRegistration, Epoch, EraHistory, Hash, Lovelace, MemoizedTransactionOutput, PREPROD_DEFAULT_PROTOCOL_PARAMETERS, PREPROD_ERA_HISTORY, Point, PoolId, PoolParams, Slot, StakeCredential, - TransactionInput, any_certificate_pointer, any_hash28, any_lovelace, any_pool_params, any_proposal_id, - any_stake_credential, + TransactionInput, any_certificate_pointer, any_hash28, any_lovelace, any_pool_params, any_stake_credential, }; + #[cfg(not(target_os = "windows"))] + use amaru_ledger::store::columns::proposals; use amaru_ledger::{ epoch_transition::{GovernanceActivity, GovernanceUpdates, pools_updates::PoolCertificates}, state::volatile::Resettable, store::{ Columns, ReadStore, Store, StoreError, TransactionalContext, apply_governance_updates, columns::{ - accounts, cc_members, dreps, proposals, + accounts, cc_members, dreps, slots::tests::any_slot, utxo::tests::{any_memoized_transaction_output, any_txin}, }, diff --git a/crates/amaru-stores/src/rocksdb/consensus/tests.rs b/crates/amaru-stores/src/rocksdb/consensus/tests.rs index a7f9188204..a2ebe2343a 100644 --- a/crates/amaru-stores/src/rocksdb/consensus/tests.rs +++ b/crates/amaru-stores/src/rocksdb/consensus/tests.rs @@ -1613,6 +1613,7 @@ fn iterator_over_chain() { // HELPERS +#[cfg(not(target_os = "windows"))] const SAMPLE_HASH: &str = "4b1f95026700f5b3df8432b3f93b023f3cbdf13c85704e0f71b0089e6e81c947"; #[derive(Clone)] diff --git a/crates/amaru-tui/AGENTS.md b/crates/amaru-tui/AGENTS.md index 43049cb96e..e3c9b12489 100644 --- a/crates/amaru-tui/AGENTS.md +++ b/crates/amaru-tui/AGENTS.md @@ -30,18 +30,14 @@ Do not make the TUI: The normal data sources are: -- tracing events captured by `TracingLayer` -- shared metrics received through the metrics subscription +- embedded mode: in-process tracing capture plus a single callback for local + `MetricsEvent`s - immutable startup data passed through `StartupContext` ## Architectural boundaries Keep this separation sharp: -- `capture.rs` - - converts tracing records into TUI messages -- `metrics.rs` - - subscribes to `amaru-metrics` - `session.rs` - owns the thread, terminal loop, and lifecycle - `startup.rs` @@ -88,11 +84,13 @@ generates a typed accessor. Update: 1. the producer-side metric event shape -2. `src/metrics.rs` -3. the metric reducer paths in the model -4. tests that assert the derived UI state +2. the metric reducer paths in the model +3. tests that assert the derived UI state -Do not add a second metrics transport just for the TUI. +Embedded mode may reuse the single local metrics callback already wired through +observability, but avoid adding broader bespoke metric channels. Process memory +footprint should come from Amaru's exported system metrics, not from a TUI-local +sampler. ### When changing layout only diff --git a/crates/amaru-tui/Cargo.toml b/crates/amaru-tui/Cargo.toml index 3074f4f03e..69f915d882 100644 --- a/crates/amaru-tui/Cargo.toml +++ b/crates/amaru-tui/Cargo.toml @@ -23,7 +23,6 @@ clap.workspace = true crossterm.workspace = true ratatui.workspace = true signal-hook.workspace = true -sysinfo.workspace = true tracing.workspace = true tracing-subscriber.workspace = true diff --git a/crates/amaru-tui/README.md b/crates/amaru-tui/README.md index c2fa2ff320..f1545a525f 100644 --- a/crates/amaru-tui/README.md +++ b/crates/amaru-tui/README.md @@ -1,27 +1,24 @@ # Amaru TUI -`amaru-tui` is the embedded terminal UI used by Amaru when running in an -interactive terminal. +`amaru-tui` is the terminal UI used by Amaru when running in an interactive +terminal. It is launched automatically by `amaru node run` when `--no-tui` is +not set. The architectural rationale lives in [EDR 030](../../engineering-decision-records/030-embedded-terminal-observability-ui.md). ## Crate layout -- `src/capture.rs`: turns structured tracing into TUI messages -- `src/host_metrics.rs`: samples host-wide RAM and aggregate disk I/O for the TUI only -- `src/metrics.rs`: subscribes to the shared metrics stream +- `src/capture.rs`: captures in-process tracing records - `src/session.rs`: owns the terminal thread and runtime wiring - `src/startup.rs`: static startup context and config sections - `src/model.rs`: root app model and tests - `src/model/`: bounded submodels plus reducer slices grouped by concern - `interaction.rs`: keyboard, mouse, focus, pane toggles, paging - - `telemetry_update.rs`: folds structured tracing into model state - - `metrics_update.rs`: folds shared metrics into time-series state + - `telemetry_update.rs`: folds OTLP-derived telemetry into model state + - `metrics_update.rs`: folds OTLP-derived metrics into UI state - `queries.rs`: derived read-only views over bounded model state - `src/ui/mod.rs`: thin compositor and shell chrome -- `src/ui/screens/`: page-level layout for `Amaru`, `Cardano`, `Config`, and `Splash` -- `src/ui/components/`: reusable panel renderers and widget-level view logic - `src/ui/common.rs`: shared border, scrollbar, gradient, and layout helpers - `src/ui/format.rs` and `src/ui/theme.rs`: shared formatting and styling primitives @@ -36,16 +33,23 @@ The crate should not become a backdoor into Amaru internals. Prefer deriving new state from telemetry or shared metrics over introducing direct database or store access. -Shared metrics should remain about the Amaru process itself. When the TUI needs -host-wide context that would be too expensive or too product-specific to export -globally, keep that sampling local to `amaru-tui`. +Embedded sessions use the same model and reducers. The only extra wiring is: -The TUI keeps bounded fixed-capacity samples rather than user-selectable rolling -time windows. Throughput, rollback, and peer timing widgets therefore summarize -the most recent retained samples, and the log pane keeps only the lines that -match the currently active filter from the moment that filter is selected. -Process and host resource gauges are simpler: they render from the latest merged -`SystemSample` snapshot rather than keeping historical TUI-local copies. +- an in-process tracing layer that forwards structured telemetry into the TUI +- a single in-process callback that mirrors local `MetricsEvent`s + +Shared metrics should remain about the Amaru process itself and the small set of +host totals that are genuinely useful to export globally. Process memory +footprint is sampled by Amaru itself and exported through the normal system +metrics payload, so the TUI consumes one metrics stream. + +The TUI keeps bounded state rather than user-selectable rolling time windows. +Throughput and peer timing widgets use exponential moving averages, rollback +widgets keep a bounded recent history, and the log pane keeps one ordered stream +backed by per-severity retention buckets plus a pre-filtered view for the active +level and target filters. Process and host resource gauges are simpler: they +render from the latest merged `SystemSample` snapshot rather than keeping +historical TUI-local copies. For telemetry, prefer the schema-generated helpers exported by `amaru-observability` for both event matching and field decoding. Avoid raw @@ -64,3 +68,4 @@ When changing the TUI itself: - `cargo check -p amaru-tui -p amaru` - `cargo test -p amaru-tui` +- `cargo run -p amaru -- node run --network preview` diff --git a/crates/amaru-tui/src/capture.rs b/crates/amaru-tui/src/capture.rs index 06b9ced57e..4f3ccdd1e5 100644 --- a/crates/amaru-tui/src/capture.rs +++ b/crates/amaru-tui/src/capture.rs @@ -26,7 +26,7 @@ use tracing::{ }; use tracing_subscriber::{Layer, layer::Context, registry::LookupSpan}; -use crate::events::{FieldValue, Message, TelemetryKind, TelemetryRecord}; +use crate::events::{FieldValue, Message, TelemetryRecord}; const TAG_FIELD_PREFIX: &str = "amaru.tag."; @@ -40,10 +40,6 @@ impl TracingLayer { Self { tx } } - pub fn sender(&self) -> SyncSender { - self.tx.clone() - } - fn emit(&self, record: TelemetryRecord) { match self.tx.try_send(Message::Telemetry(record)) { Ok(()) => {} @@ -72,7 +68,6 @@ where event.record(&mut visitor); self.emit(TelemetryRecord { - kind: TelemetryKind::Event, level: *event.metadata().level(), target: event.metadata().target().to_string(), name: event.metadata().name().to_string(), @@ -124,7 +119,6 @@ where }; self.emit(TelemetryRecord { - kind: TelemetryKind::SpanClose, level: state.level, target: state.target, name: state.name, @@ -192,6 +186,8 @@ impl Visit for FieldVisitor { mod tests { use std::sync::mpsc::sync_channel; + use amaru_kernel::{Epoch, NULL_HASH32, Slot, TransactionId}; + use amaru_observability::{amaru::ledger, info, info_span}; use tracing_subscriber::prelude::*; use super::*; @@ -202,16 +198,20 @@ mod tests { let subscriber = tracing_subscriber::registry().with(TracingLayer::new(tx)); tracing::subscriber::with_default(subscriber, || { - tracing::info!(message = "hello", transaction_id = "abc", "amaru.tag.cpu" = true); + info!(ledger::transaction::VALIDATE, transaction_id = TransactionId::new(NULL_HASH32),); }); let Message::Telemetry(record) = rx.recv().expect("telemetry event") else { panic!("expected telemetry event") }; - assert_eq!(record.kind, TelemetryKind::Event); - assert_eq!(record.fields.get("transaction_id"), Some(&FieldValue::String("abc".into()))); - assert!(!record.fields.contains_key("amaru.tag.cpu")); + assert_eq!(record.target, ledger::transaction::VALIDATE::TARGET); + assert_eq!(record.name, ledger::transaction::VALIDATE::NAME); + assert_eq!( + record.fields.get(ledger::transaction::VALIDATE::FIELD_TRANSACTION_ID), + Some(&FieldValue::String(TransactionId::new(NULL_HASH32).to_string())) + ); + assert!(record.fields.keys().all(|name| !name.starts_with(TAG_FIELD_PREFIX))); } #[test] @@ -220,14 +220,26 @@ mod tests { let subscriber = tracing_subscriber::registry().with(TracingLayer::new(tx)); tracing::subscriber::with_default(subscriber, || { - let span = tracing::info_span!("test_span", slot = 42u64, "amaru.tag.cpu" = true); + let span = info_span!( + ledger::tip::UPDATE, + slot = Slot::from(42u64), + header_hash = NULL_HASH32, + block_height = 1u64, + tx_count = 2usize, + epoch = Epoch::from(3u64), + slot_in_epoch = Slot::from(4u64), + density = 0.5f64, + current_kes_period = 5u64, + remaining_kes_periods = 6u64, + ); let _guard = span.enter(); }); let Message::Telemetry(record) = rx.recv().expect("telemetry span") else { panic!("expected telemetry span") }; - assert_eq!(record.kind, TelemetryKind::SpanClose); - assert_eq!(record.fields.get("slot"), Some(&FieldValue::U64(42))); - assert!(!record.fields.contains_key("amaru.tag.cpu")); + assert_eq!(record.target, ledger::tip::UPDATE::TARGET); + assert_eq!(record.name, ledger::tip::UPDATE::NAME); + assert_eq!(record.fields.get(ledger::tip::UPDATE::FIELD_SLOT), Some(&FieldValue::String("42".to_string()))); + assert!(record.fields.keys().all(|name| !name.starts_with(TAG_FIELD_PREFIX))); } } diff --git a/crates/amaru-tui/src/config.rs b/crates/amaru-tui/src/config.rs index c608fe9416..b01963812a 100644 --- a/crates/amaru-tui/src/config.rs +++ b/crates/amaru-tui/src/config.rs @@ -14,9 +14,14 @@ use std::time::Duration; +use tracing::Level; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Config { - pub log_capacity: usize, + pub debug_log_capacity: usize, + pub info_log_capacity: usize, + pub warn_log_capacity: usize, + pub error_log_capacity: usize, pub block_sample_capacity: usize, pub transaction_sample_capacity: usize, pub rollback_sample_capacity: usize, @@ -32,7 +37,10 @@ pub struct Config { impl Default for Config { fn default() -> Self { Self { - log_capacity: 4_096, + debug_log_capacity: 1_000, + info_log_capacity: 500, + warn_log_capacity: 100, + error_log_capacity: 100, block_sample_capacity: 100, transaction_sample_capacity: 100, rollback_sample_capacity: 100, @@ -46,3 +54,14 @@ impl Default for Config { } } } + +impl Config { + pub fn log_capacity_for(&self, level: Level) -> usize { + match level { + Level::TRACE | Level::DEBUG => self.debug_log_capacity, + Level::INFO => self.info_log_capacity, + Level::WARN => self.warn_log_capacity, + Level::ERROR => self.error_log_capacity, + } + } +} diff --git a/crates/amaru-tui/src/events.rs b/crates/amaru-tui/src/events.rs index 15812f66e7..81052d8d5b 100644 --- a/crates/amaru-tui/src/events.rs +++ b/crates/amaru-tui/src/events.rs @@ -13,14 +13,12 @@ // limitations under the License. mod field_value; -mod host_sample; mod message; mod metric_record; mod system_sample; -mod telemetry_kind; mod telemetry_record; pub use self::{ - field_value::FieldValue, host_sample::HostSample, message::Message, metric_record::MetricRecord, - system_sample::SystemSample, telemetry_kind::TelemetryKind, telemetry_record::TelemetryRecord, + field_value::FieldValue, message::Message, metric_record::MetricRecord, system_sample::SystemSample, + telemetry_record::TelemetryRecord, }; diff --git a/crates/amaru-tui/src/events/field_value.rs b/crates/amaru-tui/src/events/field_value.rs index 9b379e7ced..5b5255cacd 100644 --- a/crates/amaru-tui/src/events/field_value.rs +++ b/crates/amaru-tui/src/events/field_value.rs @@ -14,6 +14,8 @@ use std::fmt; +use tracing::Level; + #[derive(Debug, Clone, PartialEq)] pub enum FieldValue { Bool(bool), @@ -58,6 +60,65 @@ impl FieldValue { Self::Bool(_) | Self::I64(_) | Self::U64(_) | Self::F64(_) => None, } } + + pub fn as_level(&self) -> Option { + match self.as_str()? { + "TRACE" => Some(Level::TRACE), + "DEBUG" => Some(Level::DEBUG), + "INFO" => Some(Level::INFO), + "WARN" => Some(Level::WARN), + "ERROR" => Some(Level::ERROR), + _ => None, + } + } +} + +impl From for FieldValue { + fn from(value: bool) -> Self { + Self::Bool(value) + } +} + +impl From for FieldValue { + fn from(value: f64) -> Self { + Self::F64(value) + } +} + +impl From for FieldValue { + fn from(value: i64) -> Self { + Self::I64(value) + } +} + +impl From for FieldValue { + fn from(value: u32) -> Self { + Self::U64(u64::from(value)) + } +} + +impl From for FieldValue { + fn from(value: u64) -> Self { + Self::U64(value) + } +} + +impl From for FieldValue { + fn from(value: usize) -> Self { + Self::U64(value as u64) + } +} + +impl From<&str> for FieldValue { + fn from(value: &str) -> Self { + Self::String(value.into()) + } +} + +impl From for FieldValue { + fn from(value: String) -> Self { + Self::String(value) + } } impl fmt::Display for FieldValue { diff --git a/crates/amaru-tui/src/events/host_sample.rs b/crates/amaru-tui/src/events/host_sample.rs deleted file mode 100644 index 0fe188a570..0000000000 --- a/crates/amaru-tui/src/events/host_sample.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2026 PRAGMA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::time::{Duration, Instant}; - -#[derive(Debug, Clone, PartialEq)] -pub struct HostSample { - pub at: Instant, - pub interval: Duration, - pub process_memory_bytes: Option, - pub memory_used_bytes: u64, - pub memory_total_bytes: u64, - pub host_live_read_bytes: u64, - pub host_live_write_bytes: u64, -} - -impl HostSample { - pub fn host_live_read_bytes_per_second(&self) -> u64 { - bytes_per_second(self.host_live_read_bytes, self.interval) - } - - pub fn host_live_write_bytes_per_second(&self) -> u64 { - bytes_per_second(self.host_live_write_bytes, self.interval) - } -} - -fn bytes_per_second(bytes: u64, interval: Duration) -> u64 { - let seconds = interval.as_secs_f64(); - if seconds == 0.0 { 0 } else { (bytes as f64 / seconds).round() as u64 } -} diff --git a/crates/amaru-tui/src/events/message.rs b/crates/amaru-tui/src/events/message.rs index 74eea4fba8..870cff353d 100644 --- a/crates/amaru-tui/src/events/message.rs +++ b/crates/amaru-tui/src/events/message.rs @@ -12,11 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use super::{HostSample, MetricRecord, TelemetryRecord}; +use super::{MetricRecord, TelemetryRecord}; #[derive(Debug, Clone, PartialEq)] pub enum Message { Telemetry(TelemetryRecord), Metrics(MetricRecord), - HostSample(HostSample), } diff --git a/crates/amaru-tui/src/events/telemetry_record.rs b/crates/amaru-tui/src/events/telemetry_record.rs index ca98be9f39..90063f8ecb 100644 --- a/crates/amaru-tui/src/events/telemetry_record.rs +++ b/crates/amaru-tui/src/events/telemetry_record.rs @@ -20,11 +20,10 @@ use std::{ use amaru_observability::RecordFields; use tracing::Level; -use super::{FieldValue, TelemetryKind}; +use super::FieldValue; #[derive(Debug, Clone, PartialEq)] pub struct TelemetryRecord { - pub kind: TelemetryKind, pub level: Level, pub target: String, pub name: String, diff --git a/crates/amaru-tui/src/host_metrics.rs b/crates/amaru-tui/src/host_metrics.rs deleted file mode 100644 index ff1eb0d464..0000000000 --- a/crates/amaru-tui/src/host_metrics.rs +++ /dev/null @@ -1,249 +0,0 @@ -// Copyright 2026 PRAGMA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#[cfg(any(target_os = "linux", target_os = "macos"))] -use std::process::Command; -use std::{ - io, - sync::mpsc::{self, Receiver, RecvTimeoutError, Sender, SyncSender}, - thread::{self, JoinHandle}, - time::{Duration, Instant}, -}; - -use sysinfo::{DiskRefreshKind, Disks, MemoryRefreshKind, RefreshKind, System, get_current_pid}; - -use crate::events::{HostSample, Message}; - -const POLL_DELAY: Duration = Duration::from_secs(5); - -pub struct Sampler { - shutdown_tx: Sender<()>, - join: Option>>, -} - -impl Sampler { - pub fn spawn(tx: SyncSender) -> io::Result { - let (shutdown_tx, shutdown_rx) = mpsc::channel(); - let join = thread::Builder::new() - .name("amaru-tui-host-metrics".into()) - .spawn(move || run(tx, shutdown_rx)) - .map_err(|err| io::Error::other(format!("failed to spawn host metrics thread: {err}")))?; - - Ok(Self { shutdown_tx, join: Some(join) }) - } - - pub fn shutdown(mut self) -> io::Result<()> { - self.shutdown_inner() - } - - fn shutdown_inner(&mut self) -> io::Result<()> { - let _ = self.shutdown_tx.send(()); - if let Some(join) = self.join.take() { - join.join().map_err(|_| io::Error::other("host metrics thread panicked"))? - } else { - Ok(()) - } - } -} - -impl Drop for Sampler { - fn drop(&mut self) { - let _ = self.shutdown_inner(); - } -} - -fn run(tx: SyncSender, shutdown_rx: Receiver<()>) -> io::Result<()> { - let mut sys = - System::new_with_specifics(RefreshKind::nothing().with_memory(MemoryRefreshKind::nothing().with_ram())); - let mut disks = Disks::new_with_refreshed_list_specifics(DiskRefreshKind::nothing().with_io_usage()); - let own_pid = get_current_pid().map_err(|err| io::Error::other(format!("failed to resolve own pid: {err}")))?; - let mut last_sampled_at = Instant::now(); - - emit_sample(&mut sys, &mut disks, &tx, own_pid, Duration::ZERO, last_sampled_at); - - loop { - match shutdown_rx.recv_timeout(POLL_DELAY) { - Ok(()) | Err(RecvTimeoutError::Disconnected) => return Ok(()), - Err(RecvTimeoutError::Timeout) => {} - } - - let now = Instant::now(); - let interval = now.duration_since(last_sampled_at); - last_sampled_at = now; - - emit_sample(&mut sys, &mut disks, &tx, own_pid, interval, now); - } -} - -fn emit_sample( - sys: &mut System, - disks: &mut Disks, - tx: &SyncSender, - own_pid: sysinfo::Pid, - interval: Duration, - at: Instant, -) { - sys.refresh_memory_specifics(MemoryRefreshKind::nothing().with_ram()); - disks.refresh_specifics(false, DiskRefreshKind::nothing().with_io_usage()); - let process_memory = sample_process_memory(own_pid); - let (host_live_read_bytes, host_live_write_bytes) = - disks.iter().fold((0u64, 0u64), |(read_total, write_total), disk| { - let usage = disk.usage(); - (read_total.saturating_add(usage.read_bytes), write_total.saturating_add(usage.written_bytes)) - }); - - let _ = tx.try_send(Message::HostSample(HostSample { - at, - interval, - process_memory_bytes: process_memory, - memory_used_bytes: sys.used_memory(), - memory_total_bytes: sys.total_memory(), - host_live_read_bytes, - host_live_write_bytes, - })); -} - -#[cfg(target_os = "macos")] -fn sample_process_memory(pid: sysinfo::Pid) -> Option { - let output = - Command::new("top").args(["-l", "1", "-pid", &pid.as_u32().to_string(), "-stats", "pid,mem"]).output().ok()?; - - if !output.status.success() { - return None; - } - - parse_top_process_memory(&String::from_utf8_lossy(&output.stdout), pid.as_u32()) -} - -#[cfg(target_os = "linux")] -fn sample_process_memory(pid: sysinfo::Pid) -> Option { - let output = Command::new("top").args(["-b", "-n", "1", "-p", &pid.as_u32().to_string()]).output().ok()?; - - if !output.status.success() { - return None; - } - - parse_linux_top_process_memory(&String::from_utf8_lossy(&output.stdout), pid.as_u32()) -} - -#[cfg(not(any(target_os = "linux", target_os = "macos")))] -fn sample_process_memory(_pid: sysinfo::Pid) -> Option { - None -} - -#[cfg(any(target_os = "macos", test))] -fn parse_top_process_memory(output: &str, pid: u32) -> Option { - output.lines().rev().find_map(|line| { - let mut fields = line.split_whitespace(); - let line_pid = fields.next()?.parse::().ok()?; - if line_pid != pid { - return None; - } - - parse_top_bytes(fields.next()?, PlainUnit::Bytes) - }) -} - -#[cfg(any(target_os = "linux", test))] -fn parse_linux_top_process_memory(output: &str, pid: u32) -> Option { - output.lines().rev().find_map(|line| { - let mut fields = line.split_whitespace(); - let line_pid = fields.next()?.parse::().ok()?; - if line_pid != pid { - return None; - } - - for _ in 0..4 { - fields.next()?; - } - - parse_top_bytes(fields.next()?, PlainUnit::KiB) - }) -} - -#[cfg(any(target_os = "linux", target_os = "macos", test))] -#[allow(dead_code)] -enum PlainUnit { - Bytes, - KiB, -} - -#[cfg(any(target_os = "linux", target_os = "macos", test))] -fn parse_top_bytes(value: &str, plain_unit: PlainUnit) -> Option { - let value = value.trim_end_matches('+'); - let suffix = value.chars().last()?; - - let multiplier = match suffix { - 'K' => 1_024f64, - 'M' => 1_024f64 * 1_024f64, - 'G' => 1_024f64 * 1_024f64 * 1_024f64, - 'T' => 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64, - 'P' => 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64, - 'k' => 1_024f64, - 'm' => 1_024f64 * 1_024f64, - 'g' => 1_024f64 * 1_024f64 * 1_024f64, - 't' => 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64, - 'p' => 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64 * 1_024f64, - '0'..='9' => { - let amount = value.parse::().ok()?; - return Some(match plain_unit { - PlainUnit::Bytes => amount, - PlainUnit::KiB => amount.saturating_mul(1_024), - }); - } - _ => return None, - }; - - let amount = value[..value.len() - 1].parse::().ok()?; - Some((amount * multiplier).round() as u64) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_top_memory_suffixes() { - assert_eq!(parse_top_bytes("1201K", PlainUnit::Bytes), Some(1_229_824)); - assert_eq!(parse_top_bytes("1.5M", PlainUnit::Bytes), Some(1_572_864)); - assert_eq!(parse_top_bytes("2.0G", PlainUnit::Bytes), Some(2_147_483_648)); - assert_eq!(parse_top_bytes("42", PlainUnit::Bytes), Some(42)); - assert_eq!(parse_top_bytes("42", PlainUnit::KiB), Some(43_008)); - assert_eq!(parse_top_bytes("1.5g", PlainUnit::KiB), Some(1_610_612_736)); - } - - #[test] - fn parses_top_process_sample_for_expected_pid() { - let output = "\ -Processes: 1 total\n\ -PID MEM\n\ -73194 1.5G\n"; - - assert_eq!(parse_top_process_memory(output, 73_194), Some(1_610_612_736)); - } - - #[test] - fn parses_linux_top_process_sample_for_expected_pid() { - let output = "\ -top - 12:00:00 up 1 day, 1 user, load average: 0.00, 0.00, 0.00\n\ -Tasks: 1 total, 1 running, 0 sleeping, 0 stopped, 0 zombie\n\ -%Cpu(s): 0.0 us, 0.0 sy, 0.0 ni,100.0 id, 0.0 wa, 0.0 hi, 0.0 si, 0.0 st \n\ -MiB Mem : 1024.0 total, 256.0 free, 512.0 used, 256.0 buff/cache\n\ -\n\ - PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND\n\ - 73194 user 20 0 1234567 654321 12345 S 0.0 0.1 0:00.01 amaru\n"; - - assert_eq!(parse_linux_top_process_memory(output, 73_194), Some(670_024_704)); - } -} diff --git a/crates/amaru-tui/src/lib.rs b/crates/amaru-tui/src/lib.rs index 910f5b1b91..8d54f65d9f 100644 --- a/crates/amaru-tui/src/lib.rs +++ b/crates/amaru-tui/src/lib.rs @@ -15,8 +15,6 @@ mod capture; mod config; mod events; -mod host_metrics; -mod metrics; mod model; mod session; mod settings; diff --git a/crates/amaru-tui/src/metrics.rs b/crates/amaru-tui/src/metrics.rs deleted file mode 100644 index 0f84e0d849..0000000000 --- a/crates/amaru-tui/src/metrics.rs +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2026 PRAGMA -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::{ - sync::{Arc, mpsc::SyncSender}, - time::Instant, -}; - -use amaru_metrics::{MetricsEvent, subscribe}; - -use crate::events::{Message, MetricRecord}; - -#[derive(Debug)] -pub struct Subscriber { - tx: SyncSender, -} - -impl Subscriber { - pub fn new(tx: SyncSender) -> Self { - Self { tx } - } - - pub fn emit(&self, event: &MetricsEvent) { - let _ = self.tx.try_send(Message::Metrics(MetricRecord { at: Instant::now(), event: event.clone() })); - } -} - -#[derive(Debug)] -pub struct Subscription { - _inner: amaru_metrics::Subscription, -} - -impl Subscription { - pub fn new(subscriber: Arc) -> Self { - let inner = subscribe(Arc::new(move |event| subscriber.emit(event))); - Self { _inner: inner } - } -} diff --git a/crates/amaru-tui/src/model.rs b/crates/amaru-tui/src/model.rs index 4b0427ac9e..a33eaac8f7 100644 --- a/crates/amaru-tui/src/model.rs +++ b/crates/amaru-tui/src/model.rs @@ -17,17 +17,29 @@ use std::{ time::Instant, }; +use self::{ + governance_summary::GovernanceSummary, + log_buffer::LogBuffer, + mempool_state::MempoolState, + proposal_activity::{ProposalActivity, proposal_id}, + rate_counter::RateCounter, + stake_snapshot_state::StakeSnapshotState, + telemetry_event::TelemetryEvent, + tip_state::TipState, +}; use crate::{ config::Config, events::{SystemSample, TelemetryRecord}, startup::StartupContext, }; +mod exponential_moving_average; mod governance_summary; mod initial_stake_distribution_state; mod interaction; mod interaction_mode; mod level_filter; +mod log_buffer; mod mempool_state; mod metrics_update; mod page; @@ -35,6 +47,7 @@ mod pane_mode; mod peer_state; mod proposal_activity; mod queries; +mod rate_counter; mod scroll_focus; mod stake_snapshot_state; mod target_filter; @@ -43,14 +56,6 @@ mod telemetry_update; mod terminal_event_outcome; mod tip_state; -use self::{ - governance_summary::GovernanceSummary, - mempool_state::MempoolState, - proposal_activity::{ProposalActivity, proposal_id}, - stake_snapshot_state::StakeSnapshotState, - telemetry_event::TelemetryEvent, - tip_state::TipState, -}; pub use self::{ initial_stake_distribution_state::InitialStakeDistributionState, interaction_mode::InteractionMode, level_filter::LevelFilter, page::Page, pane_mode::PaneMode, peer_state::PeerState, scroll_focus::ScrollFocus, @@ -87,10 +92,10 @@ pub struct Model { pub epoch_overlay_exists: bool, pub rewards_ready: bool, pub peers: BTreeMap, - pub logs: VecDeque, + pub logs: LogBuffer, pub system_sample: Option, - pub recent_blocks: VecDeque, - pub recent_transactions: VecDeque<(Instant, u64)>, + pub block_rate: RateCounter, + pub transaction_rate: RateCounter, pub recent_rollbacks: VecDeque<(Instant, usize)>, pub initial_stake_distribution_order: Vec, pub initial_stake_distributions: BTreeMap, @@ -131,10 +136,10 @@ impl Model { epoch_overlay_exists: false, rewards_ready: false, peers: BTreeMap::default(), - logs: VecDeque::default(), + logs: LogBuffer::default(), system_sample: None, - recent_blocks: VecDeque::default(), - recent_transactions: VecDeque::default(), + block_rate: RateCounter::new(config.block_sample_capacity), + transaction_rate: RateCounter::new(config.transaction_sample_capacity), recent_rollbacks: VecDeque::default(), initial_stake_distribution_order: Vec::default(), initial_stake_distributions: BTreeMap::default(), @@ -172,64 +177,58 @@ pub fn render_fields(record: &TelemetryRecord) -> String { mod tests { use std::time::Duration; - use amaru_metrics::{MetricsEvent, ledger::LedgerMetrics, system::SystemMetrics}; + use amaru_metrics::{MetricsEvent, system::SystemMetrics}; + use amaru_observability::amaru::{consensus, ledger, protocols}; use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use tracing::Level; use super::*; use crate::{ - events::{FieldValue, HostSample, Message, TelemetryKind, TelemetryRecord}, - model::telemetry_event::{LEDGER_TARGET, PROTOCOLS_TARGET}, + events::{FieldValue, Message, TelemetryRecord}, startup::ProcessInfo, }; - fn telemetry(target: &str, name: &str, fields: &[(&str, FieldValue)]) -> TelemetryRecord { - telemetry_at(Instant::now(), target, name, fields) - } - - fn telemetry_at(at: Instant, target: &str, name: &str, fields: &[(&str, FieldValue)]) -> TelemetryRecord { + fn telemetry_record( + at: Instant, + target: &str, + name: &str, + fields: [(&str, FieldValue); N], + ) -> TelemetryRecord { TelemetryRecord { - kind: TelemetryKind::Event, level: Level::INFO, target: target.into(), name: name.into(), at, wall_time: std::time::SystemTime::UNIX_EPOCH, - fields: fields.iter().map(|(name, value)| ((*name).into(), value.clone())).collect(), + fields: fields.into_iter().map(|(name, value)| (name.into(), value)).collect(), } } - fn metric(at: Instant, event: MetricsEvent) -> Message { - Message::Metrics(crate::events::MetricRecord { at, event }) + macro_rules! telemetry { + ($schema:path $(, $field:path => $value:expr )* $(,)?) => { + telemetry_at!(Instant::now(), $schema $(, $field => $value )*) + }; } - fn host_sample(at: Instant, interval: Duration) -> Message { - Message::HostSample(HostSample { - at, - interval, - process_memory_bytes: None, - memory_used_bytes: 100_000, - memory_total_bytes: 200_000, - host_live_read_bytes: 1_500, - host_live_write_bytes: 2_500, - }) + macro_rules! telemetry_at { + ($at:expr, $schema:path $(, $field:path => $value:expr )* $(,)?) => { + telemetry_record( + $at, + <$schema>::TARGET, + <$schema>::NAME, + [$(($field, FieldValue::from($value))),*], + ) + }; } - fn host_sample_with_process_memory(at: Instant, interval: Duration, process_memory_bytes: u64) -> Message { - Message::HostSample(HostSample { - at, - interval, - process_memory_bytes: Some(process_memory_bytes), - memory_used_bytes: 100_000, - memory_total_bytes: 200_000, - host_live_read_bytes: 1_500, - host_live_write_bytes: 2_500, - }) + fn metric(at: Instant, event: MetricsEvent) -> Message { + Message::Metrics(crate::events::MetricRecord { at, event }) } - fn startup_context() -> StartupContext { + fn fixture_startup_context() -> StartupContext { StartupContext { process: ProcessInfo { + pid: 42, network: "preview".into(), software_version: "10.11.0 (abc123)".into(), target: "darwin/aarch64".into(), @@ -243,46 +242,25 @@ mod tests { system_start_millis: 1_666_656_000_000, era_history: None, runtime_sections: Vec::default(), - global_sections: Vec::default(), protocol_sections: Vec::default(), } } #[test] fn updates_tip_from_public_event() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); - - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "tip.update", - &[ - ("slot", FieldValue::U64(1)), - ("header_hash", FieldValue::String("abc".into())), - ("block_height", FieldValue::U64(2)), - ("epoch", FieldValue::U64(3)), - ("slot_in_epoch", FieldValue::U64(4)), - ("density", FieldValue::F64(0.5)), - ("current_kes_period", FieldValue::U64(5)), - ("remaining_kes_periods", FieldValue::U64(6)), - ], + let mut model = Model::new(Config::default(), fixture_startup_context()); + + model.handle_message(Message::Telemetry(telemetry!( + ledger::tip::UPDATE, + ledger::tip::UPDATE::FIELD_SLOT => 1u64, + ledger::tip::UPDATE::FIELD_HEADER_HASH => "abc", + ledger::tip::UPDATE::FIELD_BLOCK_HEIGHT => 2u64, + ledger::tip::UPDATE::FIELD_TX_COUNT => 7u64, + ledger::tip::UPDATE::FIELD_EPOCH => 3u64, + ledger::tip::UPDATE::FIELD_SLOT_IN_EPOCH => 4u64, + ledger::tip::UPDATE::FIELD_DENSITY => 0.5f64, + ledger::tip::UPDATE::FIELD_CURRENT_KES_PERIOD => 5u64, + ledger::tip::UPDATE::FIELD_REMAINING_KES_PERIODS => 6u64, ))); let tip = model.tip.expect("tip must be recorded"); @@ -292,8 +270,8 @@ mod tests { } #[test] - fn keeps_host_process_memory_override_across_metrics_updates() { - let mut model = Model::new(Config::default(), startup_context()); + fn records_process_memory_from_system_metrics() { + let mut model = Model::new(Config::default(), fixture_startup_context()); let at = Instant::now(); model.handle_message(metric( @@ -301,134 +279,106 @@ mod tests { MetricsEvent::SystemMetrics(SystemMetrics { runtime_seconds: 1, cpu_percent: 12.5, - process_memory_bytes: 10_000, - rss_bytes: 9_000, - virtual_bytes: 12_000, + process_memory_bytes: 15_000, + process_memory_live_resident: 9_000, + process_memory_available_virtual: 12_000, + memory_used_bytes: 100_000, + memory_total_bytes: 200_000, disk_read_bytes: 300, disk_write_bytes: 400, disk_live_read_bytes: 30, disk_live_write_bytes: 40, + host_live_read_bytes: 300, + host_live_write_bytes: 500, open_files: 5, }), )); - model.handle_message(host_sample_with_process_memory( - at + Duration::from_secs(5), - Duration::from_secs(5), - 15_000, - )); model.handle_message(metric( at + Duration::from_secs(6), MetricsEvent::SystemMetrics(SystemMetrics { runtime_seconds: 2, cpu_percent: 14.5, - process_memory_bytes: 10_500, - rss_bytes: 9_500, - virtual_bytes: 12_500, + process_memory_bytes: 16_000, + process_memory_live_resident: 9_500, + process_memory_available_virtual: 12_500, + memory_used_bytes: 110_000, + memory_total_bytes: 210_000, disk_read_bytes: 350, disk_write_bytes: 450, disk_live_read_bytes: 35, disk_live_write_bytes: 45, + host_live_read_bytes: 350, + host_live_write_bytes: 550, open_files: 6, }), )); - assert_eq!(model.system_sample.as_ref().map(|sample| sample.process_memory_bytes), Some(15_000)); + assert_eq!(model.system_sample.as_ref().map(|sample| sample.process_memory_bytes), Some(16_000)); assert_eq!(model.system_sample.as_ref().map(|sample| sample.rss_bytes), Some(9_500)); } #[test] - fn records_throughput_and_system_samples_from_metrics() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); + fn records_throughput_from_roll_forwards_transaction_validations_and_system_samples_from_metrics() { + let mut model = Model::new(Config::default(), fixture_startup_context()); let at = Instant::now(); - model.handle_message(metric( + model.handle_message(Message::Telemetry(telemetry_at!(at, ledger::state::ROLL_FORWARD,))); + model.handle_message(Message::Telemetry(telemetry_at!( at, - MetricsEvent::LedgerMetrics(LedgerMetrics { - block_height: 42, - tx_count: 7, - slot: 100, - slot_in_epoch: 10, - epoch: 1, - density: 0.5, - current_kes_period: 2, - remaining_kes_periods: 3, - block_header_hash: "abc".into(), - parent_block_header_hash: "def".into(), - issuer_verification_key_hash: "ghi".into(), - }), - )); + ledger::tip::UPDATE, + ledger::tip::UPDATE::FIELD_SLOT => 100u64, + ledger::tip::UPDATE::FIELD_HEADER_HASH => "abc", + ledger::tip::UPDATE::FIELD_BLOCK_HEIGHT => 42u64, + ledger::tip::UPDATE::FIELD_TX_COUNT => 7u64, + ledger::tip::UPDATE::FIELD_EPOCH => 1u64, + ledger::tip::UPDATE::FIELD_SLOT_IN_EPOCH => 10u64, + ledger::tip::UPDATE::FIELD_DENSITY => 0.5f64, + ledger::tip::UPDATE::FIELD_CURRENT_KES_PERIOD => 2u64, + ledger::tip::UPDATE::FIELD_REMAINING_KES_PERIODS => 3u64, + ))); + for index in 0..7 { + model.handle_message(Message::Telemetry(telemetry_at!( + at + Duration::from_millis(index), + ledger::transaction::VALIDATE, + ledger::transaction::VALIDATE::FIELD_TRANSACTION_ID => format!("tx-{index}"), + ))); + } model.handle_message(metric( at, MetricsEvent::SystemMetrics(SystemMetrics { runtime_seconds: 1, cpu_percent: 12.5, - process_memory_bytes: 10_000, - rss_bytes: 9_000, - virtual_bytes: 12_000, + process_memory_bytes: 18_000, + process_memory_live_resident: 9_000, + process_memory_available_virtual: 12_000, + memory_used_bytes: 100_000, + memory_total_bytes: 200_000, disk_read_bytes: 300, disk_write_bytes: 400, disk_live_read_bytes: 30, disk_live_write_bytes: 40, + host_live_read_bytes: 300, + host_live_write_bytes: 500, open_files: 5, }), )); - model.handle_message(host_sample(at + Duration::from_secs(5), Duration::from_secs(5))); assert_eq!(model.recent_blocks_count(), 1); assert_eq!(model.recent_transactions_count(), 7); - assert_eq!(model.system_sample.as_ref().map(|sample| sample.process_memory_bytes), Some(10_000)); + assert_eq!(model.system_sample.as_ref().map(|sample| sample.process_memory_bytes), Some(18_000)); assert_eq!(model.system_sample.as_ref().map(|sample| sample.memory_total_bytes), Some(200_000)); assert_eq!(model.system_sample.as_ref().map(|sample| sample.host_live_read_bytes), Some(300)); } #[test] fn updates_peer_rtt() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); + let mut model = Model::new(Config::default(), fixture_startup_context()); - model.handle_message(Message::Telemetry(telemetry( - PROTOCOLS_TARGET, - "keepalive.peer.round_trip", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("conn_id", FieldValue::String("7".into())), - ("round_trip_micros", FieldValue::U64(12_345)), - ], + model.handle_message(Message::Telemetry(telemetry!( + protocols::keepalive::peer::ROUND_TRIP, + protocols::keepalive::peer::ROUND_TRIP::FIELD_PEER => "1.2.3.4:3001", + protocols::keepalive::peer::ROUND_TRIP::FIELD_CONN_ID => "7", + protocols::keepalive::peer::ROUND_TRIP::FIELD_ROUND_TRIP_MICROS => 12_345u64, ))); let peer = model.peers.get("1.2.3.4:3001").expect("peer must exist"); @@ -437,7 +387,7 @@ mod tests { #[test] fn sorts_peers_by_ascending_rtt() { - let mut model = Model::new(Config::default(), startup_context()); + let mut model = Model::new(Config::default(), fixture_startup_context()); let now = Instant::now(); let mut fast = PeerState::new("fast.example:3001".into(), now); @@ -463,7 +413,7 @@ mod tests { fn prunes_stale_peers_after_the_inactivity_timeout() { let mut model = Model::new( Config { peer_inactivity_timeout: Duration::from_secs(30), ..Config::default() }, - startup_context(), + fixture_startup_context(), ); let stale_at = Instant::now(); let now = stale_at + Duration::from_secs(31); @@ -474,103 +424,92 @@ mod tests { PeerState::new("recent.example:3001".into(), now - Duration::from_secs(1)), ); - model.handle_message(host_sample(now, Duration::from_secs(1))); + model.handle_message(metric( + now, + MetricsEvent::SystemMetrics(SystemMetrics { process_memory_bytes: 1, ..SystemMetrics::default() }), + )); assert!(!model.peers.contains_key("stale.example:3001")); assert!(model.peers.contains_key("recent.example:3001")); } #[test] - fn tracks_peer_header_lifecycle_means() { - let mut model = Model::new(Config::default(), startup_context()); + fn tracks_peer_header_lifecycle_emas() { + let mut model = Model::new(Config { peer_timing_capacity: 2, ..Config::default() }, fixture_startup_context()); let now = model.created_at; - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now, - "amaru::consensus", - "tip.adopt", - &[ - ("slot", FieldValue::U64(1)), - ("header_hash", FieldValue::String("abc".into())), - ("block_height", FieldValue::U64(10)), - ("max_block_height", FieldValue::U64(10)), - ("suppressed", FieldValue::U64(0)), - ], + consensus::tip::ADOPT, + consensus::tip::ADOPT::FIELD_SLOT => 1u64, + consensus::tip::ADOPT::FIELD_HEADER_HASH => "abc", + consensus::tip::ADOPT::FIELD_BLOCK_HEIGHT => 10u64, + consensus::tip::ADOPT::FIELD_MAX_BLOCK_HEIGHT => 10u64, + consensus::tip::ADOPT::FIELD_SUPPRESSED => 0u32, ))); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now, - "amaru::consensus", - "perf.header.lifecycle", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("slot_start_to_header_micros", FieldValue::U64(9_000)), - ("block_fetch_wait_micros", FieldValue::U64(2_000)), - ("block_fetch_micros", FieldValue::U64(5_000)), - ("forward_micros", FieldValue::U64(11_000)), - ], + consensus::perf::header::LIFECYCLE, + consensus::perf::header::LIFECYCLE::FIELD_PEER => "1.2.3.4:3001", + consensus::perf::header::LIFECYCLE::FIELD_OUTCOME => "valid", + consensus::perf::header::LIFECYCLE::FIELD_SLOT_START_TO_HEADER_MICROS => 9_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_WAIT_MICROS => 2_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_MICROS => 5_000u64, + consensus::perf::header::LIFECYCLE::FIELD_FORWARD_MICROS => 11_000u64, ))); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now + Duration::from_secs(1), - "amaru::consensus", - "perf.header.lifecycle", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("slot_start_to_header_micros", FieldValue::U64(15_000)), - ("block_fetch_wait_micros", FieldValue::U64(4_000)), - ("block_fetch_micros", FieldValue::U64(7_000)), - ("forward_micros", FieldValue::U64(15_000)), - ], + consensus::perf::header::LIFECYCLE, + consensus::perf::header::LIFECYCLE::FIELD_PEER => "1.2.3.4:3001", + consensus::perf::header::LIFECYCLE::FIELD_OUTCOME => "valid", + consensus::perf::header::LIFECYCLE::FIELD_SLOT_START_TO_HEADER_MICROS => 15_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_WAIT_MICROS => 4_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_MICROS => 7_000u64, + consensus::perf::header::LIFECYCLE::FIELD_FORWARD_MICROS => 15_000u64, ))); let peer = model.peers.get("1.2.3.4:3001").expect("peer must exist"); - assert_eq!(peer.mean_slot_start_to_header_micros(), Some(12_000)); - assert_eq!(peer.mean_query_header_micros(), Some(3_000)); - assert_eq!(peer.mean_get_block_micros(), Some(6_000)); + assert_eq!(peer.mean_slot_start_to_header_micros(), Some(13_000)); + assert_eq!(peer.mean_query_header_micros(), Some(3_333)); + assert_eq!(peer.mean_get_block_micros(), Some(6_333)); assert_eq!(peer.mean_adopt_block_micros(), Some(4_000)); } #[test] - fn peer_header_lifecycle_means_keep_the_last_n_samples() { + fn peer_header_lifecycle_emas_follow_the_latest_sample_when_smoothing_is_one() { let config = Config { peer_timing_capacity: 1, ..Config::default() }; - let mut model = Model::new(config, startup_context()); + let mut model = Model::new(config, fixture_startup_context()); let now = model.created_at; - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now, - "amaru::consensus", - "tip.adopt", - &[ - ("slot", FieldValue::U64(1)), - ("header_hash", FieldValue::String("abc".into())), - ("block_height", FieldValue::U64(10)), - ("max_block_height", FieldValue::U64(10)), - ("suppressed", FieldValue::U64(0)), - ], + consensus::tip::ADOPT, + consensus::tip::ADOPT::FIELD_SLOT => 1u64, + consensus::tip::ADOPT::FIELD_HEADER_HASH => "abc", + consensus::tip::ADOPT::FIELD_BLOCK_HEIGHT => 10u64, + consensus::tip::ADOPT::FIELD_MAX_BLOCK_HEIGHT => 10u64, + consensus::tip::ADOPT::FIELD_SUPPRESSED => 0u32, ))); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now, - "amaru::consensus", - "perf.header.lifecycle", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("slot_start_to_header_micros", FieldValue::U64(9_000)), - ("block_fetch_wait_micros", FieldValue::U64(2_000)), - ("block_fetch_micros", FieldValue::U64(5_000)), - ("forward_micros", FieldValue::U64(11_000)), - ], + consensus::perf::header::LIFECYCLE, + consensus::perf::header::LIFECYCLE::FIELD_PEER => "1.2.3.4:3001", + consensus::perf::header::LIFECYCLE::FIELD_OUTCOME => "valid", + consensus::perf::header::LIFECYCLE::FIELD_SLOT_START_TO_HEADER_MICROS => 9_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_WAIT_MICROS => 2_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_MICROS => 5_000u64, + consensus::perf::header::LIFECYCLE::FIELD_FORWARD_MICROS => 11_000u64, ))); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now + Duration::from_secs(10), - "amaru::consensus", - "perf.header.lifecycle", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("slot_start_to_header_micros", FieldValue::U64(15_000)), - ("block_fetch_wait_micros", FieldValue::U64(4_000)), - ("block_fetch_micros", FieldValue::U64(7_000)), - ("forward_micros", FieldValue::U64(15_000)), - ], + consensus::perf::header::LIFECYCLE, + consensus::perf::header::LIFECYCLE::FIELD_PEER => "1.2.3.4:3001", + consensus::perf::header::LIFECYCLE::FIELD_OUTCOME => "valid", + consensus::perf::header::LIFECYCLE::FIELD_SLOT_START_TO_HEADER_MICROS => 15_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_WAIT_MICROS => 4_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_MICROS => 7_000u64, + consensus::perf::header::LIFECYCLE::FIELD_FORWARD_MICROS => 15_000u64, ))); let peer = model.peers.get("1.2.3.4:3001").expect("peer must exist"); @@ -580,60 +519,50 @@ mod tests { #[test] fn slot_start_timing_is_hidden_while_catching_up() { - let mut model = Model::new(Config::default(), startup_context()); + let mut model = Model::new(Config::default(), fixture_startup_context()); let now = model.created_at; - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now, - "amaru::consensus", - "perf.header.lifecycle", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("slot_start_to_header_micros", FieldValue::U64(9_000)), - ("block_fetch_wait_micros", FieldValue::U64(2_000)), - ("block_fetch_micros", FieldValue::U64(5_000)), - ("forward_micros", FieldValue::U64(11_000)), - ], + consensus::perf::header::LIFECYCLE, + consensus::perf::header::LIFECYCLE::FIELD_PEER => "1.2.3.4:3001", + consensus::perf::header::LIFECYCLE::FIELD_OUTCOME => "valid", + consensus::perf::header::LIFECYCLE::FIELD_SLOT_START_TO_HEADER_MICROS => 9_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_WAIT_MICROS => 2_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_MICROS => 5_000u64, + consensus::perf::header::LIFECYCLE::FIELD_FORWARD_MICROS => 11_000u64, ))); let peer = model.peers.get("1.2.3.4:3001").expect("peer must exist"); assert_eq!(peer.mean_slot_start_to_header_micros(), None); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now + Duration::from_secs(1), - "amaru::consensus", - "tip.adopt", - &[ - ("slot", FieldValue::U64(1)), - ("header_hash", FieldValue::String("abc".into())), - ("block_height", FieldValue::U64(10)), - ("max_block_height", FieldValue::U64(100)), - ("suppressed", FieldValue::U64(0)), - ], + consensus::tip::ADOPT, + consensus::tip::ADOPT::FIELD_SLOT => 1u64, + consensus::tip::ADOPT::FIELD_HEADER_HASH => "abc", + consensus::tip::ADOPT::FIELD_BLOCK_HEIGHT => 10u64, + consensus::tip::ADOPT::FIELD_MAX_BLOCK_HEIGHT => 100u64, + consensus::tip::ADOPT::FIELD_SUPPRESSED => 0u32, ))); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now + Duration::from_secs(2), - "amaru::consensus", - "tip.adopt", - &[ - ("slot", FieldValue::U64(1)), - ("header_hash", FieldValue::String("abc".into())), - ("block_height", FieldValue::U64(100)), - ("max_block_height", FieldValue::U64(100)), - ("suppressed", FieldValue::U64(0)), - ], + consensus::tip::ADOPT, + consensus::tip::ADOPT::FIELD_SLOT => 1u64, + consensus::tip::ADOPT::FIELD_HEADER_HASH => "abc", + consensus::tip::ADOPT::FIELD_BLOCK_HEIGHT => 100u64, + consensus::tip::ADOPT::FIELD_MAX_BLOCK_HEIGHT => 100u64, + consensus::tip::ADOPT::FIELD_SUPPRESSED => 0u32, ))); - model.handle_message(Message::Telemetry(telemetry_at( + model.handle_message(Message::Telemetry(telemetry_at!( now + Duration::from_secs(3), - "amaru::consensus", - "perf.header.lifecycle", - &[ - ("peer", FieldValue::String("1.2.3.4:3001".into())), - ("slot_start_to_header_micros", FieldValue::U64(3_000)), - ("block_fetch_wait_micros", FieldValue::U64(1_000)), - ("block_fetch_micros", FieldValue::U64(2_000)), - ("forward_micros", FieldValue::U64(5_000)), - ], + consensus::perf::header::LIFECYCLE, + consensus::perf::header::LIFECYCLE::FIELD_PEER => "1.2.3.4:3001", + consensus::perf::header::LIFECYCLE::FIELD_OUTCOME => "valid", + consensus::perf::header::LIFECYCLE::FIELD_SLOT_START_TO_HEADER_MICROS => 3_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_WAIT_MICROS => 1_000u64, + consensus::perf::header::LIFECYCLE::FIELD_BLOCK_FETCH_MICROS => 2_000u64, + consensus::perf::header::LIFECYCLE::FIELD_FORWARD_MICROS => 5_000u64, ))); let peer = model.peers.get("1.2.3.4:3001").expect("peer must exist"); @@ -642,46 +571,26 @@ mod tests { #[test] fn waits_for_initial_stake_distributions_ready_event() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); + let mut model = Model::new(Config::default(), fixture_startup_context()); let later = model.created_at + Duration::from_secs(60); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "stake_distribution.initial_begin", - &[("epoch", FieldValue::U64(100))], + model.handle_message(Message::Telemetry(telemetry!( + ledger::stake_distribution::INITIAL_BEGIN, + ledger::stake_distribution::INITIAL_BEGIN::FIELD_EPOCH => 100u64, ))); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "stake_distribution.initial_progress", - &[("epoch", FieldValue::U64(100)), ("progress", FieldValue::F64(0.42))], + model.handle_message(Message::Telemetry(telemetry!( + ledger::stake_distribution::INITIAL_PROGRESS, + ledger::stake_distribution::INITIAL_PROGRESS::FIELD_EPOCH => 100u64, + ledger::stake_distribution::INITIAL_PROGRESS::FIELD_PROGRESS => 0.42f64, ))); assert!(!model.is_ready(later)); assert_eq!(model.initial_stake_distributions().count(), 1); assert_eq!(model.initial_stake_distributions().next().map(|state| state.progress), Some(0.42)); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "stake_distribution.initial_ready", - &[("epochs", FieldValue::String("100".into()))], + model.handle_message(Message::Telemetry(telemetry!( + ledger::stake_distribution::INITIAL_READY, + ledger::stake_distribution::INITIAL_READY::FIELD_EPOCHS => "100", ))); assert!(model.is_ready(later)); @@ -691,99 +600,63 @@ mod tests { #[test] fn initial_stake_distributions_are_ordered_by_epoch() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); + let mut model = Model::new(Config::default(), fixture_startup_context()); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "stake_distribution.initial_begin", - &[("epoch", FieldValue::U64(101))], + model.handle_message(Message::Telemetry(telemetry!( + ledger::stake_distribution::INITIAL_BEGIN, + ledger::stake_distribution::INITIAL_BEGIN::FIELD_EPOCH => 101u64, ))); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "stake_distribution.initial_begin", - &[("epoch", FieldValue::U64(99))], + + model.handle_message(Message::Telemetry(telemetry!( + ledger::stake_distribution::INITIAL_BEGIN, + ledger::stake_distribution::INITIAL_BEGIN::FIELD_EPOCH => 99u64, ))); let epochs = model.initial_stake_distributions().map(|state| state.epoch).collect::>(); + assert_eq!(epochs, vec![99, 101]); } #[test] fn proposal_drop_distinguishes_expired_dropped_and_enacted() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); + let mut model = Model::new(Config::default(), fixture_startup_context()); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "governance.enacting", - &[ - ("proposal_id", FieldValue::String("enacted".into())), - ("proposal_kind", FieldValue::String("constitution".into())), - ], + model.handle_message(Message::Telemetry(telemetry!( + ledger::governance::ENACTING, + ledger::governance::ENACTING::FIELD_PROPOSAL_ID => "enacted", + ledger::governance::ENACTING::FIELD_PROPOSAL_KIND => "constitution", + ))); + model.handle_message(Message::Telemetry(telemetry!( + ledger::proposal::DROP, + ledger::proposal::DROP::FIELD_ID => "enacted", + ledger::proposal::DROP::FIELD_EXPIRED => false, + ledger::proposal::DROP::FIELD_RATIFIED_OR_EVICTED => true, + ))); + model.handle_message(Message::Telemetry(telemetry!( + ledger::proposal::ACTIVE, + ledger::proposal::ACTIVE::FIELD_ID => "expired", + ledger::proposal::ACTIVE::FIELD_PROPOSAL_KIND => "hard-fork", + ledger::proposal::ACTIVE::FIELD_PROPOSED_IN => 10u64, + ledger::proposal::ACTIVE::FIELD_VALID_UNTIL => 12u64, ))); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "proposal.drop", - &[ - ("id", FieldValue::String("enacted".into())), - ("expired", FieldValue::Bool(false)), - ("ratified_or_evicted", FieldValue::Bool(true)), - ], + model.handle_message(Message::Telemetry(telemetry!( + ledger::proposal::DROP, + ledger::proposal::DROP::FIELD_ID => "expired", + ledger::proposal::DROP::FIELD_EXPIRED => true, + ledger::proposal::DROP::FIELD_RATIFIED_OR_EVICTED => false, ))); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "proposal.drop", - &[ - ("id", FieldValue::String("expired".into())), - ("proposal_kind", FieldValue::String("hard-fork".into())), - ("expired", FieldValue::Bool(true)), - ("ratified_or_evicted", FieldValue::Bool(false)), - ], + model.handle_message(Message::Telemetry(telemetry!( + ledger::proposal::ACTIVE, + ledger::proposal::ACTIVE::FIELD_ID => "dropped", + ledger::proposal::ACTIVE::FIELD_PROPOSAL_KIND => "treasury-withdrawal", + ledger::proposal::ACTIVE::FIELD_PROPOSED_IN => 10u64, + ledger::proposal::ACTIVE::FIELD_VALID_UNTIL => 12u64, ))); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "proposal.drop", - &[ - ("id", FieldValue::String("dropped".into())), - ("proposal_kind", FieldValue::String("treasury-withdrawal".into())), - ("expired", FieldValue::Bool(false)), - ("ratified_or_evicted", FieldValue::Bool(true)), - ], + model.handle_message(Message::Telemetry(telemetry!( + ledger::proposal::DROP, + ledger::proposal::DROP::FIELD_ID => "dropped", + ledger::proposal::DROP::FIELD_EXPIRED => false, + ledger::proposal::DROP::FIELD_RATIFIED_OR_EVICTED => true, ))); assert_eq!(model.proposals_by_id.get("enacted").map(|proposal| proposal.status.as_str()), Some("enacted")); @@ -793,42 +666,18 @@ mod tests { #[test] fn proposal_drop_keeps_enacted_status_even_when_expired_flag_is_set() { - let startup = StartupContext { - process: ProcessInfo { - network: "preview".into(), - software_version: "10.11.0 (abc123)".into(), - target: "darwin/aarch64".into(), - }, - protocol_version: "10.11".into(), - mempool_max_bytes: 180_224, - epoch_length: 86_400, - active_slot_coeff_inverse: 20, - consensus_security_param: 432, - max_lovelace_supply: 45_000_000_000_000_000, - system_start_millis: 1_666_656_000_000, - era_history: None, - runtime_sections: Vec::default(), - global_sections: Vec::default(), - protocol_sections: Vec::default(), - }; - let mut model = Model::new(Config::default(), startup); + let mut model = Model::new(Config::default(), fixture_startup_context()); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "governance.enacting", - &[ - ("proposal_id", FieldValue::String("proposal".into())), - ("proposal_kind", FieldValue::String("protocol-parameters".into())), - ], + model.handle_message(Message::Telemetry(telemetry!( + ledger::governance::ENACTING, + ledger::governance::ENACTING::FIELD_PROPOSAL_ID => "proposal", + ledger::governance::ENACTING::FIELD_PROPOSAL_KIND => "protocol-parameters", ))); - model.handle_message(Message::Telemetry(telemetry( - LEDGER_TARGET, - "proposal.drop", - &[ - ("id", FieldValue::String("proposal".into())), - ("expired", FieldValue::Bool(true)), - ("ratified_or_evicted", FieldValue::Bool(true)), - ], + model.handle_message(Message::Telemetry(telemetry!( + ledger::proposal::DROP, + ledger::proposal::DROP::FIELD_ID => "proposal", + ledger::proposal::DROP::FIELD_EXPIRED => true, + ledger::proposal::DROP::FIELD_RATIFIED_OR_EVICTED => true, ))); assert_eq!(model.proposals_by_id.get("proposal").map(|proposal| proposal.status.as_str()), Some("enacted")); @@ -838,6 +687,7 @@ mod tests { fn keepalive_rtt_marks_peer_as_outbound_when_direction_is_missing() { let startup = StartupContext { process: ProcessInfo { + pid: 42, network: "preview".into(), software_version: "10.11.0 (abc123)".into(), target: "darwin/aarch64".into(), @@ -851,15 +701,14 @@ mod tests { system_start_millis: 1_666_656_000_000, era_history: None, runtime_sections: Vec::default(), - global_sections: Vec::default(), protocol_sections: Vec::default(), }; let mut model = Model::new(Config::default(), startup); - model.handle_message(Message::Telemetry(telemetry( - PROTOCOLS_TARGET, - "keepalive.peer.round_trip", - &[("peer", FieldValue::String("1.2.3.4:3001".into())), ("round_trip_micros", FieldValue::U64(1_000))], + model.handle_message(Message::Telemetry(telemetry!( + protocols::keepalive::peer::ROUND_TRIP, + protocols::keepalive::peer::ROUND_TRIP::FIELD_PEER => "1.2.3.4:3001", + protocols::keepalive::peer::ROUND_TRIP::FIELD_ROUND_TRIP_MICROS => 1_000u64, ))); let peer = model.peers.get("1.2.3.4:3001").expect("peer must exist"); @@ -869,7 +718,7 @@ mod tests { #[test] fn keyboard_navigation_uses_arrows_for_focus_and_enter_for_pane_toggle() { - let mut model = Model::new(Config::default(), startup_context()); + let mut model = Model::new(Config::default(), fixture_startup_context()); assert_eq!(model.page, Page::Amaru); assert_eq!(model.scroll_focus, ScrollFocus::Logs); @@ -917,7 +766,7 @@ mod tests { #[test] fn config_page_uses_its_own_scroll_focus() { - let mut model = Model::new(Config::default(), startup_context()); + let mut model = Model::new(Config::default(), fixture_startup_context()); assert_eq!( model.handle_key_event(KeyEvent::new(KeyCode::Tab, KeyModifiers::NONE)), @@ -954,7 +803,7 @@ mod tests { #[test] fn shutdown_mode_ignores_follow_up_terminal_input() { - let mut model = Model::new(Config::default(), startup_context()); + let mut model = Model::new(Config::default(), fixture_startup_context()); model.enter_shutdown_mode(); assert_eq!( @@ -970,4 +819,18 @@ mod tests { assert_eq!(model.page, Page::Amaru); assert_eq!(model.scroll_focus, ScrollFocus::Logs); } + + #[test] + fn splash_screen_ignores_copy_mode_toggle() { + let mut model = Model::new(Config::default(), fixture_startup_context()); + model.initial_stake_distribution_order = vec![1000, 1001]; + + assert!(!model.is_ready(Instant::now())); + assert_eq!( + model.handle_key_event(KeyEvent::new(KeyCode::Esc, KeyModifiers::NONE)), + TerminalEventOutcome::Continue + ); + assert!(!model.is_copy_mode()); + assert_eq!(model.interaction_mode, InteractionMode::Normal); + } } diff --git a/crates/amaru-tui/src/events/telemetry_kind.rs b/crates/amaru-tui/src/model/exponential_moving_average.rs similarity index 51% rename from crates/amaru-tui/src/events/telemetry_kind.rs rename to crates/amaru-tui/src/model/exponential_moving_average.rs index 90671460a3..592b5887b1 100644 --- a/crates/amaru-tui/src/events/telemetry_kind.rs +++ b/crates/amaru-tui/src/model/exponential_moving_average.rs @@ -12,8 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TelemetryKind { - Event, - SpanClose, +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ExponentialMovingAverage { + value: Option, +} + +impl ExponentialMovingAverage { + pub fn clear(&mut self) { + self.value = None; + } + + pub fn record(&mut self, sample: f64, smoothing: usize) { + let alpha = 2.0 / (smoothing.max(1) as f64 + 1.0); + self.value = Some(match self.value { + Some(value) => alpha * sample + (1.0 - alpha) * value, + None => sample, + }); + } + + pub fn value(&self) -> Option { + self.value + } } diff --git a/crates/amaru-tui/src/model/interaction.rs b/crates/amaru-tui/src/model/interaction.rs index f9812d8be1..7241d9f755 100644 --- a/crates/amaru-tui/src/model/interaction.rs +++ b/crates/amaru-tui/src/model/interaction.rs @@ -102,14 +102,14 @@ impl Model { pub fn set_level_filter(&mut self, level: LevelFilter) { self.level_filter = level; - self.logs.retain(|record| self.level_filter.allows(record.level) && self.target_filter.allows(&record.target)); + self.logs.rebuild_filtered(self.level_filter, self.target_filter); self.log_scroll = 0; self.scroll_focus = ScrollFocus::Logs; } pub fn set_target_filter(&mut self, filter: TargetFilter) { self.target_filter = filter; - self.logs.retain(|record| self.level_filter.allows(record.level) && self.target_filter.allows(&record.target)); + self.logs.rebuild_filtered(self.level_filter, self.target_filter); self.log_scroll = 0; self.scroll_focus = ScrollFocus::Logs; } @@ -231,6 +231,9 @@ impl Model { match key.code { KeyCode::Esc => { + if !self.is_ready(std::time::Instant::now()) { + return TerminalEventOutcome::Continue; + } self.enter_copy_mode(); TerminalEventOutcome::EnterCopyMode } diff --git a/crates/amaru-tui/src/model/log_buffer.rs b/crates/amaru-tui/src/model/log_buffer.rs new file mode 100644 index 0000000000..ce4fb24217 --- /dev/null +++ b/crates/amaru-tui/src/model/log_buffer.rs @@ -0,0 +1,111 @@ +// Copyright 2026 PRAGMA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::rc::Rc; + +use tracing::Level; + +use super::{LevelFilter, TargetFilter}; +use crate::{config::Config, events::TelemetryRecord}; + +#[derive(Debug, Default)] +pub struct LogBuffer { + all: Vec>, + filtered: Vec>, + debug_count: usize, + info_count: usize, + warn_count: usize, + error_count: usize, +} + +impl LogBuffer { + pub fn filtered(&self) -> &[Rc] { + &self.filtered + } + + pub fn push( + &mut self, + record: TelemetryRecord, + config: &Config, + level_filter: LevelFilter, + target_filter: TargetFilter, + ) { + let level = record.level; + let capacity = config.log_capacity_for(level); + if capacity == 0 { + return; + } + + while self.count(level) >= capacity { + self.evict_oldest(level); + } + + let record = Rc::new(record); + self.increment(level); + if matches_filters(&record, level_filter, target_filter) { + self.filtered.push(Rc::clone(&record)); + } + self.all.push(record); + } + + pub fn rebuild_filtered(&mut self, level_filter: LevelFilter, target_filter: TargetFilter) { + self.filtered.clear(); + self.filtered + .extend(self.all.iter().filter(|record| matches_filters(record, level_filter, target_filter)).cloned()); + } + + fn evict_oldest(&mut self, level: Level) { + let Some(position) = self.all.iter().position(|record| record.level == level) else { + return; + }; + + let removed = self.all.remove(position); + self.decrement(level); + + if let Some(position) = self.filtered.iter().position(|record| Rc::ptr_eq(record, &removed)) { + self.filtered.remove(position); + } + } + + fn count(&self, level: Level) -> usize { + match level { + Level::TRACE | Level::DEBUG => self.debug_count, + Level::INFO => self.info_count, + Level::WARN => self.warn_count, + Level::ERROR => self.error_count, + } + } + + fn increment(&mut self, level: Level) { + match level { + Level::TRACE | Level::DEBUG => self.debug_count += 1, + Level::INFO => self.info_count += 1, + Level::WARN => self.warn_count += 1, + Level::ERROR => self.error_count += 1, + } + } + + fn decrement(&mut self, level: Level) { + match level { + Level::TRACE | Level::DEBUG => self.debug_count = self.debug_count.saturating_sub(1), + Level::INFO => self.info_count = self.info_count.saturating_sub(1), + Level::WARN => self.warn_count = self.warn_count.saturating_sub(1), + Level::ERROR => self.error_count = self.error_count.saturating_sub(1), + } + } +} + +fn matches_filters(record: &TelemetryRecord, level_filter: LevelFilter, target_filter: TargetFilter) -> bool { + level_filter.allows(record.level) && target_filter.allows(&record.target) +} diff --git a/crates/amaru-tui/src/model/metrics_update.rs b/crates/amaru-tui/src/model/metrics_update.rs index 3024ddd643..e4dcd771f6 100644 --- a/crates/amaru-tui/src/model/metrics_update.rs +++ b/crates/amaru-tui/src/model/metrics_update.rs @@ -14,97 +14,38 @@ use std::{collections::VecDeque, time::Instant}; -use amaru_metrics::{LedgerMetrics, MempoolMetrics, MetricsEvent, SystemMetrics}; +use amaru_metrics::MetricsEvent; use super::*; -use crate::events::{HostSample, MetricRecord, SystemSample}; +use crate::events::{MetricRecord, SystemSample}; impl Model { - pub fn set_system_sample(&mut self, sample: SystemSample) { - self.system_sample = Some(sample); - } - pub(crate) fn record_metrics(&mut self, record: MetricRecord) { - match record.event { - MetricsEvent::LedgerMetrics(metrics) => self.record_ledger_metrics(record.at, metrics), - MetricsEvent::MempoolMetrics(metrics) => self.record_mempool_metrics(record.at, metrics), - MetricsEvent::SystemMetrics(metrics) => self.record_system_metrics(record.at, metrics), - MetricsEvent::ProtocolMetrics(_) | MetricsEvent::ConsensusMetrics(_) => {} + if let MetricsEvent::SystemMetrics(metrics) = record.event { + self.system_sample = Some(SystemSample { + at: record.at, + cpu_percent: metrics.cpu_percent, + process_memory_bytes: metrics.process_memory_bytes, + rss_bytes: metrics.process_memory_live_resident, + virtual_bytes: metrics.process_memory_available_virtual, + memory_used_bytes: metrics.memory_used_bytes, + memory_total_bytes: metrics.memory_total_bytes, + disk_read_bytes: metrics.disk_read_bytes, + disk_write_bytes: metrics.disk_write_bytes, + disk_live_read_bytes: metrics.disk_live_read_bytes, + disk_live_write_bytes: metrics.disk_live_write_bytes, + host_live_read_bytes: metrics.host_live_read_bytes, + host_live_write_bytes: metrics.host_live_write_bytes, + }); } } - fn push_recent_transaction_count(&mut self, at: Instant, tx_count: u64) { - self.recent_transactions.push_back((at, tx_count)); - prune_recent_count(&mut self.recent_transactions, self.config.transaction_sample_capacity); - } - - fn record_ledger_metrics(&mut self, at: Instant, metrics: LedgerMetrics) { - self.push_recent_block(at); - self.push_recent_transaction_count(at, metrics.tx_count); - } - - fn record_mempool_metrics(&mut self, at: Instant, metrics: MempoolMetrics) { - self.mempool = MempoolState { tx_count: metrics.tx_count, size_bytes: metrics.size_bytes, updated_at: at }; - } - - fn record_system_metrics(&mut self, at: Instant, metrics: SystemMetrics) { - let ( - memory_used_bytes, - memory_total_bytes, - process_memory_bytes_override, - host_live_read_bytes, - host_live_write_bytes, - ) = self.latest_host_metrics(); - - self.set_system_sample(SystemSample { - at, - cpu_percent: metrics.cpu_percent, - process_memory_bytes: process_memory_bytes_override.unwrap_or(metrics.process_memory_bytes), - rss_bytes: metrics.rss_bytes, - virtual_bytes: metrics.virtual_bytes, - memory_used_bytes, - memory_total_bytes, - disk_read_bytes: metrics.disk_read_bytes, - disk_write_bytes: metrics.disk_write_bytes, - disk_live_read_bytes: metrics.disk_live_read_bytes, - disk_live_write_bytes: metrics.disk_live_write_bytes, - host_live_read_bytes, - host_live_write_bytes, - }); - } - - pub(crate) fn record_host_sample(&mut self, sample: HostSample) { - let ( - cpu_percent, - process_memory_bytes, - rss_bytes, - virtual_bytes, - disk_read_bytes, - disk_write_bytes, - disk_live_read_bytes, - disk_live_write_bytes, - ) = self.latest_process_metrics(); - - self.set_system_sample(SystemSample { - at: sample.at, - cpu_percent, - process_memory_bytes: sample.process_memory_bytes.unwrap_or(process_memory_bytes), - rss_bytes, - virtual_bytes, - memory_used_bytes: sample.memory_used_bytes, - memory_total_bytes: sample.memory_total_bytes, - disk_read_bytes, - disk_write_bytes, - disk_live_read_bytes, - disk_live_write_bytes, - host_live_read_bytes: sample.host_live_read_bytes_per_second(), - host_live_write_bytes: sample.host_live_write_bytes_per_second(), - }); + pub(crate) fn push_recent_transaction_count(&mut self, at: Instant, tx_count: u64) { + self.transaction_rate.record(at, tx_count); } pub(crate) fn push_recent_block(&mut self, at: Instant) { - self.recent_blocks.push_back(at); - prune_recent_count(&mut self.recent_blocks, self.config.block_sample_capacity); + self.block_rate.record(at, 1); } pub(crate) fn push_recent_rollback(&mut self, rollback_length: usize, at: Instant) { @@ -118,38 +59,3 @@ pub(crate) fn prune_recent_count(entries: &mut VecDeque, capacity: usize) entries.pop_front(); } } - -impl Model { - fn latest_host_metrics(&self) -> (u64, u64, Option, u64, u64) { - self.system_sample - .as_ref() - .map(|sample| { - ( - sample.memory_used_bytes, - sample.memory_total_bytes, - Some(sample.process_memory_bytes), - sample.host_live_read_bytes, - sample.host_live_write_bytes, - ) - }) - .unwrap_or((0, 0, None, 0, 0)) - } - - fn latest_process_metrics(&self) -> (f64, u64, u64, u64, u64, u64, u64, u64) { - self.system_sample - .as_ref() - .map(|sample| { - ( - sample.cpu_percent, - sample.process_memory_bytes, - sample.rss_bytes, - sample.virtual_bytes, - sample.disk_read_bytes, - sample.disk_write_bytes, - sample.disk_live_read_bytes, - sample.disk_live_write_bytes, - ) - }) - .unwrap_or((0.0, 0, 0, 0, 0, 0, 0, 0)) - } -} diff --git a/crates/amaru-tui/src/model/peer_state.rs b/crates/amaru-tui/src/model/peer_state.rs index 2d4ad68239..3551a05bc9 100644 --- a/crates/amaru-tui/src/model/peer_state.rs +++ b/crates/amaru-tui/src/model/peer_state.rs @@ -12,41 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{collections::VecDeque, time::Instant}; +use std::time::Instant; use amaru_observability::amaru::protocols; +use super::exponential_moving_average::ExponentialMovingAverage; use crate::events::TelemetryRecord; -#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq)] struct MeanMicros { - samples: VecDeque<(Instant, u64)>, + average: ExponentialMovingAverage, } impl MeanMicros { - fn record(&mut self, at: Instant, micros: u64, capacity: usize) { - self.samples.push_back((at, micros)); - self.prune(capacity); + fn record(&mut self, micros: u64, smoothing: usize) { + self.average.record(micros as f64, smoothing); } fn mean(&self) -> Option { - let mut total_micros = 0u128; - let sample_count = self.samples.len() as u64; - for (_, micros) in &self.samples { - total_micros += u128::from(*micros); - } - - (sample_count > 0).then(|| (total_micros / u128::from(sample_count)) as u64) + self.average.value().map(|micros| micros.round() as u64) } fn clear(&mut self) { - self.samples.clear(); - } - - fn prune(&mut self, capacity: usize) { - while self.samples.len() > capacity { - self.samples.pop_front(); - } + self.average.clear(); } } @@ -124,23 +112,23 @@ impl PeerState { pub fn record_header_lifecycle( &mut self, at: Instant, - capacity: usize, + smoothing: usize, slot_start_to_header_micros: Option, query_header_micros: Option, get_block_micros: Option, adopt_block_micros: Option, ) { if let Some(micros) = slot_start_to_header_micros { - self.slot_start_to_header.record(at, micros, capacity); + self.slot_start_to_header.record(micros, smoothing); } if let Some(micros) = query_header_micros { - self.query_header.record(at, micros, capacity); + self.query_header.record(micros, smoothing); } if let Some(micros) = get_block_micros { - self.get_block.record(at, micros, capacity); + self.get_block.record(micros, smoothing); } if let Some(micros) = adopt_block_micros { - self.adopt_block.record(at, micros, capacity); + self.adopt_block.record(micros, smoothing); } self.updated_at = at; } diff --git a/crates/amaru-tui/src/model/queries.rs b/crates/amaru-tui/src/model/queries.rs index 994c70f705..098ca0476b 100644 --- a/crates/amaru-tui/src/model/queries.rs +++ b/crates/amaru-tui/src/model/queries.rs @@ -12,20 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::time::{Duration, SystemTime}; +use std::{ + rc::Rc, + time::{Duration, SystemTime}, +}; use super::*; +use crate::events::TelemetryRecord; impl Model { - pub fn filtered_logs(&self) -> Vec<&TelemetryRecord> { - self.logs - .iter() - .filter(|record| self.level_filter.allows(record.level) && self.target_filter.allows(&record.target)) - .collect() + pub fn filtered_logs(&self) -> &[Rc] { + self.logs.filtered() } - pub fn recent_blocks_count(&self) -> usize { - self.recent_blocks.len() + pub fn recent_blocks_count(&self) -> u64 { + self.block_rate.total_count() } pub fn last_block_elapsed(&self, now: Instant) -> Option { @@ -64,7 +65,7 @@ impl Model { } pub fn recent_transactions_count(&self) -> u64 { - self.recent_transactions.iter().map(|(_, count)| *count).sum() + self.transaction_rate.total_count() } pub fn average_recent_rollback_length(&self) -> Option { @@ -77,12 +78,12 @@ impl Model { } pub fn recent_rollback_frequency(&self, now: Instant) -> Option { - let span = self.retained_span( - now, - self.recent_rollbacks.front().map(|(at, _)| *at), - self.recent_rollbacks.len(), - self.config.rollback_sample_capacity, - ); + let origin = if self.recent_rollbacks.len() >= self.config.rollback_sample_capacity { + self.recent_rollbacks.front().map(|(at, _)| *at).unwrap_or(self.created_at) + } else { + self.created_at + }; + let span = now.saturating_duration_since(origin); (span > Duration::ZERO).then_some(self.recent_rollbacks.len() as f64 / span.as_secs_f64()) } @@ -102,26 +103,11 @@ impl Model { peers } - pub(crate) fn retained_blocks_span(&self, now: Instant) -> Duration { - self.retained_span( - now, - self.recent_blocks.front().copied(), - self.recent_blocks.len(), - self.config.block_sample_capacity, - ) - } - - pub(crate) fn retained_transactions_span(&self, now: Instant) -> Duration { - self.retained_span( - now, - self.recent_transactions.front().map(|(at, _)| *at), - self.recent_transactions.len(), - self.config.transaction_sample_capacity, - ) + pub fn blocks_per_second(&self) -> f64 { + self.block_rate.rate_per_second() } - fn retained_span(&self, now: Instant, oldest: Option, len: usize, capacity: usize) -> Duration { - let origin = if len >= capacity { oldest.unwrap_or(self.created_at) } else { self.created_at }; - now.saturating_duration_since(origin) + pub fn transactions_per_second(&self) -> f64 { + self.transaction_rate.rate_per_second() } } diff --git a/crates/amaru-tui/src/model/rate_counter.rs b/crates/amaru-tui/src/model/rate_counter.rs new file mode 100644 index 0000000000..147654811c --- /dev/null +++ b/crates/amaru-tui/src/model/rate_counter.rs @@ -0,0 +1,52 @@ +// Copyright 2026 PRAGMA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Instant; + +use super::exponential_moving_average::ExponentialMovingAverage; + +#[derive(Debug, Clone, PartialEq)] +pub struct RateCounter { + smoothing: usize, + total_count: u64, + last_at: Option, + average_rate: ExponentialMovingAverage, +} + +impl RateCounter { + pub fn new(smoothing: usize) -> Self { + Self { smoothing, total_count: 0, last_at: None, average_rate: ExponentialMovingAverage::default() } + } + + pub fn record(&mut self, at: Instant, count: u64) { + self.total_count = self.total_count.saturating_add(count); + + if let Some(last_at) = self.last_at { + let elapsed = at.saturating_duration_since(last_at); + if !elapsed.is_zero() { + self.average_rate.record(count as f64 / elapsed.as_secs_f64(), self.smoothing); + } + } + + self.last_at = Some(at); + } + + pub fn total_count(&self) -> u64 { + self.total_count + } + + pub fn rate_per_second(&self) -> f64 { + self.average_rate.value().unwrap_or_default() + } +} diff --git a/crates/amaru-tui/src/model/telemetry_event.rs b/crates/amaru-tui/src/model/telemetry_event.rs index 5fa54bd6ef..00444e52e6 100644 --- a/crates/amaru-tui/src/model/telemetry_event.rs +++ b/crates/amaru-tui/src/model/telemetry_event.rs @@ -45,18 +45,24 @@ pub enum TelemetryEvent { ProtocolUpgrade, RatificationSummarize, RewardsSummarize, + RollForward, StakeDistributionInitialBegin, StakeDistributionInitialProgress, StakeDistributionInitialReady, StateSwitchToFork, StakeSnapshot, TipUpdate, + TransactionValidate, } impl TelemetryEvent { pub fn from_record(record: &TelemetryRecord) -> Option { if ledger::tip::UPDATE::matches(&record.target, &record.name) { Some(Self::TipUpdate) + } else if ledger::transaction::VALIDATE::matches(&record.target, &record.name) { + Some(Self::TransactionValidate) + } else if ledger::state::ROLL_FORWARD::matches(&record.target, &record.name) { + Some(Self::RollForward) } else if consensus::tip::ADOPT::matches(&record.target, &record.name) { Some(Self::BlockAdopt) } else if ledger::stake_distribution::INITIAL_BEGIN::matches(&record.target, &record.name) { diff --git a/crates/amaru-tui/src/model/telemetry_update.rs b/crates/amaru-tui/src/model/telemetry_update.rs index 41902afd81..f8d0a98c64 100644 --- a/crates/amaru-tui/src/model/telemetry_update.rs +++ b/crates/amaru-tui/src/model/telemetry_update.rs @@ -14,7 +14,10 @@ use std::time::Instant; -use amaru_observability::amaru::{bootstrap, consensus, ledger, mempool, protocols}; +use amaru_observability::{ + RecordFields, + amaru::{bootstrap, consensus, ledger, mempool, protocols}, +}; use super::*; use crate::events::{Message, TelemetryRecord}; @@ -30,10 +33,6 @@ impl Model { self.prune_stale_peers(record.at); self.record_metrics(record); } - Message::HostSample(sample) => { - self.prune_stale_peers(sample.at); - self.record_host_sample(sample); - } } } @@ -43,12 +42,7 @@ impl Model { fn record_telemetry(&mut self, record: TelemetryRecord) { self.update_state(TelemetryEvent::from_record(&record), &record); - if self.level_filter.allows(record.level) && self.target_filter.allows(&record.target) { - self.logs.push_back(record); - while self.logs.len() > self.config.log_capacity { - self.logs.pop_front(); - } - } + self.logs.push(record, &self.config, self.level_filter, self.target_filter); } fn update_state(&mut self, event: Option, record: &TelemetryRecord) { @@ -57,7 +51,11 @@ impl Model { }; match event { - TelemetryEvent::BlockAdopt => self.update_catch_up(record), + TelemetryEvent::BlockAdopt => { + self.update_catch_up(record); + } + TelemetryEvent::RollForward => self.push_recent_block(record.at), + TelemetryEvent::TransactionValidate => self.push_recent_transaction_count(record.at, 1), TelemetryEvent::TipUpdate => self.update_tip(record), TelemetryEvent::StakeSnapshot => self.update_stake_snapshot(record), TelemetryEvent::MempoolStateUpdate => self.update_mempool(record), @@ -264,6 +262,10 @@ impl Model { } fn update_peer_header_lifecycle(&mut self, record: &TelemetryRecord) { + if record.str(consensus::perf::header::LIFECYCLE::FIELD_OUTCOME) != Some("valid") { + return; + } + let Some(peer) = consensus::perf::header::LIFECYCLE::peer(record) else { return; }; diff --git a/crates/amaru-tui/src/model/tip_state.rs b/crates/amaru-tui/src/model/tip_state.rs index 487fbe91b6..e5950e24ef 100644 --- a/crates/amaru-tui/src/model/tip_state.rs +++ b/crates/amaru-tui/src/model/tip_state.rs @@ -23,6 +23,7 @@ pub struct TipState { pub slot: u64, pub header_hash: String, pub block_height: u64, + pub tx_count: u64, pub epoch: u64, pub slot_in_epoch: u64, pub density: f64, @@ -37,6 +38,7 @@ impl TipState { slot: ledger::tip::UPDATE::slot(record), header_hash: ledger::tip::UPDATE::header_hash(record).to_owned(), block_height: ledger::tip::UPDATE::block_height(record), + tx_count: ledger::tip::UPDATE::tx_count(record) as u64, epoch: ledger::tip::UPDATE::epoch(record), slot_in_epoch: ledger::tip::UPDATE::slot_in_epoch(record), density: ledger::tip::UPDATE::density(record), diff --git a/crates/amaru-tui/src/session.rs b/crates/amaru-tui/src/session.rs index 682da6ebea..96b5fc19e6 100644 --- a/crates/amaru-tui/src/session.rs +++ b/crates/amaru-tui/src/session.rs @@ -17,16 +17,18 @@ use std::{ sync::{ Arc, atomic::{AtomicU8, Ordering}, - mpsc::{self, Receiver, Sender}, + mpsc::{self, Receiver, RecvTimeoutError, Sender, SyncSender}, }, thread::{self, JoinHandle}, - time::Instant, + time::{Duration, Instant}, }; +use amaru_metrics::MetricsEvent; use crossterm::event; use crate::{ - Config, TracingLayer, host_metrics, metrics, + Config, TracingLayer, + events::{Message, MetricRecord}, model::{Model, TerminalEventOutcome}, startup::StartupContext, terminal_guard::TerminalGuard, @@ -34,9 +36,7 @@ use crate::{ }; pub struct Session { - layer: TracingLayer, - metrics: Arc, - host_metrics: Option, + tx: SyncSender, control_tx: Sender, join: Option>>, } @@ -45,32 +45,24 @@ impl Session { pub fn spawn(config: Config, startup: StartupContext, signal_count: Arc) -> io::Result { let (telemetry_tx, telemetry_rx) = mpsc::sync_channel(config.channel_capacity); let (control_tx, control_rx) = mpsc::channel(); - let layer = TracingLayer::new(telemetry_tx); - let metrics = Arc::new(metrics::Subscriber::new(layer.sender())); let join = thread::Builder::new() .name("amaru-tui".into()) .spawn(move || run_terminal(config, startup, telemetry_rx, control_rx, signal_count)) .map_err(|err| io::Error::other(format!("failed to spawn tui thread: {err}")))?; - let host_metrics = match host_metrics::Sampler::spawn(layer.sender()) { - Ok(host_metrics) => host_metrics, - Err(err) => { - let _ = control_tx.send(Control::Shutdown); - let _ = join.join(); - return Err(err); - } - }; - - Ok(Self { layer, metrics, host_metrics: Some(host_metrics), control_tx, join: Some(join) }) + Ok(Self { tx: telemetry_tx, control_tx, join: Some(join) }) } - pub fn layer(&self) -> TracingLayer { - self.layer.clone() + pub fn tracing_layer(&self) -> TracingLayer { + TracingLayer::new(self.tx.clone()) } - pub fn subscribe_to_metrics(&self) -> metrics::Subscription { - metrics::Subscription::new(self.metrics.clone()) + pub fn metrics_observer(&self) -> Box { + let tx = self.tx.clone(); + Box::new(move |event| { + let _ = tx.try_send(Message::Metrics(MetricRecord { at: Instant::now(), event: event.clone() })); + }) } pub fn shutdown(mut self) -> io::Result<()> { @@ -79,9 +71,6 @@ impl Session { fn shutdown_inner(&mut self) -> io::Result<()> { let _ = self.control_tx.send(Control::Shutdown); - if let Some(host_metrics) = self.host_metrics.take() { - host_metrics.shutdown()?; - } if let Some(join) = self.join.take() { join.join().map_err(|_| io::Error::other("tui thread panicked"))? } else { @@ -122,40 +111,50 @@ fn run_terminal( let mut terminal = TerminalGuard::enter()?; let mut model = Model::new(config.clone(), startup); let mut views = Views::default(); + let mut next_draw_at = Instant::now(); loop { if control_rx.try_recv().is_ok() { return Ok(()); } - while let Ok(message) = telemetry_rx.try_recv() { - model.handle_message(message); - } - if signal_count.load(Ordering::SeqCst) > 0 && !model.is_shutdown_mode() { model.enter_shutdown_mode(); terminal.set_mouse_capture(true)?; } let now = Instant::now(); - if !model.is_copy_mode() { - terminal.terminal().draw(|frame| ui::render(frame, &model, &mut views, now))?; + if now >= next_draw_at { + if !model.is_copy_mode() { + terminal.terminal().draw(|frame| ui::render(frame, &model, &mut views, now))?; + } + next_draw_at = now + config.tick_interval; } - if !event::poll(config.tick_interval)? { - continue; + while event::poll(Duration::ZERO)? { + match model.handle_terminal_event(event::read()?, &views) { + TerminalEventOutcome::Continue => {} + TerminalEventOutcome::EnterCopyMode => enter_copy_mode(&mut terminal, &model, &mut views, now)?, + TerminalEventOutcome::ExitCopyMode => terminal.set_mouse_capture(true)?, + TerminalEventOutcome::Shutdown => request_shutdown()?, + } } - match model.handle_terminal_event(event::read()?, &views) { - TerminalEventOutcome::Continue => {} - TerminalEventOutcome::EnterCopyMode => enter_copy_mode(&mut terminal, &mut model, &mut views, now)?, - TerminalEventOutcome::ExitCopyMode => terminal.set_mouse_capture(true)?, - TerminalEventOutcome::Shutdown => request_shutdown()?, + let timeout = next_draw_at.saturating_duration_since(Instant::now()); + match telemetry_rx.recv_timeout(timeout) { + Ok(message) => { + model.handle_message(message); + while let Ok(message) = telemetry_rx.try_recv() { + model.handle_message(message); + } + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => {} } } } -fn enter_copy_mode(terminal: &mut TerminalGuard, model: &mut Model, views: &mut Views, now: Instant) -> io::Result<()> { +fn enter_copy_mode(terminal: &mut TerminalGuard, model: &Model, views: &mut Views, now: Instant) -> io::Result<()> { terminal.terminal().draw(|frame| ui::render(frame, model, views, now))?; terminal.set_mouse_capture(false) } diff --git a/crates/amaru-tui/src/startup.rs b/crates/amaru-tui/src/startup.rs index 388ad31bde..45c496344e 100644 --- a/crates/amaru-tui/src/startup.rs +++ b/crates/amaru-tui/src/startup.rs @@ -33,13 +33,13 @@ pub struct StartupContext { pub system_start_millis: u64, pub era_history: Option, pub runtime_sections: Vec, - pub global_sections: Vec, pub protocol_sections: Vec, } impl StartupContext { #[allow(clippy::too_many_arguments)] pub fn new( + pid: u32, network: impl Into, software_version: impl Into, target: impl Into, @@ -50,7 +50,7 @@ impl StartupContext { runtime_sections: Vec, ) -> Self { Self { - process: ProcessInfo::new(network, software_version, target), + process: ProcessInfo::new(pid, network, software_version, target), protocol_version: protocol_parameters .map(|parameters| protocol_version::fmt(¶meters.protocol_version)) .unwrap_or_else(|| "unknown".to_string()), @@ -62,7 +62,6 @@ impl StartupContext { system_start_millis: global_parameters.system_start, era_history, runtime_sections, - global_sections: global_sections(global_parameters), protocol_sections: protocol_sections(protocol_parameters), } } @@ -94,14 +93,20 @@ impl StartupContext { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProcessInfo { + pub pid: u32, pub network: String, pub software_version: String, pub target: String, } impl ProcessInfo { - pub fn new(network: impl Into, software_version: impl Into, target: impl Into) -> Self { - Self { network: network.into(), software_version: software_version.into(), target: target.into() } + pub fn new( + pid: u32, + network: impl Into, + software_version: impl Into, + target: impl Into, + ) -> Self { + Self { pid, network: network.into(), software_version: software_version.into(), target: target.into() } } } @@ -184,21 +189,6 @@ fn is_runtime_config_arg(arg: &Arg) -> bool { !matches!(id, "help" | "version" | "help_global_parameters") && arg.get_long().is_some() } -fn global_sections(global_parameters: &GlobalParameters) -> Vec { - vec![ConfigSection::new( - "Global Parameters", - vec![ - config_entry("security param k", global_parameters.consensus_security_param.to_string()), - config_entry("epoch length factor", global_parameters.epoch_length_scale_factor.to_string()), - config_entry("active slot coeff inverse", global_parameters.active_slot_coeff_inverse.to_string()), - config_entry("max lovelace supply", global_parameters.max_lovelace_supply.to_string()), - config_entry("slots per KES period", global_parameters.slots_per_kes_period.to_string()), - config_entry("max KES evolution", global_parameters.max_kes_evolution.to_string()), - config_entry("system start", global_parameters.system_start.to_string()), - ], - )] -} - fn protocol_sections(protocol_parameters: Option<&ProtocolParameters>) -> Vec { let Some(protocol_parameters) = protocol_parameters else { return Vec::default(); @@ -272,12 +262,33 @@ mod tests { use std::time::{Duration, UNIX_EPOCH}; use amaru_kernel::{PREPROD_ERA_HISTORY, PREPROD_GLOBAL_PARAMETERS}; + use clap::Parser; - use super::StartupContext; + use super::{ConfigSection, RuntimeSettingsSource, StartupContext}; + + #[derive(Debug, Parser)] + struct FixtureSettings { + #[arg(long, help_heading = "Essential")] + network: String, + + #[arg(long, help_heading = "Network Global Parameters Overrides")] + consensus_security_param: u64, + } + + impl RuntimeSettingsSource for FixtureSettings { + fn value_for(&self, id: &str) -> Option { + match id { + "network" => Some(self.network.clone()), + "consensus_security_param" => Some(self.consensus_security_param.to_string()), + _ => None, + } + } + } #[test] fn target_slot_at_uses_era_history_slot_lengths() { let startup = StartupContext::new( + 42, "preprod", "test", "test", @@ -296,6 +307,7 @@ mod tests { #[test] fn target_slot_proximity_uses_security_param_as_lag_tolerance() { let startup = StartupContext::new( + 42, "preprod", "test", "test", @@ -326,4 +338,18 @@ mod tests { Some(false) ); } + + #[test] + fn runtime_sections_include_global_parameter_overrides() { + let settings = FixtureSettings { network: "preview".to_string(), consensus_security_param: 42 }; + let sections = ConfigSection::from_runtime_settings(&settings); + + assert_eq!(sections.len(), 2); + assert_eq!(sections[0].title, "Essential"); + assert_eq!(sections[0].entries.len(), 1); + assert_eq!(sections[0].entries[0].label, "network"); + assert_eq!(sections[1].title, "Network Global Parameters Overrides"); + assert_eq!(sections[1].entries.len(), 1); + assert_eq!(sections[1].entries[0].label, "consensus security param"); + } } diff --git a/crates/amaru-tui/src/ui/components/logs.rs b/crates/amaru-tui/src/ui/components/logs.rs index f2419fbb91..29000d95e2 100644 --- a/crates/amaru-tui/src/ui/components/logs.rs +++ b/crates/amaru-tui/src/ui/components/logs.rs @@ -15,7 +15,7 @@ use ratatui::{ Frame, layout::{Alignment, Constraint, Direction, Layout, Rect}, - style::{Modifier, Style}, + style::Modifier, text::{Line, Span}, widgets::{Block, Borders, Paragraph, Wrap}, }; @@ -26,11 +26,11 @@ use super::super::{ render_horizontal_separator, render_scrollbar, scroll_panel_border, scroll_panel_border_type, spans_width, }, format::format_log_wall_time, - theme::{accent_primary, emphasis_primary, muted, style_for_level, style_for_level_filter, style_for_target}, + theme::{emphasis_primary, muted, style_for_level, style_for_level_filter, style_for_target}, }; use crate::{ - events::{TelemetryKind, TelemetryRecord}, - model::{InteractionMode, LevelFilter, Model, ScrollFocus, TargetFilter}, + events::TelemetryRecord, + model::{LevelFilter, Model, ScrollFocus, TargetFilter}, ui::Views, }; @@ -78,8 +78,7 @@ pub(in crate::ui) fn render_logs(frame: &mut Frame<'_>, area: Rect, model: &Mode let scroll = model.log_scroll.min(logs.len().saturating_sub(visible)); let end = logs.len().saturating_sub(scroll); let start = end.saturating_sub(visible); - let lines = - logs[start..end].iter().map(|record| log_record_line(record, model.interaction_mode)).collect::>(); + let lines = logs[start..end].iter().map(|record| log_record_line(record.as_ref())).collect::>(); let paragraph = Paragraph::new(lines).wrap(Wrap { trim: false }); frame.render_widget(paragraph, layout[2]); @@ -139,7 +138,7 @@ fn log_toggle_label(model: &Model) -> &'static str { if model.log_pane_mode.is_maximized() { "-" } else { "+" } } -fn log_record_line(record: &TelemetryRecord, mode: InteractionMode) -> Line<'static> { +fn log_record_line(record: &TelemetryRecord) -> Line<'static> { let fields = crate::model::render_fields(record); let mut spans = vec![ Span::styled(format_log_wall_time(record.wall_time), muted()), @@ -151,10 +150,6 @@ fn log_record_line(record: &TelemetryRecord, mode: InteractionMode) -> Line<'sta Span::styled(record.primary_label().to_string(), super::super::theme::emphasis_white()), ]; - if record.kind == TelemetryKind::SpanClose { - spans.push(Span::styled(" close", Style::default().fg(accent_primary(mode)).add_modifier(Modifier::BOLD))); - } - if !fields.is_empty() { spans.push(Span::raw(" ")); spans.push(Span::styled(fields, muted())); diff --git a/crates/amaru-tui/src/ui/screens/amaru.rs b/crates/amaru-tui/src/ui/screens/amaru.rs index bfb5fb4ee5..955fac00c1 100644 --- a/crates/amaru-tui/src/ui/screens/amaru.rs +++ b/crates/amaru-tui/src/ui/screens/amaru.rs @@ -87,14 +87,14 @@ pub(in crate::ui) fn render_amaru(frame: &mut Frame<'_>, area: Rect, model: &Mod aligned_pair_lines(vec![ ("Network", model.startup.process.network.clone()), ("Platform", model.startup.process.target.clone()), - ("PID", std::process::id().to_string()), + ("PID", model.startup.process.pid.to_string()), ("Uptime", format_duration(now.duration_since(model.created_at))), ]), model.interaction_mode, ); - let block_rate = blocks_per_second(model, now); - let transaction_rate = transactions_per_second(model, now); + let block_rate = blocks_per_second(model); + let transaction_rate = transactions_per_second(model); render_card( frame, cards[1], @@ -257,16 +257,12 @@ fn linear_ratio_f64(current: f64, max: f64) -> f64 { if max == 0.0 { 0.0 } else { (current / max).clamp(0.0, 1.0) } } -fn blocks_per_second(model: &Model, now: Instant) -> f64 { - let blocks = model.recent_blocks_count() as f64; - let seconds = model.retained_blocks_span(now).as_secs_f64(); - if seconds == 0.0 { 0.0 } else { blocks / seconds } +fn blocks_per_second(model: &Model) -> f64 { + model.blocks_per_second() } -fn transactions_per_second(model: &Model, now: Instant) -> f64 { - let transactions = model.recent_transactions_count() as f64; - let seconds = model.retained_transactions_span(now).as_secs_f64(); - if seconds == 0.0 { 0.0 } else { transactions / seconds } +fn transactions_per_second(model: &Model) -> f64 { + model.transactions_per_second() } fn format_kib(bytes: u64) -> String { @@ -291,10 +287,11 @@ mod tests { } #[test] - fn throughput_uses_uptime_until_the_sample_buffer_is_filled() { + fn throughput_uses_exponential_moving_average() { let mut model = Model::new( crate::Config::default(), crate::startup::StartupContext::new( + 42, "preview", "test", "test", @@ -305,13 +302,13 @@ mod tests { Vec::default(), ), ); - let now = model.created_at + Duration::from_secs(10); - model.recent_blocks.push_back(model.created_at + Duration::from_secs(1)); - model.recent_blocks.push_back(model.created_at + Duration::from_secs(4)); - model.recent_transactions.push_back((model.created_at + Duration::from_secs(2), 9)); + model.block_rate.record(model.created_at + Duration::from_secs(1), 1); + model.block_rate.record(model.created_at + Duration::from_secs(4), 1); + model.transaction_rate.record(model.created_at + Duration::from_secs(2), 9); + model.transaction_rate.record(model.created_at + Duration::from_secs(4), 3); - assert_eq!(blocks_per_second(&model, now), 0.2); - assert_eq!(transactions_per_second(&model, now), 0.9); + assert_eq!(blocks_per_second(&model), 1.0 / 3.0); + assert_eq!(transactions_per_second(&model), 1.5); } #[test] diff --git a/crates/amaru-tui/src/ui/screens/config.rs b/crates/amaru-tui/src/ui/screens/config.rs index fa95fc0d71..ba1ea50080 100644 --- a/crates/amaru-tui/src/ui/screens/config.rs +++ b/crates/amaru-tui/src/ui/screens/config.rs @@ -47,13 +47,7 @@ pub(in crate::ui) fn render_config(frame: &mut Frame<'_>, area: Rect, model: &Mo .constraints([Constraint::Percentage(left_width), Constraint::Percentage(100 - left_width)]) .split(content_area); - render_section_groups( - frame, - columns[0], - &[&model.startup.runtime_sections, &model.startup.global_sections], - scroll, - model.interaction_mode, - ); + render_section_groups(frame, columns[0], &[&model.startup.runtime_sections], scroll, model.interaction_mode); render_section_groups(frame, columns[1], &[&model.startup.protocol_sections], scroll, model.interaction_mode); if overflowing { @@ -62,9 +56,7 @@ pub(in crate::ui) fn render_config(frame: &mut Frame<'_>, area: Rect, model: &Mo } pub(in crate::ui) fn page_content_height(model: &Model) -> u16 { - config_column_height(&model.startup.runtime_sections) - .saturating_add(config_column_height(&model.startup.global_sections)) - .max(config_column_height(&model.startup.protocol_sections)) + config_column_height(&model.startup.runtime_sections).max(config_column_height(&model.startup.protocol_sections)) } fn config_column_height(sections: &[crate::startup::ConfigSection]) -> u16 { diff --git a/crates/amaru/src/bin/amaru/cmd/node/run.rs b/crates/amaru/src/bin/amaru/cmd/node/run.rs index 76dee48f80..f1549a01b5 100644 --- a/crates/amaru/src/bin/amaru/cmd/node/run.rs +++ b/crates/amaru/src/bin/amaru/cmd/node/run.rs @@ -31,7 +31,7 @@ use amaru::{ }; use amaru_kernel::{EraHistory, GlobalParameters, NetworkName, PEER_SNAPSHOT_NETWORKS}; use amaru_mempool::MempoolConfig; -use amaru_metrics::METRICS_METER_NAME; +use amaru_metrics::Meter; use amaru_node::{ DEFAULT_DOWNSTREAM_PEERS, DEFAULT_PEER_REMOVAL_COOLDOWN_SECS, DEFAULT_UPSTREAM_PEERS, peer_snapshot::{embedded_configs_commit, load_embedded_peer_snapshot, load_peer_snapshot}, @@ -45,14 +45,13 @@ use amaru_protocols::tx_submission::ResponderParams; use amaru_pure_stage::{Sender, trace_buffer::TraceBuffer}; use amaru_stores::rocksdb::RocksDbConfig; use amaru_tui as tui; -use anyhow::anyhow; use clap::{self, ArgAction, Parser}; -use opentelemetry::metrics::MeterProvider; -use opentelemetry_sdk::metrics::SdkMeterProvider; use parking_lot::Mutex; use thiserror::Error; use tokio_util::sync::CancellationToken; -use tracing::{error, info, warn}; +#[cfg(unix)] +use tracing::error; +use tracing::{info, warn}; use crate::pid::optional_pid_file; @@ -282,13 +281,17 @@ impl Args { tui::Settings::new( self.no_tui, tui::StartupContext::new( + std::process::id(), self.network.to_string(), version::display_version(), format!("{}/{}", version::target_os(), version::target_arch()), MempoolConfig::default().max_bytes, &global_parameters, self.network.as_protocol_parameters(), - self.network.as_era_history().cloned().or_else(|| load_era_history(self.era_history.as_deref()).ok()), + self.network + .as_era_history() + .cloned() + .or_else(|| self.era_history.as_deref().and_then(|path| EraHistory::load(path).ok())), tui::ConfigSection::from_runtime_settings(self), ), ) @@ -305,13 +308,19 @@ impl tui::RuntimeSettingsSource for Args { match id { "network" => Some(self.network.to_string()), - "chain_dir" => { - Some(self.chain_dir.as_deref().map(path_value).unwrap_or_else(|| default_chain_dir(self.network))) - } + "chain_dir" => Some( + self.chain_dir + .as_deref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| default_chain_dir(self.network)), + ), "migrate_chain_db" => Some(self.migrate_chain_db.to_string()), - "ledger_dir" => { - Some(self.ledger_dir.as_deref().map(path_value).unwrap_or_else(|| default_ledger_dir(self.network))) - } + "ledger_dir" => Some( + self.ledger_dir + .as_deref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| default_ledger_dir(self.network)), + ), "listen_address" => Some(self.listen_address.clone()), "submit_api_address" => Some(self.submit_api_address.clone().unwrap_or_else(|| "disabled".to_string())), "no_tui" => Some(self.no_tui.to_string()), @@ -321,14 +330,25 @@ impl tui::RuntimeSettingsSource for Args { "downstream_peers" => Some(self.downstream_peers.to_string()), "max_extra_ledger_snapshots" => Some(self.max_extra_ledger_snapshots.to_string()), "peer_removal_cooldown_secs" => Some(self.peer_removal_cooldown_secs.to_string()), - "pid_file" => Some(self.pid_file.as_deref().map(path_value).unwrap_or_else(|| "disabled".to_string())), + "pid_file" => Some( + self.pid_file + .as_deref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "disabled".to_string()), + ), "trace_buffer" => Some(self.trace_buffer.clone().unwrap_or_else(|| "disabled".to_string())), - "dump_trace_buffer" => { - Some(self.dump_trace_buffer.as_deref().map(path_value).unwrap_or_else(|| "disabled".to_string())) - } - "era_history" => { - Some(self.era_history.as_deref().map(path_value).unwrap_or_else(|| era_history_value(self.network))) - } + "dump_trace_buffer" => Some( + self.dump_trace_buffer + .as_deref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "disabled".to_string()), + ), + "era_history" => Some( + self.era_history + .as_deref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| self.network.to_string()), + ), "consensus_security_param" => Some(global_parameters.consensus_security_param.to_string()), "epoch_length_scale_factor" => Some(global_parameters.epoch_length_scale_factor.to_string()), "active_slot_coeff_inverse" => Some(global_parameters.active_slot_coeff_inverse.to_string()), @@ -341,14 +361,6 @@ impl tui::RuntimeSettingsSource for Args { } } -fn era_history_value(network: NetworkName) -> String { - if network.as_era_history().is_some() { format!("builtin for {network}") } else { "not set".to_string() } -} - -fn path_value(path: &Path) -> String { - path.display().to_string() -} - fn peer_addresses_value(args: &Args) -> String { if args.peer_address.is_empty() { default_peer_for_network(args.network).to_string() @@ -359,7 +371,7 @@ fn peer_addresses_value(args: &Args) -> String { fn peer_snapshot_value(args: &Args) -> String { if let Some(path) = args.peer_snapshot.as_deref() { - return path_value(path); + return path.display().to_string(); } if PEER_SNAPSHOT_NETWORKS.contains(&args.network) { @@ -374,14 +386,10 @@ fn peer_snapshot_value(args: &Args) -> String { const SUBMIT_API_JOIN_TIMEOUT: Duration = Duration::from_secs(5); pub(crate) fn runnable(args: Args) -> Runnable { - Runnable::soft(RuntimeKind::Node, move |shutdown, meter_provider| run(args, meter_provider, shutdown)) + Runnable::soft(RuntimeKind::Node, move |shutdown, meter| run(args, meter, shutdown)) } -async fn run( - args: Args, - meter_provider: Option, - shutdown: ShutdownHandle, -) -> Result<(), Box> { +async fn run(args: Args, meter: Meter, shutdown: ShutdownHandle) -> Result<(), Box> { let _pid_file = optional_pid_file(args.pid_file.clone()); let config = parse_args(args)?; @@ -389,8 +397,8 @@ async fn run( let submit_api_address = config.submit_api_address()?; pre_flight_checks()?; - let metrics = track_system_metrics(meter_provider.clone())?; - let meter = meter_provider.map(|mp| mp.meter(METRICS_METER_NAME)); + let meter = Arc::new(meter); + let metrics = track_system_metrics(meter.clone())?; let running = build_and_run_node(config, meter)?; // Main-thread signal path can abort stages without scheduling this future. @@ -502,8 +510,12 @@ fn parse_trace_buffer_limits(s: &str) -> Result<(usize, usize), String> { fn parse_args(args: Args) -> Result> { let network = args.network; - let era_history = - network.as_era_history().cloned().map(Ok).unwrap_or_else(|| load_era_history(args.era_history.as_deref()))?; + let era_history = network.as_era_history().cloned().map(Ok).unwrap_or_else(|| { + args.era_history + .as_deref() + .ok_or_else(|| "missing era history for custom network".into()) + .and_then(|path| EraHistory::load(path).map_err(|err| -> Box { Box::new(err) })) + })?; let global_parameters = network.as_global_parameters().cloned().unwrap_or(args.global_parameters); @@ -629,13 +641,6 @@ fn parse_args(args: Args) -> Result> { }) } -fn load_era_history(path: Option<&Path>) -> Result> { - match path { - Some(path) => Ok(serde_json::from_slice(&std::fs::read(path)?)?), - None => Err(anyhow!("missing era history for custom network").into()), - } -} - fn log_loaded_snapshot(path: Option<&Path>, snapshot: &amaru_node::peer_snapshot::PeerSnapshot) { if snapshot.peers.is_empty() { warn!( @@ -657,6 +662,7 @@ fn log_loaded_snapshot(path: Option<&Path>, snapshot: &amaru_node::peer_snapshot } } +#[allow(dead_code, reason = "Debug instance is unused but useful to keep")] #[derive(Debug, Error)] pub enum PreFlightError { #[error("File descriptors limit too low: minimum required {0}, available {1}")] diff --git a/crates/amaru/src/bin/amaru/main.rs b/crates/amaru/src/bin/amaru/main.rs index 016da637fc..53bb18129d 100644 --- a/crates/amaru/src/bin/amaru/main.rs +++ b/crates/amaru/src/bin/amaru/main.rs @@ -17,7 +17,7 @@ use std::{error::Error, process::ExitCode, time::Duration}; use amaru::{ exit::install_termination_signals, lifecycle::{RUNTIME_SHUTDOWN_TIMEOUT, set_signal_stderr_enabled}, - observability::{Color, ObservabilityHints, setup_observability}, + observability::{Color, ObservabilityHints, OpenTelemetryHandle, setup_observability}, panic::panic_handler, version, }; @@ -71,45 +71,46 @@ fn try_main() -> Result<(), Box> { let runnable = cli.command.into_runnable(); let rt = runnable.build_runtime().map_err(|e| anyhow!(e).context("failed to build Tokio runtime"))?; - let tui = if skip_logging { - None + let with_tui = if !skip_logging + && let Some(settings) = tui_settings.filter(|settings| tui::should_enable(settings.no_tui, with_json_traces)) + { + let (_, config, startup) = settings.into_parts(); + set_signal_stderr_enabled(false); + Some(tui::Session::spawn(config, startup, signals.shared_count())?) } else { - tui_settings - .filter(|settings| tui::should_enable(settings.no_tui, with_json_traces)) - .map(|settings| { - let (_, config, startup) = settings.into_parts(); - tui::Session::spawn(config, startup, signals.shared_count()) - }) - .transpose()? + set_signal_stderr_enabled(true); + None }; - set_signal_stderr_enabled(tui.is_none()); - let _metrics_subscription = tui.as_ref().map(tui::Session::subscribe_to_metrics); - let (metrics, teardown) = if skip_logging { - (None, Box::new(|| Ok(())) as Box Result<(), Box> + Send>) + let OpenTelemetryHandle { meter, teardown } = if skip_logging { + OpenTelemetryHandle::default() } else { // OpenTelemetry batch exporters require a current Tokio runtime. let _enter = rt.enter(); setup_observability( with_open_telemetry, with_json_traces, + with_tui.as_ref(), color_enabled, &ListenAddressHint(listen_address.as_deref()), - tui.as_ref().map(tui::Session::layer), ) }; - let result = runnable.run_on(&rt, &signals, metrics); + let result = runnable.run_on(&rt, &signals, meter); // Keep the runtime alive while OTEL providers flush (their batch tasks were spawned on it). if let Err(err) = run_teardown_with_timeout(teardown, Duration::from_secs(10)) { eprintln!("amaru: failed to teardown tracing: {err}"); } - if let Some(tui) = tui - && let Err(err) = tui.shutdown() - { - eprintln!("amaru: failed to shutdown terminal dashboard cleanly: {err}"); + if let Some(tui) = with_tui { + if let Err(err) = tui.shutdown() { + eprintln!("amaru: failed to shutdown terminal dashboard cleanly: {err}"); + } + + if let Err(ref err) = result { + eprintln!("amaru: {err}"); + } } rt.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); diff --git a/crates/amaru/src/bin/amaru/pid.rs b/crates/amaru/src/bin/amaru/pid.rs index bb34352fcc..070bfd793c 100644 --- a/crates/amaru/src/bin/amaru/pid.rs +++ b/crates/amaru/src/bin/amaru/pid.rs @@ -80,12 +80,12 @@ impl Display for ProcessIdHandle { } #[cfg(unix)] -fn process_exists(pid: u32) -> bool { +pub fn process_exists(pid: u32) -> bool { Command::new("kill").args(["-0", &pid.to_string()]).output().map(|output| output.status.success()).unwrap_or(false) } #[cfg(windows)] -fn process_exists(pid: u32) -> bool { +pub fn process_exists(pid: u32) -> bool { Command::new("tasklist") .args(["/FI", &format!("PID eq {}", pid)]) .output() diff --git a/crates/amaru/src/lifecycle.rs b/crates/amaru/src/lifecycle.rs index 6625c96919..0111ab9735 100644 --- a/crates/amaru/src/lifecycle.rs +++ b/crates/amaru/src/lifecycle.rs @@ -27,7 +27,7 @@ use std::{ time::Duration, }; -use opentelemetry_sdk::metrics::SdkMeterProvider; +use amaru_metrics::Meter; use parking_lot::Mutex; use tokio::runtime::{Builder, Runtime}; use tokio_util::sync::CancellationToken; @@ -81,14 +81,14 @@ pub enum FirstSignal { } type CmdFuture = std::pin::Pin>> + 'static>>; -type CmdWork = Box) -> CmdFuture + Send>; +type CmdWork = Box CmdFuture + Send>; /// A fully resolved leaf subcommand ready to run under the process lifecycle. /// /// Construct via [`Runnable::soft`] / [`Runnable::exit_on_signal`], then: /// 1. [`Self::build_runtime`] — create the Tokio runtime for this command /// 2. Set up observability while that runtime is current ([`Runtime::enter`]) -/// 3. [`Self::run_on`] — bind metrics and drive the command future on that runtime +/// 3. [`Self::run_on`] — bind meter and drive the command future on that runtime pub struct Runnable { runtime: RuntimeKind, first_signal: FirstSignal, @@ -101,13 +101,13 @@ impl Runnable { /// produced by observability setup (only used by long-running commands). pub fn soft(runtime: RuntimeKind, work: F) -> Self where - F: FnOnce(ShutdownHandle, Option) -> Fut + Send + 'static, + F: FnOnce(ShutdownHandle, Meter) -> Fut + Send + 'static, Fut: Future>> + 'static, { Self { runtime, first_signal: FirstSignal::SoftShutdown, - work: Box::new(move |shutdown, metrics| Box::pin(work(shutdown, metrics))), + work: Box::new(move |shutdown, meter| Box::pin(work(shutdown, meter))), } } @@ -133,13 +133,8 @@ impl Runnable { /// Drive this command on an already-built runtime (observability must already be set up). /// /// Does **not** shut down `rt`; the caller tears down observability and then the runtime. - pub fn run_on( - self, - rt: &Runtime, - signals: &SignalState, - metrics: Option, - ) -> Result<(), Box> { - run_until_exit(rt, signals, self.first_signal, self.work, metrics) + pub fn run_on(self, rt: &Runtime, signals: &SignalState, meter: Meter) -> Result<(), Box> { + run_until_exit(rt, signals, self.first_signal, self.work, meter) } /// Build a runtime, run the command with no meter provider, then shut the runtime down. @@ -147,7 +142,7 @@ impl Runnable { /// Prefer the explicit build → observability → [`Self::run_on`] sequence in the binary. pub fn run(self, signals: &SignalState) -> Result<(), Box> { let rt = self.build_runtime()?; - let result = self.run_on(&rt, signals, None); + let result = self.run_on(&rt, signals, Meter::default()); rt.shutdown_timeout(RUNTIME_SHUTDOWN_TIMEOUT); result } @@ -233,7 +228,7 @@ pub fn run_until_exit( signals: &SignalState, first_signal: FirstSignal, work: CmdWork, - metrics: Option, + meter: Meter, ) -> Result<(), Box> { let shutdown = ShutdownHandle::new(); let shutdown_work = shutdown.clone(); @@ -243,7 +238,7 @@ pub fn run_until_exit( let worker = thread::Builder::new() .name("amaru-cmd".into()) .spawn(move || { - let result = handle.block_on(work(shutdown_work, metrics)); + let result = handle.block_on(work(shutdown_work, meter)); let _ = result_tx.send(result.map_err(|err| err.to_string())); }) .map_err(|e| format!("failed to spawn command worker thread: {e}"))?; diff --git a/crates/amaru/src/metrics.rs b/crates/amaru/src/metrics.rs index 1a9dc3b4bd..dbfa25a2d3 100644 --- a/crates/amaru/src/metrics.rs +++ b/crates/amaru/src/metrics.rs @@ -12,13 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{sync::LazyLock, time::Duration}; - -use amaru_metrics::{METRICS_METER_NAME, MetricRecorder, SystemMetrics, has_subscribers, notify_subscribers}; +use std::{ + sync::{Arc, LazyLock}, + time::Duration, +}; + +#[cfg(unix)] +use amaru_kernel::utils::process::sample_process_memory; +use amaru_metrics::{Meter, MetricRecorder, MetricsEvent, SystemMetrics}; use anyhow::anyhow; -use opentelemetry::{KeyValue, metrics::MeterProvider}; -use opentelemetry_sdk::metrics::SdkMeterProvider; -use sysinfo::{CpuRefreshKind, MemoryRefreshKind, ProcessRefreshKind, ProcessesToUpdate, RefreshKind, System}; +use opentelemetry::KeyValue; +use sysinfo::{ + CpuRefreshKind, DiskRefreshKind, Disks, MemoryRefreshKind, ProcessRefreshKind, ProcessesToUpdate, RefreshKind, + System, +}; use tokio::task::JoinHandle; use tracing::error; @@ -26,24 +33,26 @@ use crate::version; static METRICS_POLL_DELAY: LazyLock = LazyLock::new(|| Duration::from_secs(1)); -pub fn track_system_metrics( - provider: Option, -) -> Result>, Box> { - if provider.is_none() && !has_subscribers() { - return Ok(None); - } +#[cfg(unix)] +fn sampled_process_memory_bytes(pid: sysinfo::Pid, rss_bytes: u64) -> u64 { + sample_process_memory(pid.as_u32()).unwrap_or(rss_bytes) +} - if let Some(provider) = provider.as_ref() { - record_build_info(provider); - } +#[cfg(not(unix))] +fn sampled_process_memory_bytes(_pid: sysinfo::Pid, rss_bytes: u64) -> u64 { + rss_bytes +} + +pub fn track_system_metrics(meter: Arc) -> Result>, Box> { + record_build_info(&meter); let mut sys = System::new_with_specifics( RefreshKind::nothing() .with_cpu(CpuRefreshKind::everything().without_frequency()) - .with_memory(MemoryRefreshKind::everything().without_swap()), + .with_memory(MemoryRefreshKind::nothing().with_ram()), ); + let mut disks = Disks::new_with_refreshed_list_specifics(DiskRefreshKind::nothing().with_io_usage()); let number_of_cpus = sys.cpus().len() as u64; - let meter = provider.as_ref().map(|provider| provider.meter(METRICS_METER_NAME)); let own_pid = sysinfo::get_current_pid().map_err(|err| anyhow!("unable to retrieve own pid: {err}"))?; @@ -56,36 +65,50 @@ pub fn track_system_metrics( true, ProcessRefreshKind::nothing().with_cpu().with_disk_usage().with_memory(), ); + sys.refresh_memory_specifics(MemoryRefreshKind::nothing().with_ram()); + disks.refresh_specifics(false, DiskRefreshKind::nothing().with_io_usage()); match sys.process(own_pid) { None => error!("unable to find amaru's own process (pid={own_pid}) ?!"), Some(process) => { let disk_usage = process.disk_usage(); - let metrics = SystemMetrics { + let process_memory_live_resident = process.memory(); + let (host_live_read_bytes, host_live_write_bytes) = + disks.iter().fold((0u64, 0u64), |(read_total, write_total), disk| { + let usage = disk.usage(); + ( + read_total.saturating_add(usage.read_bytes), + write_total.saturating_add(usage.written_bytes), + ) + }); + let event = MetricsEvent::SystemMetrics(SystemMetrics { runtime_seconds: process.run_time(), cpu_percent: process.cpu_usage() as f64 / number_of_cpus as f64, - process_memory_bytes: process.memory(), - rss_bytes: process.memory(), - virtual_bytes: process.virtual_memory(), + process_memory_bytes: sampled_process_memory_bytes(own_pid, process_memory_live_resident), + process_memory_live_resident, + process_memory_available_virtual: process.virtual_memory(), + memory_used_bytes: sys.used_memory(), + memory_total_bytes: sys.total_memory(), disk_read_bytes: disk_usage.total_read_bytes, disk_write_bytes: disk_usage.total_written_bytes, disk_live_read_bytes: disk_usage.read_bytes, disk_live_write_bytes: disk_usage.written_bytes, + host_live_read_bytes, + host_live_write_bytes, open_files: process.open_files().map_or(0, |files| files as u64), - }; + }); - if let Some(meter) = meter.as_ref() { - metrics.record_to_meter(meter); - } - notify_subscribers(&metrics.into()); + event.record_to_meter(&meter); } } } }))) } -fn record_build_info(provider: &SdkMeterProvider) { - let meter = provider.meter(METRICS_METER_NAME); +fn record_build_info(meter: &Meter) { + let Some(meter) = meter.get() else { + return; + }; let build_info = meter .u64_gauge("cardano_node_metrics_cardano_build_info") diff --git a/crates/amaru/src/observability.rs b/crates/amaru/src/observability.rs index 3caccd10eb..9f507f0c2b 100644 --- a/crates/amaru/src/observability.rs +++ b/crates/amaru/src/observability.rs @@ -22,11 +22,17 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +use amaru_metrics::{METRICS_METER_NAME, Meter}; use amaru_observability::{info, warn}; use amaru_tui::TracingLayer as TuiTracingLayer; -use opentelemetry::trace::TracerProvider; +use opentelemetry::{Key, KeyValue, metrics::MeterProvider, trace::TracerProvider}; use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; -use opentelemetry_sdk::{logs::SdkLoggerProvider, metrics::SdkMeterProvider, trace::SdkTracerProvider}; +use opentelemetry_sdk::{ + Resource, + logs::SdkLoggerProvider, + metrics::{SdkMeterProvider, Temporality}, + trace::SdkTracerProvider, +}; use opentelemetry_semantic_conventions::resource::{SERVICE_INSTANCE_ID, SERVICE_NAME}; use tracing::{ Metadata, Subscriber, @@ -80,10 +86,10 @@ type JsonLayer = Layered, S>; type JsonFilter = Filtered, ThrottledEnvFilter, S>; -type TuiLayer = Layered, S>; - type TuiFilter = Filtered; +type TuiLayer = Layered, S>; + type DelayedWarning = Option>; // ----------------------------------------------------------------------------- @@ -282,51 +288,47 @@ impl TracingSubscriber { Self::Registry(registry) => { *self = TracingSubscriber::WithOpenTelemetry(registry.with(layer).with(log_bridge)); } - _ => panic!("'with_open_telemetry' called after 'with_json' or 'with_tui'"), + _ => panic!("'with_open_telemetry' called after 'with_json' or another terminal layer"), } } #[expect(clippy::panic)] #[expect(clippy::wildcard_enum_match_arm)] - pub fn with_json(&mut self, registry_layer: F, otel_layer: G) -> DelayedWarning - where - F: FnOnce() -> (JsonFilter, DelayedWarning), - G: FnOnce() -> (JsonFilter>, DelayedWarning), - { + pub fn with_tui(&mut self, layer: TuiTracingLayer) -> DelayedWarning { + let (default_filter, warning) = new_trace_filter(); + match std::mem::take(self) { Self::Registry(registry) => { - let (layer, warning) = registry_layer(); - *self = TracingSubscriber::WithJson(registry.with(layer)); - warning + *self = TracingSubscriber::WithTui(registry.with(layer.with_filter(default_filter))); } Self::WithOpenTelemetry(layered) => { - let (layer, warning) = otel_layer(); - *self = TracingSubscriber::WithJsonAndOpenTelemetry(layered.with(layer)); - warning + *self = TracingSubscriber::WithTuiAndOpenTelemetry(layered.with(layer.with_filter(default_filter))); } - _ => panic!("'with_json' called after as third layer"), + _ => panic!("'with_tui' called after as third layer"), } + + warning } #[expect(clippy::panic)] #[expect(clippy::wildcard_enum_match_arm)] - pub fn with_tui(&mut self, registry_layer: F, otel_layer: G) -> DelayedWarning + pub fn with_json(&mut self, registry_layer: F, otel_layer: G) -> DelayedWarning where - F: FnOnce() -> (TuiFilter, DelayedWarning), - G: FnOnce() -> (TuiFilter>, DelayedWarning), + F: FnOnce() -> (JsonFilter, DelayedWarning), + G: FnOnce() -> (JsonFilter>, DelayedWarning), { match std::mem::take(self) { Self::Registry(registry) => { let (layer, warning) = registry_layer(); - *self = TracingSubscriber::WithTui(registry.with(layer)); + *self = TracingSubscriber::WithJson(registry.with(layer)); warning } Self::WithOpenTelemetry(layered) => { let (layer, warning) = otel_layer(); - *self = TracingSubscriber::WithTuiAndOpenTelemetry(layered.with(layer)); + *self = TracingSubscriber::WithJsonAndOpenTelemetry(layered.with(layer)); warning } - _ => panic!("'with_tui' called after as third layer"), + _ => panic!("'with_json' called after as third layer"), } } @@ -345,7 +347,7 @@ impl TracingSubscriber { .with_filter(default_filter), ) .init(); - warning + return warning; } TracingSubscriber::WithOpenTelemetry(layered) => { let (default_filter, warning) = new_log_filter(); @@ -359,46 +361,24 @@ impl TracingSubscriber { .with_filter(default_filter), ) .init(); - warning + return warning; } TracingSubscriber::WithTui(layered) => { layered.init(); - None } TracingSubscriber::WithTuiAndOpenTelemetry(layered) => { layered.init(); - None } TracingSubscriber::WithJson(layered) => { layered.init(); - None } TracingSubscriber::WithJsonAndOpenTelemetry(layered) => { layered.init(); - None } } - } -} - -// ----------------------------------------------------------------------------- -// TUI TRACES -// ----------------------------------------------------------------------------- - -pub fn setup_tui_traces(subscriber: &mut TracingSubscriber, layer: TuiTracingLayer) -> DelayedWarning { - let registry_layer = { - let layer = layer.clone(); - move || { - let (default_filter, warning) = new_trace_filter(); - (layer.clone().with_filter(default_filter), warning) - } - }; - let otel_layer = move || { - let (default_filter, warning) = new_trace_filter(); - (layer.with_filter(default_filter), warning) - }; - subscriber.with_tui(registry_layer, otel_layer) + None + } } // ----------------------------------------------------------------------------- @@ -441,14 +421,14 @@ pub fn setup_json_traces(subscriber: &mut TracingSubscriber) -> Delaye // ----------------------------------------------------------------------------- pub struct OpenTelemetryHandle { - pub metrics: Option, + pub meter: Meter, pub teardown: Box Result<(), Box> + Send>, } impl Default for OpenTelemetryHandle { fn default() -> Self { OpenTelemetryHandle { - metrics: None::, + meter: Meter::default(), teardown: Box::new(|| Ok(())) as Box Result<(), Box> + Send>, } } @@ -463,14 +443,7 @@ pub trait ObservabilityHints { fn listen_address(&self) -> Option<&str>; } -#[expect(clippy::panic)] -pub fn setup_open_telemetry( - subscriber: &mut TracingSubscriber, - hints: &impl ObservabilityHints, -) -> (OpenTelemetryHandle, DelayedWarning) { - use opentelemetry::KeyValue; - use opentelemetry_sdk::{Resource, metrics::Temporality}; - +pub fn new_resource(hints: &impl ObservabilityHints) -> Resource { // Build the SDK-default resource to discover attributes already set via // OTEL_RESOURCE_ATTRIBUTES. This is used only to guard our *fallback* values; // the dedicated OTEL_SERVICE_NAME / OTEL_SERVICE_INSTANCE_ID env vars always @@ -480,6 +453,7 @@ pub fn setup_open_telemetry( let explicit_service_instance_id = var("OTEL_SERVICE_INSTANCE_ID").ok().map(|v| v.trim().to_string()).filter(|v| !v.is_empty()); + let service_instance_id: Option = explicit_service_instance_id.clone().or_else(|| { let listen_addr = hints.listen_address()?; let hostname = sysinfo::System::host_name().unwrap_or_else(|| "localhost".to_string()); @@ -492,10 +466,23 @@ pub fn setup_open_telemetry( if let Some(instance_id) = service_instance_id { attributes.push(KeyValue::new(SERVICE_INSTANCE_ID, instance_id)); } - let resource = Resource::builder().with_attributes(attributes).build(); - // Traces & span - let opentelemetry_provider = SdkTracerProvider::builder() + Resource::builder().with_attributes(attributes).build() +} + +#[expect(clippy::panic)] +#[expect(clippy::expect_used)] +pub fn setup_open_telemetry( + subscriber: &mut TracingSubscriber, + resource: Resource, +) -> (OpenTelemetryHandle, DelayedWarning) { + let service_name = resource + .get(&Key::from(SERVICE_NAME)) + .expect("missing 'service_name' on the provided OTLP resource") + .as_str() + .to_string(); + + let traces_provider = SdkTracerProvider::builder() .with_resource(resource.clone()) .with_batch_exporter( opentelemetry_otlp::SpanExporter::builder() @@ -505,46 +492,45 @@ pub fn setup_open_telemetry( ) .build(); - // Metrics - // NOTE: We use the http exporter here because not every OTLP receiver supports gRPC for metrics. + let logs_provider = SdkLoggerProvider::builder() + .with_resource(resource.clone()) + .with_batch_exporter( + opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .build() + .unwrap_or_else(|e| panic!("failed to setup opentelemetry log exporter: {e}")), + ) + .build(); + let metric_exporter = opentelemetry_otlp::MetricExporter::builder() - .with_http() + .with_tonic() .with_temporality(Temporality::default()) .build() .unwrap_or_else(|e| panic!("unable to create metric exporter: {e:?}")); - let metric_reader = opentelemetry_sdk::metrics::PeriodicReader::builder(metric_exporter).build(); - - let metrics_provider = opentelemetry_sdk::metrics::SdkMeterProvider::builder() - .with_reader(metric_reader) + let meter_provider = SdkMeterProvider::builder() .with_resource(resource.clone()) + .with_reader(opentelemetry_sdk::metrics::PeriodicReader::builder(metric_exporter).build()) .build(); - opentelemetry::global::set_meter_provider(metrics_provider.clone()); - // Subscriber - let opentelemetry_tracer = opentelemetry_provider.tracer(service_name); let (default_filter, warning) = new_trace_filter(); - let opentelemetry_layer = - tracing_opentelemetry::layer().with_tracer(opentelemetry_tracer).with_level(true).with_filter(default_filter); + let opentelemetry_layer = tracing_opentelemetry::layer() + .with_tracer(traces_provider.tracer(service_name)) + .with_level(true) + .with_target(true) + .with_filter(default_filter); // Logs - let logs_exporter = opentelemetry_otlp::LogExporter::builder() - .with_tonic() - .build() - .unwrap_or_else(|e| panic!("failed to setup opentelemetry log exporter: {e}")); - let logs_provider = SdkLoggerProvider::builder().with_resource(resource).with_batch_exporter(logs_exporter).build(); - let log_bridge = OpenTelemetryTracingBridge::new(&logs_provider); - let (log_bridge_filter, _) = new_trace_filter(); - let log_bridge = log_bridge.with_filter(log_bridge_filter); + let log_bridge = OpenTelemetryTracingBridge::new(&logs_provider).with_filter(new_trace_filter().0); subscriber.with_open_telemetry(opentelemetry_layer, log_bridge); ( OpenTelemetryHandle { - metrics: Some(metrics_provider.clone()), - teardown: Box::new(|| teardown_open_telemetry(opentelemetry_provider, metrics_provider, logs_provider)), + meter: Meter::from(meter_provider.meter(METRICS_METER_NAME)), + teardown: Box::new(|| teardown_open_telemetry(traces_provider, meter_provider, logs_provider)), }, warning, ) @@ -552,13 +538,13 @@ pub fn setup_open_telemetry( fn teardown_open_telemetry( tracing: SdkTracerProvider, - metrics: SdkMeterProvider, + meter: SdkMeterProvider, logs: SdkLoggerProvider, ) -> Result<(), Box> { // Shut down the providers so that it flushes any remaining spans. // The process lifecycle layer applies an outer timeout around this teardown. tracing.shutdown()?; - metrics.shutdown()?; + meter.shutdown()?; logs.shutdown()?; Ok(()) @@ -693,34 +679,36 @@ fn new_trace_filter() -> (ThrottledEnvFilter, DelayedWarning) { pub fn setup_observability( with_open_telemetry: bool, with_json_traces: bool, + with_tui: Option<&amaru_tui::Session>, color: bool, hints: &impl ObservabilityHints, - tui_layer: Option, -) -> (Option, Box Result<(), Box> + Send>) { +) -> OpenTelemetryHandle { let mut subscriber = TracingSubscriber::new(); - let (OpenTelemetryHandle { metrics, teardown }, warning_otlp) = if with_open_telemetry { - setup_open_telemetry(&mut subscriber, hints) + let (OpenTelemetryHandle { mut meter, teardown }, warning_otlp) = if with_open_telemetry { + setup_open_telemetry(&mut subscriber, new_resource(hints)) } else { (OpenTelemetryHandle::default(), None) }; - let warning_tui = tui_layer.and_then(|layer| setup_tui_traces(&mut subscriber, layer)); + let warning_tui = if let Some(tui) = with_tui { + meter.set_local_observer(tui.metrics_observer()); + subscriber.with_tui(tui.tracing_layer()) + } else { + None + }; + let warning_json = if with_json_traces { setup_json_traces(&mut subscriber) } else { None }; let warning_log = subscriber.init(color); - // NOTE: Both warnings are bound to the same ENV var, so `.or` prevents from logging it twice. - if let Some(notify) = warning_log.or(warning_tui) { - notify(); - } - if let Some(notify) = warning_otlp.or(warning_json) { + for notify in [warning_otlp, warning_tui, warning_json, warning_log].into_iter().flatten() { notify(); } info!(setup::observability::INIT, with_open_telemetry, with_json_traces, with_colors = color,); - (metrics, teardown) + OpenTelemetryHandle { meter, teardown } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/engineering-decision-records/030-embedded-terminal-observability-ui.md b/engineering-decision-records/030-embedded-terminal-observability-ui.md index e1ac7e53d1..5893e8b84e 100644 --- a/engineering-decision-records/030-embedded-terminal-observability-ui.md +++ b/engineering-decision-records/030-embedded-terminal-observability-ui.md @@ -28,25 +28,25 @@ maintain: ## Decision -Amaru shall provide an embedded terminal UI through a dedicated `amaru-tui` -crate. +Amaru shall provide a terminal UI through a dedicated `amaru-tui` crate. + +That UI is launched automatically by `amaru node run` in an interactive +terminal. ### The TUI is a consumer, not a control plane The TUI derives its state from the same observability streams exposed to other -consumers: - -- structured tracing events, captured through a tracing layer -- shared runtime metrics, captured through the existing metrics publication - mechanism +consumers and runtime producers: -Shared runtime metrics remain focused on the Amaru process itself. When the -dashboard needs host-wide context that would be too expensive or too -product-specific to export as a normal metric stream, `amaru-tui` samples it -locally. This currently covers: +- structured tracing events, consumed from an in-process tracing layer +- runtime metrics, consumed from a single in-process callback for local + `MetricsEvent`s -- total and used system RAM -- aggregate live disk I/O observed across all refreshed processes +Runtime metrics remain focused on the Amaru process and the host-level totals +that are cheap and meaningful to expose globally. Process memory footprint as +reported by local host tooling is sampled by Amaru itself and exported through +the normal system metric stream, so the TUI does not maintain a separate +process sampler. The TUI may additionally receive a small startup context for static or bootstrapping-time information such as: @@ -85,20 +85,26 @@ Where possible, that telemetry contract should be consumed through schema-generated identity constants and field accessors rather than raw string keys, so schema drift causes compile failures instead of silently blank widgets. +For the terminal UI specifically, observability setup installs: + +- an in-process tracing layer that forwards typed telemetry records into the + TUI thread +- a single local metrics callback that mirrors `MetricsEvent`s into the TUI + +This keeps the embedded mode cheap and direct without introducing a broader +internal control plane. + ### Model the UI as a fold over events The TUI maintains a bounded in-memory model that is updated from: - telemetry records - metric records -- TUI-local host samples - local terminal events such as keyboard, mouse, focus, and scrolling This keeps the architecture simple: -- `capture` converts tracing into TUI messages -- `metrics` subscribes to the shared metrics stream -- `host_metrics` samples slower host-wide context for the TUI only +- `capture` converts in-process tracing records into TUI messages - `model` owns bounded UI state and folds incoming messages through focused reducer slices - `ui` renders from the model through a thin shell plus page and component views diff --git a/monitoring/Makefile b/monitoring/Makefile index 2e996198d1..8961a318f5 100644 --- a/monitoring/Makefile +++ b/monitoring/Makefile @@ -18,7 +18,6 @@ run-amaru: ## Run Amaru with OpenTelemetry enabled cd $(AMARU_DIR) && \ OTEL_SERVICE_NAME=amaru \ OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \ - OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics \ OTEL_METRIC_EXPORT_INTERVAL=1000 \ amaru --with-open-telemetry node run diff --git a/monitoring/README.md b/monitoring/README.md index edb5288501..72dcdda8b6 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -79,7 +79,7 @@ docker compose -f monitoring/docker-compose.yml up -d The stack includes: -- **OpenTelemetry Collector** on `localhost:4317` (OTLP/gRPC) and `localhost:4318` (OTLP/HTTP) +- **OpenTelemetry Collector** on `localhost:4317` (OTLP/gRPC) - **Tempo** for spans and span-derived metrics - **Prometheus** for application and span-derived metrics - **Loki** for OpenTelemetry logs and their structured metadata @@ -132,14 +132,10 @@ Amaru recognizes standard OpenTelemetry env variable for its configuration: - `OTEL_SERVICE_NAME`: Sets the [service.name](https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/#service-name) key used to identify metrics, logs, and spans. Defaults to `amaru`. - `OTEL_SERVICE_INSTANCE_ID`: Sets the [service.instance.id](https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/#service-instance-id) key used to identify this specific amaru instance -- `OTEL_EXPORTER_OTLP_ENDPOINT`: Sets the endpoint used to send logs and spans, defaults to `http://localhost:4317` -- `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`: Sets the endpoint used to send metrics, defaults to `http://localhost:4318/v1/metrics` +- `OTEL_EXPORTER_OTLP_ENDPOINT`: Sets the OTLP/gRPC endpoint used to send metrics, logs, and spans. Defaults to `http://localhost:4317` -Note that two different transports are used internally: - -- OTLP/gRPC for logs and spans -- OTLP/HTTP for metrics - -This helps maximize compatibility with 3rd party tools receiving those data. +If you override signal-specific OTLP endpoint variables such as `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, +`OTEL_EXPORTER_OTLP_LOGS_ENDPOINT`, or `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, they should also point to the same +OTLP/gRPC endpoint and should not include a `/v1/...` suffix. One can find more available env variables [here](https://opentelemetry.io/docs/specs/otel/configuration/sdk-environment-variables/) and [here](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md). diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index cf4a47fca0..7c4cea0951 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -12,7 +12,6 @@ services: - tempo ports: - "4317:4317" - - "4318:4318" - "8889:8889" prometheus: diff --git a/monitoring/otlp-collector.yml b/monitoring/otlp-collector.yml index 1973a8919d..eebaeb6a11 100644 --- a/monitoring/otlp-collector.yml +++ b/monitoring/otlp-collector.yml @@ -3,8 +3,6 @@ receivers: protocols: grpc: endpoint: "0.0.0.0:4317" - http: - endpoint: "0.0.0.0:4318" processors: batch: