From a94044cd50c49809aa0154179c955f3b862f50e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Tue, 31 Mar 2026 15:37:02 +0200 Subject: [PATCH 1/5] metrics --- crates/blockchain/blockchain.rs | 21 +- crates/blockchain/metrics/api.rs | 11 +- crates/blockchain/metrics/fullsync.rs | 267 ++++++++++++++++++++++++++ crates/blockchain/metrics/mod.rs | 2 + crates/networking/p2p/sync/full.rs | 64 +++++- 5 files changed, 356 insertions(+), 9 deletions(-) create mode 100644 crates/blockchain/metrics/fullsync.rs diff --git a/crates/blockchain/blockchain.rs b/crates/blockchain/blockchain.rs index 367af3ff637..795a1184604 100644 --- a/crates/blockchain/blockchain.rs +++ b/crates/blockchain/blockchain.rs @@ -114,6 +114,8 @@ use vm::StoreVmDatabase; #[cfg(feature = "metrics")] use ethrex_metrics::blocks::METRICS_BLOCKS; +#[cfg(feature = "metrics")] +use ethrex_metrics::fullsync::METRICS_FULLSYNC; #[cfg(feature = "c-kzg")] use ethrex_common::types::BlobsBundle; @@ -2178,7 +2180,8 @@ impl Blockchain { let mut total_gas_used = 0; let mut transactions_count = 0; - let interval = Instant::now(); + let interval = std::time::Instant::now(); + let exec_start = std::time::Instant::now(); for (i, block) in blocks.iter().enumerate() { if cancellation_token.is_cancelled() { info!("Received shutdown signal, aborting"); @@ -2222,9 +2225,13 @@ impl Blockchain { tokio::task::yield_now().await; } + let exec_ms = exec_start.elapsed().as_millis(); + + let transitions_start = std::time::Instant::now(); let account_updates = vm .get_state_transitions() .map_err(|err| (ChainError::EvmError(err), None))?; + let transitions_ms = transitions_start.elapsed().as_millis(); let last_block = blocks .last() @@ -2234,11 +2241,13 @@ impl Blockchain { let last_block_gas_limit = last_block.header.gas_limit; // Apply the account updates over all blocks and compute the new state root + let merkle_start = std::time::Instant::now(); let account_updates_list = self .storage .apply_account_updates_batch(first_block_header.parent_hash, &account_updates) .map_err(|e| (e.into(), None))? .ok_or((ChainError::ParentStateNotFound, None))?; + let merkle_ms = merkle_start.elapsed().as_millis(); let new_state_root = account_updates_list.state_trie_hash; let state_updates = account_updates_list.state_updates; @@ -2248,6 +2257,7 @@ impl Blockchain { // Check state root matches the one in block header validate_state_root(&last_block.header, new_state_root).map_err(|e| (e, None))?; + let store_start = std::time::Instant::now(); let update_batch = UpdateBatch { account_updates: state_updates, storage_updates: accounts_updates, @@ -2260,6 +2270,12 @@ impl Blockchain { self.storage .store_block_updates(update_batch) .map_err(|e| (e.into(), None))?; + let store_ms = store_start.elapsed().as_millis(); + + info!( + "[FULLSYNC TIMING] Batch {}: exec={}ms, transitions={}ms, merkle={}ms, store={}ms, total={}ms", + blocks_len, exec_ms, transitions_ms, merkle_ms, store_ms, interval.elapsed().as_millis() + ); let elapsed_seconds = interval.elapsed().as_secs_f64(); let throughput = if elapsed_seconds > 0.0 && total_gas_used != 0 { @@ -2275,6 +2291,9 @@ impl Blockchain { // Set the latest gas used as the average gas used per block in the batch METRICS_BLOCKS.set_latest_gas_used(total_gas_used as f64 / blocks_len as f64); METRICS_BLOCKS.set_latest_gigagas(throughput); + METRICS_FULLSYNC.set_batch_execution_ms(exec_ms as f64); + METRICS_FULLSYNC.set_batch_merkle_ms(merkle_ms as f64); + METRICS_FULLSYNC.set_batch_store_ms(store_ms as f64); ); if self.options.perf_logs_enabled { diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index bda3b98f735..f4440a58b11 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -1,8 +1,9 @@ use axum::{Router, routing::get}; use crate::{ - MetricsApiError, blocks::METRICS_BLOCKS, gather_default_metrics, node::METRICS_NODE, - p2p::METRICS_P2P, process::METRICS_PROCESS, transactions::METRICS_TX, + MetricsApiError, blocks::METRICS_BLOCKS, fullsync::METRICS_FULLSYNC, + gather_default_metrics, node::METRICS_NODE, p2p::METRICS_P2P, process::METRICS_PROCESS, + transactions::METRICS_TX, }; pub async fn start_prometheus_metrics_api( @@ -54,6 +55,12 @@ pub(crate) async fn get_metrics() -> String { Err(_) => tracing::error!("Failed to gather METRICS_P2P"), }; + ret_string.push('\n'); + match METRICS_FULLSYNC.gather_metrics() { + Ok(s) => ret_string.push_str(&s), + Err(_) => tracing::error!("Failed to gather METRICS_FULLSYNC"), + }; + ret_string.push('\n'); if let Some(node_metrics) = METRICS_NODE.get() { match node_metrics.gather_metrics() { diff --git a/crates/blockchain/metrics/fullsync.rs b/crates/blockchain/metrics/fullsync.rs new file mode 100644 index 00000000000..78b91865add --- /dev/null +++ b/crates/blockchain/metrics/fullsync.rs @@ -0,0 +1,267 @@ +use prometheus::{Encoder, Gauge, IntCounter, IntGauge, Registry, TextEncoder}; +use std::sync::LazyLock; + +use crate::MetricsError; + +pub static METRICS_FULLSYNC: LazyLock = LazyLock::new(MetricsFullSync::default); + +#[derive(Debug, Clone)] +pub struct MetricsFullSync { + // Phase & progress + stage: IntGauge, + target_block: IntGauge, + lowest_header: IntGauge, + headers_downloaded: IntCounter, + bodies_downloaded: IntCounter, + blocks_executed: IntGauge, + blocks_total: IntGauge, + + // Rates (in-process, instant) + headers_per_second: Gauge, + bodies_per_second: Gauge, + blocks_per_second: Gauge, + + // Batch timing + batch_body_download_ms: Gauge, + batch_execution_ms: Gauge, + batch_merkle_ms: Gauge, + batch_store_ms: Gauge, + batch_total_ms: Gauge, + batch_size: IntGauge, + + // Reliability + header_failures: IntCounter, + body_failures: IntCounter, + cycles_started: IntCounter, + cycles_completed: IntCounter, +} + +impl Default for MetricsFullSync { + fn default() -> Self { + Self::new() + } +} + +impl MetricsFullSync { + pub fn new() -> Self { + MetricsFullSync { + stage: IntGauge::new( + "fullsync_stage", + "Current full sync stage: 0=idle, 1=downloading_headers, 2=downloading_bodies, 3=executing_blocks", + ) + .expect("Failed to create fullsync_stage metric"), + target_block: IntGauge::new( + "fullsync_target_block", + "Chain tip block number (sync target)", + ) + .expect("Failed to create fullsync_target_block metric"), + lowest_header: IntGauge::new( + "fullsync_lowest_header", + "Lowest block number whose header has been downloaded (decreases during header walk)", + ) + .expect("Failed to create fullsync_lowest_header metric"), + headers_downloaded: IntCounter::new( + "fullsync_headers_downloaded", + "Total headers downloaded", + ) + .expect("Failed to create fullsync_headers_downloaded metric"), + bodies_downloaded: IntCounter::new( + "fullsync_bodies_downloaded", + "Total bodies downloaded", + ) + .expect("Failed to create fullsync_bodies_downloaded metric"), + blocks_executed: IntGauge::new( + "fullsync_blocks_executed", + "Highest block number executed so far", + ) + .expect("Failed to create fullsync_blocks_executed metric"), + blocks_total: IntGauge::new( + "fullsync_blocks_total", + "Total blocks to execute in current cycle", + ) + .expect("Failed to create fullsync_blocks_total metric"), + + // Rates + headers_per_second: Gauge::new( + "fullsync_headers_per_second", + "Headers downloaded per second (averaged over last batch)", + ) + .expect("Failed to create fullsync_headers_per_second metric"), + bodies_per_second: Gauge::new( + "fullsync_bodies_per_second", + "Bodies downloaded per second (averaged over last batch)", + ) + .expect("Failed to create fullsync_bodies_per_second metric"), + blocks_per_second: Gauge::new( + "fullsync_blocks_per_second", + "Blocks executed per second (averaged over last batch)", + ) + .expect("Failed to create fullsync_blocks_per_second metric"), + + // Batch timing + batch_body_download_ms: Gauge::new( + "fullsync_batch_body_download_ms", + "Body download time for last batch in milliseconds", + ) + .expect("Failed to create fullsync_batch_body_download_ms metric"), + batch_execution_ms: Gauge::new( + "fullsync_batch_execution_ms", + "EVM execution time for last batch in milliseconds", + ) + .expect("Failed to create fullsync_batch_execution_ms metric"), + batch_merkle_ms: Gauge::new( + "fullsync_batch_merkle_ms", + "Merkleization time for last batch in milliseconds", + ) + .expect("Failed to create fullsync_batch_merkle_ms metric"), + batch_store_ms: Gauge::new( + "fullsync_batch_store_ms", + "Storage write time for last batch in milliseconds", + ) + .expect("Failed to create fullsync_batch_store_ms metric"), + batch_total_ms: Gauge::new( + "fullsync_batch_total_ms", + "Total time for last batch in milliseconds", + ) + .expect("Failed to create fullsync_batch_total_ms metric"), + batch_size: IntGauge::new( + "fullsync_batch_size", + "Number of blocks in last batch", + ) + .expect("Failed to create fullsync_batch_size metric"), + + // Reliability + header_failures: IntCounter::new( + "fullsync_header_failures", + "Total header fetch failures", + ) + .expect("Failed to create fullsync_header_failures metric"), + body_failures: IntCounter::new( + "fullsync_body_failures", + "Total body fetch failures", + ) + .expect("Failed to create fullsync_body_failures metric"), + cycles_started: IntCounter::new( + "fullsync_cycles_started", + "Number of sync cycles initiated", + ) + .expect("Failed to create fullsync_cycles_started metric"), + cycles_completed: IntCounter::new( + "fullsync_cycles_completed", + "Number of sync cycles that completed successfully", + ) + .expect("Failed to create fullsync_cycles_completed metric"), + } + } + + // Phase & progress setters + pub fn set_stage(&self, stage: i64) { + self.stage.set(stage); + } + pub fn set_target_block(&self, block: u64) { + self.target_block.set(block.cast_signed()); + } + pub fn set_lowest_header(&self, block: u64) { + self.lowest_header.set(block.cast_signed()); + } + pub fn inc_headers_downloaded(&self, count: u64) { + self.headers_downloaded.inc_by(count); + } + pub fn inc_bodies_downloaded(&self, count: u64) { + self.bodies_downloaded.inc_by(count); + } + pub fn set_blocks_executed(&self, block: u64) { + self.blocks_executed.set(block.cast_signed()); + } + pub fn set_blocks_total(&self, total: u64) { + self.blocks_total.set(total.cast_signed()); + } + + // Rate setters + pub fn set_headers_per_second(&self, rate: f64) { + self.headers_per_second.set(rate); + } + pub fn set_bodies_per_second(&self, rate: f64) { + self.bodies_per_second.set(rate); + } + pub fn set_blocks_per_second(&self, rate: f64) { + self.blocks_per_second.set(rate); + } + + // Batch timing setters + pub fn set_batch_body_download_ms(&self, ms: f64) { + self.batch_body_download_ms.set(ms); + } + pub fn set_batch_execution_ms(&self, ms: f64) { + self.batch_execution_ms.set(ms); + } + pub fn set_batch_merkle_ms(&self, ms: f64) { + self.batch_merkle_ms.set(ms); + } + pub fn set_batch_store_ms(&self, ms: f64) { + self.batch_store_ms.set(ms); + } + pub fn set_batch_total_ms(&self, ms: f64) { + self.batch_total_ms.set(ms); + } + pub fn set_batch_size(&self, size: i64) { + self.batch_size.set(size); + } + + // Reliability setters + pub fn inc_header_failures(&self) { + self.header_failures.inc(); + } + pub fn inc_body_failures(&self) { + self.body_failures.inc(); + } + pub fn inc_cycles_started(&self) { + self.cycles_started.inc(); + } + pub fn inc_cycles_completed(&self) { + self.cycles_completed.inc(); + } + + pub fn gather_metrics(&self) -> Result { + let r = Registry::new(); + + let metrics: Vec> = vec![ + Box::new(self.stage.clone()), + Box::new(self.target_block.clone()), + Box::new(self.lowest_header.clone()), + Box::new(self.headers_downloaded.clone()), + Box::new(self.bodies_downloaded.clone()), + Box::new(self.blocks_executed.clone()), + Box::new(self.blocks_total.clone()), + Box::new(self.headers_per_second.clone()), + Box::new(self.bodies_per_second.clone()), + Box::new(self.blocks_per_second.clone()), + Box::new(self.batch_body_download_ms.clone()), + Box::new(self.batch_execution_ms.clone()), + Box::new(self.batch_merkle_ms.clone()), + Box::new(self.batch_store_ms.clone()), + Box::new(self.batch_total_ms.clone()), + Box::new(self.batch_size.clone()), + Box::new(self.header_failures.clone()), + Box::new(self.body_failures.clone()), + Box::new(self.cycles_started.clone()), + Box::new(self.cycles_completed.clone()), + ]; + + for metric in metrics { + r.register(metric) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + } + + let encoder = TextEncoder::new(); + let metric_families = r.gather(); + + let mut buffer = Vec::new(); + encoder + .encode(&metric_families, &mut buffer) + .map_err(|e| MetricsError::PrometheusErr(e.to_string()))?; + + let res = String::from_utf8(buffer)?; + Ok(res) + } +} diff --git a/crates/blockchain/metrics/mod.rs b/crates/blockchain/metrics/mod.rs index 47f188a09ca..484d93fb560 100644 --- a/crates/blockchain/metrics/mod.rs +++ b/crates/blockchain/metrics/mod.rs @@ -2,6 +2,8 @@ pub mod api; #[cfg(any(feature = "api", feature = "metrics"))] pub mod blocks; +#[cfg(any(feature = "api", feature = "metrics"))] +pub mod fullsync; #[cfg(feature = "api")] pub mod l2; #[cfg(feature = "api")] diff --git a/crates/networking/p2p/sync/full.rs b/crates/networking/p2p/sync/full.rs index aaf6c145fa4..b0481a528a0 100644 --- a/crates/networking/p2p/sync/full.rs +++ b/crates/networking/p2p/sync/full.rs @@ -9,6 +9,17 @@ use std::time::Duration; use ethrex_blockchain::{BatchBlockProcessingFailure, Blockchain, error::ChainError}; use ethrex_common::{H256, types::Block}; +#[cfg(feature = "metrics")] +use ethrex_metrics::fullsync::METRICS_FULLSYNC; + +macro_rules! fullsync_metrics { + ($($code:tt)*) => { + #[cfg(feature = "metrics")] + { + $($code)* + } + }; +} use ethrex_storage::Store; use tokio::time::Instant; use tokio_util::sync::CancellationToken; @@ -32,6 +43,10 @@ pub async fn sync_cycle_full( store: Store, ) -> Result<(), SyncError> { info!("Syncing to sync_head {:?}", sync_head); + fullsync_metrics!( + METRICS_FULLSYNC.set_stage(1); + METRICS_FULLSYNC.inc_cycles_started(); + ); // Check if the sync_head is a pending block, if so, gather all pending blocks belonging to its chain let mut pending_blocks = vec![]; @@ -55,6 +70,8 @@ pub async fn sync_cycle_full( let mut attempts = 0; // Request and store all block headers from the advertised sync head + #[cfg(feature = "metrics")] + let mut header_batch_start = Instant::now(); loop { let Some(mut block_headers) = peers .request_block_headers_from_hash(sync_head, BlockRequestOrder::NewToOld) @@ -67,6 +84,7 @@ pub async fn sync_cycle_full( return Ok(()); } attempts += 1; + fullsync_metrics!(METRICS_FULLSYNC.inc_header_failures()); warn!( "Failed to fetch headers for sync head (attempt {attempts}/{MAX_HEADER_FETCH_ATTEMPTS}), retrying in 5s" ); @@ -87,6 +105,15 @@ pub async fn sync_cycle_full( end_block_number = end_block_number.max(first_header.number); start_block_number = last_header.number; + fullsync_metrics!( + let header_batch_secs = header_batch_start.elapsed().as_secs_f64().max(0.001); + METRICS_FULLSYNC.set_target_block(end_block_number); + METRICS_FULLSYNC.set_lowest_header(start_block_number); + METRICS_FULLSYNC.inc_headers_downloaded(block_headers.len() as u64); + METRICS_FULLSYNC.set_headers_per_second(block_headers.len() as f64 / header_batch_secs); + header_batch_start = Instant::now(); + ); + sync_head = last_header.parent_hash; if store.is_canonical_sync(sync_head)? || sync_head.is_zero() { // Incoming chain merged with current chain @@ -115,6 +142,9 @@ pub async fn sync_cycle_full( } end_block_number += 1; start_block_number = start_block_number.max(1); + fullsync_metrics!( + METRICS_FULLSYNC.set_blocks_total(end_block_number - start_block_number); + ); // Download block bodies and execute full blocks in batches for start in (start_block_number..end_block_number).step_by(*EXECUTE_BATCH_SIZE) { @@ -131,13 +161,17 @@ pub async fn sync_cycle_full( } let mut blocks = Vec::new(); // Request block bodies - // Download block bodies + fullsync_metrics!(METRICS_FULLSYNC.set_stage(2)); + let body_download_start = Instant::now(); + let mut body_request_count = 0u32; while !headers.is_empty() { let header_batch = &headers[..min(MAX_BLOCK_BODIES_TO_REQUEST, headers.len())]; let bodies = peers .request_block_bodies(header_batch) .await? .ok_or(SyncError::BodiesNotFound)?; + body_request_count += 1; + fullsync_metrics!(METRICS_FULLSYNC.inc_bodies_downloaded(bodies.len() as u64)); debug!("Obtained: {} block bodies", bodies.len()); let block_batch = headers .drain(..bodies.len()) @@ -145,13 +179,20 @@ pub async fn sync_cycle_full( .map(|(header, body)| Block { header, body }); blocks.extend(block_batch); } + let body_download_ms = body_download_start.elapsed().as_millis(); if !blocks.is_empty() { - // Execute blocks + let body_download_secs = (body_download_ms as f64 / 1000.0).max(0.001); info!( - "Executing {} blocks for full sync. First block hash: {:#?} Last block hash: {:#?}", + "[FULLSYNC TIMING] Body download: {}ms for {} blocks ({} requests, {:.0} blocks/s)", + body_download_ms, blocks.len(), - blocks.first().ok_or(SyncError::NoBlocks)?.hash(), - blocks.last().ok_or(SyncError::NoBlocks)?.hash() + body_request_count, + blocks.len() as f64 / body_download_secs + ); + fullsync_metrics!( + METRICS_FULLSYNC.set_batch_body_download_ms(body_download_ms as f64); + METRICS_FULLSYNC.set_bodies_per_second(blocks.len() as f64 / body_download_secs); + METRICS_FULLSYNC.set_stage(3); ); add_blocks_in_batch( blockchain.clone(), @@ -183,6 +224,10 @@ pub async fn sync_cycle_full( } store.clear_fullsync_headers().await?; + fullsync_metrics!( + METRICS_FULLSYNC.set_stage(0); + METRICS_FULLSYNC.inc_cycles_completed(); + ); Ok(()) } @@ -248,7 +293,14 @@ async fn add_blocks_in_batch( .await?; let execution_time: f64 = execution_start.elapsed().as_millis() as f64 / 1000.0; - let blocks_per_second = blocks_len as f64 / execution_time; + let blocks_per_second = blocks_len as f64 / execution_time.max(0.001); + + fullsync_metrics!( + METRICS_FULLSYNC.set_blocks_executed(last_block_number); + METRICS_FULLSYNC.set_blocks_per_second(blocks_per_second); + METRICS_FULLSYNC.set_batch_total_ms(execution_start.elapsed().as_millis() as f64); + METRICS_FULLSYNC.set_batch_size(blocks_len as i64); + ); info!( "[SYNCING] Executed & stored {} blocks in {:.3} seconds.\n\ From 88a37f2280144e75696b6b747c9caa4e0f6d518c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Tue, 31 Mar 2026 23:40:12 +0200 Subject: [PATCH 2/5] moar metrics --- crates/blockchain/metrics/fullsync.rs | 53 +++++++++++++++++++++++++++ crates/networking/p2p/sync/full.rs | 3 ++ 2 files changed, 56 insertions(+) diff --git a/crates/blockchain/metrics/fullsync.rs b/crates/blockchain/metrics/fullsync.rs index 78b91865add..492103f3250 100644 --- a/crates/blockchain/metrics/fullsync.rs +++ b/crates/blockchain/metrics/fullsync.rs @@ -29,6 +29,10 @@ pub struct MetricsFullSync { batch_total_ms: Gauge, batch_size: IntGauge, + // Timestamps (Unix epoch seconds, for Grafana elapsed time calculation) + header_stage_start_timestamp: Gauge, + execution_stage_start_timestamp: Gauge, + // Reliability header_failures: IntCounter, body_failures: IntCounter, @@ -130,6 +134,18 @@ impl MetricsFullSync { ) .expect("Failed to create fullsync_batch_size metric"), + // Timestamps + header_stage_start_timestamp: Gauge::new( + "fullsync_header_stage_start_timestamp", + "Unix timestamp (seconds) when header download stage began", + ) + .expect("Failed to create fullsync_header_stage_start_timestamp metric"), + execution_stage_start_timestamp: Gauge::new( + "fullsync_execution_stage_start_timestamp", + "Unix timestamp (seconds) when block execution stage began", + ) + .expect("Failed to create fullsync_execution_stage_start_timestamp metric"), + // Reliability header_failures: IntCounter::new( "fullsync_header_failures", @@ -208,6 +224,41 @@ impl MetricsFullSync { self.batch_size.set(size); } + // Timestamp setters + pub fn set_header_stage_start_now(&self) { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + self.header_stage_start_timestamp.set(now); + } + pub fn set_execution_stage_start_now(&self) { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs_f64(); + self.execution_stage_start_timestamp.set(now); + } + + /// Reset gauges at the start of a new sync cycle so stale data doesn't persist + pub fn reset_cycle(&self) { + self.blocks_executed.set(0); + self.blocks_total.set(0); + self.blocks_per_second.set(0.0); + self.bodies_per_second.set(0.0); + self.headers_per_second.set(0.0); + self.lowest_header.set(0); + self.target_block.set(0); + self.batch_body_download_ms.set(0.0); + self.batch_execution_ms.set(0.0); + self.batch_merkle_ms.set(0.0); + self.batch_store_ms.set(0.0); + self.batch_total_ms.set(0.0); + self.batch_size.set(0); + self.header_stage_start_timestamp.set(0.0); + self.execution_stage_start_timestamp.set(0.0); + } + // Reliability setters pub fn inc_header_failures(&self) { self.header_failures.inc(); @@ -244,6 +295,8 @@ impl MetricsFullSync { Box::new(self.batch_size.clone()), Box::new(self.header_failures.clone()), Box::new(self.body_failures.clone()), + Box::new(self.header_stage_start_timestamp.clone()), + Box::new(self.execution_stage_start_timestamp.clone()), Box::new(self.cycles_started.clone()), Box::new(self.cycles_completed.clone()), ]; diff --git a/crates/networking/p2p/sync/full.rs b/crates/networking/p2p/sync/full.rs index b0481a528a0..54be627ef5e 100644 --- a/crates/networking/p2p/sync/full.rs +++ b/crates/networking/p2p/sync/full.rs @@ -44,7 +44,9 @@ pub async fn sync_cycle_full( ) -> Result<(), SyncError> { info!("Syncing to sync_head {:?}", sync_head); fullsync_metrics!( + METRICS_FULLSYNC.reset_cycle(); METRICS_FULLSYNC.set_stage(1); + METRICS_FULLSYNC.set_header_stage_start_now(); METRICS_FULLSYNC.inc_cycles_started(); ); @@ -144,6 +146,7 @@ pub async fn sync_cycle_full( start_block_number = start_block_number.max(1); fullsync_metrics!( METRICS_FULLSYNC.set_blocks_total(end_block_number - start_block_number); + METRICS_FULLSYNC.set_execution_stage_start_now(); ); // Download block bodies and execute full blocks in batches From bdfef9e7c1a92b7fddafd8f234aae38d8cb03002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Mon, 13 Apr 2026 19:16:40 +0200 Subject: [PATCH 3/5] add grafana dasbhoard for full sync --- .../common_dashboards/fullsync_dashboard.json | 385 ++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 metrics/provisioning/grafana/dashboards/common_dashboards/fullsync_dashboard.json diff --git a/metrics/provisioning/grafana/dashboards/common_dashboards/fullsync_dashboard.json b/metrics/provisioning/grafana/dashboards/common_dashboards/fullsync_dashboard.json new file mode 100644 index 00000000000..480082c3df2 --- /dev/null +++ b/metrics/provisioning/grafana/dashboards/common_dashboards/fullsync_dashboard.json @@ -0,0 +1,385 @@ +{ + "annotations": { "list": [] }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "title": "Sync Overview", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "options": { "0": { "text": "Idle" }, "1": { "text": "Downloading Headers" }, "2": { "text": "Downloading Bodies" }, "3": { "text": "Executing Blocks" } }, "type": "value" } + ], + "thresholds": { "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 1 }, { "color": "orange", "value": 2 }, { "color": "blue", "value": 3 }] }, + "color": { "mode": "thresholds" } + } + }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 }, + "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "center", "textMode": "value", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "fullsync_stage{instance=~\"$instance\"}", "legendFormat": "Stage" }], + "title": "Sync Stage", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "min": 0, "max": 100, "unit": "percent", + "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 33 }, { "color": "green", "value": 66 }] } + } + }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true }, + "targets": [{ "expr": "block_number{instance=~\"$instance\"} / fullsync_target_block{instance=~\"$instance\"} * 100", "legendFormat": "Progress" }], + "title": "Overall Progress", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0 } }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "ethrex_p2p_peer_count{instance=~\"$instance\"}", "legendFormat": "Peers" }], + "title": "Peers", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none", "decimals": 2 } }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 }, + "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "center", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "gigagas{instance=~\"$instance\"}", "legendFormat": "Ggas/s" }], + "title": "Throughput (Ggas/s)", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, + "title": "Header Download", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "min": 0, "max": 100, "unit": "percent", "decimals": 1, + "thresholds": { "steps": [{ "color": "yellow", "value": null }, { "color": "green", "value": 95 }] }, + "color": { "mode": "thresholds" } + } + }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 6 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "(1 - fullsync_lowest_header{instance=~\"$instance\"} / clamp_min(fullsync_target_block{instance=~\"$instance\"}, 1)) * 100", "legendFormat": "Headers Downloaded" }], + "title": "Header Download Progress", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "min": 0, "unit": "none", "decimals": 0, + "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 500 }, { "color": "green", "value": 1500 }] }, + "color": { "mode": "thresholds" } + } + }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 6 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "rate(fullsync_headers_downloaded{instance=~\"$instance\"}[5m])", "legendFormat": "hdr/s (5m avg)" }], + "title": "Headers per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 6 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "fullsync_lowest_header{instance=~\"$instance\"} / clamp_min(rate(fullsync_headers_downloaded{instance=~\"$instance\"}[5m]), 0.001)", "legendFormat": "Header ETA" } + ], + "title": "Header Download ETA", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 6 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "time() - fullsync_header_stage_start_timestamp{instance=~\"$instance\"} and fullsync_header_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" } + ], + "title": "Header Stage Elapsed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0, "unit": "none" } }, + "gridPos": { "h": 2, "w": 6, "x": 0, "y": 11 }, + "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "horizontal" }, + "targets": [ + { "expr": "fullsync_target_block{instance=~\"$instance\"} - fullsync_lowest_header{instance=~\"$instance\"}", "legendFormat": "Downloaded" }, + { "expr": "fullsync_target_block{instance=~\"$instance\"}", "legendFormat": "Total" } + ], + "title": "", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, + "title": "Block Execution", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "min": 0, "max": 100, "unit": "percent", "decimals": 1, + "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 33 }, { "color": "green", "value": 66 }] }, + "color": { "mode": "thresholds" } + } + }, + "gridPos": { "h": 5, "w": 6, "x": 0, "y": 14 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "fullsync_blocks_executed{instance=~\"$instance\"} / clamp_min(fullsync_target_block{instance=~\"$instance\"}, 1) * 100", "legendFormat": "Blocks Executed" }], + "title": "Block Execution Progress", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "min": 0, "unit": "none", "decimals": 0, + "thresholds": { "steps": [{ "color": "red", "value": null }, { "color": "orange", "value": 50 }, { "color": "green", "value": 100 }] }, + "color": { "mode": "thresholds" } + } + }, + "gridPos": { "h": 5, "w": 6, "x": 6, "y": 14 }, + "options": { "reduceOptions": { "calcs": ["lastNotNull"] }, "showThresholdLabels": false, "showThresholdMarkers": true, "orientation": "horizontal" }, + "targets": [{ "expr": "rate(fullsync_blocks_executed{instance=~\"$instance\"}[5m])", "legendFormat": "blk/s (5m avg)" }], + "title": "Blocks per Second", + "type": "gauge" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 12, "y": 14 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "(fullsync_target_block{instance=~\"$instance\"} - fullsync_blocks_executed{instance=~\"$instance\"}) / clamp_min(rate(fullsync_blocks_executed{instance=~\"$instance\"}[5m]), 0.001) and rate(fullsync_blocks_executed{instance=~\"$instance\"}[5m]) > 0", "legendFormat": "Execution ETA" } + ], + "title": "Block Execution ETA", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "s", "decimals": 0 } }, + "gridPos": { "h": 5, "w": 6, "x": 18, "y": 14 }, + "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [ + { "expr": "time() - fullsync_execution_stage_start_timestamp{instance=~\"$instance\"} and fullsync_execution_stage_start_timestamp{instance=~\"$instance\"} > 0", "legendFormat": "Elapsed" } + ], + "title": "Execution Stage Elapsed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "decimals": 0, "unit": "none" } }, + "gridPos": { "h": 2, "w": 6, "x": 0, "y": 19 }, + "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "center", "textMode": "value_and_name", "reduceOptions": { "calcs": ["lastNotNull"] }, "orientation": "horizontal" }, + "targets": [ + { "expr": "block_number{instance=~\"$instance\"}", "legendFormat": "Executed" }, + { "expr": "fullsync_target_block{instance=~\"$instance\"}", "legendFormat": "Target" } + ], + "title": "", + "type": "stat" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, + "title": "Batch Performance", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "unit": "ms", + "custom": { + "drawStyle": "bars", + "barAlignment": 0, + "fillOpacity": 80, + "stacking": { "mode": "normal", "group": "A" }, + "lineWidth": 0 + } + } + }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, + "options": { "tooltip": { "mode": "multi", "sort": "desc" } }, + "targets": [ + { "expr": "fullsync_batch_body_download_ms{instance=~\"$instance\"}", "legendFormat": "Body Download" }, + { "expr": "fullsync_batch_execution_ms{instance=~\"$instance\"}", "legendFormat": "Execution" }, + { "expr": "fullsync_batch_merkle_ms{instance=~\"$instance\"}", "legendFormat": "Merkle" }, + { "expr": "fullsync_batch_store_ms{instance=~\"$instance\"}", "legendFormat": "Storage" } + ], + "title": "Batch Time Breakdown (stacked bars)", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { "expr": "fullsync_blocks_per_second{instance=~\"$instance\"}", "legendFormat": "Blocks/s" }, + { "expr": "gigagas{instance=~\"$instance\"}", "legendFormat": "Ggas/s" } + ], + "title": "Execution Throughput", + "type": "timeseries" + }, + + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 }, + "title": "Network & Reliability", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [{ "expr": "ethrex_p2p_peer_count{instance=~\"$instance\"}", "legendFormat": "Connected Peers" }], + "title": "Peer Count", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, + "options": { "tooltip": { "mode": "multi" } }, + "targets": [ + { "expr": "rate(fullsync_header_failures{instance=~\"$instance\"}[5m]) * 60", "legendFormat": "Header Failures/min" }, + { "expr": "rate(fullsync_body_failures{instance=~\"$instance\"}[5m]) * 60", "legendFormat": "Body Failures/min" } + ], + "title": "Failures per Minute", + "type": "timeseries" + }, + + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 39 }, + "title": "Detailed Rates", + "type": "row", + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 40 }, + "targets": [ + { "expr": "fullsync_headers_per_second{instance=~\"$instance\"}", "legendFormat": "Instant" }, + { "expr": "rate(fullsync_headers_downloaded{instance=~\"$instance\"}[5m])", "legendFormat": "Avg (5m)" } + ], + "title": "Header Download Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 40 }, + "targets": [ + { "expr": "fullsync_bodies_per_second{instance=~\"$instance\"}", "legendFormat": "Instant" }, + { "expr": "rate(fullsync_bodies_downloaded{instance=~\"$instance\"}[5m])", "legendFormat": "Avg (5m)" } + ], + "title": "Body Download Rate", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 40 }, + "targets": [ + { "expr": "fullsync_batch_size{instance=~\"$instance\"}", "legendFormat": "Batch Size" }, + { "expr": "fullsync_batch_total_ms{instance=~\"$instance\"} / 1000", "legendFormat": "Batch Duration (s)" } + ], + "title": "Batch Stats", + "type": "timeseries" + } + ] + }, + { + "collapsed": true, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 48 }, + "title": "Cycle Statistics", + "type": "row", + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 4, "w": 6, "x": 0, "y": 49 }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "fullsync_cycles_started{instance=~\"$instance\"}", "legendFormat": "Started" }], + "title": "Cycles Started", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 4, "w": 6, "x": 6, "y": 49 }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "fullsync_cycles_completed{instance=~\"$instance\"}", "legendFormat": "Completed" }], + "title": "Cycles Completed", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 49 }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "fullsync_header_failures{instance=~\"$instance\"}", "legendFormat": "Total" }], + "title": "Total Header Failures", + "type": "stat" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "unit": "none" } }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 49 }, + "options": { "colorMode": "value", "graphMode": "none", "reduceOptions": { "calcs": ["lastNotNull"] } }, + "targets": [{ "expr": "fullsync_body_failures{instance=~\"$instance\"}", "legendFormat": "Total" }], + "title": "Total Body Failures", + "type": "stat" + } + ] + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": ["ethrex", "fullsync"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": "ethrex-mainnet-8:3701", "value": "ethrex-mainnet-8:3701" }, + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "definition": "label_values(fullsync_stage, instance)", + "name": "instance", + "query": "label_values(fullsync_stage, instance)", + "refresh": 2, + "type": "query" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "ethrex Full Sync", + "uid": "ethrex-fullsync" +} From 47eda9430f2eba7b948e2fbf44fcba954a7d2983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Thu, 16 Apr 2026 17:55:19 +0200 Subject: [PATCH 4/5] format --- crates/blockchain/blockchain.rs | 7 ++++++- crates/blockchain/metrics/api.rs | 5 ++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/blockchain/blockchain.rs b/crates/blockchain/blockchain.rs index 795a1184604..cc9f86b3f25 100644 --- a/crates/blockchain/blockchain.rs +++ b/crates/blockchain/blockchain.rs @@ -2274,7 +2274,12 @@ impl Blockchain { info!( "[FULLSYNC TIMING] Batch {}: exec={}ms, transitions={}ms, merkle={}ms, store={}ms, total={}ms", - blocks_len, exec_ms, transitions_ms, merkle_ms, store_ms, interval.elapsed().as_millis() + blocks_len, + exec_ms, + transitions_ms, + merkle_ms, + store_ms, + interval.elapsed().as_millis() ); let elapsed_seconds = interval.elapsed().as_secs_f64(); diff --git a/crates/blockchain/metrics/api.rs b/crates/blockchain/metrics/api.rs index f4440a58b11..cd94c17731c 100644 --- a/crates/blockchain/metrics/api.rs +++ b/crates/blockchain/metrics/api.rs @@ -1,9 +1,8 @@ use axum::{Router, routing::get}; use crate::{ - MetricsApiError, blocks::METRICS_BLOCKS, fullsync::METRICS_FULLSYNC, - gather_default_metrics, node::METRICS_NODE, p2p::METRICS_P2P, process::METRICS_PROCESS, - transactions::METRICS_TX, + MetricsApiError, blocks::METRICS_BLOCKS, fullsync::METRICS_FULLSYNC, gather_default_metrics, + node::METRICS_NODE, p2p::METRICS_P2P, process::METRICS_PROCESS, transactions::METRICS_TX, }; pub async fn start_prometheus_metrics_api( From 28f3e58677e453f14c05c72562a601b422f70b5b Mon Sep 17 00:00:00 2001 From: MrAzteca Date: Thu, 16 Apr 2026 20:54:44 +0200 Subject: [PATCH 5/5] fix(l1): defer Amsterdam block gas overflow check to post-execution (#6486) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - The Hive consume-engine Amsterdam tests for EIP-7778 and EIP-8037 were failing because ethrex's per-tx gas limit checks were incompatible with Amsterdam's new gas accounting rules. - **EIP-7778** uses pre-refund gas for block accounting, so cumulative pre-refund gas can exceed the block gas limit even when a block builder correctly included all transactions. - **EIP-8037** introduces 2D gas accounting (`block_gas = max(regular, state)`), meaning cumulative total gas (regular + state) can legally exceed the block gas limit. - The fix skips the per-tx cumulative gas check for Amsterdam and adds a **post-execution** block-level overflow check using `max(sum_regular, sum_state)` in all three execution paths (sequential, pipeline, parallel). ## Local test results - **200/201** EIP-7778 + EIP-8037 Hive consume-engine tests pass - **105/105** EIP-7778 + EIP-8037 EF blockchain tests pass (4 + 101) - The single remaining Hive failure (`test_block_regular_gas_limit[exceed=True]`) expects `TransactionException.GAS_ALLOWANCE_EXCEEDED` but we return `BlockException.GAS_USED_OVERFLOW` — the block is correctly rejected, just with a different error classification. ## Test plan - [x] All EIP-7778 EF blockchain tests pass locally - [x] All EIP-8037 EF blockchain tests pass locally - [x] 200/201 Hive consume-engine Amsterdam tests pass locally - [ ] Full CI Amsterdam Hive suite passes --------- Co-authored-by: Claude Sonnet 4.5 --- crates/blockchain/payload.rs | 36 ++++++++-- crates/vm/backends/levm/mod.rs | 118 ++++++++++++++++++++------------- 2 files changed, 103 insertions(+), 51 deletions(-) diff --git a/crates/blockchain/payload.rs b/crates/blockchain/payload.rs index c3cc0a0dc52..1748a07cc2a 100644 --- a/crates/blockchain/payload.rs +++ b/crates/blockchain/payload.rs @@ -833,10 +833,35 @@ pub fn apply_plain_transaction( // EIP-8037 (Amsterdam+): track regular and state gas separately let tx_state_gas = report.state_gas_used; let tx_regular_gas = report.gas_used.saturating_sub(tx_state_gas); - context.block_regular_gas_used = context + + // Compute new totals before committing them + let new_regular = context .block_regular_gas_used .saturating_add(tx_regular_gas); - context.block_state_gas_used = context.block_state_gas_used.saturating_add(tx_state_gas); + let new_state = context.block_state_gas_used.saturating_add(tx_state_gas); + + // EIP-8037 (Amsterdam+): post-execution block gas overflow check + // Reject the transaction if adding it would cause max(regular, state) to exceed the gas limit + if context.is_amsterdam && new_regular.max(new_state) > context.payload.header.gas_limit { + // Rollback transaction state before returning error: + // 1. Undo DB mutations (nonce, balance, storage, etc.) + // 2. Revert cumulative gas counter inflation + // This ensures the next transaction executes against clean state. + context.vm.undo_last_tx()?; + context.cumulative_gas_spent -= report.gas_spent; + + return Err(EvmError::Custom(format!( + "block gas limit exceeded (state gas overflow): \ + max({new_regular}, {new_state}) = {} > gas_limit {}", + new_regular.max(new_state), + context.payload.header.gas_limit + )) + .into()); + } + + // Commit the new totals + context.block_regular_gas_used = new_regular; + context.block_state_gas_used = new_state; if context.is_amsterdam { debug!( @@ -852,15 +877,14 @@ pub fn apply_plain_transaction( } // Update remaining_gas for block gas limit checks. - // EIP-8037 (Amsterdam+): per-tx check only validates regular gas against block limit. - // State gas is NOT checked per-tx; block-end validation enforces - // max(block_regular, block_state) <= gas_limit. + // EIP-8037 (Amsterdam+): remaining_gas reflects both regular and state gas dimensions. + // For pre-tx heuristic checks, this ensures we reject txs when either dimension is full. if context.is_amsterdam { context.remaining_gas = context .payload .header .gas_limit - .saturating_sub(context.block_regular_gas_used); + .saturating_sub(new_regular.max(new_state)); } else { context.remaining_gas = context.remaining_gas.saturating_sub(report.gas_used); } diff --git a/crates/vm/backends/levm/mod.rs b/crates/vm/backends/levm/mod.rs index 9500c2c83b7..70381b50650 100644 --- a/crates/vm/backends/levm/mod.rs +++ b/crates/vm/backends/levm/mod.rs @@ -31,7 +31,6 @@ use ethrex_levm::account::{AccountStatus, LevmAccount}; use ethrex_levm::call_frame::Stack; use ethrex_levm::constants::{ POST_OSAKA_GAS_LIMIT_CAP, STACK_LIMIT, SYS_CALL_GAS_LIMIT, TX_BASE_COST, - TX_MAX_GAS_LIMIT_AMSTERDAM, }; use ethrex_levm::db::Database; use ethrex_levm::db::gen_db::{CacheDB, GeneralizedDatabase}; @@ -126,18 +125,13 @@ impl LEVM { })?; for (tx_idx, (tx, tx_sender)) in transactions_with_sender.into_iter().enumerate() { - // Pre-tx gas limit guard per EIP-8037/EIP-7825: - // Amsterdam: check min(TX_MAX_GAS_LIMIT, tx.gas) against regular gas only. - // State gas is NOT checked per-tx; block-end validation enforces - // max(block_regular, block_state) <= gas_limit. - // Pre-Amsterdam: check tx.gas against cumulative_gas_used (post-refund sum). - if is_amsterdam { - check_gas_limit( - block_regular_gas_used, - tx.gas_limit().min(TX_MAX_GAS_LIMIT_AMSTERDAM), - block.header.gas_limit, - )?; - } else { + // Pre-tx gas limit guard: + // Pre-Amsterdam: reject tx if cumulative post-refund gas + tx.gas > block limit. + // Amsterdam+: skip — EIP-8037's 2D gas model means cumulative gas (regular + + // state) can legally exceed the block gas limit as long as + // max(sum_regular, sum_state) stays within it. Block-level overflow is + // detected post-execution. + if !is_amsterdam { check_gas_limit(cumulative_gas_used, tx.gas_limit(), block.header.gas_limit)?; } @@ -175,6 +169,20 @@ impl LEVM { report.gas_used, report.gas_spent, ); + + // DoS protection: early exit if either regular or state gas exceeds the limit. + // Since block_gas_used = max(regular, state), if either component exceeds + // the limit, we know the block is invalid and can safely reject without + // violating EIP-8037 semantics. + if block_regular_gas_used > block.header.gas_limit + || block_state_gas_used > block.header.gas_limit + { + return Err(EvmError::Transaction(format!( + "Gas allowance exceeded: Block gas used overflow: \ + block_gas_used {block_gas_used} > block_gas_limit {}", + block.header.gas_limit + ))); + } } else { block_gas_used = block_gas_used.saturating_add(report.gas_used); } @@ -189,6 +197,17 @@ impl LEVM { receipts.push(receipt); } + // EIP-7778 (Amsterdam+): block-level gas overflow check. + // Per-tx checks are skipped for Amsterdam because block gas is computed + // from pre-refund values; overflow can only be detected after execution. + if is_amsterdam && block_gas_used > block.header.gas_limit { + return Err(EvmError::Transaction(format!( + "Gas allowance exceeded: Block gas used overflow: \ + block_gas_used {block_gas_used} > block_gas_limit {}", + block.header.gas_limit + ))); + } + // Set BAL index for post-execution phase (requests + withdrawals, uint16) // Order must match geth: requests (system calls) BEFORE withdrawals. if is_amsterdam { @@ -424,18 +443,13 @@ impl LEVM { let mut tx_since_last_flush = 2; for (tx_idx, (tx, tx_sender)) in transactions_with_sender.into_iter().enumerate() { - // Pre-tx gas limit guard per EIP-8037/EIP-7825: - // Amsterdam: check min(TX_MAX_GAS_LIMIT, tx.gas) against regular gas only. - // State gas is NOT checked per-tx; block-end validation enforces - // max(block_regular, block_state) <= gas_limit. - // Pre-Amsterdam: check tx.gas against cumulative_gas_used (post-refund sum). - if is_amsterdam { - check_gas_limit( - block_regular_gas_used, - tx.gas_limit().min(TX_MAX_GAS_LIMIT_AMSTERDAM), - block.header.gas_limit, - )?; - } else { + // Pre-tx gas limit guard: + // Pre-Amsterdam: reject tx if cumulative post-refund gas + tx.gas > block limit. + // Amsterdam+: skip — EIP-8037's 2D gas model means cumulative gas (regular + + // state) can legally exceed the block gas limit as long as + // max(sum_regular, sum_state) stays within it. Block-level overflow is + // detected post-execution. + if !is_amsterdam { check_gas_limit(cumulative_gas_used, tx.gas_limit(), block.header.gas_limit)?; } @@ -483,6 +497,20 @@ impl LEVM { if is_amsterdam { // Amsterdam+: block gas = max(regular_sum, state_sum) block_gas_used = block_regular_gas_used.max(block_state_gas_used); + + // DoS protection: early exit if either regular or state gas exceeds the limit. + // Since block_gas_used = max(regular, state), if either component exceeds + // the limit, we know the block is invalid and can safely reject without + // violating EIP-8037 semantics. + if block_regular_gas_used > block.header.gas_limit + || block_state_gas_used > block.header.gas_limit + { + return Err(EvmError::Transaction(format!( + "Gas allowance exceeded: Block gas used overflow: \ + block_gas_used {block_gas_used} > block_gas_limit {}", + block.header.gas_limit + ))); + } } else { block_gas_used = block_gas_used.saturating_add(report.gas_used); } @@ -497,6 +525,17 @@ impl LEVM { receipts.push(receipt); } + // EIP-7778 (Amsterdam+): block-level gas overflow check. + // Per-tx checks are skipped for Amsterdam because block gas is computed + // from pre-refund values; overflow can only be detected after execution. + if is_amsterdam && block_gas_used > block.header.gas_limit { + return Err(EvmError::Transaction(format!( + "Gas allowance exceeded: Block gas used overflow: \ + block_gas_used {block_gas_used} > block_gas_limit {}", + block.header.gas_limit + ))); + } + #[cfg(feature = "perf_opcode_timings")] { let mut timings = OPCODE_TIMINGS.lock().expect("poison"); @@ -972,32 +1011,21 @@ impl LEVM { // balance in the BAL won't match execution that ran all txs). let mut block_regular_gas_used = 0_u64; let mut block_state_gas_used = 0_u64; - for (tx_idx, _, report, _, _, _) in &exec_results { - // Per-tx check: only regular gas is checked per-tx (EIP-8037/EIP-7825). - // State gas is validated at block end via max(regular, state) <= gas_limit. - let tx_gas_limit = txs_with_sender[*tx_idx].0.gas_limit(); - check_gas_limit( - block_regular_gas_used, - tx_gas_limit.min(TX_MAX_GAS_LIMIT_AMSTERDAM), - header.gas_limit, - )?; + for (_, _, report, _, _, _) in &exec_results { let tx_state_gas = report.state_gas_used; let tx_regular_gas = report.gas_used.saturating_sub(tx_state_gas); block_regular_gas_used = block_regular_gas_used.saturating_add(tx_regular_gas); block_state_gas_used = block_state_gas_used.saturating_add(tx_state_gas); - // Post-tx check: needed because all txs are already executed — if the last tx - // pushes actual gas over the limit, there's no next iteration to catch it - // like the sequential path does. - let running_block_gas_after = block_regular_gas_used.max(block_state_gas_used); - if running_block_gas_after > header.gas_limit { - return Err(EvmError::Transaction(format!( - "Gas allowance exceeded: \ - used {running_block_gas_after} > block limit {}", - header.gas_limit - ))); - } } let block_gas_used = block_regular_gas_used.max(block_state_gas_used); + // EIP-7778: block-level overflow check using pre-refund gas. + if block_gas_used > header.gas_limit { + return Err(EvmError::Transaction(format!( + "Gas allowance exceeded: Block gas used overflow: \ + block_gas_used {block_gas_used} > block_gas_limit {}", + header.gas_limit + ))); + } // 4. Per-tx BAL validation — now safe to run after gas limit is confirmed OK. // Also mark off storage_reads that appear in per-tx execution state.