Skip to content

fix(l1): prune Docker build cache and handle disk-full errors in multisync monitor #6500

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
avilagaston9 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix(l1): prune Docker build cache and handle disk-full errors in multisync monitor #6500

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
28f3e58
fix(l1): defer Amsterdam block gas overflow check to post-execution (…
azteca1998 Apr 16, 2026
0b69085
Prune Docker build cache after each image build and handle disk-full
avilagaston9 Apr 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 30 additions & 6 deletions crates/blockchain/payload.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -833,10 +833,35 @@ pub fn apply_plain_transaction(
// EIP-8037 (Amsterdam+): track regular and state gas separately
let tx_state_gas = report.state_gas_used;
let tx_regular_gas = report.gas_used.saturating_sub(tx_state_gas);
context.block_regular_gas_used = context

// Compute new totals before committing them
let new_regular = context
.block_regular_gas_used
.saturating_add(tx_regular_gas);
context.block_state_gas_used = context.block_state_gas_used.saturating_add(tx_state_gas);
let new_state = context.block_state_gas_used.saturating_add(tx_state_gas);

// EIP-8037 (Amsterdam+): post-execution block gas overflow check
// Reject the transaction if adding it would cause max(regular, state) to exceed the gas limit
if context.is_amsterdam && new_regular.max(new_state) > context.payload.header.gas_limit {
// Rollback transaction state before returning error:
// 1. Undo DB mutations (nonce, balance, storage, etc.)
// 2. Revert cumulative gas counter inflation
// This ensures the next transaction executes against clean state.
context.vm.undo_last_tx()?;
context.cumulative_gas_spent -= report.gas_spent;

return Err(EvmError::Custom(format!(
"block gas limit exceeded (state gas overflow): \
max({new_regular}, {new_state}) = {} > gas_limit {}",
new_regular.max(new_state),
context.payload.header.gas_limit
))
.into());
}

// Commit the new totals
context.block_regular_gas_used = new_regular;
context.block_state_gas_used = new_state;

if context.is_amsterdam {
debug!(
Expand All @@ -852,15 +877,14 @@ pub fn apply_plain_transaction(
}

// Update remaining_gas for block gas limit checks.
// EIP-8037 (Amsterdam+): per-tx check only validates regular gas against block limit.
// State gas is NOT checked per-tx; block-end validation enforces
// max(block_regular, block_state) <= gas_limit.
// EIP-8037 (Amsterdam+): remaining_gas reflects both regular and state gas dimensions.
// For pre-tx heuristic checks, this ensures we reject txs when either dimension is full.
if context.is_amsterdam {
context.remaining_gas = context
.payload
.header
.gas_limit
.saturating_sub(context.block_regular_gas_used);
.saturating_sub(new_regular.max(new_state));
} else {
context.remaining_gas = context.remaining_gas.saturating_sub(report.gas_used);
}
Expand Down
118 changes: 73 additions & 45 deletions crates/vm/backends/levm/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ use ethrex_levm::account::{AccountStatus, LevmAccount};
use ethrex_levm::call_frame::Stack;
use ethrex_levm::constants::{
POST_OSAKA_GAS_LIMIT_CAP, STACK_LIMIT, SYS_CALL_GAS_LIMIT, TX_BASE_COST,
TX_MAX_GAS_LIMIT_AMSTERDAM,
};
use ethrex_levm::db::Database;
use ethrex_levm::db::gen_db::{CacheDB, GeneralizedDatabase};
Expand Down Expand Up @@ -126,18 +125,13 @@ impl LEVM {
})?;

for (tx_idx, (tx, tx_sender)) in transactions_with_sender.into_iter().enumerate() {
// Pre-tx gas limit guard per EIP-8037/EIP-7825:
// Amsterdam: check min(TX_MAX_GAS_LIMIT, tx.gas) against regular gas only.
// State gas is NOT checked per-tx; block-end validation enforces
// max(block_regular, block_state) <= gas_limit.
// Pre-Amsterdam: check tx.gas against cumulative_gas_used (post-refund sum).
if is_amsterdam {
check_gas_limit(
block_regular_gas_used,
tx.gas_limit().min(TX_MAX_GAS_LIMIT_AMSTERDAM),
block.header.gas_limit,
)?;
} else {
// Pre-tx gas limit guard:
// Pre-Amsterdam: reject tx if cumulative post-refund gas + tx.gas > block limit.
// Amsterdam+: skip — EIP-8037's 2D gas model means cumulative gas (regular +
// state) can legally exceed the block gas limit as long as
// max(sum_regular, sum_state) stays within it. Block-level overflow is
// detected post-execution.
if !is_amsterdam {
check_gas_limit(cumulative_gas_used, tx.gas_limit(), block.header.gas_limit)?;
}

Expand Down Expand Up @@ -175,6 +169,20 @@ impl LEVM {
report.gas_used,
report.gas_spent,
);

// DoS protection: early exit if either regular or state gas exceeds the limit.
// Since block_gas_used = max(regular, state), if either component exceeds
// the limit, we know the block is invalid and can safely reject without
// violating EIP-8037 semantics.
if block_regular_gas_used > block.header.gas_limit
|| block_state_gas_used > block.header.gas_limit
{
return Err(EvmError::Transaction(format!(
"Gas allowance exceeded: Block gas used overflow: \
block_gas_used {block_gas_used} > block_gas_limit {}",
block.header.gas_limit
)));
}
} else {
block_gas_used = block_gas_used.saturating_add(report.gas_used);
}
Expand All @@ -189,6 +197,17 @@ impl LEVM {
receipts.push(receipt);
}

// EIP-7778 (Amsterdam+): block-level gas overflow check.
// Per-tx checks are skipped for Amsterdam because block gas is computed
// from pre-refund values; overflow can only be detected after execution.
if is_amsterdam && block_gas_used > block.header.gas_limit {
return Err(EvmError::Transaction(format!(
"Gas allowance exceeded: Block gas used overflow: \
block_gas_used {block_gas_used} > block_gas_limit {}",
block.header.gas_limit
)));
}

// Set BAL index for post-execution phase (requests + withdrawals, uint16)
// Order must match geth: requests (system calls) BEFORE withdrawals.
if is_amsterdam {
Expand Down Expand Up @@ -424,18 +443,13 @@ impl LEVM {
let mut tx_since_last_flush = 2;

for (tx_idx, (tx, tx_sender)) in transactions_with_sender.into_iter().enumerate() {
// Pre-tx gas limit guard per EIP-8037/EIP-7825:
// Amsterdam: check min(TX_MAX_GAS_LIMIT, tx.gas) against regular gas only.
// State gas is NOT checked per-tx; block-end validation enforces
// max(block_regular, block_state) <= gas_limit.
// Pre-Amsterdam: check tx.gas against cumulative_gas_used (post-refund sum).
if is_amsterdam {
check_gas_limit(
block_regular_gas_used,
tx.gas_limit().min(TX_MAX_GAS_LIMIT_AMSTERDAM),
block.header.gas_limit,
)?;
} else {
// Pre-tx gas limit guard:
// Pre-Amsterdam: reject tx if cumulative post-refund gas + tx.gas > block limit.
// Amsterdam+: skip — EIP-8037's 2D gas model means cumulative gas (regular +
// state) can legally exceed the block gas limit as long as
// max(sum_regular, sum_state) stays within it. Block-level overflow is
// detected post-execution.
if !is_amsterdam {
check_gas_limit(cumulative_gas_used, tx.gas_limit(), block.header.gas_limit)?;
}

Expand Down Expand Up @@ -483,6 +497,20 @@ impl LEVM {
if is_amsterdam {
// Amsterdam+: block gas = max(regular_sum, state_sum)
block_gas_used = block_regular_gas_used.max(block_state_gas_used);

// DoS protection: early exit if either regular or state gas exceeds the limit.
// Since block_gas_used = max(regular, state), if either component exceeds
// the limit, we know the block is invalid and can safely reject without
// violating EIP-8037 semantics.
if block_regular_gas_used > block.header.gas_limit
|| block_state_gas_used > block.header.gas_limit
{
return Err(EvmError::Transaction(format!(
"Gas allowance exceeded: Block gas used overflow: \
block_gas_used {block_gas_used} > block_gas_limit {}",
block.header.gas_limit
)));
}
} else {
block_gas_used = block_gas_used.saturating_add(report.gas_used);
}
Expand All @@ -497,6 +525,17 @@ impl LEVM {
receipts.push(receipt);
}

// EIP-7778 (Amsterdam+): block-level gas overflow check.
// Per-tx checks are skipped for Amsterdam because block gas is computed
// from pre-refund values; overflow can only be detected after execution.
if is_amsterdam && block_gas_used > block.header.gas_limit {
return Err(EvmError::Transaction(format!(
"Gas allowance exceeded: Block gas used overflow: \
block_gas_used {block_gas_used} > block_gas_limit {}",
block.header.gas_limit
)));
}

#[cfg(feature = "perf_opcode_timings")]
{
let mut timings = OPCODE_TIMINGS.lock().expect("poison");
Expand Down Expand Up @@ -972,32 +1011,21 @@ impl LEVM {
// balance in the BAL won't match execution that ran all txs).
let mut block_regular_gas_used = 0_u64;
let mut block_state_gas_used = 0_u64;
for (tx_idx, _, report, _, _, _) in &exec_results {
// Per-tx check: only regular gas is checked per-tx (EIP-8037/EIP-7825).
// State gas is validated at block end via max(regular, state) <= gas_limit.
let tx_gas_limit = txs_with_sender[*tx_idx].0.gas_limit();
check_gas_limit(
block_regular_gas_used,
tx_gas_limit.min(TX_MAX_GAS_LIMIT_AMSTERDAM),
header.gas_limit,
)?;
for (_, _, report, _, _, _) in &exec_results {
let tx_state_gas = report.state_gas_used;
let tx_regular_gas = report.gas_used.saturating_sub(tx_state_gas);
block_regular_gas_used = block_regular_gas_used.saturating_add(tx_regular_gas);
block_state_gas_used = block_state_gas_used.saturating_add(tx_state_gas);
// Post-tx check: needed because all txs are already executed — if the last tx
// pushes actual gas over the limit, there's no next iteration to catch it
// like the sequential path does.
let running_block_gas_after = block_regular_gas_used.max(block_state_gas_used);
if running_block_gas_after > header.gas_limit {
return Err(EvmError::Transaction(format!(
"Gas allowance exceeded: \
used {running_block_gas_after} > block limit {}",
header.gas_limit
)));
}
}
let block_gas_used = block_regular_gas_used.max(block_state_gas_used);
// EIP-7778: block-level overflow check using pre-refund gas.
if block_gas_used > header.gas_limit {
return Err(EvmError::Transaction(format!(
"Gas allowance exceeded: Block gas used overflow: \
block_gas_used {block_gas_used} > block_gas_limit {}",
header.gas_limit
)));
}

// 4. Per-tx BAL validation — now safe to run after gas limit is confirmed OK.
// Also mark off storage_reads that appear in per-tx execution state.
Expand Down
54 changes: 38 additions & 16 deletions tooling/sync/docker_monitor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,19 @@ def build_docker_image(profile: str, image_tag: str, ethrex_dir: str) -> bool:
check=True
)
print("✅ Docker image built successfully")
# Prune build cache to prevent unbounded disk growth across runs
try:
result = subprocess.run(
["docker", "builder", "prune", "-f"],
capture_output=True, text=True, timeout=120
)
if result.returncode == 0:
reclaimed = result.stdout.strip().split("\n")[-1] if result.stdout.strip() else ""
print(f"🧹 Build cache pruned. {reclaimed}")
else:
print(f"⚠️ Build cache prune failed: {result.stderr.strip()}")
except Exception as e:
print(f"⚠️ Build cache prune error: {e}")
return True
except subprocess.CalledProcessError as e:
print(f"❌ Failed to build Docker image: {e}")
Expand Down Expand Up @@ -410,15 +423,18 @@ def save_container_logs(container: str, run_id: str, suffix: str = ""):
def save_all_logs(instances: list[Instance], run_id: str, compose_file: str):
"""Save logs for all containers (ethrex + consensus)."""
print(f"\n📁 Saving logs for run {run_id}...")

for inst in instances:
# Save ethrex logs
save_container_logs(inst.container, run_id)
# Save consensus logs (convention: consensus-{network})
consensus_container = inst.container.replace("ethrex-", "consensus-")
save_container_logs(consensus_container, run_id)

print(f"📁 Logs saved to {LOGS_DIR}/run_{run_id}/\n")

try:
for inst in instances:
# Save ethrex logs
save_container_logs(inst.container, run_id)
# Save consensus logs (convention: consensus-{network})
consensus_container = inst.container.replace("ethrex-", "consensus-")
save_container_logs(consensus_container, run_id)

print(f"📁 Logs saved to {LOGS_DIR}/run_{run_id}/\n")
except OSError as e:
print(f"⚠️ Failed to save some logs (disk full?): {e}", flush=True)
Comment on lines +427 to +437
Copy link
Copy Markdown

@greptile-apps greptile-apps Bot Apr 17, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 except OSError in save_all_logs has incomplete coverage

The OSError guard here only catches errors from log_file.parent.mkdir() in save_container_logs, because that call sits outside save_container_logs's inner try block. However, write-time ENOSPC from log_file.write_text(logs) is swallowed by save_container_logs's broad except Exception handler, so the user-friendly "disk full?" message in save_all_logs is never printed for the most common failure scenario. The monitor won't crash either way (both paths print a warning), but the coverage is inconsistent.

A straightforward fix is to re-raise OSError from inside save_container_logs so it can propagate to the caller:

# save_container_logs – change the broad handler to only silence non-OSError exceptions
except OSError:
    raise  # let the caller (save_all_logs) handle disk-full
except Exception as e:
    print(f"  ⚠️ Error saving logs for {container}: {e}")
    return False
Prompt To Fix With AI 
This is a comment left during a code review.
Path: tooling/sync/docker_monitor.py
Line: 427-437

Comment:
**`except OSError` in `save_all_logs` has incomplete coverage**

The `OSError` guard here only catches errors from `log_file.parent.mkdir()` in `save_container_logs`, because that call sits *outside* `save_container_logs`'s inner `try` block. However, write-time `ENOSPC` from `log_file.write_text(logs)` is swallowed by `save_container_logs`'s broad `except Exception` handler, so the user-friendly `"disk full?"` message in `save_all_logs` is never printed for the most common failure scenario. The monitor won't crash either way (both paths print a warning), but the coverage is inconsistent.

A straightforward fix is to re-raise `OSError` from inside `save_container_logs` so it can propagate to the caller:

```python
# save_container_logs – change the broad handler to only silence non-OSError exceptions
except OSError:
    raise  # let the caller (save_all_logs) handle disk-full
except Exception as e:
    print(f"  ⚠️ Error saving logs for {container}: {e}")
    return False
```

How can I resolve this? If you propose a fix, please make it concise.



def log_run_result(run_id: str, run_count: int, instances: list[Instance], hostname: str, branch: str, commit: str, build_profile: str = ""):
Expand Down Expand Up @@ -479,13 +495,19 @@ def log_run_result(run_id: str, run_count: int, instances: list[Instance], hostn

lines.append("")
# Append to log file
with open(RUN_LOG_FILE, "a") as f:
f.write("\n".join(lines) + "\n")
print(f"📝 Run logged to {RUN_LOG_FILE}")
# Also write summary to the run folder
summary_file = LOGS_DIR / f"run_{run_id}" / "summary.txt"
summary_file.parent.mkdir(parents=True, exist_ok=True)
summary_file.write_text("\n".join(lines))
text = "\n".join(lines) + "\n"
try:
with open(RUN_LOG_FILE, "a") as f:
f.write(text)
print(f"📝 Run logged to {RUN_LOG_FILE}")
# Also write summary to the run folder
summary_file = LOGS_DIR / f"run_{run_id}" / "summary.txt"
summary_file.parent.mkdir(parents=True, exist_ok=True)
summary_file.write_text(text)
except OSError as e:
print(f"⚠️ Failed to write run log (disk full?): {e}", flush=True)
# Print to stdout so the result isn't lost entirely
print(text, flush=True)


def generate_run_id() -> str:
Expand Down
Loading