From 4fc1bd0023b03e2283b530a250f7514c56ae0346 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Tue, 3 Mar 2026 18:36:56 +0000 Subject: [PATCH] First draft of sol vs fe gas benchmarking --- Cargo.lock | 2 + crates/fe/Cargo.toml | 2 + crates/fe/bench_fixtures/bitwise_u256.fe | 34 ++ crates/fe/bench_fixtures/bitwise_u256.sol | 24 + crates/fe/bench_fixtures/bitwise_u256.toml | 21 + crates/fe/bench_fixtures/loops.fe | 71 +++ crates/fe/bench_fixtures/loops.sol | 64 +++ crates/fe/bench_fixtures/loops.toml | 25 + crates/fe/bench_fixtures/math_u256.fe | 34 ++ crates/fe/bench_fixtures/math_u256.sol | 24 + crates/fe/bench_fixtures/math_u256.toml | 21 + crates/fe/bench_fixtures/storage_simple.fe | 28 + crates/fe/bench_fixtures/storage_simple.sol | 19 + crates/fe/bench_fixtures/storage_simple.toml | 13 + crates/fe/src/bench.rs | 512 +++++++++++++++++++ crates/fe/src/main.rs | 33 ++ crates/solc-runner/src/lib.rs | 81 ++- 17 files changed, 992 insertions(+), 16 deletions(-) create mode 100644 crates/fe/bench_fixtures/bitwise_u256.fe create mode 100644 crates/fe/bench_fixtures/bitwise_u256.sol create mode 100644 crates/fe/bench_fixtures/bitwise_u256.toml create mode 100644 crates/fe/bench_fixtures/loops.fe create mode 100644 crates/fe/bench_fixtures/loops.sol create mode 100644 crates/fe/bench_fixtures/loops.toml create mode 100644 crates/fe/bench_fixtures/math_u256.fe create mode 100644 crates/fe/bench_fixtures/math_u256.sol create mode 100644 crates/fe/bench_fixtures/math_u256.toml create mode 100644 crates/fe/bench_fixtures/storage_simple.fe create mode 100644 crates/fe/bench_fixtures/storage_simple.sol create mode 100644 crates/fe/bench_fixtures/storage_simple.toml create mode 100644 crates/fe/src/bench.rs diff --git a/Cargo.lock b/Cargo.lock index 4a92adef17..0001c45947 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2080,6 +2080,7 @@ dependencies = [ "cranelift-entity 0.115.1", "crossbeam-channel", "dir-test", + "ethers-core", "fe-codegen", "fe-common", "fe-contract-harness", @@ -2096,6 +2097,7 @@ dependencies = [ "petgraph", "rustc-hash 2.1.1", "scip", + "serde", "serde_json", "similar", "smol_str 0.1.24", diff --git a/crates/fe/Cargo.toml b/crates/fe/Cargo.toml index d8a88e3650..7d8c66c9f9 100644 --- a/crates/fe/Cargo.toml +++ b/crates/fe/Cargo.toml @@ -33,6 +33,8 @@ similar = "2" colored = "2" hex = "0.4" toml = "0.8.8" +serde = { version = "1", features = ["derive"] } +ethers-core = "2" glob.workspace = true crossbeam-channel.workspace = true scip = "0.6.1" diff --git a/crates/fe/bench_fixtures/bitwise_u256.fe b/crates/fe/bench_fixtures/bitwise_u256.fe new file mode 100644 index 0000000000..6ce571d842 --- /dev/null +++ b/crates/fe/bench_fixtures/bitwise_u256.fe @@ -0,0 +1,34 @@ +use std::abi::sol + +pub msg BenchMsg { + #[selector = sol("and_(uint256,uint256)")] + And { a: u256, b: u256 } -> u256, + #[selector = sol("or_(uint256,uint256)")] + Or { a: u256, b: u256 } -> u256, + #[selector = sol("xor_(uint256,uint256)")] + Xor { a: u256, b: u256 } -> u256, + #[selector = sol("shl_(uint256,uint256)")] + Shl { a: u256, b: u256 } -> u256, + #[selector = sol("shr_(uint256,uint256)")] + Shr { a: u256, b: u256 } -> u256, +} + +pub contract Bench { + recv BenchMsg { + And { a, b } -> u256 { + a & b + } + Or { a, b } -> u256 { + a | b + } + Xor { a, b } -> u256 { + a ^ b + } + Shl { a, b } -> u256 { + a << b + } + Shr { a, b } -> u256 { + a >> b + } + } +} diff --git a/crates/fe/bench_fixtures/bitwise_u256.sol b/crates/fe/bench_fixtures/bitwise_u256.sol new file mode 100644 index 0000000000..84856442af --- /dev/null +++ b/crates/fe/bench_fixtures/bitwise_u256.sol @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.20; + +contract Bench { + function and_(uint256 a, uint256 b) public pure returns (uint256) { + return a & b; + } + + function or_(uint256 a, uint256 b) public pure returns (uint256) { + return a | b; + } + + function xor_(uint256 a, uint256 b) public pure returns (uint256) { + return a ^ b; + } + + function shl_(uint256 a, uint256 b) public pure returns (uint256) { + return a << b; + } + + function shr_(uint256 a, uint256 b) public pure returns (uint256) { + return a >> b; + } +} diff --git a/crates/fe/bench_fixtures/bitwise_u256.toml b/crates/fe/bench_fixtures/bitwise_u256.toml new file mode 100644 index 0000000000..4dd30cee5d --- /dev/null +++ b/crates/fe/bench_fixtures/bitwise_u256.toml @@ -0,0 +1,21 @@ +contract = "Bench" + +[[calls]] +function = "and_(uint256,uint256)" +args = ["255", "170"] + +[[calls]] +function = "or_(uint256,uint256)" +args = ["255", "170"] + +[[calls]] +function = "xor_(uint256,uint256)" +args = ["255", "170"] + +[[calls]] +function = "shl_(uint256,uint256)" +args = ["1", "8"] + +[[calls]] +function = "shr_(uint256,uint256)" +args = ["256", "4"] diff --git a/crates/fe/bench_fixtures/loops.fe b/crates/fe/bench_fixtures/loops.fe new file mode 100644 index 0000000000..d884d0c79f --- /dev/null +++ b/crates/fe/bench_fixtures/loops.fe @@ -0,0 +1,71 @@ +use std::abi::sol + +pub msg BenchMsg { + #[selector = sol("counter(uint256)")] + Counter { n: u256 } -> u256, + #[selector = sol("sum(uint256)")] + Sum { n: u256 } -> u256, + #[selector = sol("xorLoop(uint256,uint256)")] + XorLoop { n: u256, seed: u256 } -> u256, + #[selector = sol("counterUnchecked(uint256)")] + CounterUnchecked { n: u256 } -> u256, + #[selector = sol("sumUnchecked(uint256)")] + SumUnchecked { n: u256 } -> u256, + #[selector = sol("xorLoopUnchecked(uint256,uint256)")] + XorLoopUnchecked { n: u256, seed: u256 } -> u256, +} + +pub contract Bench { + recv BenchMsg { + Counter { n } -> u256 { + let mut i: u256 = 0 + while i < n { + i = i + 1 + } + i + } + Sum { n } -> u256 { + let mut total: u256 = 0 + let mut i: u256 = 1 + while i <= n { + total = total + i + i = i + 1 + } + total + } + XorLoop { n, seed } -> u256 { + let mut acc: u256 = seed + let mut i: u256 = 0 + while i < n { + acc = acc ^ (seed ^ i) + i = i + 1 + } + acc + } + CounterUnchecked { n } -> u256 { + let mut i: u256 = 0 + while i < n { + i = i + 1 + } + i + } + SumUnchecked { n } -> u256 { + let mut total: u256 = 0 + let mut i: u256 = 1 + while i <= n { + total = total + i + i = i + 1 + } + total + } + XorLoopUnchecked { n, seed } -> u256 { + let mut acc: u256 = seed + let mut i: u256 = 0 + while i < n { + acc = acc ^ (seed ^ i) + i = i + 1 + } + acc + } + } +} diff --git a/crates/fe/bench_fixtures/loops.sol b/crates/fe/bench_fixtures/loops.sol new file mode 100644 index 0000000000..fd06f1f365 --- /dev/null +++ b/crates/fe/bench_fixtures/loops.sol @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.20; + +contract Bench { + function counter(uint256 n) public pure returns (uint256) { + uint256 i = 0; + while (i < n) { + i = i + 1; + } + return i; + } + + function sum(uint256 n) public pure returns (uint256) { + uint256 total = 0; + uint256 i = 1; + while (i <= n) { + total = total + i; + i = i + 1; + } + return total; + } + + function xorLoop(uint256 n, uint256 seed) public pure returns (uint256) { + uint256 acc = seed; + uint256 i = 0; + while (i < n) { + acc = acc ^ (seed ^ i); + i = i + 1; + } + return acc; + } + + function counterUnchecked(uint256 n) public pure returns (uint256) { + uint256 i = 0; + while (i < n) { + unchecked { i = i + 1; } + } + return i; + } + + function sumUnchecked(uint256 n) public pure returns (uint256) { + uint256 total = 0; + uint256 i = 1; + while (i <= n) { + unchecked { + total = total + i; + i = i + 1; + } + } + return total; + } + + function xorLoopUnchecked(uint256 n, uint256 seed) public pure returns (uint256) { + uint256 acc = seed; + uint256 i = 0; + while (i < n) { + unchecked { + acc = acc ^ (seed ^ i); + i = i + 1; + } + } + return acc; + } +} diff --git a/crates/fe/bench_fixtures/loops.toml b/crates/fe/bench_fixtures/loops.toml new file mode 100644 index 0000000000..a977ff1ad9 --- /dev/null +++ b/crates/fe/bench_fixtures/loops.toml @@ -0,0 +1,25 @@ +contract = "Bench" + +[[calls]] +function = "counter(uint256)" +args = ["100"] + +[[calls]] +function = "sum(uint256)" +args = ["100"] + +[[calls]] +function = "xorLoop(uint256,uint256)" +args = ["100", "42"] + +[[calls]] +function = "counterUnchecked(uint256)" +args = ["100"] + +[[calls]] +function = "sumUnchecked(uint256)" +args = ["100"] + +[[calls]] +function = "xorLoopUnchecked(uint256,uint256)" +args = ["100", "42"] diff --git a/crates/fe/bench_fixtures/math_u256.fe b/crates/fe/bench_fixtures/math_u256.fe new file mode 100644 index 0000000000..e54ea8b1d3 --- /dev/null +++ b/crates/fe/bench_fixtures/math_u256.fe @@ -0,0 +1,34 @@ +use std::abi::sol + +pub msg BenchMsg { + #[selector = sol("add(uint256,uint256)")] + Add { a: u256, b: u256 } -> u256, + #[selector = sol("sub(uint256,uint256)")] + Sub { a: u256, b: u256 } -> u256, + #[selector = sol("mul(uint256,uint256)")] + Mul { a: u256, b: u256 } -> u256, + #[selector = sol("div(uint256,uint256)")] + Div { a: u256, b: u256 } -> u256, + #[selector = sol("mod_(uint256,uint256)")] + Mod { a: u256, b: u256 } -> u256, +} + +pub contract Bench { + recv BenchMsg { + Add { a, b } -> u256 { + a + b + } + Sub { a, b } -> u256 { + a - b + } + Mul { a, b } -> u256 { + a * b + } + Div { a, b } -> u256 { + a / b + } + Mod { a, b } -> u256 { + a % b + } + } +} diff --git a/crates/fe/bench_fixtures/math_u256.sol b/crates/fe/bench_fixtures/math_u256.sol new file mode 100644 index 0000000000..a756b7d9f1 --- /dev/null +++ b/crates/fe/bench_fixtures/math_u256.sol @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.20; + +contract Bench { + function add(uint256 a, uint256 b) public pure returns (uint256) { + return a + b; + } + + function sub(uint256 a, uint256 b) public pure returns (uint256) { + return a - b; + } + + function mul(uint256 a, uint256 b) public pure returns (uint256) { + return a * b; + } + + function div(uint256 a, uint256 b) public pure returns (uint256) { + return a / b; + } + + function mod_(uint256 a, uint256 b) public pure returns (uint256) { + return a % b; + } +} diff --git a/crates/fe/bench_fixtures/math_u256.toml b/crates/fe/bench_fixtures/math_u256.toml new file mode 100644 index 0000000000..398c844874 --- /dev/null +++ b/crates/fe/bench_fixtures/math_u256.toml @@ -0,0 +1,21 @@ +contract = "Bench" + +[[calls]] +function = "add(uint256,uint256)" +args = ["100", "200"] + +[[calls]] +function = "sub(uint256,uint256)" +args = ["300", "100"] + +[[calls]] +function = "mul(uint256,uint256)" +args = ["50", "4"] + +[[calls]] +function = "div(uint256,uint256)" +args = ["1000", "7"] + +[[calls]] +function = "mod_(uint256,uint256)" +args = ["1000", "7"] diff --git a/crates/fe/bench_fixtures/storage_simple.fe b/crates/fe/bench_fixtures/storage_simple.fe new file mode 100644 index 0000000000..2f798dc08b --- /dev/null +++ b/crates/fe/bench_fixtures/storage_simple.fe @@ -0,0 +1,28 @@ +use std::abi::sol +use std::evm::StorageMap + +pub msg BenchMsg { + #[selector = sol("store(uint256,uint256)")] + Store { key: u256, value: u256 }, + #[selector = sol("load(uint256)")] + Load { key: u256 } -> u256, + #[selector = sol("storeAndLoad(uint256,uint256)")] + StoreAndLoad { key: u256, value: u256 } -> u256, +} + +pub contract Bench { + mut data: StorageMap, + + recv BenchMsg { + Store { key, value } uses (mut data) { + data.set(key, value) + } + Load { key } -> u256 uses (data) { + data.get(key) + } + StoreAndLoad { key, value } -> u256 uses (mut data) { + data.set(key, value) + data.get(key) + } + } +} diff --git a/crates/fe/bench_fixtures/storage_simple.sol b/crates/fe/bench_fixtures/storage_simple.sol new file mode 100644 index 0000000000..8ae39573c9 --- /dev/null +++ b/crates/fe/bench_fixtures/storage_simple.sol @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.20; + +contract Bench { + mapping(uint256 => uint256) public data; + + function store(uint256 key, uint256 value) public { + data[key] = value; + } + + function load(uint256 key) public view returns (uint256) { + return data[key]; + } + + function storeAndLoad(uint256 key, uint256 value) public returns (uint256) { + data[key] = value; + return data[key]; + } +} diff --git a/crates/fe/bench_fixtures/storage_simple.toml b/crates/fe/bench_fixtures/storage_simple.toml new file mode 100644 index 0000000000..7462118dec --- /dev/null +++ b/crates/fe/bench_fixtures/storage_simple.toml @@ -0,0 +1,13 @@ +contract = "Bench" + +[[calls]] +function = "store(uint256,uint256)" +args = ["42", "12345"] + +[[calls]] +function = "load(uint256)" +args = ["42"] + +[[calls]] +function = "storeAndLoad(uint256,uint256)" +args = ["99", "67890"] diff --git a/crates/fe/src/bench.rs b/crates/fe/src/bench.rs new file mode 100644 index 0000000000..e82f0ec7c9 --- /dev/null +++ b/crates/fe/src/bench.rs @@ -0,0 +1,512 @@ +//! Gas benchmarking: compares Fe (Yul + Sonatina) against Solidity. +//! +//! Discovers paired `.fe` / `.sol` fixture files with a `.toml` manifest, +//! compiles each through all backends, deploys and calls them, and reports +//! per-function gas consumption. + +use std::fmt::Write as _; +use std::fs; + +use camino::{Utf8Path, Utf8PathBuf}; +use codegen::OptLevel; +use common::InputDb; +use contract_harness::{ExecutionOptions, RuntimeInstance}; +use driver::DriverDataBase; +use ethers_core::abi::AbiParser; +use url::Url; + +/// A single benchmark fixture: paired Fe + Solidity sources with a call manifest. +struct BenchFixture { + name: String, + fe_source: String, + sol_source: String, + contract_name: String, + calls: Vec, +} + +/// A function call to benchmark. +struct BenchCall { + /// Solidity-style function signature, e.g. `"add(uint256,uint256)"`. + signature: String, + /// Hex-encoded argument values (no 0x prefix), each padded to 32 bytes. + args: Vec, +} + +/// Gas measurement for a single function call across all backends. +struct BenchResult { + fixture: String, + function: String, + fe_yul_gas: Option, + fe_yul_opt_gas: Option, + fe_sonatina_gas: Option, + sol_gas: Option, + sol_opt_gas: Option, +} + +/// TOML manifest deserialized from `.toml`. +#[derive(serde::Deserialize)] +struct Manifest { + contract: String, + #[serde(default)] + calls: Vec, +} + +#[derive(serde::Deserialize)] +struct ManifestCall { + function: String, + #[serde(default)] + args: Vec, +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +pub fn run_benchmarks( + path: &Utf8Path, + filter: Option<&str>, + solc: Option<&str>, + output: Option<&Utf8Path>, +) -> Result<(), String> { + let path = resolve_fixtures_dir(path)?; + let fixtures = discover_fixtures(&path, filter)?; + if fixtures.is_empty() { + return Err(format!("no benchmark fixtures found in {path}")); + } + + println!("Found {} benchmark fixture(s)\n", fixtures.len()); + + let mut all_results: Vec = Vec::new(); + + for fixture in &fixtures { + println!("--- {} ---", fixture.name); + + // 1. Compile Solidity (unoptimized + optimized) + let sol_bytecode = compile_solidity(&fixture.sol_source, &fixture.contract_name, false, solc); + let sol_opt_bytecode = compile_solidity(&fixture.sol_source, &fixture.contract_name, true, solc); + + // 2. Compile Fe via Yul backend (unoptimized + optimized) + let fe_yul = compile_fe_yul(&fixture.fe_source, &fixture.name); + let fe_yul_bytecode = fe_yul.as_ref().and_then(|yul| { + compile_yul_to_bytecode(yul, &fixture.contract_name, false, solc) + }); + let fe_yul_opt_bytecode = fe_yul.as_ref().and_then(|yul| { + compile_yul_to_bytecode(yul, &fixture.contract_name, true, solc) + }); + + // 3. Compile Fe via Sonatina backend + let fe_sonatina_bytecode = compile_fe_sonatina(&fixture.fe_source, &fixture.name, &fixture.contract_name); + + // 4. Deploy all variants and measure gas per call + for call in &fixture.calls { + let calldata = match encode_calldata(&call.signature, &call.args) { + Ok(cd) => cd, + Err(err) => { + eprintln!(" skip {}: {err}", call.signature); + continue; + } + }; + + let fe_yul_gas = measure_call(&fe_yul_bytecode, &calldata); + let fe_yul_opt_gas = measure_call(&fe_yul_opt_bytecode, &calldata); + let fe_sonatina_gas = measure_call_bytes(&fe_sonatina_bytecode, &calldata); + let sol_gas = measure_call(&sol_bytecode, &calldata); + let sol_opt_gas = measure_call(&sol_opt_bytecode, &calldata); + + let fn_name = call.signature.split('(').next().unwrap_or(&call.signature); + all_results.push(BenchResult { + fixture: fixture.name.clone(), + function: fn_name.to_string(), + fe_yul_gas, + fe_yul_opt_gas, + fe_sonatina_gas, + sol_gas, + sol_opt_gas, + }); + } + } + + // Print results + print_table(&all_results); + + // Write CSV if requested + if let Some(out_dir) = output { + write_csv(&all_results, out_dir)?; + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// Fixture discovery +// --------------------------------------------------------------------------- + +/// If `path` doesn't exist, try common locations relative to the repo root. +fn resolve_fixtures_dir(path: &Utf8Path) -> Result { + if path.exists() { + return Ok(path.to_path_buf()); + } + + // When running from the repo root, check crates/fe/ + let under_crate = Utf8PathBuf::from("crates/fe").join(path); + if under_crate.exists() { + return Ok(under_crate); + } + + Err(format!("fixtures directory does not exist: {path}")) +} + +fn discover_fixtures( + dir: &Utf8Path, + filter: Option<&str>, +) -> Result, String> { + if !dir.exists() { + return Err(format!("fixtures directory does not exist: {dir}")); + } + + let mut fixtures = Vec::new(); + let mut toml_files: Vec<_> = fs::read_dir(dir.as_std_path()) + .map_err(|e| format!("failed to read {dir}: {e}"))? + .filter_map(Result::ok) + .filter(|e| { + e.path() + .extension() + .is_some_and(|ext| ext == "toml") + }) + .collect(); + toml_files.sort_by_key(|e| e.file_name()); + + for entry in toml_files { + let path = Utf8PathBuf::from_path_buf(entry.path()) + .map_err(|p| format!("non-utf8 path: {}", p.display()))?; + let stem = path + .file_stem() + .ok_or_else(|| format!("no file stem: {path}"))?; + + if let Some(f) = filter { + if !stem.contains(f) { + continue; + } + } + + let fe_path = dir.join(format!("{stem}.fe")); + let sol_path = dir.join(format!("{stem}.sol")); + + if !fe_path.exists() { + eprintln!("warning: skipping {stem} — missing {fe_path}"); + continue; + } + if !sol_path.exists() { + eprintln!("warning: skipping {stem} — missing {sol_path}"); + continue; + } + + let manifest_str = + fs::read_to_string(path.as_std_path()).map_err(|e| format!("read {path}: {e}"))?; + let manifest: Manifest = + toml::from_str(&manifest_str).map_err(|e| format!("parse {path}: {e}"))?; + + let fe_source = + fs::read_to_string(fe_path.as_std_path()).map_err(|e| format!("read {fe_path}: {e}"))?; + let sol_source = + fs::read_to_string(sol_path.as_std_path()).map_err(|e| format!("read {sol_path}: {e}"))?; + + let calls = manifest + .calls + .into_iter() + .map(|c| BenchCall { + signature: c.function, + args: c.args, + }) + .collect(); + + fixtures.push(BenchFixture { + name: stem.to_string(), + fe_source, + sol_source, + contract_name: manifest.contract, + calls, + }); + } + + Ok(fixtures) +} + +// --------------------------------------------------------------------------- +// Compilation helpers +// --------------------------------------------------------------------------- + +/// Compile Solidity source to deploy bytecode hex string. +fn compile_solidity( + source: &str, + contract_name: &str, + optimize: bool, + solc_path: Option<&str>, +) -> Option { + match solc_runner::compile_solidity(contract_name, source, optimize, solc_path) { + Ok(bc) => Some(bc.bytecode), + Err(e) => { + let label = if optimize { "sol+opt" } else { "sol" }; + eprintln!(" {label} compile error: {}", e.0); + None + } + } +} + +/// Set up a temp ingot and run a callback with `(db, ingot)`. +/// +/// Writes `fe.toml` + `src/lib.fe` to a temp directory, then calls `init_ingot` +/// so that `std` and `core` are resolved correctly. +fn with_fe_ingot( + fe_source: &str, + name: &str, + f: impl for<'db> FnOnce(&'db DriverDataBase, hir::Ingot<'db>) -> T, +) -> Option { + // Write source to a real temp directory so the resolver can find std/core. + let tmp = std::env::temp_dir().join(format!("fe_bench_{name}")); + let _ = std::fs::remove_dir_all(&tmp); + std::fs::create_dir_all(tmp.join("src")).ok()?; + std::fs::write( + tmp.join("fe.toml"), + format!("[ingot]\nname = \"{name}\"\nversion = \"0.1.0\"\n"), + ) + .ok()?; + std::fs::write(tmp.join("src").join("lib.fe"), fe_source).ok()?; + + let ingot_url = Url::from_directory_path(&tmp).ok()?; + let mut db = DriverDataBase::default(); + let had_errors = driver::init_ingot(&mut db, &ingot_url); + if had_errors { + eprintln!(" fe ingot init errors for {name}"); + return None; + } + + let ingot = db.workspace().containing_ingot(&db, ingot_url)?; + Some(f(&db, ingot)) +} + +/// Compile Fe source to Yul IR string. +fn compile_fe_yul(fe_source: &str, name: &str) -> Option { + with_fe_ingot(fe_source, name, |db, ingot| { + let diags = db.run_on_ingot(ingot); + if !diags.is_empty() { + eprintln!(" fe/yul diagnostics for {name}:"); + diags.emit(db); + return None; + } + + match codegen::emit_ingot_yul(db, ingot) { + Ok(yul) => Some(yul), + Err(err) => { + eprintln!(" fe/yul emit error for {name}: {err}"); + None + } + } + }) + .flatten() +} + +/// Compile Yul IR to deploy bytecode hex via solc. +fn compile_yul_to_bytecode( + yul: &str, + contract_name: &str, + optimize: bool, + solc_path: Option<&str>, +) -> Option { + match solc_runner::compile_single_contract_with_solc(contract_name, yul, optimize, true, solc_path) { + Ok(bc) => Some(bc.bytecode), + Err(e) => { + let label = if optimize { "fe/yul+opt" } else { "fe/yul" }; + eprintln!(" {label} solc error: {}", e.0); + None + } + } +} + +/// Compile Fe source to bytecode via Sonatina backend. Returns deploy bytecode as raw bytes. +fn compile_fe_sonatina(fe_source: &str, name: &str, contract_name: &str) -> Option> { + let contract_name = contract_name.to_string(); + with_fe_ingot(fe_source, name, move |db, ingot| { + let diags = db.run_on_ingot(ingot); + if !diags.is_empty() { + eprintln!(" fe/sonatina diagnostics for {name}:"); + diags.emit(db); + return None; + } + + match codegen::emit_ingot_sonatina_bytecode(db, ingot, OptLevel::O1, Some(&contract_name)) { + Ok(mut map) => map.remove(&contract_name).map(|bc| bc.deploy), + Err(err) => { + eprintln!(" fe/sonatina emit error for {name}: {err}"); + None + } + } + }) + .flatten() +} + +// --------------------------------------------------------------------------- +// Execution / measurement +// --------------------------------------------------------------------------- + +/// Encode calldata from a function signature and hex-string arguments. +fn encode_calldata(signature: &str, args: &[String]) -> Result, String> { + let function = AbiParser::default() + .parse_function(signature) + .map_err(|e| format!("bad signature `{signature}`: {e}"))?; + + // Build tokens from hex-encoded args + let tokens: Vec = args + .iter() + .zip(function.inputs.iter()) + .map(|(val, param)| parse_arg(val, ¶m.kind)) + .collect::, _>>()?; + + let encoded = function + .encode_input(&tokens) + .map_err(|e| format!("encode error: {e}"))?; + Ok(encoded) +} + +/// Parse a string argument into an ABI token based on the expected param type. +fn parse_arg( + val: &str, + kind: ðers_core::abi::ParamType, +) -> Result { + use ethers_core::abi::{ParamType, Token}; + match kind { + ParamType::Uint(_) => { + let n: ethers_core::types::U256 = val + .parse() + .map_err(|e| format!("cannot parse `{val}` as uint: {e}"))?; + Ok(Token::Uint(n)) + } + ParamType::Int(_) => { + let n: i128 = val + .parse() + .map_err(|e| format!("cannot parse `{val}` as int: {e}"))?; + // ABI-encode signed ints as U256 with two's complement + let u = if n < 0 { + ethers_core::types::U256::from(n as u128 | (u128::MAX << 64 >> 64 << 64)) + } else { + ethers_core::types::U256::from(n as u128) + }; + Ok(Token::Int(u)) + } + ParamType::Bool => { + let b: bool = val + .parse() + .map_err(|e| format!("cannot parse `{val}` as bool: {e}"))?; + Ok(Token::Bool(b)) + } + ParamType::Address => { + let addr: ethers_core::types::Address = val + .parse() + .map_err(|e| format!("cannot parse `{val}` as address: {e}"))?; + Ok(Token::Address(addr)) + } + _ => Err(format!("unsupported param type: {kind}")), + } +} + +/// Deploy a contract from hex-encoded init bytecode and call it. +fn measure_call(bytecode_hex: &Option, calldata: &[u8]) -> Option { + let hex = bytecode_hex.as_ref()?; + let mut instance = RuntimeInstance::deploy(hex).ok()?; + let result = instance.call_raw(calldata, ExecutionOptions::default()).ok()?; + Some(result.gas_used) +} + +/// Deploy a contract from raw bytes and call it. +fn measure_call_bytes(bytecode: &Option>, calldata: &[u8]) -> Option { + let bytes = bytecode.as_ref()?; + let hex_str = hex::encode(bytes); + let mut instance = RuntimeInstance::deploy(&hex_str).ok()?; + let result = instance.call_raw(calldata, ExecutionOptions::default()).ok()?; + Some(result.gas_used) +} + +// --------------------------------------------------------------------------- +// Reporting +// --------------------------------------------------------------------------- + +fn print_table(results: &[BenchResult]) { + if results.is_empty() { + println!("No results."); + return; + } + + // Header + println!( + "{:<20} {:<12} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}", + "Fixture", "Function", "Fe/Yul", "Fe/Yul+O", "Sonatina", "Sol", "Sol+O", "vs Sol+O" + ); + println!("{}", "-".repeat(94)); + + for r in results { + let delta = match (r.fe_yul_opt_gas, r.sol_opt_gas) { + (Some(fe), Some(sol)) if sol > 0 => { + let pct = ((fe as f64 - sol as f64) / sol as f64) * 100.0; + format!("{:+.1}%", pct) + } + _ => "-".to_string(), + }; + println!( + "{:<20} {:<12} {:>10} {:>10} {:>10} {:>10} {:>10} {:>10}", + r.fixture, + r.function, + fmt_gas(r.fe_yul_gas), + fmt_gas(r.fe_yul_opt_gas), + fmt_gas(r.fe_sonatina_gas), + fmt_gas(r.sol_gas), + fmt_gas(r.sol_opt_gas), + delta, + ); + } +} + +fn fmt_gas(gas: Option) -> String { + match gas { + Some(g) => g.to_string(), + None => "-".to_string(), + } +} + +fn write_csv(results: &[BenchResult], out_dir: &Utf8Path) -> Result<(), String> { + fs::create_dir_all(out_dir.as_std_path()) + .map_err(|e| format!("create dir {out_dir}: {e}"))?; + + let path = out_dir.join("gas_benchmark.csv"); + let mut csv = String::new(); + writeln!( + csv, + "fixture,function,fe_yul,fe_yul_opt,fe_sonatina,sol,sol_opt,delta_fe_yul_opt_vs_sol_opt_pct" + ) + .unwrap(); + + for r in results { + let delta = match (r.fe_yul_opt_gas, r.sol_opt_gas) { + (Some(fe), Some(sol)) if sol > 0 => { + format!("{:.2}", ((fe as f64 - sol as f64) / sol as f64) * 100.0) + } + _ => String::new(), + }; + writeln!( + csv, + "{},{},{},{},{},{},{},{}", + r.fixture, + r.function, + fmt_gas(r.fe_yul_gas), + fmt_gas(r.fe_yul_opt_gas), + fmt_gas(r.fe_sonatina_gas), + fmt_gas(r.sol_gas), + fmt_gas(r.sol_opt_gas), + delta, + ) + .unwrap(); + } + + fs::write(path.as_std_path(), &csv).map_err(|e| format!("write csv: {e}"))?; + println!("\nCSV report written to {}", out_dir.join("gas_benchmark.csv")); + Ok(()) +} diff --git a/crates/fe/src/main.rs b/crates/fe/src/main.rs index f70fb824ba..1878533796 100644 --- a/crates/fe/src/main.rs +++ b/crates/fe/src/main.rs @@ -1,4 +1,5 @@ #![allow(clippy::print_stderr, clippy::print_stdout)] +mod bench; mod build; mod check; mod cli; @@ -265,6 +266,21 @@ pub enum Command { #[arg(long)] call_trace: bool, }, + /// Run gas benchmarks comparing Fe (Yul + Sonatina) against Solidity. + Bench { + /// Path to benchmark fixtures directory. + #[arg(value_name = "PATH", default_value = "bench_fixtures")] + path: Utf8PathBuf, + /// Filter benchmarks by name. + #[arg(short, long)] + filter: Option, + /// solc binary to use (overrides FE_SOLC_PATH). + #[arg(long)] + solc: Option, + /// Output directory for CSV reports. + #[arg(long, short)] + output: Option, + }, /// Create a new ingot or workspace. New { /// Path to create the ingot or workspace in. @@ -538,6 +554,23 @@ pub fn run(opts: &Options) { } } } + Command::Bench { + path, + filter, + solc, + output, + } => match bench::run_benchmarks( + path.as_path(), + filter.as_deref(), + solc.as_deref(), + output.as_ref().map(|p| p.as_path()), + ) { + Ok(()) => {} + Err(err) => { + eprintln!("Error: {err}"); + std::process::exit(1); + } + }, Command::New { path, workspace, diff --git a/crates/solc-runner/src/lib.rs b/crates/solc-runner/src/lib.rs index 46049ac62c..9a1e0c3fe7 100644 --- a/crates/solc-runner/src/lib.rs +++ b/crates/solc-runner/src/lib.rs @@ -8,7 +8,7 @@ use std::{ const SOLC_ENV: &str = "FE_SOLC_PATH"; -/// Error wrapper used throughout the Yul compilation pipeline. +/// Error wrapper used throughout the compilation pipeline. #[derive(Debug, Clone)] pub struct YulcError(pub String); @@ -64,18 +64,30 @@ pub fn compile_single_contract_with_solc( verify_runtime_bytecode: bool, solc_path: Option<&str>, ) -> Result { - let input_json = build_standard_json(yul_src, optimize)?; + let input_json = build_standard_json_yul(yul_src, optimize)?; let solc_output = run_solc_with_path(&input_json, solc_path)?; - parse_contract_output(name, &solc_output, verify_runtime_bytecode) + parse_contract_output_for_source(name, &solc_output, "input.yul", verify_runtime_bytecode) } -/// Builds the standard JSON input description expected by `solc`. +/// Compiles a Solidity source file to bytecode using `solc`. /// -/// * `yul_src` - Yul program fed into the compiler. -/// * `optimize` - Toggles optimizer support in the generated JSON. -/// -/// Returns a serialized JSON string or a [`YulcError`] if serialization fails. -fn build_standard_json(yul_src: &str, optimize: bool) -> Result { +/// * `name` - Contract name as declared in the Solidity source. +/// * `solidity_src` - Solidity source code. +/// * `optimize` - Enables `solc`'s optimizer when `true`. +/// * `solc_path` - Optional path to `solc` binary; falls back to `FE_SOLC_PATH` / `solc`. +pub fn compile_solidity( + name: &str, + solidity_src: &str, + optimize: bool, + solc_path: Option<&str>, +) -> Result { + let input_json = build_standard_json_solidity(solidity_src, optimize)?; + let solc_output = run_solc_with_path(&input_json, solc_path)?; + parse_contract_output_for_source(name, &solc_output, "input.sol", true) +} + +/// Builds the standard JSON input for Yul compilation. +fn build_standard_json_yul(yul_src: &str, optimize: bool) -> Result { let value = json!({ "language": "Yul", "sources": { @@ -102,9 +114,36 @@ fn build_standard_json(yul_src: &str, optimize: bool) -> Result Result { + let value = json!({ + "language": "Solidity", + "sources": { + "input.sol": { "content": solidity_src } + }, + "settings": { + "optimizer": { + "enabled": optimize, + "runs": 200 + }, + "evmVersion": "cancun", + "outputSelection": { + "*": { + "*": [ + "evm.bytecode.object", + "evm.deployedBytecode.object" + ] + } + } + } + }); + + serde_json::to_string(&value).map_err(|err| YulcError(format!("failed to encode json: {err}"))) +} + /// Invokes the `solc` binary with the provided standard JSON input. /// -/// * `input` - Serialized standard JSON payload describing the Yul compilation. +/// * `input` - Serialized standard JSON payload. /// /// Returns the raw stdout emitted by `solc`, or a [`YulcError`] if the process fails or produces /// invalid UTF-8. @@ -156,12 +195,12 @@ fn run_solc_with_path(input: &str, solc_path: Option<&str>) -> Result Result { let value: Value = @@ -183,8 +222,8 @@ fn parse_contract_output( let contracts = value .get("contracts") - .and_then(|contracts| contracts.get("input.yul")) - .ok_or_else(|| YulcError("solc output missing `contracts.input.yul`".into()))?; + .and_then(|contracts| contracts.get(source_key)) + .ok_or_else(|| YulcError(format!("solc output missing `contracts.{source_key}`")))?; let contract = contracts .get(name) @@ -247,13 +286,23 @@ mod tests { } #[test] fn build_standard_json_contains_fields() { - let json_str = build_standard_json("{ sstore(0, 0) }", false).unwrap(); + let json_str = build_standard_json_yul("{ sstore(0, 0) }", false).unwrap(); let value: Value = serde_json::from_str(&json_str).unwrap(); assert_eq!(value["language"], "Yul"); assert_eq!(value["settings"]["optimizer"]["enabled"], false); assert_eq!(value["sources"]["input.yul"]["content"], "{ sstore(0, 0) }"); } + #[test] + fn build_standard_json_solidity_contains_fields() { + let json_str = + build_standard_json_solidity("contract Foo {}", false).unwrap(); + let value: Value = serde_json::from_str(&json_str).unwrap(); + assert_eq!(value["language"], "Solidity"); + assert_eq!(value["settings"]["optimizer"]["enabled"], false); + assert_eq!(value["sources"]["input.sol"]["content"], "contract Foo {}"); + } + #[test] fn executes_contract_function() { if !solc_available() {