Skip to content

feat(benchmarks): highlights chart #1802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zerosnacks merged 4 commits into from
Apr 28, 2026
+292 −47
Merged

feat(benchmarks): highlights chart #1802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c883666
feat(benchmarks): add per-category highlights bar graph
Apr 28, 2026
0a5f05a
feat(benchmarks): move highlights bar graph to top of page
Apr 28, 2026
dfd961f
feat(bench): pretty performance chart
mablr Apr 28, 2026
1d9d620
fix: lints
mablr Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 175 additions & 46 deletions scripts/fetch-benchmarks.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,19 +31,19 @@ async function fetchBenchmarks(): Promise<string> {

function parseTimeToSeconds(timeStr: string): number | null {
if (!timeStr || timeStr === '–' || timeStr === '-') return null;

const minutesMatch = timeStr.match(/(\d+)m\s*([\d.]+)s/);
if (minutesMatch) {
const minutes = parseFloat(minutesMatch[1]);
const seconds = parseFloat(minutesMatch[2]);
return minutes * 60 + seconds;
}

const secondsMatch = timeStr.match(/([\d.]+)\s*s/);
if (secondsMatch) {
return parseFloat(secondsMatch[1]);
}

return null;
}

Expand All @@ -57,21 +57,21 @@ function parseMarkdown(markdown: string): Map<string, BenchmarkData> {
const lines = markdown.split('\n');
const data = new Map<string, BenchmarkData>();
const repos = new Set<string>();

let currentSection = '';

for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();

if (line.startsWith('## ')) {
currentSection = line.substring(3).trim();
continue;
}

if (!line.includes('|') || line.includes('---') || line.includes('Repository')) {
continue;
}

const cells = line.split('|').map(cell => cell.trim()).filter(cell => cell);
if (cells.length >= 3) {
const repo = cells[0];
Expand All @@ -80,7 +80,7 @@ function parseMarkdown(markdown: string): Map<string, BenchmarkData> {
}
}
}

repos.forEach(repo => {
data.set(repo, {
repository: repo,
Expand All @@ -92,27 +92,27 @@ function parseMarkdown(markdown: string): Map<string, BenchmarkData> {
forgeCoverage: { old: '–', new: '–' }
});
});

currentSection = '';
for (let i = 0; i < lines.length; i++) {
const line = lines[i].trim();

if (line.startsWith('## ')) {
currentSection = line.substring(3).trim();
continue;
}

if (!line.includes('|') || line.includes('---') || line.includes('Repository')) {
continue;
}

const cells = line.split('|').map(cell => cell.trim()).filter(cell => cell);
if (cells.length < 3) continue;

const [repo, oldValue, newValue] = cells;
const benchData = data.get(repo);
if (!benchData) continue;

Comment thread
zerosnacks marked this conversation as resolved.
switch (currentSection) {
case 'Forge Test':
benchData.forgeTest = { old: oldValue, new: newValue };
Expand All @@ -134,7 +134,7 @@ function parseMarkdown(markdown: string): Map<string, BenchmarkData> {
break;
}
}

return data;
}

Expand All @@ -145,22 +145,22 @@ function getRepositoryUrl(repoName: string): string {
'Uniswap-v4-core': 'https://github.com/Uniswap/v4-core',
'sparkdotfi-spark-psm': 'https://github.com/sparkdotfi/spark-psm'
};

return repoMap[repoName] || `https://github.com/search?q=${encodeURIComponent(repoName)}`;
}

function formatBenchmark(bench: { old: string; new: string }): { oldTime: string; newTime: string; change: string; color: string; bgColor: string } | null {
if (bench.new === '–' || bench.new === '-' || bench.old === '–' || bench.old === '-') return null;

const oldTime = parseTimeToSeconds(bench.old);
const newTime = parseTimeToSeconds(bench.new);

if (oldTime === null || newTime === null) {
return { oldTime: bench.old.replace(/\s+s/g, 's'), newTime: bench.new.replace(/\s+s/g, 's'), change: '', color: 'inherit', bgColor: 'transparent' };
}

const { value, display } = calculatePercentageChange(oldTime, newTime);

let color: string;
let bgColor: string;
let arrow: string;
Expand All @@ -177,7 +177,7 @@ function formatBenchmark(bench: { old: string; new: string }): { oldTime: string
bgColor = 'transparent';
arrow = value === 0 ? '' : '↑';
}

return {
oldTime: bench.old.replace(/\s+s/g, 's'),
newTime: bench.new.replace(/\s+s/g, 's'),
Expand All @@ -187,6 +187,131 @@ function formatBenchmark(bench: { old: string; new: string }): { oldTime: string
};
}

interface CategoryAggregate {
label: string;
key: keyof Omit<BenchmarkData, 'repository'>;
oldSumSec: number;
newSumSec: number;
repoCount: number;
absDelta: number; // positive => improvement (old - new)
pctDelta: number; // positive => improvement
}

function aggregateCategory(
data: Map<string, BenchmarkData>,
key: keyof Omit<BenchmarkData, 'repository'>,
label: string
): CategoryAggregate | null {
let oldSum = 0;
let newSum = 0;
let count = 0;

for (const [, benchData] of data) {
const bench = benchData[key];
if (!bench || bench.old === '–' || bench.old === '-' || bench.new === '–' || bench.new === '-') continue;

const oldSec = parseTimeToSeconds(bench.old);
const newSec = parseTimeToSeconds(bench.new);
if (oldSec === null || newSec === null) continue;

oldSum += oldSec;
newSum += newSec;
count += 1;
}

if (count === 0 || oldSum === 0) return null;

const absDelta = oldSum - newSum;
const pctDelta = (absDelta / oldSum) * 100;

return {
label,
key,
oldSumSec: oldSum,
newSumSec: newSum,
repoCount: count,
absDelta,
pctDelta,
};
}

function generateBenchmarkBarGraph(data: Map<string, BenchmarkData>): string {
const categories: { key: keyof Omit<BenchmarkData, 'repository'>; label: string }[] = [
{ key: 'forgeTest', label: 'Forge Test' },
{ key: 'forgeFuzzTest', label: 'Forge Fuzz Test' },
{ key: 'forgeTestIsolated', label: 'Forge Test (Isolated)' },
{ key: 'forgeBuildNoCache', label: 'Forge Build (No Cache)' },
{ key: 'forgeBuildWithCache', label: 'Forge Build (With Cache)' },
{ key: 'forgeCoverage', label: 'Forge Coverage' },
];

const aggregates: CategoryAggregate[] = [];
for (const { key, label } of categories) {
const a = aggregateCategory(data, key, label);
if (a) aggregates.push(a);
}
if (aggregates.length === 0) return '';

// Scale so that the longest bar (baseline or latest, across all categories)
// fills 100% of the available width. All baseline bars share the same width.
// baselineWidth = 100 / max(1, maxLatestRatio)
// latestWidth_i = ratio_i * baselineWidth
const maxRatio = Math.max(
1,
...aggregates.map(a => a.newSumSec / a.oldSumSec),
);
const BASELINE_WIDTH_PCT = 100 / maxRatio;

let mdx = '';
mdx += `<div style={{ marginBottom: '2rem' }}>\n`;
mdx += ` <style>{\`@keyframes bench-bar-grow{from{width:var(--bench-bar-from)}to{width:var(--bench-bar-to)}}\`}</style>\n`;
mdx += ` <h2 style={{ fontSize: '1.25rem', fontWeight: 600, marginBottom: '0.25rem' }}>Highlights</h2>\n`;
mdx += ` <p style={{ fontSize: '0.875rem', color: 'var(--vocs-color_text3)', marginBottom: '1rem' }}>Aggregated change per benchmark category (sum of wallclock time across all repositories).</p>\n`;
mdx += ` <div style={{ display: 'flex', flexDirection: 'column', gap: '1rem', border: '1px solid rgba(255,255,255,0.1)', borderRadius: '8px', padding: '1rem', background: 'rgba(255,255,255,0.02)' }}>\n`;

for (const a of aggregates) {
// Treat changes within ±2% as neutral noise.
const NEUTRAL_PCT_THRESHOLD = 2;
const isNeutral = Math.abs(a.pctDelta) < NEUTRAL_PCT_THRESHOLD;
const improved = !isNeutral && a.absDelta > 0;
const regressed = !isNeutral && a.absDelta < 0;
const accentColor = improved ? '#22c55e' : regressed ? '#ef4444' : 'var(--vocs-color_text3)';
const newBg = improved ? 'rgba(34, 197, 94, 0.85)' : regressed ? 'rgba(239, 68, 68, 0.85)' : 'rgba(148, 163, 184, 0.6)';
const baselineBg = 'rgba(148, 163, 184, 0.25)';

const ratio = a.newSumSec / a.oldSumSec;
const newWidthPct = Math.min(ratio * BASELINE_WIDTH_PCT, 100);

const arrow = a.absDelta > 0 ? '↓' : a.absDelta < 0 ? '↑' : '';
const pctDisplay = `${arrow}${Math.abs(a.pctDelta).toFixed(1)}%`;

mdx += ` <div>\n`;
mdx += ` <div style={{ fontSize: '0.875rem', fontWeight: 600, marginBottom: '0.375rem' }}>${a.label} <span style={{ color: 'var(--vocs-color_text3)', fontWeight: 400 }}>· ${a.repoCount} ${a.repoCount === 1 ? 'repo' : 'repos'}</span></div>\n`;
mdx += ` <div style={{ display: 'flex', alignItems: 'center', gap: '0.75rem' }}>\n`;
mdx += ` <div style={{ flex: 1, display: 'flex', flexDirection: 'column', gap: '0.25rem', minWidth: 0 }}>\n`;
// Baseline bar (uniform width across categories)
mdx += ` <div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>\n`;
mdx += ` <div style={{ flex: 1, height: '0.75rem', borderRadius: '4px', overflow: 'hidden' }}>\n`;
mdx += ` <div style={{ width: '${BASELINE_WIDTH_PCT.toFixed(2)}%', height: '100%', background: '${baselineBg}', borderRadius: '4px' }} />\n`;
mdx += ` </div>\n`;
mdx += ` </div>\n`;
// Latest bar (proportional to baseline)
mdx += ` <div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>\n`;
mdx += ` <div style={{ flex: 1, height: '0.75rem', borderRadius: '4px', overflow: 'hidden' }}>\n`;
mdx += ` <div style={{ ['--bench-bar-from']: '${BASELINE_WIDTH_PCT.toFixed(2)}%', ['--bench-bar-to']: '${newWidthPct.toFixed(2)}%', width: 'var(--bench-bar-to)', height: '100%', background: '${newBg}', borderRadius: '4px', animation: 'bench-bar-grow 800ms ease-out' }} />\n`;
mdx += ` </div>\n`;
mdx += ` </div>\n`;
mdx += ` </div>\n`;
mdx += ` <span style={{ fontSize: '0.75rem', fontWeight: 600, color: '${accentColor}', background: '${improved ? 'rgba(34,197,94,0.15)' : regressed ? 'rgba(239,68,68,0.15)' : 'rgba(148,163,184,0.18)'}', padding: '0.125rem 0.5rem', borderRadius: '9999px', fontVariantNumeric: 'tabular-nums', flexShrink: 0 }}>${pctDisplay}</span>\n`;
mdx += ` </div>\n`;
mdx += ` </div>\n`;
}

mdx += ` </div>\n`;
mdx += `</div>\n`;
return mdx;
}

function generateBenchmarkCards(data: Map<string, BenchmarkData>): string {
const benchmarkTypes = [
{ key: 'forgeTest', label: 'Test' },
Expand All @@ -198,15 +323,15 @@ function generateBenchmarkCards(data: Map<string, BenchmarkData>): string {
] as const;

const sortedRepos = Array.from(data.keys()).sort();

let mdx = `<div style={{ display: 'flex', flexDirection: 'column', gap: '1rem' }}>\n`;

for (const repo of sortedRepos) {
const benchData = data.get(repo);
if (!benchData) continue;

const repoUrl = getRepositoryUrl(repo);

mdx += ` <div style={{ border: '1px solid rgba(255,255,255,0.1)', borderRadius: '8px', overflow: 'hidden' }}>\n`;
mdx += ` <div style={{ padding: '0.75rem 1rem', borderBottom: '1px solid rgba(255,255,255,0.1)', background: 'rgba(255,255,255,0.03)' }}>\n`;
mdx += ` <a href="${repoUrl}" style={{ fontWeight: 600, color: 'var(--vocs-color_text)', textDecoration: 'none' }}>${benchData.repository} <span style={{ opacity: 0.5, fontSize: '0.875em' }}>↗</span></a>\n`;
Expand All @@ -218,10 +343,10 @@ function generateBenchmarkCards(data: Map<string, BenchmarkData>): string {
const result = formatBenchmark(benchData[key]);
const isLastInRow = (i + 1) % 6 === 0 || i === benchmarkTypes.length - 1;
const borderRight = isLastInRow ? '' : 'borderRight: \'1px solid var(--vocs-color_border)\', ';

mdx += ` <div style={{ ${borderRight}padding: '0.875rem 1rem', textAlign: 'center' }}>\n`;
mdx += ` <div style={{ fontSize: '0.7rem', color: 'var(--vocs-color_text3)', textTransform: 'uppercase', letterSpacing: '0.05em', marginBottom: '0.5rem' }}>${label}</div>\n`;

if (result) {
mdx += ` <div style={{ fontSize: '1.1rem', fontWeight: 600, fontVariantNumeric: 'tabular-nums', marginBottom: '0.25rem' }}>${result.newTime}</div>\n`;
mdx += ` <div style={{ fontSize: '0.7rem', color: 'var(--vocs-color_text4)', marginBottom: '0.375rem' }}>${result.oldTime}</div>\n`;
Expand All @@ -231,7 +356,7 @@ function generateBenchmarkCards(data: Map<string, BenchmarkData>): string {
} else {
mdx += ` <div style={{ fontSize: '1.1rem', color: 'var(--vocs-color_text3)' }}>–</div>\n`;
}

mdx += ` </div>\n`;
}

Expand All @@ -247,35 +372,37 @@ async function main() {
try {
console.log('Fetching benchmark data from Foundry repository...');
const markdown = await fetchBenchmarks();

console.log('Parsing benchmark data...');
const benchmarkData = parseMarkdown(markdown);

const dateMatch = markdown.match(/# Foundry Benchmarks \[(.*?)\]/);
const benchmarkDate = dateMatch ? dateMatch[1] : new Date().toLocaleDateString();

const versionLines = markdown.match(/forge (?:Version: )?[0-9.]+-.+/g) || [];

let baselineVersion = 'v1.2.3';
let latestVersionDisplay: string;
let latestVersionUrl: string;

if (versionLines.length >= 1) {
const baselineMatch = versionLines[0].match(/([0-9.]+)-(v[0-9.]+)/);

const baselineLine = versionLines[0];
if (baselineLine) {
const baselineMatch = baselineLine.match(/([0-9.]+)-(v[0-9.]+)/);
if (baselineMatch) {
baselineVersion = baselineMatch[2];
baselineVersion = baselineMatch[2] ?? baselineVersion;
}
}

if (versionLines.length >= 2) {
const releaseMatch = versionLines[1].match(/([0-9.]+)-(v[0-9.]+)/);
if (releaseMatch) {

const latestLine = versionLines[1];
if (latestLine) {
const releaseMatch = latestLine.match(/([0-9.]+)-(v[0-9.]+)/);
if (releaseMatch && releaseMatch[2]) {
const latestRelease = releaseMatch[2];
latestVersionDisplay = latestRelease;
latestVersionUrl = `https://github.com/foundry-rs/foundry/releases/tag/${latestRelease}`;
} else {
const nightlyMatch = versionLines[1].match(/[0-9.]+-nightly \(([a-f0-9]+) /);
if (nightlyMatch) {
const nightlyMatch = latestLine.match(/[0-9.]+-nightly \(([a-f0-9]+) /);
if (nightlyMatch && nightlyMatch[1]) {
const latestCommit = nightlyMatch[1];
latestVersionDisplay = `nightly-${latestCommit}`;
latestVersionUrl = `https://github.com/foundry-rs/foundry/commit/${latestCommit}`;
Expand All @@ -288,11 +415,13 @@ async function main() {
latestVersionDisplay = 'master';
latestVersionUrl = 'https://github.com/foundry-rs/foundry/tree/master';
}

console.log('Generating benchmark cards...');
let output = `{/* Auto-generated from ${benchmarkDate}. Do not edit manually. */}\n\n`;
output += `# Benchmarks\n\n`;
output += `Performance comparison between Foundry releases.\n\n`;
output += generateBenchmarkBarGraph(benchmarkData);
output += `\n`;
output += `<div style={{ display: 'flex', gap: '2rem', marginBottom: '1.5rem', flexWrap: 'wrap' }}>\n`;
output += ` <div>\n`;
output += ` <span style={{ fontSize: '0.75rem', color: 'var(--vocs-color_text3)', textTransform: 'uppercase', letterSpacing: '0.05em' }}>Baseline</span>\n`;
Expand All @@ -304,12 +433,12 @@ async function main() {
output += ` </div>\n`;
output += `</div>\n\n`;
output += generateBenchmarkCards(benchmarkData);

mkdirSync(OUTPUT_DIR, { recursive: true });

console.log(`Writing benchmark cards to ${OUTPUT_FILE}...`);
writeFileSync(OUTPUT_FILE, output);

console.log('✅ Successfully generated benchmark cards');
} catch (error) {
console.error('Failed to fetch and process benchmarks:', error);
Expand Down
Loading