diff --git a/scripts/fetch-benchmarks.ts b/scripts/fetch-benchmarks.ts index 7aaa77b7a..7af5dbfd7 100644 --- a/scripts/fetch-benchmarks.ts +++ b/scripts/fetch-benchmarks.ts @@ -31,19 +31,19 @@ async function fetchBenchmarks(): Promise { function parseTimeToSeconds(timeStr: string): number | null { if (!timeStr || timeStr === '–' || timeStr === '-') return null; - + const minutesMatch = timeStr.match(/(\d+)m\s*([\d.]+)s/); if (minutesMatch) { const minutes = parseFloat(minutesMatch[1]); const seconds = parseFloat(minutesMatch[2]); return minutes * 60 + seconds; } - + const secondsMatch = timeStr.match(/([\d.]+)\s*s/); if (secondsMatch) { return parseFloat(secondsMatch[1]); } - + return null; } @@ -57,21 +57,21 @@ function parseMarkdown(markdown: string): Map { const lines = markdown.split('\n'); const data = new Map(); const repos = new Set(); - + let currentSection = ''; - + for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); - + if (line.startsWith('## ')) { currentSection = line.substring(3).trim(); continue; } - + if (!line.includes('|') || line.includes('---') || line.includes('Repository')) { continue; } - + const cells = line.split('|').map(cell => cell.trim()).filter(cell => cell); if (cells.length >= 3) { const repo = cells[0]; @@ -80,7 +80,7 @@ function parseMarkdown(markdown: string): Map { } } } - + repos.forEach(repo => { data.set(repo, { repository: repo, @@ -92,27 +92,27 @@ function parseMarkdown(markdown: string): Map { forgeCoverage: { old: '–', new: '–' } }); }); - + currentSection = ''; for (let i = 0; i < lines.length; i++) { const line = lines[i].trim(); - + if (line.startsWith('## ')) { currentSection = line.substring(3).trim(); continue; } - + if (!line.includes('|') || line.includes('---') || line.includes('Repository')) { continue; } - + const cells = line.split('|').map(cell => cell.trim()).filter(cell => cell); if (cells.length < 3) continue; - + const [repo, oldValue, newValue] = cells; const benchData = data.get(repo); if (!benchData) continue; - + switch (currentSection) { case 'Forge Test': benchData.forgeTest = { old: oldValue, new: newValue }; @@ -134,7 +134,7 @@ function parseMarkdown(markdown: string): Map { break; } } - + return data; } @@ -145,22 +145,22 @@ function getRepositoryUrl(repoName: string): string { 'Uniswap-v4-core': 'https://github.com/Uniswap/v4-core', 'sparkdotfi-spark-psm': 'https://github.com/sparkdotfi/spark-psm' }; - + return repoMap[repoName] || `https://github.com/search?q=${encodeURIComponent(repoName)}`; } function formatBenchmark(bench: { old: string; new: string }): { oldTime: string; newTime: string; change: string; color: string; bgColor: string } | null { if (bench.new === '–' || bench.new === '-' || bench.old === '–' || bench.old === '-') return null; - + const oldTime = parseTimeToSeconds(bench.old); const newTime = parseTimeToSeconds(bench.new); - + if (oldTime === null || newTime === null) { return { oldTime: bench.old.replace(/\s+s/g, 's'), newTime: bench.new.replace(/\s+s/g, 's'), change: '', color: 'inherit', bgColor: 'transparent' }; } - + const { value, display } = calculatePercentageChange(oldTime, newTime); - + let color: string; let bgColor: string; let arrow: string; @@ -177,7 +177,7 @@ function formatBenchmark(bench: { old: string; new: string }): { oldTime: string bgColor = 'transparent'; arrow = value === 0 ? '' : '↑'; } - + return { oldTime: bench.old.replace(/\s+s/g, 's'), newTime: bench.new.replace(/\s+s/g, 's'), @@ -187,6 +187,131 @@ function formatBenchmark(bench: { old: string; new: string }): { oldTime: string }; } +interface CategoryAggregate { + label: string; + key: keyof Omit; + oldSumSec: number; + newSumSec: number; + repoCount: number; + absDelta: number; // positive => improvement (old - new) + pctDelta: number; // positive => improvement +} + +function aggregateCategory( + data: Map, + key: keyof Omit, + label: string +): CategoryAggregate | null { + let oldSum = 0; + let newSum = 0; + let count = 0; + + for (const [, benchData] of data) { + const bench = benchData[key]; + if (!bench || bench.old === '–' || bench.old === '-' || bench.new === '–' || bench.new === '-') continue; + + const oldSec = parseTimeToSeconds(bench.old); + const newSec = parseTimeToSeconds(bench.new); + if (oldSec === null || newSec === null) continue; + + oldSum += oldSec; + newSum += newSec; + count += 1; + } + + if (count === 0 || oldSum === 0) return null; + + const absDelta = oldSum - newSum; + const pctDelta = (absDelta / oldSum) * 100; + + return { + label, + key, + oldSumSec: oldSum, + newSumSec: newSum, + repoCount: count, + absDelta, + pctDelta, + }; +} + +function generateBenchmarkBarGraph(data: Map): string { + const categories: { key: keyof Omit; label: string }[] = [ + { key: 'forgeTest', label: 'Forge Test' }, + { key: 'forgeFuzzTest', label: 'Forge Fuzz Test' }, + { key: 'forgeTestIsolated', label: 'Forge Test (Isolated)' }, + { key: 'forgeBuildNoCache', label: 'Forge Build (No Cache)' }, + { key: 'forgeBuildWithCache', label: 'Forge Build (With Cache)' }, + { key: 'forgeCoverage', label: 'Forge Coverage' }, + ]; + + const aggregates: CategoryAggregate[] = []; + for (const { key, label } of categories) { + const a = aggregateCategory(data, key, label); + if (a) aggregates.push(a); + } + if (aggregates.length === 0) return ''; + + // Scale so that the longest bar (baseline or latest, across all categories) + // fills 100% of the available width. All baseline bars share the same width. + // baselineWidth = 100 / max(1, maxLatestRatio) + // latestWidth_i = ratio_i * baselineWidth + const maxRatio = Math.max( + 1, + ...aggregates.map(a => a.newSumSec / a.oldSumSec), + ); + const BASELINE_WIDTH_PCT = 100 / maxRatio; + + let mdx = ''; + mdx += `
\n`; + mdx += ` \n`; + mdx += `

Highlights

\n`; + mdx += `

Aggregated change per benchmark category (sum of wallclock time across all repositories).

\n`; + mdx += `
\n`; + + for (const a of aggregates) { + // Treat changes within ±2% as neutral noise. + const NEUTRAL_PCT_THRESHOLD = 2; + const isNeutral = Math.abs(a.pctDelta) < NEUTRAL_PCT_THRESHOLD; + const improved = !isNeutral && a.absDelta > 0; + const regressed = !isNeutral && a.absDelta < 0; + const accentColor = improved ? '#22c55e' : regressed ? '#ef4444' : 'var(--vocs-color_text3)'; + const newBg = improved ? 'rgba(34, 197, 94, 0.85)' : regressed ? 'rgba(239, 68, 68, 0.85)' : 'rgba(148, 163, 184, 0.6)'; + const baselineBg = 'rgba(148, 163, 184, 0.25)'; + + const ratio = a.newSumSec / a.oldSumSec; + const newWidthPct = Math.min(ratio * BASELINE_WIDTH_PCT, 100); + + const arrow = a.absDelta > 0 ? '↓' : a.absDelta < 0 ? '↑' : ''; + const pctDisplay = `${arrow}${Math.abs(a.pctDelta).toFixed(1)}%`; + + mdx += `
\n`; + mdx += `
${a.label} · ${a.repoCount} ${a.repoCount === 1 ? 'repo' : 'repos'}
\n`; + mdx += `
\n`; + mdx += `
\n`; + // Baseline bar (uniform width across categories) + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + // Latest bar (proportional to baseline) + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + mdx += `
\n`; + mdx += ` ${pctDisplay}\n`; + mdx += `
\n`; + mdx += `
\n`; + } + + mdx += `
\n`; + mdx += `
\n`; + return mdx; +} + function generateBenchmarkCards(data: Map): string { const benchmarkTypes = [ { key: 'forgeTest', label: 'Test' }, @@ -198,15 +323,15 @@ function generateBenchmarkCards(data: Map): string { ] as const; const sortedRepos = Array.from(data.keys()).sort(); - + let mdx = `
\n`; for (const repo of sortedRepos) { const benchData = data.get(repo); if (!benchData) continue; - + const repoUrl = getRepositoryUrl(repo); - + mdx += `
\n`; mdx += `
\n`; mdx += ` ${benchData.repository} ↗\n`; @@ -218,10 +343,10 @@ function generateBenchmarkCards(data: Map): string { const result = formatBenchmark(benchData[key]); const isLastInRow = (i + 1) % 6 === 0 || i === benchmarkTypes.length - 1; const borderRight = isLastInRow ? '' : 'borderRight: \'1px solid var(--vocs-color_border)\', '; - + mdx += `
\n`; mdx += `
${label}
\n`; - + if (result) { mdx += `
${result.newTime}
\n`; mdx += `
${result.oldTime}
\n`; @@ -231,7 +356,7 @@ function generateBenchmarkCards(data: Map): string { } else { mdx += `
–
\n`; } - + mdx += `
\n`; } @@ -247,35 +372,37 @@ async function main() { try { console.log('Fetching benchmark data from Foundry repository...'); const markdown = await fetchBenchmarks(); - + console.log('Parsing benchmark data...'); const benchmarkData = parseMarkdown(markdown); - + const dateMatch = markdown.match(/# Foundry Benchmarks \[(.*?)\]/); const benchmarkDate = dateMatch ? dateMatch[1] : new Date().toLocaleDateString(); - + const versionLines = markdown.match(/forge (?:Version: )?[0-9.]+-.+/g) || []; - + let baselineVersion = 'v1.2.3'; let latestVersionDisplay: string; let latestVersionUrl: string; - - if (versionLines.length >= 1) { - const baselineMatch = versionLines[0].match(/([0-9.]+)-(v[0-9.]+)/); + + const baselineLine = versionLines[0]; + if (baselineLine) { + const baselineMatch = baselineLine.match(/([0-9.]+)-(v[0-9.]+)/); if (baselineMatch) { - baselineVersion = baselineMatch[2]; + baselineVersion = baselineMatch[2] ?? baselineVersion; } } - - if (versionLines.length >= 2) { - const releaseMatch = versionLines[1].match(/([0-9.]+)-(v[0-9.]+)/); - if (releaseMatch) { + + const latestLine = versionLines[1]; + if (latestLine) { + const releaseMatch = latestLine.match(/([0-9.]+)-(v[0-9.]+)/); + if (releaseMatch && releaseMatch[2]) { const latestRelease = releaseMatch[2]; latestVersionDisplay = latestRelease; latestVersionUrl = `https://github.com/foundry-rs/foundry/releases/tag/${latestRelease}`; } else { - const nightlyMatch = versionLines[1].match(/[0-9.]+-nightly \(([a-f0-9]+) /); - if (nightlyMatch) { + const nightlyMatch = latestLine.match(/[0-9.]+-nightly \(([a-f0-9]+) /); + if (nightlyMatch && nightlyMatch[1]) { const latestCommit = nightlyMatch[1]; latestVersionDisplay = `nightly-${latestCommit}`; latestVersionUrl = `https://github.com/foundry-rs/foundry/commit/${latestCommit}`; @@ -288,11 +415,13 @@ async function main() { latestVersionDisplay = 'master'; latestVersionUrl = 'https://github.com/foundry-rs/foundry/tree/master'; } - + console.log('Generating benchmark cards...'); let output = `{/* Auto-generated from ${benchmarkDate}. Do not edit manually. */}\n\n`; output += `# Benchmarks\n\n`; output += `Performance comparison between Foundry releases.\n\n`; + output += generateBenchmarkBarGraph(benchmarkData); + output += `\n`; output += `
\n`; output += `
\n`; output += ` Baseline\n`; @@ -304,12 +433,12 @@ async function main() { output += `
\n`; output += `
\n\n`; output += generateBenchmarkCards(benchmarkData); - + mkdirSync(OUTPUT_DIR, { recursive: true }); - + console.log(`Writing benchmark cards to ${OUTPUT_FILE}...`); writeFileSync(OUTPUT_FILE, output); - + console.log('✅ Successfully generated benchmark cards'); } catch (error) { console.error('Failed to fetch and process benchmarks:', error); diff --git a/src/pages/benchmarks.mdx b/src/pages/benchmarks.mdx index 427ce3614..b3eb23abd 100644 --- a/src/pages/benchmarks.mdx +++ b/src/pages/benchmarks.mdx @@ -1,9 +1,125 @@ -{/* Auto-generated from 4/27/2026. Do not edit manually. */} +{/* Auto-generated from 4/28/2026. Do not edit manually. */} # Benchmarks Performance comparison between Foundry releases. +
+ +

Highlights

+

Aggregated change per benchmark category (sum of wallclock time across all repositories).

+
+
+
Forge Test · 2 repos
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ↓17.7% +
+
+
+
Forge Fuzz Test · 5 repos
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ↓12.9% +
+
+
+
Forge Test (Isolated) · 2 repos
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ↓8.0% +
+
+
+
Forge Build (No Cache) · 5 repos
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ↑0.6% +
+
+
+
Forge Build (With Cache) · 5 repos
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ↑2.8% +
+
+
+
Forge Coverage · 3 repos
+
+
+
+
+
+
+
+
+
+
+
+
+
+ ↓3.3% +
+
+
+
+
Baseline