diff --git a/.github/workflows/version-benchmark-comment.yaml b/.github/workflows/version-benchmark-comment.yaml new file mode 100644 index 00000000..d7ba9de8 --- /dev/null +++ b/.github/workflows/version-benchmark-comment.yaml @@ -0,0 +1,90 @@ +name: Comment the impit version chart + +on: + workflow_run: + workflows: [Compare impit versions] + types: [completed] + +permissions: + contents: read + actions: read # to download the artifact from the triggering workflow_run + +jobs: + comment: + name: Comment on the pull request + # A workflow_run job always runs the copy of this file from the default branch, + # never from the triggering run's head ref - unlike a second job gated by + # `if: github.event_name == 'pull_request'` directly in version-benchmark.yaml, + # which pull_request runs the workflow file from the pull request's own branch. + # Since that workflow's `paths` filter includes its own file, a pull request + # editing it would otherwise control the steps that see this job's push token. + # See https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: master + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + - name: Download the chart + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: version-chart + path: downloaded + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Find the pull request + id: pr + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + number=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${HEAD_SHA}/pulls" \ + --jq '[.[] | select(.state == "open")][0].number') + echo "number=$number" >> "$GITHUB_OUTPUT" + + - name: Publish the chart and comment on the pull request + if: steps.pr.outputs.number + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + RUN_ID: ${{ github.event.workflow_run.id }} + run: | + test -f downloaded/version-chart.png || { echo 'the artifact did not unpack where expected'; exit 1; } + + versions=$(jq '.results | length' downloaded/results-node-versions.json) + + # The chart is published to its own branch instead of the pull request, + # so a raw.githubusercontent.com URL can be embedded in a comment - + # GitHub strips data: URIs from comment markdown, and the pull request + # itself should not carry a generated PNG as one of its files. + git config user.name 'apify-service-account' + git config user.email 'apify-service-account@users.noreply.github.com' + git fetch origin benchmarks/version-chart-assets || true + if git show-ref --verify --quiet refs/remotes/origin/benchmarks/version-chart-assets; then + git checkout -B benchmarks/version-chart-assets origin/benchmarks/version-chart-assets + else + git checkout --orphan benchmarks/version-chart-assets + git rm -rf . > /dev/null + fi + + filename="pr-${PR_NUMBER}.png" + mv downloaded/version-chart.png "$filename" + git add "$filename" + git commit -m "chore: publish version benchmark chart for #${PR_NUMBER}" + git push origin benchmarks/version-chart-assets + + url="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/benchmarks/version-chart-assets/${filename}?run=${RUN_ID}" + { + echo '### impit version comparison' + echo + echo "Median throughput of the last $versions npm and PyPI releases of impit." + echo + echo "![impit version comparison]($url)" + } > comment.md + + gh pr comment "$PR_NUMBER" --body-file comment.md --edit-last --create-if-none \ + || gh pr comment "$PR_NUMBER" --body-file comment.md diff --git a/.github/workflows/version-benchmark.yaml b/.github/workflows/version-benchmark.yaml new file mode 100644 index 00000000..b14c5a34 --- /dev/null +++ b/.github/workflows/version-benchmark.yaml @@ -0,0 +1,85 @@ +name: Compare impit versions + +on: + schedule: + # First of the month, offset from the client comparison so they don't compete for runners. + - cron: '0 5 1 * *' + workflow_dispatch: + inputs: + versions: + description: Versions per ecosystem to compare + default: '5' + requests: + description: Requests per run + default: '2000' + runs: + description: Runs per version, the median is reported + default: '11' + # Exercise the harness whenever it changes. version-benchmark-comment.yaml posts the + # resulting chart as a pull request comment - see that file for why it's a separate + # workflow rather than a second job here. + pull_request: + paths: + - benchmarks/** + - .github/workflows/version-benchmark.yaml + +permissions: + contents: read + +concurrency: + # Runs on different PRs shouldn't block each other, but a new push to the same PR + # makes its own previous run's comment stale, so that one is cancelled outright. + group: version-benchmark-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + VERSIONS: ${{ inputs.versions || '5' }} + REQUESTS: ${{ inputs.requests || '2000' }} + RUNS: ${{ inputs.runs || '11' }} + +jobs: + benchmark: + name: Measure + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + # No credentials in this job: it installs and runs several unpinned releases + # of impit, so it must have nothing worth stealing. + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Setup uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + + - name: Set up a Python venv with matplotlib + run: | + uv venv --seed --python 3.12 benchmarks/python/.venv + uv pip install --python benchmarks/python/.venv/bin/python matplotlib + + - name: Benchmark the npm releases + run: node benchmarks/node/bench-versions.mjs --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS" + + - name: Benchmark the PyPI releases + run: benchmarks/python/.venv/bin/python benchmarks/python/bench_versions.py --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS" + + - name: Render the chart + run: benchmarks/python/.venv/bin/python benchmarks/chart-versions.py + + - name: Upload the chart and raw measurements + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: version-chart + path: | + benchmarks/version-chart.png + benchmarks/results-node-versions.json + benchmarks/results-python-versions.json + benchmarks/results-python-async-versions.json diff --git a/.gitignore b/.gitignore index 003ba457..73c9b15b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,5 +17,6 @@ _build/ # Benchmarks /benchmarks/.cert /benchmarks/results-*.json +/benchmarks/version-chart.png /benchmarks/node/node_modules /benchmarks/node/package-lock.json diff --git a/benchmarks/README.md b/benchmarks/README.md index 04dd9d6d..b52a3b16 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -53,3 +53,25 @@ resets and take a `GOAWAY` mid-run. Right for a public origin, wrong for a bench Add an entry to `CLIENTS`: how to build it, how to issue one request, and how to count its profiles. `update-readme.mjs` takes care of ordering and the caption. + +## Comparing impit's own releases + +[`node/bench-versions.mjs`](node/bench-versions.mjs) and +[`python/bench_versions.py`](python/bench_versions.py) run the same measurement across the last N +published releases of impit itself (from npm and PyPI respectively), to track throughput across +versions rather than against other clients. The Python script measures both the sync `Client` and +the async `AsyncClient`, since `AsyncClient` bridges each call through an asyncio event loop the way +Node's Promise-returning `fetch()` does - separating the two shows how much of the npm/PyPI gap is +that bridge rather than the underlying Rust client: + +```bash +node node/bench-versions.mjs # writes results-node-versions.json +python/.venv/bin/python python/bench_versions.py # writes results-python-versions.json (sync) and + # results-python-async-versions.json (async) +python/.venv/bin/python chart-versions.py # writes version-chart.png +``` + +`--versions` picks how many releases to compare (default 5); `--requests`, `--runs` and `--warmup` +work as above. `chart-versions.py` needs `matplotlib` (`uv pip install matplotlib`); its output is a +CI artifact posted as a comment on the pull request that triggered it, not a file committed to the +repository. diff --git a/benchmarks/chart-versions.py b/benchmarks/chart-versions.py new file mode 100644 index 00000000..514f4f82 --- /dev/null +++ b/benchmarks/chart-versions.py @@ -0,0 +1,118 @@ +"""Renders the version-history throughput chart from bench-versions.mjs / bench_versions.py. + +Reads results-node-versions.json, results-python-versions.json (the sync `Client`) and +results-python-async-versions.json (the async `AsyncClient`), and plots median req/s +against release date, one line per client. Node's `fetch()` and Python's `AsyncClient` +both bridge each call through an event loop; the sync `Client` doesn't - splitting +Python's two clients out shows how much of the npm/PyPI gap that bridge accounts for. +The PNG is a CI artifact, not a committed file - see ../.github/workflows/version-benchmark.yaml. +""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime +from pathlib import Path + +import matplotlib + +matplotlib.use('Agg') + +import matplotlib.dates as mdates # noqa: E402 +import matplotlib.pyplot as plt # noqa: E402 +import matplotlib.ticker as mticker # noqa: E402 + +HERE = Path(__file__).resolve().parent + +INK = '#0b0b0b' +SECONDARY_INK = '#52514e' +MUTED = '#898781' +GRIDLINE = '#e1e0d9' +SURFACE = '#fcfcfb' +SERIES = { + ('node', None): {'label': 'npm (Node.js)', 'color': '#2a78d6'}, + ('python', 'sync'): {'label': 'PyPI (Python, sync)', 'color': '#eb6834'}, + ('python', 'async'): {'label': 'PyPI (Python, async)', 'color': '#1baf7a'}, +} + + +def load(path: Path) -> dict | None: + if not path.exists(): + return None + report = json.loads(path.read_text()) + if not report['results']: + return None + return report + + +def plot(reports: list[dict], out: Path) -> None: + fig, ax = plt.subplots(figsize=(8, 4.5), dpi=200, facecolor=SURFACE) + ax.set_facecolor(SURFACE) + + max_rate = max(point['rpsMedian'] for report in reports for point in report['results']) + + for report in reports: + series = SERIES[(report['ecosystem'], report.get('variant'))] + points = sorted(report['results'], key=lambda r: r['publishedAt']) + dates = [datetime.fromisoformat(p['publishedAt'].replace('Z', '+00:00')) for p in points] + rates = [p['rpsMedian'] for p in points] + + ax.plot(dates, rates, color=series['color'], linewidth=2, solid_capstyle='round', + marker='o', markersize=8, markerfacecolor=series['color'], + markeredgecolor=SURFACE, markeredgewidth=2, label=series['label']) + + for point, date, rate in zip(points, dates, rates): + ax.annotate(point['version'], (date, rate), textcoords='offset points', + xytext=(0, 10), ha='center', fontsize=8, color=MUTED) + + last_date, last_rate = dates[-1], rates[-1] + ax.annotate(f'{last_rate:,.0f} req/s', (last_date, last_rate), textcoords='offset points', + xytext=(10, -4), ha='left', fontsize=9, color=SECONDARY_INK, fontweight='bold') + + ax.set_title('impit throughput by release', fontsize=13, color=INK, loc='left', pad=14) + ax.set_ylabel('req/s (median)', fontsize=10, color=SECONDARY_INK) + ax.yaxis.set_major_formatter(mticker.FuncFormatter(lambda value, _: f'{value:,.0f}')) + # Headroom above the highest point so its label never collides with the legend. + ax.set_ylim(0, max_rate * 1.3) + + ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=3, maxticks=6)) + ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(ax.xaxis.get_major_locator())) + fig.autofmt_xdate(rotation=0, ha='center') + + ax.grid(axis='y', color=GRIDLINE, linewidth=1) + ax.set_axisbelow(True) + for spine in ('top', 'right', 'left'): + ax.spines[spine].set_visible(False) + ax.spines['bottom'].set_color('#c3c2b7') + ax.tick_params(axis='both', colors=MUTED, labelsize=9, length=0) + + legend = ax.legend(loc='upper left', frameon=False, fontsize=9, labelcolor=SECONDARY_INK) + legend.set_zorder(10) + + fig.tight_layout() + fig.savefig(out, facecolor=SURFACE) + plt.close(fig) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--node', type=Path, default=HERE / 'results-node-versions.json') + parser.add_argument('--python', type=Path, default=HERE / 'results-python-versions.json') + parser.add_argument('--python-async', type=Path, + default=HERE / 'results-python-async-versions.json') + parser.add_argument('--out', type=Path, default=HERE / 'version-chart.png') + args = parser.parse_args() + + reports = [report for report in (load(args.node), load(args.python), load(args.python_async)) + if report is not None] + if not reports: + raise SystemExit('neither results file has any results; nothing to chart') + + plot(reports, args.out) + print(f'wrote {args.out}') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/harness.mjs b/benchmarks/harness.mjs index 8df089d0..1d44ee3c 100644 --- a/benchmarks/harness.mjs +++ b/benchmarks/harness.mjs @@ -1,7 +1,10 @@ import { spawn } from 'node:child_process'; import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); export function parseArgs(argv, defaults) { const out = { ...defaults }; @@ -64,15 +67,21 @@ function run(command, args, options = {}) { }); } +/** `npm install`s `pkg@version` into a fresh temp dir and returns its path. */ +export async function installPackage(pkg, version) { + const dir = await mkdtemp(join(tmpdir(), 'impit-bench-install-')); + await run('npm', [ + 'install', `${pkg}@${version}`, + '--prefix', dir, + '--no-save', '--no-audit', '--no-fund', '--loglevel', 'error', + ]); + return dir; +} + /** Bytes a fresh `npm install ` drops on disk, transitive dependencies included. */ export async function installSize(pkg, version) { - const dir = await mkdtemp(join(tmpdir(), 'impit-bench-size-')); + const dir = await installPackage(pkg, version); try { - await run('npm', [ - 'install', `${pkg}@${version}`, - '--prefix', dir, - '--no-save', '--no-audit', '--no-fund', '--loglevel', 'error', - ]); return await treeSize(join(dir, 'node_modules')); } finally { await rm(dir, { recursive: true, force: true }); @@ -82,3 +91,20 @@ export async function installSize(pkg, version) { export function formatMB(bytes) { return `${(bytes / 1e6).toFixed(1)} MB`; } + +/** Spawns the shared HTTP/2 origin (../server.mjs) in its own process and resolves once it prints its URL. */ +export function spawnOrigin(bodyBytes) { + const child = spawn(process.execPath, [join(here, 'server.mjs')], { + env: { ...process.env, PORT: '0', BODY_BYTES: String(bodyBytes) }, + stdio: ['ignore', 'pipe', 'inherit'], + }); + return new Promise((resolve, reject) => { + let buffered = ''; + child.stdout.on('data', (chunk) => { + buffered += chunk; + const newline = buffered.indexOf('\n'); + if (newline !== -1) resolve({ child, url: buffered.slice(0, newline) }); + }); + child.on('exit', (code) => reject(new Error(`server exited with ${code} before listening`))); + }); +} diff --git a/benchmarks/node/bench-versions.mjs b/benchmarks/node/bench-versions.mjs new file mode 100644 index 00000000..e6cad65a --- /dev/null +++ b/benchmarks/node/bench-versions.mjs @@ -0,0 +1,101 @@ +/** + * Throughput of the last N published npm releases of impit, against each other. + * + * Unlike bench.mjs, which compares impit to other clients, this installs several + * versions of impit itself into isolated temp dirs and benchmarks them in turn. + */ +import { createRequire } from 'node:module'; +import { rm, writeFile } from 'node:fs/promises'; +import { arch, platform } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { installPackage, measure, parseArgs, spawnOrigin } from '../harness.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); + +async function recentVersions(pkg, count) { + const response = await fetch(`https://registry.npmjs.org/${pkg}`); + if (!response.ok) throw new Error(`npm registry returned ${response.status} for ${pkg}`); + const manifest = await response.json(); + return Object.keys(manifest.versions) + // Stable releases only, no `-beta.1`-style prereleases. + .filter((version) => /^\d+\.\d+\.\d+$/.test(version)) + .sort((a, b) => new Date(manifest.time[a]) - new Date(manifest.time[b])) + .slice(-count) + .map((version) => ({ version, publishedAt: manifest.time[version] })); +} + +async function benchmarkVersion({ version, publishedAt }, url, options) { + const dir = await installPackage('impit', version); + try { + const { Impit } = require(join(dir, 'node_modules', 'impit')); + const client = new Impit({ browser: 'chrome', ignoreTlsErrors: true }); + const request = async () => { + const response = await client.fetch(url); + return { body: await response.text(), alpn: response.headers.get('x-alpn') }; + }; + + const probe = await request(); + if (probe.body.length !== options.bodyBytes) { + throw new Error(`expected a ${options.bodyBytes} byte body, got ${probe.body.length}`); + } + + const timings = await measure(request, options); + return { version, publishedAt, alpn: probe.alpn, ...timings }; + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +const options = parseArgs(process.argv.slice(2), { + versions: 5, + requests: 2000, + runs: 11, + warmup: 200, + bodyBytes: 1024, + out: join(here, '..', 'results-node-versions.json'), +}); + +const versions = await recentVersions('impit', options.versions); +const { child, url } = await spawnOrigin(options.bodyBytes); + +const results = []; +const failures = []; +try { + for (const entry of versions) { + process.stderr.write(`impit@${entry.version}: `); + try { + results.push(await benchmarkVersion(entry, url, options)); + const { rpsMedian, alpn } = results.at(-1); + process.stderr.write(`${rpsMedian.toFixed(0)} req/s over ${alpn}\n`); + } catch (error) { + failures.push(`${entry.version}: ${error.message}`); + process.stderr.write(`FAILED (${error.message})\n`); + } + } +} finally { + child.kill(); +} + +await writeFile(options.out, `${JSON.stringify({ + ecosystem: 'node', + package: 'impit', + runtime: `Node.js ${process.version}`, + platform: `${platform()}-${arch()}`, + measuredAt: new Date().toISOString(), + options: { + requests: options.requests, + runs: options.runs, + warmup: options.warmup, + bodyBytes: options.bodyBytes, + }, + results, +}, null, 2)}\n`); + +process.stderr.write(`wrote ${options.out}\n`); +if (failures.length > 0) { + process.stderr.write(`${failures.length} version(s) failed:\n${failures.join('\n')}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/node/bench.mjs b/benchmarks/node/bench.mjs index 28ce1a99..a22a5c7a 100644 --- a/benchmarks/node/bench.mjs +++ b/benchmarks/node/bench.mjs @@ -1,10 +1,9 @@ -import { spawn } from 'node:child_process'; import { readFile, writeFile } from 'node:fs/promises'; import { arch, platform } from 'node:os'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { installSize, measure, parseArgs } from '../harness.mjs'; +import { installSize, measure, parseArgs, spawnOrigin } from '../harness.mjs'; const here = dirname(fileURLToPath(import.meta.url)); @@ -124,22 +123,6 @@ const CLIENTS = [ }, ]; -function startServer(bodyBytes) { - const child = spawn(process.execPath, [join(here, '..', 'server.mjs')], { - env: { ...process.env, PORT: '0', BODY_BYTES: String(bodyBytes) }, - stdio: ['ignore', 'pipe', 'inherit'], - }); - return new Promise((resolve, reject) => { - let buffered = ''; - child.stdout.on('data', (chunk) => { - buffered += chunk; - const newline = buffered.indexOf('\n'); - if (newline !== -1) resolve({ child, url: buffered.slice(0, newline) }); - }); - child.on('exit', (code) => reject(new Error(`server exited with ${code} before listening`))); - }); -} - const options = parseArgs(process.argv.slice(2), { requests: 2000, runs: 11, @@ -154,7 +137,7 @@ const selected = options.only : CLIENTS; if (selected.length === 0) throw new Error(`--only matched no client: ${options.only}`); -const { child, url } = await startServer(options.bodyBytes); +const { child, url } = await spawnOrigin(options.bodyBytes); const results = []; const failures = []; diff --git a/benchmarks/python/bench_versions.py b/benchmarks/python/bench_versions.py new file mode 100644 index 00000000..072ee1fc --- /dev/null +++ b/benchmarks/python/bench_versions.py @@ -0,0 +1,184 @@ +"""Throughput of the last N published PyPI releases of impit, against each other. + +Unlike bench.py, which compares impit to other clients, this pip installs several +versions of impit itself into isolated dirs and benchmarks them in turn. Both the +sync `Client` and the async `AsyncClient` are measured, so the version chart can +show the cost of bridging each call through an asyncio event loop - the same kind +of bridge the Node.js binding's Promise-returning `fetch()` pays on every call. See +../README.md for how the numbers are taken. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import platform +import re +import subprocess +import sys +import tempfile +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from bench import ARCH_ALIASES, measure, start_server + +HERE = Path(__file__).resolve().parent + + +def recent_versions(pkg: str, count: int) -> list[dict[str, str]]: + with urllib.request.urlopen(f'https://pypi.org/pypi/{pkg}/json') as response: # noqa: S310 + manifest = json.load(response) + + entries = [] + for version, files in manifest['releases'].items(): + # Stable releases only, no `0.9.0rc1`-style prereleases. + if not re.fullmatch(r'\d+\.\d+\.\d+', version): + continue + upload_times = [f['upload_time_iso_8601'] for f in files if not f.get('yanked')] + if upload_times: + entries.append((version, min(upload_times))) + + entries.sort(key=lambda entry: entry[1]) + return [{'version': version, 'publishedAt': published} for version, published in entries[-count:]] + + +def _purge_impit_modules() -> None: + for name in [name for name in sys.modules if name == 'impit' or name.startswith('impit.')]: + del sys.modules[name] + + +async def measure_async(request, *, requests: int, runs: int, warmup: int) -> dict[str, float]: + """Async twin of bench.measure: same batches-of-sequential-awaits algorithm.""" + for _ in range(warmup): + await request() + + rates = [] + for _ in range(runs): + started = time.perf_counter() + for _ in range(requests): + await request() + rates.append(requests / (time.perf_counter() - started)) + rates.sort() + return {'rps': rates[-1], 'rpsMedian': rates[len(rates) // 2], 'rpsWorst': rates[0]} + + +def benchmark_version(entry: dict[str, str], url: str, *, requests: int, runs: int, warmup: int, + body_bytes: int) -> dict[str, dict[str, Any]]: + """Measures both impit.Client (sync) and impit.AsyncClient for one installed version.""" + version = entry['version'] + with tempfile.TemporaryDirectory() as target: + subprocess.run( + [sys.executable, '-m', 'pip', 'install', '--quiet', '--disable-pip-version-check', + '--target', target, f'impit=={version}'], + check=True, + ) + + sys.path.insert(0, target) + try: + _purge_impit_modules() + import impit # noqa: PLC0415 + + client = impit.Client(browser='chrome', verify=False) + + def request() -> tuple[bytes, str | None]: + response = client.get(url) + return response.content, response.headers.get('x-alpn') + + body, alpn = request() + if len(body) != body_bytes: + raise RuntimeError(f'expected a {body_bytes} byte body, got {len(body)}') + + sync_timings = measure(request, requests=requests, runs=runs, warmup=warmup) + + async def run_async_client() -> dict[str, Any]: + async_client = impit.AsyncClient(browser='chrome', verify=False) + + async def async_request() -> tuple[bytes, str | None]: + response = await async_client.get(url) + return response.content, response.headers.get('x-alpn') + + async_body, async_alpn = await async_request() + if len(async_body) != body_bytes: + raise RuntimeError(f'expected a {body_bytes} byte body, got {len(async_body)}') + + async_timings = await measure_async(async_request, requests=requests, runs=runs, warmup=warmup) + return {**entry, 'alpn': async_alpn, **async_timings} + + return { + 'sync': {**entry, 'alpn': alpn, **sync_timings}, + 'async': asyncio.run(run_async_client()), + } + finally: + _purge_impit_modules() + sys.path.remove(target) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--versions', type=int, default=5) + parser.add_argument('--requests', type=int, default=2000) + parser.add_argument('--runs', type=int, default=11) + parser.add_argument('--warmup', type=int, default=200) + parser.add_argument('--body-bytes', type=int, default=1024) + parser.add_argument('--out', type=Path, default=HERE.parent / 'results-python-versions.json') + parser.add_argument('--out-async', type=Path, + default=HERE.parent / 'results-python-async-versions.json') + args = parser.parse_args() + + versions = recent_versions('impit', args.versions) + process, url = start_server(args.body_bytes) + + sync_results: list[dict[str, Any]] = [] + async_results: list[dict[str, Any]] = [] + failures: list[str] = [] + try: + for entry in versions: + print(f'impit=={entry["version"]}: ', end='', flush=True, file=sys.stderr) + try: + timings = benchmark_version( + entry, url, requests=args.requests, runs=args.runs, warmup=args.warmup, + body_bytes=args.body_bytes, + ) + sync_results.append(timings['sync']) + async_results.append(timings['async']) + print(f'{timings["sync"]["rpsMedian"]:.0f} req/s sync, ' + f'{timings["async"]["rpsMedian"]:.0f} req/s async', file=sys.stderr) + except Exception as exc: # noqa: BLE001 + failures.append(f'{entry["version"]}: {exc}') + print(f'FAILED ({exc})', file=sys.stderr) + finally: + process.kill() + + def write_report(out: Path, variant: str, results: list[dict[str, Any]]) -> None: + out.write_text(json.dumps({ + 'ecosystem': 'python', + 'variant': variant, + 'package': 'impit', + 'runtime': f'CPython {platform.python_version()}', + 'platform': f'{sys.platform}-{ARCH_ALIASES.get(platform.machine(), platform.machine())}', + 'measuredAt': datetime.now(timezone.utc).isoformat(), + 'options': { + 'requests': args.requests, + 'runs': args.runs, + 'warmup': args.warmup, + 'bodyBytes': args.body_bytes, + }, + 'results': results, + }, indent=2) + '\n') + print(f'wrote {out}', file=sys.stderr) + + write_report(args.out, 'sync', sync_results) + write_report(args.out_async, 'async', async_results) + + if failures: + print(f'{len(failures)} version(s) failed:', *failures, sep='\n', file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main())