From 20f728cbc9fb89d32c37e467ee145de0fce8701e Mon Sep 17 00:00:00 2001 From: Dennis Jeong <3719829+w0nche0l@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:26:39 -0400 Subject: [PATCH 1/3] perf(sdk): characterize and reduce import memory (ECO-2747) Co-authored-by: Cursor --- .speakeasy/gen.lock | 29 +- .speakeasy/gen.yaml | 2 +- .speakeasy/workflow.lock | 2 +- benchmarks/characterize/README.md | 29 ++ benchmarks/characterize/RESULTS.md | 46 +++ .../characterize/entries/root-import.ts | 14 + .../characterize/entries/sse-fragmentation.ts | 69 +++++ benchmarks/characterize/entries/transport.ts | 66 +++++ benchmarks/characterize/entries/validation.ts | 35 +++ benchmarks/characterize/index.mjs | 213 ++++++++++++++ benchmarks/characterize/runner.mjs | 61 ++++ jsr.json | 2 +- package.json | 17 +- pnpm-lock.yaml | 3 + src/lib/base64-codec.ts | 23 ++ src/lib/base64.ts | 27 +- src/lib/config.ts | 4 +- src/lib/encodings.ts | 2 +- src/lib/sdks.ts | 2 +- tests/unit/base64-codec.test.ts | 36 +++ tests/unit/event-streams.test.ts | 265 ++++++++++++++++++ 21 files changed, 895 insertions(+), 52 deletions(-) create mode 100644 benchmarks/characterize/README.md create mode 100644 benchmarks/characterize/RESULTS.md create mode 100644 benchmarks/characterize/entries/root-import.ts create mode 100644 benchmarks/characterize/entries/sse-fragmentation.ts create mode 100644 benchmarks/characterize/entries/transport.ts create mode 100644 benchmarks/characterize/entries/validation.ts create mode 100644 benchmarks/characterize/index.mjs create mode 100644 benchmarks/characterize/runner.mjs create mode 100644 src/lib/base64-codec.ts create mode 100644 tests/unit/base64-codec.test.ts create mode 100644 tests/unit/event-streams.test.ts diff --git a/.speakeasy/gen.lock b/.speakeasy/gen.lock index f76cfdc8d..31096de94 100644 --- a/.speakeasy/gen.lock +++ b/.speakeasy/gen.lock @@ -5,15 +5,15 @@ management: docVersion: 1.0.0 speakeasyVersion: 1.787.0 generationVersion: 2.914.0 - releaseVersion: 1.2.5 - configChecksum: 92f44d5cd212efdb767342ff76962bd6 + releaseVersion: 1.2.6 + configChecksum: 5bbe4aa9e30b0165d6e6567fea50b5d9 repoURL: https://github.com/OpenRouterTeam/typescript-sdk.git installationURL: https://github.com/OpenRouterTeam/typescript-sdk published: true persistentEdits: - generation_id: f55ce102-f99e-4b95-95fe-fc5a307169c6 - pristine_commit_hash: 5cada18aa7c5670817e1eaf2108927122b4492a8 - pristine_tree_hash: e52e77fb09a094942f8e01dd300ea7d5129e8845 + generation_id: 68bae83e-9b68-4ce0-8ad6-a6c6068ccca0 + pristine_commit_hash: 8dcd85ae1a6350ccf72b5dd7ec3b2cac77a41fda + pristine_tree_hash: 837dd36abd8e397b97ac852545e0e1748dfdfd45 features: typescript: acceptHeaders: 2.81.2 @@ -12741,12 +12741,12 @@ trackedFiles: pristine_git_object: 410efafd6a7f50d91ccb87131fedbe0c3d47e15a jsr.json: id: 7f6ab7767282 - last_write_checksum: sha1:4487b371ab1978bd91d9e6c7321afcd2f549bda4 - pristine_git_object: 94a3242648365f303b2bf03cfe6f79e246465536 + last_write_checksum: sha1:e363c1cc9b383dfc3a887bb91b8f100d8c5a13e0 + pristine_git_object: bbd839f7fa3076463efedccc5456daf7bfba2420 package.json: id: 7030d0b2f71b - last_write_checksum: sha1:df3f8fe2c207300a30a0a9344d9f2d288178e78c - pristine_git_object: f7255ec02372915d4db5057979796cd4ce6729b1 + last_write_checksum: sha1:f93265930913ef79f0badce5a5962821e98c7aab + pristine_git_object: e442edfb393489b5692228a32bf8cb9d4fcc1dd6 src/core.ts: id: f431fdbcd144 last_write_checksum: sha1:5aa66b0b6a5964f3eea7f3098c2eb3c0ee9c0131 @@ -13145,12 +13145,12 @@ trackedFiles: pristine_git_object: bb0c15148be25feb935e2d50c35c072b516cbcb5 src/lib/base64.ts: id: "598522066688" - last_write_checksum: sha1:26b234d589cc15afab76ac7aaba1dd1bd4b4a84c + last_write_checksum: sha1:5e8eb1f050e47f489cc4698107bbfe3e26c43f3d pristine_git_object: a187e58707bdb726ca2aff74941efe7493422d4e src/lib/config.ts: id: 320761608fb3 - last_write_checksum: sha1:17c88f470a6039ad7159eb5ef120fe6295829b8f - pristine_git_object: e3446dbd670ebb474e18d2aee29dc75df62fa034 + last_write_checksum: sha1:db03c1bac38d342aa0e1d3d3e4ee2391543489c3 + pristine_git_object: 6b5918f2c141a9872c49b357751cfc44cd96dd89 src/lib/dlv.ts: id: b1988214835a last_write_checksum: sha1:eaac763b22717206a6199104e0403ed17a4e2711 @@ -13158,7 +13158,7 @@ trackedFiles: deleted: true src/lib/encodings.ts: id: 3bd8ead98afd - last_write_checksum: sha1:a74725064d06b6994b95873c037975d2f2e63467 + last_write_checksum: sha1:80c84d1404e35b723e20648398f3f13e3e1d72dc pristine_git_object: 49f15904923362434dfcdd02476c0487210d2f1a src/lib/env.ts: id: c52972a3b198 @@ -13203,7 +13203,7 @@ trackedFiles: pristine_git_object: 35b0fb3e60638aa3d9ed11c19da774255cb05052 src/lib/sdks.ts: id: 8a6d91f1218d - last_write_checksum: sha1:2ad4fe931d24de5dd737424cc82cd8ddab1c9d66 + last_write_checksum: sha1:c847a6f3659887742f372368f568d873af59cfc4 pristine_git_object: fed3c8256d26f6a8f119255cd231d4af2c16cf77 src/lib/security.ts: id: 0502afa7922e @@ -17936,4 +17936,3 @@ examples: "500": application/json: {"error": {"code": 500, "message": "Internal Server Error"}} examplesVersion: 1.0.2 -releaseNotes: "## Typescript SDK Changes:\n* `openrouter.analytics.getUserActivity()`: \n * `request` **Changed**\n * `response.data[].workspaceId` **Added**\n* `openrouter.generations.getGeneration()`: `response.data.workspaceId` **Added**\n" diff --git a/.speakeasy/gen.yaml b/.speakeasy/gen.yaml index a0dda55e6..4f7f1f612 100644 --- a/.speakeasy/gen.yaml +++ b/.speakeasy/gen.yaml @@ -37,7 +37,7 @@ generation: documentation: mintlify preApplyUnionDiscriminators: true typescript: - version: 1.2.5 + version: 1.2.6 acceptHeaderEnum: false additionalDependencies: dependencies: diff --git a/.speakeasy/workflow.lock b/.speakeasy/workflow.lock index 22785161c..2e87a8a50 100644 --- a/.speakeasy/workflow.lock +++ b/.speakeasy/workflow.lock @@ -14,7 +14,7 @@ targets: sourceRevisionDigest: sha256:673a878a4627855ecde98dd43bcfd5d5fe70dba8fba956cfed66be2d033368d6 sourceBlobDigest: sha256:ad52a5d986bf19682306eff8bb475c11074dc16593c4e4937117d9d72e07f297 codeSamplesNamespace: open-router-chat-completions-api-typescript-code-samples - codeSamplesRevisionDigest: sha256:03ded13c05d37b77c8555988abc536a03fbc521a4ea9d9bc1a983f595d14b19f + codeSamplesRevisionDigest: sha256:75bf5a2ea7bbb8f1b7ef892dff80f423feea021e70e825d3713abccb50822e40 workflow: workflowVersion: 1.0.0 speakeasyVersion: 1.787.0 diff --git a/benchmarks/characterize/README.md b/benchmarks/characterize/README.md new file mode 100644 index 000000000..9fa4198bf --- /dev/null +++ b/benchmarks/characterize/README.md @@ -0,0 +1,29 @@ +# SDK bundle and memory characterization + +This harness measures four isolated SDK paths without network access: + +- importing and constructing the root SDK; +- creating authenticated transport requests; +- validating a representative chat request; +- parsing 1,000 deterministically fragmented SSE events. + +Run it from the repository root: + +```sh +pnpm benchmark:characterize --runs=5 +pnpm benchmark:characterize --runs=7 --json +``` + +Each case is bundled independently with esbuild. The harness explicitly enables bundling, +minification, and tree shaking and reports the resulting raw, gzip level 9, and Brotli quality 11 +byte counts. Dependencies are included in each bundle. + +Memory measurements run each bundle in multiple fresh `node --expose-gc` processes and report the +median. `import heap` and `import RSS` are retained deltas after forced garbage collection. +`scenario peak` is the largest sampled heap increase while the deterministic workload runs, and +`scenario retained` is the post-workload heap delta after forced garbage collection. The absolute +post-import heap and RSS values are also present in the JSON report. + +Use the same machine, Node version, run count, and source revision for before/after comparisons. +Memory figures can vary across operating systems and Node/V8 releases; bundle byte counts are the +more stable cross-machine signal. diff --git a/benchmarks/characterize/RESULTS.md b/benchmarks/characterize/RESULTS.md new file mode 100644 index 000000000..984c680f0 --- /dev/null +++ b/benchmarks/characterize/RESULTS.md @@ -0,0 +1,46 @@ +# ECO-2747 SDK characterization + +Measured on macOS arm64 with Node 24.18.0, esbuild 0.25.11, and five fresh processes per +case. For a controlled comparison, the baseline restored the three optimized generated +files from `cea17fa4` while holding the harness and package metadata constant. Both +measurements used: + +```sh +node benchmarks/characterize/index.mjs --runs=5 --json +``` + +Every bundle used ESM output, bundled dependencies, minification, and explicit +`treeShaking: true`, targeting Node 22. + +## Before and after + +- Root import bundle: raw 661,969 → 661,787 bytes; gzip 125,630 → 125,589; Brotli + 97,006 → 96,995. Median retained import heap delta was 118,906,904 → 118,955,320 + bytes and RSS delta was 229,212,160 → 230,113,280 bytes. +- Transport bundle: raw 118,279 → 118,098 bytes; gzip 32,898 → 32,878; Brotli + 28,565 → 28,524. Median retained import heap delta was 973,592 → 965,016 bytes and + RSS delta was 4,505,600 → 4,292,608 bytes. +- Validation bundle: raw 135,048 → 135,048 bytes; gzip 36,803 → 36,803; Brotli + 32,110 → 32,110. Median retained import heap delta was 13,414,304 → 13,410,560 + bytes and RSS delta was 30,736,384 → 29,835,264 bytes. +- SSE fragmentation bundle: raw 2,750 → 2,750 bytes; gzip 1,386 → 1,386; Brotli + 1,251 → 1,251. Median retained import heap delta was 373,760 → 373,760 bytes and + RSS delta was 2,244,608 → 2,310,144 bytes. + +The deterministic outputs were identical before and after: root methods remained +functions; transport produced the same URL, headers, authorization, cookie, and body; +validation produced the same outbound JSON shape; and SSE parsing produced 1,000 events +and 23,890 characters. Small memory differences outside transport are measurement noise. + +## Generator constraint + +The pinned Speakeasy CLI 1.787.0 accepts `useIndexModules: false`, but this specification +cannot currently regenerate with it. The generated direct imports collide with local +operation wrapper names, for example `ListScimGroupsResponse`, and the generator's compile +step fails with `TS2440`, `TS2395`, `TS2448`, and `TS2454`. Keeping that output would +require generated-code alias patches or public model renames, neither of which is a safe +or generator-owned SDK optimization. + +The retained optimization splits codec-only base64 helpers from the Zod adapters and +redirects transport imports to the codec module. Speakeasy persistent edits preserved +all three generated-file import changes during a successful pinned regeneration. diff --git a/benchmarks/characterize/entries/root-import.ts b/benchmarks/characterize/entries/root-import.ts new file mode 100644 index 000000000..a1b6926f8 --- /dev/null +++ b/benchmarks/characterize/entries/root-import.ts @@ -0,0 +1,14 @@ +import { OpenRouter } from '../../../src/index.ts'; + +export function run({ sample }: { sample: () => void }) { + const sdk = new OpenRouter({ + apiKey: 'benchmark-key', + serverURL: 'https://benchmark.invalid/api/v1', + }); + sample(); + + return { + callModel: typeof sdk.callModel, + chatSend: typeof sdk.chat.send, + }; +} diff --git a/benchmarks/characterize/entries/sse-fragmentation.ts b/benchmarks/characterize/entries/sse-fragmentation.ts new file mode 100644 index 000000000..2971f0a71 --- /dev/null +++ b/benchmarks/characterize/entries/sse-fragmentation.ts @@ -0,0 +1,69 @@ +import { EventStream } from '../../../src/lib/event-streams.ts'; + +const eventCount = 1_000; +const widths = [ + 1, + 2, + 3, + 5, + 8, + 13, +]; + +function fragmentedSource(bytes: Uint8Array): ReadableStream { + let offset = 0; + let chunkIndex = 0; + return new ReadableStream( + { + pull(controller) { + if (offset === bytes.length) { + controller.close(); + return; + } + const end = Math.min(bytes.length, offset + widths[chunkIndex % widths.length]); + controller.enqueue(bytes.slice(offset, end)); + offset = end; + chunkIndex++; + }, + }, + { + highWaterMark: 0, + }, + ); +} + +export async function run({ sample }: { sample: () => void }) { + const payload = Array.from( + { + length: eventCount, + }, + (_, index) => `data: event-${index} 👋\r\ndata: second line\r\n\r\n`, + ).join(''); + const source = fragmentedSource(new TextEncoder().encode(payload)); + const stream = new EventStream(source, (message) => ({ + done: false, + value: message.data ?? '', + })); + let count = 0; + let characterCount = 0; + + for await (const event of stream) { + const expected = `event-${count} 👋\nsecond line`; + if (event !== expected) { + throw new Error(`event ${count} changed: ${JSON.stringify(event)}`); + } + characterCount += event.length; + count++; + if (count % 25 === 0) { + sample(); + } + } + + if (count !== eventCount) { + throw new Error(`expected ${eventCount} events, received ${count}`); + } + return { + characterCount, + eventCount: count, + }; +} diff --git a/benchmarks/characterize/entries/transport.ts b/benchmarks/characterize/entries/transport.ts new file mode 100644 index 000000000..6594d0bff --- /dev/null +++ b/benchmarks/characterize/entries/transport.ts @@ -0,0 +1,66 @@ +import { ClientSDK } from '../../../src/lib/sdks.ts'; + +export async function run({ sample }: { sample: () => void }) { + const client = new ClientSDK({ + serverURL: 'https://benchmark.invalid/api/v1', + }); + const context = {}; + let lastRequest: Request | undefined; + + for (let iteration = 0; iteration < 2_000; iteration++) { + const result = client._createRequest( + context, + { + body: JSON.stringify({ + iteration, + prompt: 'deterministic benchmark', + }), + headers: { + 'content-type': 'application/json', + }, + method: 'POST', + path: 'chat/completions', + security: { + basic: { + password: 'password', + username: 'benchmark', + }, + cookies: { + session: 'session-id', + }, + headers: { + 'x-api-key': 'benchmark-key', + }, + oauth2: { + type: 'none', + }, + queryParams: { + source: 'characterization', + }, + }, + }, + { + headers: { + 'x-request-id': `request-${iteration}`, + }, + }, + ); + if (!result.ok) { + throw result.error; + } + lastRequest = result.value; + if (iteration % 50 === 0) { + sample(); + } + } + + if (!lastRequest) { + throw new Error('transport scenario produced no request'); + } + return { + authorization: lastRequest.headers.get('authorization'), + body: await lastRequest.text(), + cookie: lastRequest.headers.get('cookie'), + url: lastRequest.url, + }; +} diff --git a/benchmarks/characterize/entries/validation.ts b/benchmarks/characterize/entries/validation.ts new file mode 100644 index 000000000..cc4f6059c --- /dev/null +++ b/benchmarks/characterize/entries/validation.ts @@ -0,0 +1,35 @@ +import { SendChatCompletionRequestRequest$outboundSchema } from '../../../src/models/operations/sendchatcompletionrequest.ts'; + +const request = { + appCategories: 'benchmark,memory', + appTitle: 'SDK characterization', + chatRequest: { + maxCompletionTokens: 64, + messages: [ + { + content: 'Return a deterministic short response.', + role: 'user', + }, + ], + model: 'openai/gpt-5', + stream: false, + temperature: 0, + }, + httpReferer: 'https://benchmark.invalid', +}; + +export function run({ sample }: { sample: () => void }) { + let output: ReturnType | undefined; + + for (let iteration = 0; iteration < 1_000; iteration++) { + output = SendChatCompletionRequestRequest$outboundSchema.parse(request); + if (iteration % 25 === 0) { + sample(); + } + } + + if (!output) { + throw new Error('validation scenario produced no output'); + } + return output; +} diff --git a/benchmarks/characterize/index.mjs b/benchmarks/characterize/index.mjs new file mode 100644 index 000000000..9297f65a1 --- /dev/null +++ b/benchmarks/characterize/index.mjs @@ -0,0 +1,213 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { brotliCompressSync, constants, gzipSync } from 'node:zlib'; + +import { build, version as esbuildVersion } from 'esbuild'; + +const execFileAsync = promisify(execFile); +const benchmarkDirectory = fileURLToPath(new URL('.', import.meta.url)); +const runnerPath = join(benchmarkDirectory, 'runner.mjs'); +const cases = [ + [ + 'root-import', + 'entries/root-import.ts', + ], + [ + 'transport', + 'entries/transport.ts', + ], + [ + 'validation', + 'entries/validation.ts', + ], + [ + 'sse-fragmentation', + 'entries/sse-fragmentation.ts', + ], +]; + +function parseRuns(args) { + const argument = args.find((value) => value.startsWith('--runs=')); + const value = Number(argument?.slice('--runs='.length) ?? process.env.BENCHMARK_RUNS ?? 5); + if (!Number.isInteger(value) || value < 1) { + throw new Error(`--runs must be a positive integer, received ${value}`); + } + return value; +} + +function median(values) { + const sorted = [ + ...values, + ].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + if (sorted.length % 2 === 1) { + return sorted[middle]; + } + return Math.round((sorted[middle - 1] + sorted[middle]) / 2); +} + +function formatBytes(value) { + const sign = value < 0 ? '-' : ''; + const absolute = Math.abs(value); + if (absolute < 1024) { + return `${value} B`; + } + return `${sign}${(absolute / 1024).toFixed(1)} KiB`; +} + +function summarizeMemory(samples) { + const fields = [ + 'afterImportHeapBytes', + 'afterImportRssBytes', + 'importHeapDeltaBytes', + 'importRssDeltaBytes', + 'scenarioPeakHeapDeltaBytes', + 'scenarioRetainedHeapDeltaBytes', + ]; + return Object.fromEntries( + fields.map((field) => [ + field, + median(samples.map((sample) => sample[field])), + ]), + ); +} + +async function buildBundle(name, relativeEntry, outputDirectory) { + const result = await build({ + absWorkingDir: benchmarkDirectory, + bundle: true, + entryPoints: [ + relativeEntry, + ], + format: 'esm', + legalComments: 'none', + minify: true, + outfile: join(outputDirectory, `${name}.mjs`), + packages: 'bundle', + platform: 'node', + sourcemap: false, + target: 'node22', + treeShaking: true, + write: false, + }); + const output = result.outputFiles?.find((file) => basename(file.path) === `${name}.mjs`); + if (!output) { + throw new Error(`esbuild did not produce ${name}.mjs`); + } + + const artifactPath = join(outputDirectory, `${name}.mjs`); + await writeFile(artifactPath, output.contents); + return { + artifactPath, + sizes: { + rawBytes: output.contents.byteLength, + gzipBytes: gzipSync(output.contents, { + level: 9, + }).byteLength, + brotliBytes: brotliCompressSync(output.contents, { + params: { + [constants.BROTLI_PARAM_QUALITY]: 11, + }, + }).byteLength, + }, + }; +} + +async function measureBundle(artifactPath, runs) { + const samples = []; + for (let run = 0; run < runs; run++) { + const { stdout } = await execFileAsync( + process.execPath, + [ + '--expose-gc', + runnerPath, + artifactPath, + ], + { + maxBuffer: 1024 * 1024, + }, + ); + samples.push(JSON.parse(stdout)); + } + + const results = new Set(samples.map((sample) => JSON.stringify(sample.result))); + if (results.size !== 1) { + throw new Error(`scenario output changed between fresh processes for ${artifactPath}`); + } + + return { + memory: summarizeMemory(samples), + result: samples[0].result, + }; +} + +async function main() { + const runs = parseRuns(process.argv.slice(2)); + const outputDirectory = await mkdtemp(join(tmpdir(), 'openrouter-sdk-characterize-')); + const measurements = []; + + try { + for (const [name, relativeEntry] of cases) { + const bundle = await buildBundle(name, relativeEntry, outputDirectory); + const runtime = await measureBundle(bundle.artifactPath, runs); + measurements.push({ + name, + bundle: bundle.sizes, + ...runtime, + }); + } + } finally { + await rm(outputDirectory, { + force: true, + recursive: true, + }); + } + + const report = { + metadata: { + architecture: process.arch, + bundle: { + format: 'esm', + minify: true, + packages: 'bundle', + platform: 'node', + target: 'node22', + treeShaking: true, + }, + esbuild: esbuildVersion, + node: process.version, + platform: process.platform, + runs, + }, + measurements, + }; + + if (process.argv.includes('--json')) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + return; + } + + console.log( + `Node ${report.metadata.node}; esbuild ${esbuildVersion}; ${runs} fresh processes per case`, + ); + console.log('All bundles: minified, treeShaking=true, dependencies bundled'); + console.table( + measurements.map((measurement) => ({ + case: measurement.name, + raw: formatBytes(measurement.bundle.rawBytes), + gzip: formatBytes(measurement.bundle.gzipBytes), + brotli: formatBytes(measurement.bundle.brotliBytes), + 'import heap': formatBytes(measurement.memory.importHeapDeltaBytes), + 'import RSS': formatBytes(measurement.memory.importRssDeltaBytes), + 'scenario peak': formatBytes(measurement.memory.scenarioPeakHeapDeltaBytes), + 'scenario retained': formatBytes(measurement.memory.scenarioRetainedHeapDeltaBytes), + })), + ); + console.log(JSON.stringify(report, null, 2)); +} + +await main(); diff --git a/benchmarks/characterize/runner.mjs b/benchmarks/characterize/runner.mjs new file mode 100644 index 000000000..db57273db --- /dev/null +++ b/benchmarks/characterize/runner.mjs @@ -0,0 +1,61 @@ +import { pathToFileURL } from 'node:url'; + +const artifactPath = process.argv[2]; +if (!artifactPath) { + throw new Error('expected a bundle path'); +} +if (typeof global.gc !== 'function') { + throw new Error('runner requires node --expose-gc'); +} + +async function collectGarbage() { + for (let iteration = 0; iteration < 3; iteration++) { + global.gc(); + await new Promise((resolve) => setImmediate(resolve)); + } +} + +function snapshot() { + const memory = process.memoryUsage(); + return { + heapUsed: memory.heapUsed, + rss: memory.rss, + }; +} + +await collectGarbage(); +const beforeImport = snapshot(); +const scenario = await import(pathToFileURL(artifactPath).href); +await collectGarbage(); +const afterImport = snapshot(); +let peak = afterImport; + +function sample() { + const current = snapshot(); + peak = { + heapUsed: Math.max(peak.heapUsed, current.heapUsed), + rss: Math.max(peak.rss, current.rss), + }; +} + +if (typeof scenario.run !== 'function') { + throw new Error(`${artifactPath} must export a run function`); +} +const result = await scenario.run({ + sample, +}); +sample(); +await collectGarbage(); +const afterScenario = snapshot(); + +process.stdout.write( + `${JSON.stringify({ + afterImportHeapBytes: afterImport.heapUsed, + afterImportRssBytes: afterImport.rss, + importHeapDeltaBytes: afterImport.heapUsed - beforeImport.heapUsed, + importRssDeltaBytes: afterImport.rss - beforeImport.rss, + result, + scenarioPeakHeapDeltaBytes: peak.heapUsed - afterImport.heapUsed, + scenarioRetainedHeapDeltaBytes: afterScenario.heapUsed - afterImport.heapUsed, + })}\n`, +); diff --git a/jsr.json b/jsr.json index 94a324264..bbd839f7f 100644 --- a/jsr.json +++ b/jsr.json @@ -2,7 +2,7 @@ { "name": "@openrouter/sdk", - "version": "1.2.5", + "version": "1.2.6", "exports": { ".": "./src/index.ts", "./models/errors": "./src/models/errors/index.ts", diff --git a/package.json b/package.json index 999120018..5002a53ba 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openrouter/sdk", - "version": "1.2.5", + "version": "1.2.6", "author": "OpenRouter", "description": "The OpenRouter TypeScript SDK is a type-safe toolkit for building AI applications with access to 400+ language models through a unified API.", "keywords": [ @@ -70,24 +70,25 @@ "url": "https://github.com/OpenRouterTeam/typescript-sdk.git" }, "scripts": { + "benchmark:characterize": "node benchmarks/characterize/index.mjs", "lint": "eslint --cache --max-warnings=0 src", "build": "tsc", "prepublishOnly": "npm run build", + "compile": "tsc", "postinstall": "node scripts/check-types.js || true", - "prepare": "npm run build", - "test": "vitest --run --project unit", - "test:watch": "vitest --watch --project unit", + "test:e2e": "vitest --run --project e2e", "typecheck": "tsc --noEmit", "typecheck:transit": "exit 0", - "compile": "tsc", - "test:e2e": "vitest --run --project e2e", - "test:transit": "exit 0" + "prepare": "npm run build", + "test": "vitest --run --project unit", + "test:transit": "exit 0", + "test:watch": "vitest --watch --project unit" }, - "peerDependencies": {}, "devDependencies": { "@eslint/js": "^9.26.0", "@types/node": "^22.13.12", "dotenv": "^16.4.7", + "esbuild": "0.25.11", "eslint": "^9.26.0", "globals": "^15.14.0", "typescript": "~5.8.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6fac9ead9..92cf5313d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,6 +21,9 @@ importers: dotenv: specifier: ^16.4.7 version: 16.6.1 + esbuild: + specifier: 0.25.11 + version: 0.25.11 eslint: specifier: ^9.26.0 version: 9.38.0 diff --git a/src/lib/base64-codec.ts b/src/lib/base64-codec.ts new file mode 100644 index 000000000..0790abcf3 --- /dev/null +++ b/src/lib/base64-codec.ts @@ -0,0 +1,23 @@ +export function bytesToBase64(u8arr: Uint8Array): string { + return btoa(String.fromCodePoint(...u8arr)); +} + +export function bytesFromBase64(encoded: string): Uint8Array { + return Uint8Array.from(atob(encoded), (character) => character.charCodeAt(0)); +} + +export function stringToBytes(value: string): Uint8Array { + return new TextEncoder().encode(value); +} + +export function stringFromBytes(value: Uint8Array): string { + return new TextDecoder().decode(value); +} + +export function stringToBase64(value: string): string { + return bytesToBase64(stringToBytes(value)); +} + +export function stringFromBase64(value: string): string { + return stringFromBytes(bytesFromBase64(value)); +} diff --git a/src/lib/base64.ts b/src/lib/base64.ts index a187e5870..c6bc74f00 100644 --- a/src/lib/base64.ts +++ b/src/lib/base64.ts @@ -5,29 +5,12 @@ import * as z from "zod/v4"; -export function bytesToBase64(u8arr: Uint8Array): string { - return btoa(String.fromCodePoint(...u8arr)); -} +import { + bytesFromBase64, + stringToBytes, +} from "./base64-codec.js"; -export function bytesFromBase64(encoded: string): Uint8Array { - return Uint8Array.from(atob(encoded), (c) => c.charCodeAt(0)); -} - -export function stringToBytes(str: string): Uint8Array { - return new TextEncoder().encode(str); -} - -export function stringFromBytes(u8arr: Uint8Array): string { - return new TextDecoder().decode(u8arr); -} - -export function stringToBase64(str: string): string { - return bytesToBase64(stringToBytes(str)); -} - -export function stringFromBase64(b64str: string): string { - return stringFromBytes(bytesFromBase64(b64str)); -} +export * from "./base64-codec.js"; export const zodOutbound = z.custom(x => x instanceof Uint8Array) .or(z.string().transform(stringToBytes)); diff --git a/src/lib/config.ts b/src/lib/config.ts index e3446dbd6..6b5918f2c 100644 --- a/src/lib/config.ts +++ b/src/lib/config.ts @@ -75,7 +75,7 @@ export function serverURLFromOptions(options: SDKOptions): URL | null { export const SDK_METADATA = { language: "typescript", openapiDocVersion: "1.0.0", - sdkVersion: "1.2.5", + sdkVersion: "1.2.6", genVersion: "2.914.0", - userAgent: "speakeasy-sdk/typescript 1.2.5 2.914.0 1.0.0 @openrouter/sdk", + userAgent: "speakeasy-sdk/typescript 1.2.6 2.914.0 1.0.0 @openrouter/sdk", } as const; diff --git a/src/lib/encodings.ts b/src/lib/encodings.ts index 49f159049..4d39b1d00 100644 --- a/src/lib/encodings.ts +++ b/src/lib/encodings.ts @@ -3,7 +3,7 @@ * @generated-id: 3bd8ead98afd */ -import { bytesToBase64 } from "./base64.js"; +import { bytesToBase64 } from "./base64-codec.js"; import { isPlainObject } from "./primitives.js"; export class EncodingError extends Error { diff --git a/src/lib/sdks.ts b/src/lib/sdks.ts index fed3c8256..db98c47e3 100644 --- a/src/lib/sdks.ts +++ b/src/lib/sdks.ts @@ -13,7 +13,7 @@ import { UnexpectedClientError, } from "../models/errors/httpclienterrors.js"; import { ERR, OK, Result } from "../types/fp.js"; -import { stringToBase64 } from "./base64.js"; +import { stringToBase64 } from "./base64-codec.js"; import { SDK_METADATA, SDKOptions, serverURLFromOptions } from "./config.js"; import { encodeForm } from "./encodings.js"; import { env, fillGlobals } from "./env.js"; diff --git a/tests/unit/base64-codec.test.ts b/tests/unit/base64-codec.test.ts new file mode 100644 index 000000000..8fdeb2406 --- /dev/null +++ b/tests/unit/base64-codec.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest'; + +import { + bytesFromBase64, + bytesToBase64, + stringFromBase64, + stringFromBytes, + stringToBase64, + stringToBytes, +} from '../../src/lib/base64-codec.js'; +import { zodInbound, zodOutbound } from '../../src/lib/base64.js'; + +describe('base64 codec', () => { + it('preserves exact byte and string output', () => { + const bytes = Uint8Array.of(0, 16, 127, 128, 255); + + expect(bytesToBase64(bytes)).toBe('ABB/gP8='); + expect(bytesFromBase64('ABB/gP8=')).toEqual(bytes); + expect(stringToBase64('hello 👋')).toBe('aGVsbG8g8J+Riw=='); + expect(stringFromBase64('aGVsbG8g8J+Riw==')).toBe('hello 👋'); + expect(stringFromBytes(stringToBytes('café 漢字'))).toBe('café 漢字'); + }); + + it('preserves the malformed base64 error', () => { + expect(() => bytesFromBase64('%%%')).toThrow( + expect.objectContaining({ + name: 'InvalidCharacterError', + }), + ); + }); + + it('keeps the Zod adapters behavior unchanged', () => { + expect(zodOutbound.parse('hello 👋')).toEqual(stringToBytes('hello 👋')); + expect(zodInbound.parse('aGVsbG8g8J+Riw==')).toEqual(stringToBytes('hello 👋')); + }); +}); diff --git a/tests/unit/event-streams.test.ts b/tests/unit/event-streams.test.ts new file mode 100644 index 000000000..9a19a458c --- /dev/null +++ b/tests/unit/event-streams.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { EventStream, type SseMessage } from '../../src/lib/event-streams.js'; + +const encoder = new TextEncoder(); + +function chunkBytes(input: string, splitPoints: number[]): Uint8Array[] { + const bytes = encoder.encode(input); + const points = [ + 0, + ...splitPoints, + bytes.length, + ]; + return points.slice(0, -1).map((start, index) => { + return bytes.slice(start, points[index + 1]); + }); +} + +function sourceFromChunks( + chunks: Uint8Array[], + onCancel = vi.fn<(reason?: unknown) => void>(), +): ReadableStream { + let index = 0; + return new ReadableStream( + { + pull(controller) { + const chunk = chunks[index++]; + if (chunk) { + controller.enqueue(chunk); + } else { + controller.close(); + } + }, + cancel: onCancel, + }, + { + highWaterMark: 0, + }, + ); +} + +async function collect( + chunks: Uint8Array[], + parse: (message: SseMessage) => IteratorResult, undefined> = ( + message, + ) => ({ + done: false, + value: message, + }), +): Promise[]> { + const events: SseMessage[] = []; + const stream = new EventStream(sourceFromChunks(chunks), parse); + for await (const event of stream) { + events.push(event); + } + return events; +} + +describe('EventStream', () => { + it('produces identical events for every two-part and byte-by-byte fragmentation', async () => { + const input = [ + ': comment\r\n', + 'event: update\r\n', + 'id: event-7\r\n', + 'retry: 2500\r\n', + 'data: first line\r\n', + 'data: second 👋 café 漢字\r\n', + '\r\n', + 'data: next\r\n', + '\r\n', + ].join(''); + const bytes = encoder.encode(input); + const expected = await collect([ + bytes, + ]); + + expect(expected).toEqual([ + { + data: 'first line\nsecond 👋 café 漢字', + event: 'update', + id: 'event-7', + retry: 2500, + }, + { + data: 'next', + id: 'event-7', + }, + ]); + + for (let split = 1; split < bytes.length; split++) { + expect( + await collect( + chunkBytes(input, [ + split, + ]), + ), + ).toEqual(expected); + } + + const everyByte = Array.from(bytes, (byte) => Uint8Array.of(byte)); + expect(await collect(everyByte)).toEqual(expected); + }); + + it.each([ + [ + 'CRLF + CRLF', + '\r\n\r\n', + ], + [ + 'CRLF + CR', + '\r\n\r', + ], + [ + 'CRLF + LF', + '\r\n\n', + ], + [ + 'CR + CRLF', + '\r\r\n', + ], + [ + 'LF + CRLF', + '\n\r\n', + ], + [ + 'CR + CR', + '\r\r', + ], + [ + 'LF + CR', + '\n\r', + ], + [ + 'LF + LF', + '\n\n', + ], + ])('accepts the %s event boundary across every split', async (_name, boundary) => { + const input = `data: value${boundary}`; + const bytes = encoder.encode(input); + + for (let split = 1; split < bytes.length; split++) { + expect( + await collect( + chunkBytes(input, [ + split, + ]), + ), + ).toEqual([ + { + data: 'value', + id: undefined, + }, + ]); + } + }); + + it('ignores malformed fields while preserving exact field semantics', async () => { + const input = [ + 'id: persisted\n\n', + 'id: ignored\0value\n', + 'retry: 1.5\n', + 'unknown: ignored\n', + 'data\n\n', + ].join(''); + + expect( + await collect([ + encoder.encode(input), + ]), + ).toEqual([ + { + data: '', + id: 'persisted', + }, + ]); + }); + + it('drops a truncated final event without invoking the parser', async () => { + const parse = vi.fn((message: SseMessage) => ({ + done: false as const, + value: message, + })); + + expect( + await collect( + [ + encoder.encode('data: incomplete'), + ], + parse, + ), + ).toEqual([]); + expect(parse).not.toHaveBeenCalled(); + }); + + it('propagates parser errors and cancels the upstream with the same error', async () => { + const error = new Error('malformed event payload'); + const onCancel = vi.fn<(reason?: unknown) => void>(); + const stream = new EventStream( + sourceFromChunks( + [ + encoder.encode('data: value\n\n'), + ], + onCancel, + ), + () => { + throw error; + }, + ); + + await expect(stream.getReader().read()).rejects.toBe(error); + expect(onCancel).toHaveBeenCalledOnce(); + expect(onCancel).toHaveBeenCalledWith(error); + }); + + it('cancels upstream when the parser returns done', async () => { + const onCancel = vi.fn<(reason?: unknown) => void>(); + const stream = new EventStream( + sourceFromChunks( + [ + encoder.encode('data: [DONE]\n\n'), + ], + onCancel, + ), + () => ({ + done: true, + value: undefined, + }), + ); + + await expect(stream.getReader().read()).resolves.toEqual({ + done: true, + value: undefined, + }); + expect(onCancel).toHaveBeenCalledWith('done'); + }); + + it('forwards downstream cancellation to the upstream reader', async () => { + const onCancel = vi.fn<(reason?: unknown) => void>(); + const stream = new EventStream( + sourceFromChunks( + [ + encoder.encode('data: first\n\n'), + encoder.encode('data: second\n\n'), + ], + onCancel, + ), + (message) => ({ + done: false, + value: message, + }), + ); + const reader = stream.getReader(); + + await expect(reader.read()).resolves.toEqual({ + done: false, + value: { + data: 'first', + id: undefined, + }, + }); + await reader.cancel('consumer stopped'); + + expect(onCancel).toHaveBeenCalledWith('consumer stopped'); + }); +}); From a2a8cd1a9f0ec0b2a56e552d9135eea25b7eb316 Mon Sep 17 00:00:00 2001 From: Dennis Jeong <3719829+w0nche0l@users.noreply.github.com> Date: Mon, 3 Aug 2026 15:53:44 -0400 Subject: [PATCH 2/3] bench(sdk): isolate Responses schema memory Co-authored-by: Cursor --- benchmarks/characterize/RESULTS.md | 25 +++++++++++++++++++ .../entries/completed-event-schema.ts | 7 ++++++ .../characterize/entries/core-import.ts | 11 ++++++++ .../entries/responses-request-schema.ts | 19 ++++++++++++++ .../entries/responses-send-import.ts | 7 ++++++ .../entries/stream-events-schema.ts | 22 ++++++++++++++++ .../characterize/entries/text-delta-schema.ts | 15 +++++++++++ benchmarks/characterize/index.mjs | 24 ++++++++++++++++++ 8 files changed, 130 insertions(+) create mode 100644 benchmarks/characterize/entries/completed-event-schema.ts create mode 100644 benchmarks/characterize/entries/core-import.ts create mode 100644 benchmarks/characterize/entries/responses-request-schema.ts create mode 100644 benchmarks/characterize/entries/responses-send-import.ts create mode 100644 benchmarks/characterize/entries/stream-events-schema.ts create mode 100644 benchmarks/characterize/entries/text-delta-schema.ts diff --git a/benchmarks/characterize/RESULTS.md b/benchmarks/characterize/RESULTS.md index 984c680f0..623a9e148 100644 --- a/benchmarks/characterize/RESULTS.md +++ b/benchmarks/characterize/RESULTS.md @@ -44,3 +44,28 @@ or generator-owned SDK optimization. The retained optimization splits codec-only base64 helpers from the Zod adapters and redirects transport imports to the codec module. Speakeasy persistent edits preserved all three generated-file import changes during a successful pinned regeneration. + +## Twenty-run Responses attribution + +The expanded matrix ran each minified, tree-shaken bundle in 20 fresh processes. +Median retained import heap deltas were: + +- `OpenRouterCore`: 965,552 bytes. +- `responsesSend`: 41,988,580 bytes. +- `ResponsesRequest$outboundSchema`: 21,757,296 bytes. +- `StreamEvents$inboundSchema`: 21,580,420 bytes. +- `TextDeltaEvent$inboundSchema`: 1,087,232 bytes. +- `StreamEventsResponseCompleted$inboundSchema`: 18,010,468 bytes. + +This isolates the impactful remaining target: the transport and SSE framing runtime +are small, while the generated Responses sender eagerly constructs both broad request +and stream-event schema graphs. A text-delta-specific schema uses about 95% less retained +import heap than the all-event union, but the completion schema still loads the full +response/output graph. + +Lazy event dispatch alone would improve startup but not invocation peak because the +completion event eventually loads its 18 MB graph. A material peak reduction requires +generator-owned operation-private validators that dispatch request tools, stream events, +and completed output items by discriminator without constructing every unused branch. +That remains a generator architecture change; it must pass the exact schema/error +equivalence suite before replacing the public generated schemas. diff --git a/benchmarks/characterize/entries/completed-event-schema.ts b/benchmarks/characterize/entries/completed-event-schema.ts new file mode 100644 index 000000000..55e8a1cbb --- /dev/null +++ b/benchmarks/characterize/entries/completed-event-schema.ts @@ -0,0 +1,7 @@ +import { StreamEventsResponseCompleted$inboundSchema } from '../../../src/models/streameventsresponsecompleted.ts'; + +export function run() { + return { + schema: StreamEventsResponseCompleted$inboundSchema.constructor.name, + }; +} diff --git a/benchmarks/characterize/entries/core-import.ts b/benchmarks/characterize/entries/core-import.ts new file mode 100644 index 000000000..aa7e196c8 --- /dev/null +++ b/benchmarks/characterize/entries/core-import.ts @@ -0,0 +1,11 @@ +import { OpenRouterCore } from '../../../src/core.ts'; + +export function run() { + const client = new OpenRouterCore({ + apiKey: 'benchmark-key', + serverURL: 'https://benchmark.invalid/api/v1', + }); + return { + baseURL: client._baseURL?.toString(), + }; +} diff --git a/benchmarks/characterize/entries/responses-request-schema.ts b/benchmarks/characterize/entries/responses-request-schema.ts new file mode 100644 index 000000000..62b445aff --- /dev/null +++ b/benchmarks/characterize/entries/responses-request-schema.ts @@ -0,0 +1,19 @@ +import { ResponsesRequest$outboundSchema } from '../../../src/models/responsesrequest.ts'; + +const request = { + model: 'openai/gpt-5.6-luna', + input: 'Return a deterministic response.', + maxOutputTokens: 64, + stream: true, +}; + +export function run({ sample }: { sample: () => void }) { + let output: ReturnType | undefined; + for (let iteration = 0; iteration < 1_000; iteration++) { + output = ResponsesRequest$outboundSchema.parse(request); + if (iteration % 50 === 0) { + sample(); + } + } + return output; +} diff --git a/benchmarks/characterize/entries/responses-send-import.ts b/benchmarks/characterize/entries/responses-send-import.ts new file mode 100644 index 000000000..3a315c219 --- /dev/null +++ b/benchmarks/characterize/entries/responses-send-import.ts @@ -0,0 +1,7 @@ +import { responsesSend } from '../../../src/funcs/responsesSend.ts'; + +export function run() { + return { + responsesSend: typeof responsesSend, + }; +} diff --git a/benchmarks/characterize/entries/stream-events-schema.ts b/benchmarks/characterize/entries/stream-events-schema.ts new file mode 100644 index 000000000..6d6e57096 --- /dev/null +++ b/benchmarks/characterize/entries/stream-events-schema.ts @@ -0,0 +1,22 @@ +import { StreamEvents$inboundSchema } from '../../../src/models/streamevents.ts'; + +const event = { + type: 'response.output_text.delta', + sequence_number: 1, + item_id: 'message_1', + output_index: 0, + content_index: 0, + delta: 'deterministic delta', + logprobs: [], +}; + +export function run({ sample }: { sample: () => void }) { + let output: ReturnType | undefined; + for (let iteration = 0; iteration < 10_000; iteration++) { + output = StreamEvents$inboundSchema.parse(event); + if (iteration % 100 === 0) { + sample(); + } + } + return output?.type; +} diff --git a/benchmarks/characterize/entries/text-delta-schema.ts b/benchmarks/characterize/entries/text-delta-schema.ts new file mode 100644 index 000000000..019f7ec8e --- /dev/null +++ b/benchmarks/characterize/entries/text-delta-schema.ts @@ -0,0 +1,15 @@ +import { TextDeltaEvent$inboundSchema } from '../../../src/models/textdeltaevent.ts'; + +const event = { + type: 'response.output_text.delta', + sequence_number: 1, + item_id: 'message_1', + output_index: 0, + content_index: 0, + delta: 'deterministic delta', + logprobs: [], +}; + +export function run() { + return TextDeltaEvent$inboundSchema.parse(event).type; +} diff --git a/benchmarks/characterize/index.mjs b/benchmarks/characterize/index.mjs index 9297f65a1..4e9a97ba7 100644 --- a/benchmarks/characterize/index.mjs +++ b/benchmarks/characterize/index.mjs @@ -12,6 +12,30 @@ const execFileAsync = promisify(execFile); const benchmarkDirectory = fileURLToPath(new URL('.', import.meta.url)); const runnerPath = join(benchmarkDirectory, 'runner.mjs'); const cases = [ + [ + 'core-import', + 'entries/core-import.ts', + ], + [ + 'responses-send-import', + 'entries/responses-send-import.ts', + ], + [ + 'responses-request-schema', + 'entries/responses-request-schema.ts', + ], + [ + 'stream-events-schema', + 'entries/stream-events-schema.ts', + ], + [ + 'text-delta-schema', + 'entries/text-delta-schema.ts', + ], + [ + 'completed-event-schema', + 'entries/completed-event-schema.ts', + ], [ 'root-import', 'entries/root-import.ts', From 96f774ed1ab641dbece493d44955c1a94d89041c Mon Sep 17 00:00:00 2001 From: Dennis Jeong <3719829+w0nche0l@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:05:10 -0400 Subject: [PATCH 3/3] docs(sdk): record rejected lazy-error prototype Co-authored-by: Cursor --- benchmarks/characterize/RESULTS.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/benchmarks/characterize/RESULTS.md b/benchmarks/characterize/RESULTS.md index 623a9e148..58da94d71 100644 --- a/benchmarks/characterize/RESULTS.md +++ b/benchmarks/characterize/RESULTS.md @@ -69,3 +69,17 @@ generator-owned operation-private validators that dispatch request tools, stream and completed output items by discriminator without constructing every unused branch. That remains a generator architecture change; it must pass the exact schema/error equivalence suite before replacing the public generated schemas. + +## Rejected: dynamic status-error imports + +A persistent-edit prototype replaced the fourteen eager Responses HTTP error +schemas with status-specific dynamic imports. Across 20 fresh processes it +regressed the `responsesSend` import from 41,988,580 to 99,969,304 median heap +bytes and increased the minified bundle from 213,200 to 357,667 bytes. + +With the current non-splitting Worker/Node bundle, esbuild retained the dynamic +module graph and its initialization wrappers instead of providing an isolated +status chunk. The prototype was fully reverted. Status-lazy schemas are only +viable if the deployment produces real code-split modules and measures their +combined Worker startup/first-error behavior; they are not a safe optimization +for the current bundle.