diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7dda8f3..9b4f67f8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: version: '0.11.32' - name: Install deps - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile && pnpm peers check - name: Check publish credential isolation run: pnpm run check:publish-workflow @@ -131,7 +131,7 @@ jobs: cache-dependency-path: pnpm-lock.yaml - name: Install package set - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile && pnpm peers check - name: Build current agent-runtime run: pnpm run build @@ -161,7 +161,7 @@ jobs: python-version: '3.12' - name: Install package set - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile && pnpm peers check - name: Verify packed Runtime with official GEPA and SkillOpt run: pnpm run verify:official-optimizers diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index b2d5e83e..04a125cc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -118,7 +118,7 @@ jobs: exit 1 - name: Install deps - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile && pnpm peers check - name: Lint (biome) run: pnpm run lint @@ -344,7 +344,7 @@ jobs: cache-dependency-path: pnpm-lock.yaml - name: Install deps - run: pnpm install --frozen-lockfile + run: pnpm install --frozen-lockfile && pnpm peers check - name: Wait for published agent-bench dependencies timeout-minutes: 25 diff --git a/CHANGELOG.md b/CHANGELOG.md index efb7788d..2e163262 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## 0.209.0 + +Budgets accept caller-named resources with explicit units, atomic reservations, measured refunds, and durable usage accounting. +Choose integer units such as GPU milliseconds, storage bytes, or API requests to preserve exact conservation across workers and restarts. +Missing, conflicting, or unknown enforced measurements close further admission while retaining the available evidence. +Streams, nested execution, cancellation, and replay share the same resource accounting contract. +Router transports can report the same receipts through `usage.resources`; retried calls retain incomplete measurement explicitly. +The package cohort uses Eval 0.180.0 and Knowledge 15.0.3. +Eval supervisor reports preserve individual resource receipts and their sources. + +## 0.208.1 + +Retained and interactive runs use session identities accepted by Sandbox storage. + ## 0.208.0 Provider executors select caller-declared child placements, preserve exact profiles, and bind configuration identity across retained recovery. diff --git a/api-surface.json b/api-surface.json index 061dd675..f321732e 100644 --- a/api-surface.json +++ b/api-surface.json @@ -777,10 +777,10 @@ "BridgeHarnessStore": "type ca55cbc26b63", "BridgeModelCredential": "type 82ad4d863f5c", "BridgeSeam": "type 3ee6c139b2d1", - "Budget": "type 198591da0ea4", - "BudgetPool": "type 9cb3e3daa900", + "Budget": "type d3e20424e44d", + "BudgetPool": "type 32b97a2266d1", "BudgetPoolRestore": "type 09960d15dfe8", - "BudgetReadout": "type 626458ab8582", + "BudgetReadout": "type 0e2ea9923511", "BusEvent": "type 736dd037425e", "BusRecord": "type fccfdf943e88", "BusStats": "type 10ba4be00459", @@ -1105,14 +1105,16 @@ "RepairStop": "type dfe122fcef11", "ReproductionCheck": "type d866d16ccd22", "ReservationRejection": "type 95613aa46396", - "ReservationTicket": "type e7df35bc9166", + "ReservationTicket": "type 58a154a39d5e", "ResolveDriveHarness": "type 14e24449c8f2", "ResolveSandboxClientOptions": "type bc4eb1021d43", "ResolveSupervisorTools": "type 1c7000abe533", "ResolvedMcpServerLaunch": "type 38c483055998", "ResolvedSupervisorProfile": "type a1c298f11bc0", "ResolvedWorkerSpawnRetry": "type b7269d19dba0", + "ResourceLimit": "type c0a8ca7a5438", "ResourceRequest": "type 20dbb23e8101", + "ResourceSpend": "type dfe0748e77c5", "ResultBlobStore": "type 69e6e03b75ab", "ResumedKeyState": "type 018905da27e4", "ResumedWork": "type 57b7c709155a", @@ -1173,11 +1175,11 @@ "SandboxSteeringOptions": "type 526a58a22f51", "SandboxToolPartState": "type 26cdd7a50965", "SandboxUsageLedger": "type e2058a39dd8b", - "Scope": "type f0ce3d119b61", + "Scope": "type edc3d5f40e17", "ScopeAnalyst": "type 1cd5ae8d0d15", "ScopeAnalyzeInput": "type 98b6562915c6", "ScopeArgs": "type e739f4591d93", - "ScopeProgressInput": "type cf86c0b0aa0d", + "ScopeProgressInput": "type 5add6a25b8f5", "ScopeWidenGate": "type 6c09e47c6dd2", "ScoreOf": "type 652f59845272", "SelectionReceipt": "type c2a6434f17c9", @@ -1205,8 +1207,8 @@ "SpawnRefusal": "type e0fea783ffe7", "SpawnRefusalCause": "type a9399e96f43a", "SpawnRejection": "type 0f740e25e598", - "Spend": "type 6e558e6744c8", - "SpendChannel": "type 978ffae337cd", + "Spend": "type d7119cca63e2", + "SpendChannel": "type 3ca973d6c931", "SpendGap": "type d24924855c35", "StartRetainedInteractiveRunOptions": "type 29aca25d53e3", "StartRetainedRunInEnvironmentOptions": "type f52dea3534df", @@ -1264,7 +1266,7 @@ "TerminalDecision": "type 1a822915f49f", "TokenUsageProvenance": "type 4ce5b69b2e4d", "ToolLoopCallContext": "type 1c845a95cad0", - "ToolLoopChat": "type 898c9ad10190", + "ToolLoopChat": "type d25f10a0b0e1", "ToolLoopCompaction": "type 1848f98b6067", "ToolLoopCompactionOptions": "type 39c5e1c62e09", "ToolLoopMessageRecord": "type b757f8750838", @@ -1283,7 +1285,7 @@ "UnconfirmedTeardown": "type 791f75c80508", "UnknownMaterializationReason": "type 4da80f596ffe", "UntrackedCopyStats": "type 3d60a38b88e2", - "UsageEvent": "type 326dc5abb5da", + "UsageEvent": "type 4b3e1f2098da", "VERIFY_TAIL_CHARS": "value 95999f4bd438", "ValidationCtx": "type 45da43f45d39", "Validator": "type c67346b2cb69", @@ -1317,7 +1319,7 @@ "WorkerInteractiveProviderSource": "type b738c5faa346", "WorkerInteractiveSession": "type 5090ed4fc91d", "WorkerInteractiveUnavailableReason": "type 2692319a7ec2", - "WorkerProgress": "type bfbefaec0759", + "WorkerProgress": "type ffc0e787cacc", "WorkerResumeContext": "type f96033ae9376", "WorkerSpawnContext": "type a0f0b5934f6b", "WorkerSpawnRetryAttempt": "type ee327ef375b6", @@ -1947,7 +1949,7 @@ "SuperviseTestOptions": "type 1fd4a3f892e4", "SupervisorAgentTestDeps": "type f39aa3b16149", "ToolLoopCallContext": "type 1c845a95cad0", - "ToolLoopChat": "type 898c9ad10190", + "ToolLoopChat": "type d25f10a0b0e1", "driverAgent": "value fba667f488cd", "loadAgentImprovementProposalFixture": "value 2f33e49d4be9", "loadAgentProfileImprovementFixture": "value 3bf4543ae6f7", diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 8b19a406..2e135560 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.11.3 + +Uses Runtime 0.209.0, Eval 0.180.0, and Knowledge 15.0.3 together, including named resource accounting and source-attributed resource reports. +SWE-bench setup, prompts, and patch extraction use the writable session workspace instead of a root-level directory. +Official grading behavior is unchanged. + ## 0.11.2 Supports SWE-bench 5.x by detecting removed cache and namespace evaluator flags while retaining the 4.x path. diff --git a/bench/package.json b/bench/package.json index 28817c74..79ee7e65 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.11.2", + "version": "0.11.3", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/bench/src/benchmarks/swe-bench.test.mts b/bench/src/benchmarks/swe-bench.test.mts index 3486d81a..44d707e4 100644 --- a/bench/src/benchmarks/swe-bench.test.mts +++ b/bench/src/benchmarks/swe-bench.test.mts @@ -1,5 +1,9 @@ import assert from 'node:assert/strict' import test from 'node:test' +import { execFileSync } from 'node:child_process' +import { mkdtempSync, mkdirSync, writeFileSync, rmSync, existsSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' import { createSweBenchAdapter, scoreSweReport, sweEvaluationArgv } from './swe-bench' const taskId = 'django__django-12345' @@ -59,3 +63,48 @@ test('SWE evaluation command preserves the requested instance image', () => { /invalid cacheLevel/, ) }) + +test('SWE setup and extraction stay in the session workspace and exclude test edits', () => { + const root = mkdtempSync(join(tmpdir(), 'swe-workspace-')) + try { + const origin = join(root, 'origin') + const workspace = join(root, 'session') + mkdirSync(origin) + mkdirSync(workspace) + const git = (args: string[], input?: string) => execFileSync('git', args, { cwd: origin, input, encoding: 'utf8' }).trim() + git(['init', '--quiet']) + const blob = git(['hash-object', '-w', '--stdin'], 'before\n') + const tree = git(['mktree'], `100644 blob ${blob}\tsource.py\n`) + // Construct fixture history without changing the developer's Git identity or configuration. + const base = git(['hash-object', '-t', 'commit', '-w', '--stdin'], + `tree ${tree}\nauthor Fixture 1 +0000\ncommitter Fixture 1 +0000\n\nfixture\n`) + git(['update-ref', 'HEAD', base]) + const task = { id: taskId, prompt: 'fix', metadata: { repo: 'fixture/repo', base_commit: base } } + const adapter = createSweBenchAdapter() + const setup = adapter.boxSetup!(task) + const extract = adapter.boxExtract!(task) + assert.match(setup.command, /^rm -rf '\.\//) + assert.equal(setup.cwd, undefined) + assert.equal(extract.cwd, undefined) + const env = { + ...process.env, + GIT_CONFIG_COUNT: '2', + GIT_CONFIG_KEY_0: `url.file://${origin}.insteadOf`, + GIT_CONFIG_VALUE_0: 'https://github.com/fixture/repo', + GIT_CONFIG_KEY_1: 'protocol.file.allow', + GIT_CONFIG_VALUE_1: 'always', + } + execFileSync('sh', ['-c', setup.command], { cwd: workspace, env }) + const repo = join(workspace, 'swe-bench-repo') + assert.ok(existsSync(join(repo, '.git'))) + writeFileSync(join(repo, 'source.py'), 'after\n') + mkdirSync(join(repo, 'tests')) + writeFileSync(join(repo, 'tests', 'test_fix.py'), 'hidden-test-edit\n') + const patch = execFileSync('sh', ['-c', extract.command], { cwd: workspace, env, encoding: 'utf8' }) + assert.match(patch, /diff --git a\/source.py b\/source.py/) + assert.match(patch, /\+after/) + assert.doesNotMatch(patch, /hidden-test-edit|test_fix/) + } finally { + rmSync(root, { recursive: true, force: true }) + } +}) diff --git a/bench/src/benchmarks/swe-bench.ts b/bench/src/benchmarks/swe-bench.ts index 919feecf..4226f84a 100644 --- a/bench/src/benchmarks/swe-bench.ts +++ b/bench/src/benchmarks/swe-bench.ts @@ -25,13 +25,8 @@ import { } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' -/** - * Fixed in-box path the agent clones the instance repo into. It is the SINGLE - * source of truth shared by the prompt template (which tells the agent to clone - * here) and `boxExtract` (which runs `git diff` here after the shot) — so the - * harness always knows exactly where the agent's edits live, for any instance. - */ -const SWE_REPO_DIR = '/work' +/** Root-level directories are not writable in every sandbox; use the session workspace. */ +const SWE_REPO_DIR = './swe-bench-repo' /** * The SWE deliverable's FALLBACK parser, from the agent's event STREAM. @@ -286,10 +281,10 @@ print(json.dumps(out)) prompt: [ `Repository: ${r.repo} @ ${r.base_commit}`, '', - `The repository is ALREADY cloned at ${SWE_REPO_DIR}, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`, + `The repository is ALREADY cloned at ${SWE_REPO_DIR} relative to your initial session workspace, checked out at commit ${r.base_commit}. Work there directly (\`cd ${SWE_REPO_DIR}\`); do not re-clone.`, '', 'Resolve this issue by editing the repository SOURCE so the failing tests pass without breaking the passing ones. Do NOT edit test files — the evaluation runs hidden tests on a fresh checkout, so editing tests does not count. Keep the change minimal and confined to the cloned repo.', - 'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your committed edits directly from the repo.', + 'Work iteratively: reproduce the issue, implement the fix in the source, and re-run the relevant tests until they pass. You do NOT need to print the diff — the harness reads your source edits directly from the repo, including uncommitted changes.', '', '--- Issue ---', String(r.problem_statement ?? ''), diff --git a/docs/agent-managed-compute/evidence/interactive-2026-09-09/README.md b/docs/agent-managed-compute/evidence/interactive-2026-09-09/README.md new file mode 100644 index 00000000..e4fdcbbb --- /dev/null +++ b/docs/agent-managed-compute/evidence/interactive-2026-09-09/README.md @@ -0,0 +1,63 @@ +# Live retained terminal evidence + +Two bounded Sandbox runs exercised the maintained Runtime retained interactive path on 2026-09-09. +Neither run sent a model prompt. +Both used Codex with no initial prompt and terminal input `/help`. +These observations establish partial Sandbox conformance for issue [#773](https://github.com/tangle-network/agent-runtime/issues/773). + +| Observation | First run | Second run | +|---|---|---| +| Initial output frames | 168 | 192 | +| Replay frames | 168 | 192 | +| Identical sequence and SHA-256 pairs | 168/168 | 192/192 | +| Contiguous sequences | 1–168 | 1–192 | +| Process running after detach | Yes | Yes | +| Terminal close acknowledgement | `unknown` | Not called | +| Interactive stop acknowledgement | Not called | `accepted`, effect `stopped` | +| Status after interactive stop | Not checked | `exited` | +| Sandbox deleted and absence verified | Yes | Yes | +| Model prompts | 0 | 0 | +| Process exit code | 0 | 0 | + +Both runs returned an exact preparation receipt with matching authored and effective profile digests. +Both resized the terminal from 80×24 to 100×30 and then 120×40. +Both reconstructed the retained handle from its exact reference and acquired fresh control before replay. +The second run stopped through that reconstructed handle. +Terminal close alone did not establish native-process termination in the first run. + +## Scope and limits + +Runtime source revision: `2f13fc7b`. +Packages: Sandbox `0.38.2`, provider-tangle `1.1.6`, and agent-interface `2.6.0`. +Each run requested 1 CPU, 1024 MB RAM, 1 GB disk, blocked egress, and a 180-second maximum lifetime. +Each requested a 60-second idle timeout and used a 180-second client abort timer. +These resource values are requests, not independent measurements of platform enforcement. +No model token usage or billing measurement was collected; no model prompt API was invoked. + +This proof does not test worker native-child binding, a live Braid coordinator restart, or environment expiry semantics. +Local CLI Bridge conformance remains blocked by [cli-bridge#183](https://github.com/drewstone/cli-bridge/issues/183). +The inspected Bridge revision `30038a6` lacked the PTY and duplex attach contract. +These records do not satisfy every #773 acceptance condition and do not justify closing that issue. + +## Artifacts and reproduction + +[close-result.json](./close-result.json) and [stop-result.json](./stop-result.json) preserve public identities, preparation digests, and output frame digests. +Terminal bytes, credentials, account email, and transport URLs are excluded. +The command recorded in each result names its original scratch script. + +[close-proof.mts](./close-proof.mts) and [stop-proof.mts](./stop-proof.mts) preserve the executed logic with portable source locations and sanitized output. +The preserved scripts remove account email and error messages from output and restrict admissions to public identifiers. +They were syntax-checked after these recording changes; they were not rerun against another sandbox. + +Copy the chosen script into a scratch directory and install the pinned packages there: + +```sh +npm install --save-exact @tangle-network/sandbox@0.38.2 @tangle-network/agent-provider-tangle@1.1.6 @tangle-network/agent-interface@2.6.0 +``` + +Set `RUNTIME_SOURCE` to the Runtime checkout and `SANDBOX_CLI_SOURCE` to the maintained Sandbox CLI `src` directory. +Set `EXPECTED_SANDBOX_EMAIL` to the authorized account email. +The scripts read existing Sandbox credentials through the maintained CLI configuration and verify account ownership through its read-only account API. +They require the production Sandbox origin and never print credentials. +Run `tsx stop-proof.mts` or `tsx close-proof.mts` from the scratch directory. +Each script creates a sandbox and deletes it in `finally`, then checks that it is absent. diff --git a/docs/agent-managed-compute/evidence/interactive-2026-09-09/close-proof.mts b/docs/agent-managed-compute/evidence/interactive-2026-09-09/close-proof.mts new file mode 100644 index 00000000..80954f40 --- /dev/null +++ b/docs/agent-managed-compute/evidence/interactive-2026-09-09/close-proof.mts @@ -0,0 +1,208 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { writeFile } from 'node:fs/promises' +import { setTimeout as delay } from 'node:timers/promises' +import { Sandbox } from '@tangle-network/sandbox' +import { createTangleProvider } from '@tangle-network/agent-provider-tangle' +const { getApiKey, getBaseUrl, resolveProductProfile } = await import( + `${process.env.SANDBOX_CLI_SOURCE}/lib/config.ts` +) +const { fetchAuthenticatedAccount } = await import(`${process.env.SANDBOX_CLI_SOURCE}/lib/auth.ts`) +const { startRetainedInteractiveRun, reconnectRetainedInteractiveRun } = await import( + `${process.env.RUNTIME_SOURCE}/src/runtime/retained-interactive.ts` +) +const { claimRetainedInteractiveControl } = await import( + `${process.env.RUNTIME_SOURCE}/src/runtime/retained-interactive-control.ts` +) + +const name = `runtime-773-no-inference-${Date.now()}` +const admissions = [] +const evidence = { + name, + sdk: '0.38.2', + provider: '1.1.6', + modelPrompts: 0, + startedAt: new Date().toISOString(), + steps: [], +} +const profileName = resolveProductProfile('sandbox') +const apiKey = getApiKey(undefined, profileName) +const baseUrl = getBaseUrl(undefined, profileName) +assert.equal(new URL(baseUrl).origin, 'https://sandbox.tangle.tools') +const client = new Sandbox({ apiKey, baseUrl, timeoutMs: 15000 }) +const controller = new AbortController() +const timeout = setTimeout(() => controller.abort('180-second proof limit'), 180000) +let environmentId +let terminal +async function record(step, data = {}) { + evidence.steps.push({ step, ...data }) + console.log(JSON.stringify({ step })) + await writeFile('./evidence.json', JSON.stringify(evidence, null, 2)) +} +const provider = createTangleProvider({ + client, + defaultBackend: 'codex', + readyTimeoutMs: 120000, + mapCreateInput: (input) => ({ + name, + backend: { type: 'codex', profile: input.profile }, + resources: { cpuCores: 1, memoryMB: 1024, diskGB: 1 }, + egressPolicy: { mode: 'blocked' }, + idempotencyKey: input.idempotencyKey, + metadata: input.metadata, + maxLifetimeSeconds: 180, + idleTimeoutSeconds: 60, + }), +}) +async function main() { + try { + const account = await fetchAuthenticatedAccount({ apiKey, baseUrl, timeoutMs: 10000 }) + assert.equal(account.email, process.env.EXPECTED_SANDBOX_EMAIL) + await record('sandbox-account-verified', {}) + const caps = await provider.capabilities() + assert.equal(caps.interactiveAgent?.start, true) + await record('provider-client-capability-confirmed') + const handle = await startRetainedInteractiveRun({ + provider, + environment: { + name, + idempotencyKey: name, + profile: { name: 'terminal-conformance-no-inference', harness: 'codex' }, + backend: 'codex', + resources: { cpu: 1, memoryMb: 1024, diskMb: 1024 }, + egress: { mode: 'blocked' }, + }, + interactiveIdempotencyKey: `${name}-session`, + cols: 80, + rows: 24, + signal: controller.signal, + onAdmission: async (admission) => { + admissions.push({ + phase: admission.phase, + ...(admission.environmentId ? { environmentId: admission.environmentId } : {}), + }) + if (admission.phase === 'interactive_environment') environmentId = admission.environmentId + await writeFile('./admissions.json', JSON.stringify(admissions, null, 2)) + await record('admission', { + phase: admission.phase, + ...(environmentId ? { environmentId } : {}), + }) + }, + }) + environmentId = handle.ref.environmentId ?? handle.ref.run.environmentId + await record('interactive-started', { environmentId, ref: handle.ref }) + const control = await claimRetainedInteractiveControl({ + handle, + holderId: name, + signal: controller.signal, + }) + terminal = await handle.attach({ control, cols: 80, rows: 24 }, { signal: controller.signal }) + const frames = [] + const readerCtl = new AbortController() + const reader = (async () => { + try { + for await (const event of terminal.events({ signal: readerCtl.signal })) { + if (event.type === 'output') + frames.push({ + seq: event.seq, + digest: createHash('sha256').update(event.data).digest('hex'), + }) + } + } catch (error) { + if (!readerCtl.signal.aborted) throw error + } + })() + await delay(1000) + await terminal.resize({ cols: 100, rows: 30 }, { signal: controller.signal }) + await terminal.resize({ cols: 120, rows: 40 }, { signal: controller.signal }) + await terminal.input({ data: '/help\r' }, { signal: controller.signal }) + await delay(1500) + const detached = await terminal.detach({ signal: controller.signal }) + readerCtl.abort() + await reader + assert.equal(detached.status, 'detached') + assert.equal((await handle.status()).state, 'running') + await record('detached-process-running', { outputFrames: frames.length }) + const recovered = await reconnectRetainedInteractiveRun({ + provider, + ref: handle.ref, + signal: controller.signal, + }) + assert.ok(recovered) + const nextControl = await claimRetainedInteractiveControl({ + handle: recovered, + holderId: `${name}-reconnect`, + signal: controller.signal, + }) + terminal = await recovered.attach( + { control: nextControl, cols: 120, rows: 40 }, + { signal: controller.signal }, + ) + const replay = [] + const replayCtl = new AbortController() + const replayTimer = setTimeout(() => replayCtl.abort(), 2000) + try { + for await (const event of terminal.events({ signal: replayCtl.signal })) { + if (event.type === 'output') + replay.push({ + seq: event.seq, + digest: createHash('sha256').update(event.data).digest('hex'), + }) + } + } catch (error) { + if (!replayCtl.signal.aborted) throw error + } finally { + clearTimeout(replayTimer) + } + assert.ok(frames.length > 0) + assert.ok(replay.length > 0) + for (let i = 1; i < replay.length; i++) assert.ok(replay[i].seq > replay[i - 1].seq) + const prior = new Map(frames.map((frame) => [frame.seq, frame.digest])) + for (const frame of replay) + if (prior.has(frame.seq)) assert.equal(prior.get(frame.seq), frame.digest) + await record('reconnected-ordered-history', { + firstFrames: frames.length, + replayFrames: replay.length, + frames, + replay, + }) + await record('terminal-close', { + acknowledgement: await terminal.close({ signal: controller.signal }), + }) + } catch (error) { + await record('failure', { name: error.name, status: error.status ?? error.statusCode }) + if (error.cleanupHandle?.id) environmentId ??= error.cleanupHandle.id + process.exitCode = 1 + } finally { + clearTimeout(timeout) + if (!environmentId) { + try { + const matches = (await client.list({ limit: 100 })).filter((box) => box.name === name) + if (matches.length === 1) environmentId = matches[0].id + await record('cleanup-inventory', { matchingSandboxes: matches.length }) + } catch (error) { + await record('cleanup-inventory-failed', { name: error.name }) + } + } + if (environmentId) { + try { + const box = await client.get(environmentId) + if (box) await box.delete() + let gone = false + try { + gone = (await client.get(environmentId)) === null + } catch (error) { + if (error.status === 404 || error.statusCode === 404 || error.name === 'NotFoundError') + gone = true + else throw error + } + await record('cleanup-confirmed', { environmentId, gone }) + assert.ok(gone) + } catch (error) { + await record('cleanup-failed', { environmentId, name: error.name }) + process.exitCode = 1 + } + } + } +} +main() diff --git a/docs/agent-managed-compute/evidence/interactive-2026-09-09/close-result.json b/docs/agent-managed-compute/evidence/interactive-2026-09-09/close-result.json new file mode 100644 index 00000000..1e7c300d --- /dev/null +++ b/docs/agent-managed-compute/evidence/interactive-2026-09-09/close-result.json @@ -0,0 +1,1470 @@ +{ + "name": "runtime-773-no-inference-1788987147360", + "sdk": "0.38.2", + "provider": "1.1.6", + "modelPrompts": 0, + "startedAt": "2026-09-09T20:52:27.360Z", + "steps": [ + { + "step": "sandbox-account-verified" + }, + { + "step": "provider-client-capability-confirmed" + }, + { + "step": "admission", + "phase": "interactive_intent" + }, + { + "step": "admission", + "phase": "interactive_environment", + "environmentId": "sandbox-1fd5bc901d7a" + }, + { + "step": "admission", + "phase": "interactive_started", + "environmentId": "sandbox-1fd5bc901d7a" + }, + { + "step": "interactive-started", + "environmentId": "sandbox-1fd5bc901d7a", + "ref": { + "run": { + "runId": "interactive-run-318fee415a3ac14ff72fe63188e59dfc885f7cfb3640a1a8bf8fa828f61f02f3", + "provider": "tangle-sandbox", + "environmentId": "sandbox-1fd5bc901d7a", + "sessionId": "retained-session:runtime-773-no-inference-1788987147360:runtime-773-no-inference-1788987147360-session", + "executionId": "retained-execution:runtime-773-no-inference-1788987147360:runtime-773-no-inference-1788987147360-session", + "requestDigest": "sha256:318fee415a3ac14ff72fe63188e59dfc885f7cfb3640a1a8bf8fa828f61f02f3" + }, + "preparationReceipt": { + "kind": "agent-execution-preparation", + "schemaVersion": 1, + "preparationId": "interactive-preparation:fa978ebb-2252-487a-8594-ea49119e8f32", + "requestDigest": "sha256:318fee415a3ac14ff72fe63188e59dfc885f7cfb3640a1a8bf8fa828f61f02f3", + "authoredProfileDigest": "sha256:670375e14fc065a0b3e2c1020829beecfd261be768ec3c2ce5f97de40a0fa41a", + "effectiveProfileDigest": "sha256:670375e14fc065a0b3e2c1020829beecfd261be768ec3c2ce5f97de40a0fa41a", + "backend": "codex", + "harness": "codex", + "harnessVersion": "sidecar-registry", + "resolvedModel": { + "requested": "openai/gpt-5.6-luna", + "resolved": "openai/gpt-5.6-luna" + }, + "workspace": { + "leaseId": "interactive:retained-session:runtime-773-no-inference-1788987147360:runtime-773-no-inference-1788987147360-session:fa978ebb-2252-487a-8594-ea49119e8f32", + "provider": "tangle-sidecar", + "identityDigest": "sha256:39a0ca5c2ddeec6e90c99101deb17bbc17ebaa5eec5fa3266b511ee1eece1004", + "isolation": "per-run", + "sourceSnapshotDigest": "sha256:0714270120738257c578fb2470aa1bd09a7d6fba86e3ab70c5fd8cced852f854", + "sourceSnapshotPolicy": { + "kind": "provider-declared", + "name": "sidecar-interactive-workspace", + "version": 1, + "digest": "sha256:3c1690cf4d29a6098aeac0b1b5e457044dc6f928caa6780de536f1cc23344739" + }, + "preparedWorkspaceDigest": "sha256:57d9d22ca2a7eb3d9a1889d3aa8976bcf4a7a23c0d76022f19e629ca3ebd9625", + "profileActivationDigest": "sha256:369b26d8aee7b2a3d257cb4ecdae63f0ee4475660ad4e5c34b86d541aacfacd3" + }, + "axisResults": [], + "executionPlanDigest": "sha256:4d77c3c772a6e4e51147810e5ddf060b6f676dfd8025697db53fd10ce00a7657", + "materializer": { + "name": "@tangle-network/agent-profile-materialize", + "version": "workspace" + }, + "expiresAtMs": 1789073553295, + "digest": "sha256:399880c89f052ce7396fe150555d8f2508580447641da647a2c49c4baa25db44" + }, + "incarnationId": "fa978ebb-2252-487a-8594-ea49119e8f32", + "startedAt": "2026-09-09T20:52:33.295Z" + } + }, + { + "step": "detached-process-running", + "outputFrames": 168 + }, + { + "step": "reconnected-ordered-history", + "firstFrames": 168, + "replayFrames": 168, + "frames": [ + { + "seq": 1, + "digest": "e2dc73553e21d9e51f57370421432c070eceaed4a75ed84f323ae88d6b939d5f" + }, + { + "seq": 2, + "digest": "6ac905f9d618e1c34f5fdad7f3d881ee94251830798dc1fbfd68401a806e5515" + }, + { + "seq": 3, + "digest": "a3fa018c1a36d8e3439aba7a81f1508255c152bca3dbc38cda2b430104a18121" + }, + { + "seq": 4, + "digest": "84db741a32ec8f5776493d1f18bca0fe447d7e695f3d60949d20fec7c5184970" + }, + { + "seq": 5, + "digest": "64a6f16b3c4faba9f8009120d7d13443d1306af8f26576da9171ca652c74f6cf" + }, + { + "seq": 6, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 7, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 8, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 9, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 10, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 11, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 12, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 13, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 14, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 15, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 16, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 17, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 18, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 19, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 20, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 21, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 22, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 23, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 24, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 25, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 26, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 27, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 28, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 29, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 30, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 31, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 32, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 33, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 34, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 35, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 36, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 37, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 38, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 39, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 40, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 41, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 42, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 43, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 44, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 45, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 46, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 47, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 48, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 49, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 50, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 51, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 52, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 53, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 54, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 55, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 56, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 57, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 58, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 59, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 60, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 61, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 62, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 63, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 64, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 65, + "digest": "27c3232c04a6274399deba5709c34fc533c7a0d0874069dbb38fb786cca80410" + }, + { + "seq": 66, + "digest": "258a452f167bd26e28b2f3a1737b6e2115d934282635d2a822c7a605c91860dd" + }, + { + "seq": 67, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 68, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 69, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 70, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 71, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 72, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 73, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 74, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 75, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 76, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 77, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 78, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 79, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 80, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 81, + "digest": "b1d44050e44aac1ed94ab6ebb0dea35dd070036dcd6ecf313f576ea26767d9fe" + }, + { + "seq": 82, + "digest": "091c7f2448bfba5e1179a5ef8169eb2e9cf3ad869ee776bc6e82f4cf28a4ba42" + }, + { + "seq": 83, + "digest": "893f76c9811c522336ad6100904bcdab7359caa5e9183dbd5e11bc2f0d0e1843" + }, + { + "seq": 84, + "digest": "93ddce95bec43ce4b04684764c798fcd764cd0a7df56d78b48a01f09af08efbe" + }, + { + "seq": 85, + "digest": "dc4da566826a383bccd361848c4586d4fac73df95848300f5ae2a35e443e6fa2" + }, + { + "seq": 86, + "digest": "1129fd7f26688d814d3b1a9590c29d6c6c4ce13924cee4545b938b85eaf44ad8" + }, + { + "seq": 87, + "digest": "23f74364c345926d3d39df9eb557d3a9f4accb010fc75ef6e93b9c009dc18fb8" + }, + { + "seq": 88, + "digest": "9488c81df85323069e49624a52ead0d1695ed81b68be652a8ff9f07e30f8e8d6" + }, + { + "seq": 89, + "digest": "0856ec4adabbf231bcf1c1a647525f0b7267c841d6d6b5a17b6d374586f2d60b" + }, + { + "seq": 90, + "digest": "84c32c6983ccfa80c50cdb968358982223ff51a943632932bf0685a4d7b65ad0" + }, + { + "seq": 91, + "digest": "0cad09547877955f65fbaa4b5ee258bbd542230acffe05f11e674cb6ee539fca" + }, + { + "seq": 92, + "digest": "22736bb74ba98b0892b98ff78105b14ce7ccb164e37a8fd358b6c20cd34a83d4" + }, + { + "seq": 93, + "digest": "924a874ae3b873234c9184db683963cc7358a00b1e9ce8d8a1e23f9f0c51eff2" + }, + { + "seq": 94, + "digest": "469c5612212a13f5f7b8d06d42bff789a443b89d558f7df30e09edf7b46a71c9" + }, + { + "seq": 95, + "digest": "9fe6e2b902c7ab996ece9f388b1256a02dffc31ecd178cbf85918946f502896a" + }, + { + "seq": 96, + "digest": "414a52b52cb124c076bdb041b52840cf862f81e2335538e69a06a7678549b616" + }, + { + "seq": 97, + "digest": "c244ac4e31909b0ebe3bc09922c874ab71906ce72aa1e07fde3eb7250c93fe1f" + }, + { + "seq": 98, + "digest": "0cf7685989517c70ab72f5a95b81981b69b36645061804179faf2bbde846ee00" + }, + { + "seq": 99, + "digest": "0d7e4d13d74766c70042ef041b66c70c40f86178fc41ff1f737a5752eb9e8ead" + }, + { + "seq": 100, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 101, + "digest": "97286c21880a38583f7fef609f5161f1f7a2ea6ef8c78264af701f7371c5a20c" + }, + { + "seq": 102, + "digest": "2000c4f0b234cce1234f628c351c9bee336e7131c672a647f8d8cee0280dafb9" + }, + { + "seq": 103, + "digest": "33a2aebfc6a6b03fc7bacc6349178f724f28396be79ff360637efec5a4d022e4" + }, + { + "seq": 104, + "digest": "965bada94e3e652ec836ad58a19c982572674cc0e84585f1dcf508733e80ee63" + }, + { + "seq": 105, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 106, + "digest": "cf5695ab9373c64bde821bde01c82b38e7ab05c9482ae47c0657c28295018df0" + }, + { + "seq": 107, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 108, + "digest": "d7c8cfa2bb32597580c4eb2507d5305c5ce47cf332f38da481611f6249d24c4d" + }, + { + "seq": 109, + "digest": "b4b46b29c74c659383a8e61de3d48ab841b38f704b25833e297ef63ffa256066" + }, + { + "seq": 110, + "digest": "6f8376c298f8b1be71d98e7ed09055e76cf24fc27a9bdc850100c1032be2ba6a" + }, + { + "seq": 111, + "digest": "82c2f25b5061bee056ec9bb84df049dd6319300831c9b010d29e09ddeb6a96a4" + }, + { + "seq": 112, + "digest": "2a9f50f57cc8dc0a5f502a5f20ad3e6a96ad5ee9c8d754af90ee047beffd7991" + }, + { + "seq": 113, + "digest": "a84d626533723885f773c990ca626a3e1796eba12d3c9575fe2930a6c73627e5" + }, + { + "seq": 114, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 115, + "digest": "aed1655cae9150520c876fd9b6970690e76f5e7668f527aa4b00fde327065303" + }, + { + "seq": 116, + "digest": "50bb9b47edd9804ee5926045cf2430a5d9e65c9e1401a78a754c6970ddef8c0f" + }, + { + "seq": 117, + "digest": "0b79a867b846de91bba730509a77f158254381a0e7856bfa9b7bbdcd5a9ef0ca" + }, + { + "seq": 118, + "digest": "f1acc2ef36617ebf4aae9ecb0748eec97115733a7649fa99a49c150abd31903f" + }, + { + "seq": 119, + "digest": "f6fcb0d0a3fadf1c46644783cacb5541185bbe832e0918dae7686ed3977424ae" + }, + { + "seq": 120, + "digest": "2475f62fba3b27b986a8b2552710fa9f5b3169a3fff478229baa735f4b82ff2e" + }, + { + "seq": 121, + "digest": "311e8479904bcd203e3986a386e0029f2284f10b3f9a0e605e180e101c448b72" + }, + { + "seq": 122, + "digest": "41c5cd6e35bad8467ddafc558ae5445ca44ab9145621614d9c7f92abb7075422" + }, + { + "seq": 123, + "digest": "b61daeb47c9176ebfdea34c9b4622aa499a857a8d25c7692ff61dd0e88993251" + }, + { + "seq": 124, + "digest": "bf6b9971d0e12d0fc40c999573b7ba1911e76466d552dcc0b0050baf30801a6b" + }, + { + "seq": 125, + "digest": "2f860fb5b5f703c5be7ffa63323b27a935639e269c659e35b71680c25046fe01" + }, + { + "seq": 126, + "digest": "1981999076a47eee0924064f0ce51bc78ed8e2715dc0ca4071122db722b1ee7c" + }, + { + "seq": 127, + "digest": "df152bfcab10e61cc729362516068868fbcfe43f8c9aa1150f773fd661106f17" + }, + { + "seq": 128, + "digest": "e6db7c9c5f88e55bbdfc455c087616b9c592eb4efc14797582ee9add5ef15964" + }, + { + "seq": 129, + "digest": "8df24185d6a2d7fbf5db031e5037251970046d0df0c997401e854d376501451b" + }, + { + "seq": 130, + "digest": "4197690d9005d30333bc6c676b464ffd49ab309a399a4a873e40f519750e5594" + }, + { + "seq": 131, + "digest": "77a5bebec180fb84bab3ac3fbf5fbd74e052ee0b61d5da5ab05697c0b4314fc8" + }, + { + "seq": 132, + "digest": "54b5a5138704d736937975ae6a63cdc516e85c68fe9c7d6484b624140ddc00c4" + }, + { + "seq": 133, + "digest": "1fc2880c38ccc599525c6e65592067e9d0678e4a85564a65a1f0ae44f9ce8273" + }, + { + "seq": 134, + "digest": "37293e18a980f63ac186e1a1fdb2ea0058304a6ee760b8eb45d6e9a48c25f539" + }, + { + "seq": 135, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 136, + "digest": "62d2147917de6f5d122d0dd04b99f487592e1ed3f2464698a9968da3718c0c07" + }, + { + "seq": 137, + "digest": "a2df80b19491c5ea923d226b5aa834a8119aa92ea9248a66287fd310c5b9b995" + }, + { + "seq": 138, + "digest": "2896f1de2319850b7e5d9e13eb3eebf854cb3605aa58d9f65ae43cf74cb5b16b" + }, + { + "seq": 139, + "digest": "96094cfb58b32316f84012d775b01da601a271f6ff366a2613218b0c0f1ff541" + }, + { + "seq": 140, + "digest": "e937c91eb950623051ac598bfa7f98d955a4034c15a9d67d203e2cc612498c57" + }, + { + "seq": 141, + "digest": "0996376a632df35da3751d53b59bc3d7c87ad8850cd4a618fbf78368ae61ecd1" + }, + { + "seq": 142, + "digest": "bbf9b38232422eb2ac60f7eb63c31fbc8c78dfad82169ab5782b89c957bb107d" + }, + { + "seq": 143, + "digest": "8103affe397953ca9ef964149778085a3592045ed7aca4babd7b0aca088c271b" + }, + { + "seq": 144, + "digest": "3713ff52950c7d1c722424124fcadfdfbcfab6e685779156b5a2a1f42eff8858" + }, + { + "seq": 145, + "digest": "64d73bbb15722c80e3ea871758d3663e1379af2e1f2489ada9fbbf9cd6f3971c" + }, + { + "seq": 146, + "digest": "68abf2b7d0b22556d17196e6a2a72caf00f78586dfd2133a587b09d2ddde72d2" + }, + { + "seq": 147, + "digest": "4ef9eaa750be382c87d3658b023d493880051dad257677f51b1b3599d577418c" + }, + { + "seq": 148, + "digest": "f0bc2c5a2c55a9daf6174d2585905b08f7fc6f8e0f0773c13adfc860811c4178" + }, + { + "seq": 149, + "digest": "ed1ba2be16b0643ed7d3d8906d38603313c29d168e48a0a678174253e3fd3ca8" + }, + { + "seq": 150, + "digest": "5a63915716f62f25152fd39a3bbe96c12185bc1ed80625bbe5e3b9ba559bdb39" + }, + { + "seq": 151, + "digest": "4004904b161afb1cb3498ad4ffd8e0aa5451d5469f4a60263d3a9e60bc6f9a3f" + }, + { + "seq": 152, + "digest": "131c228b903e1618d7951d7ac799f4c7d1791f4f30505b2636e7b96586db5ea0" + }, + { + "seq": 153, + "digest": "24a90676a89a9ac8bc7cf026c8e5df676e25eb6ccc8ae3c99a8e900b17ef2720" + }, + { + "seq": 154, + "digest": "d6ff7e31e96bf3f1720106395d85cfd989cce910d694511c6a9aa9408df17c77" + }, + { + "seq": 155, + "digest": "c243ad9aa5e2acb50a05f309b58c106bc9e961005e7ee6ef2bf1230120344c6d" + }, + { + "seq": 156, + "digest": "5239d998874aa2876e1c7dd6271777dd308589b2a2d43497201bdbb4a77b62fd" + }, + { + "seq": 157, + "digest": "0cbe121e2dc16629aa198e400fe82df70454ccf544520790eaff4e9580c5d2da" + }, + { + "seq": 158, + "digest": "641dabb7e022ff6fff535fa45f4ecd0575e8c471665f1d8606dde54fc41be893" + }, + { + "seq": 159, + "digest": "273818b42c7a14358058326d98305e090db661fb097ff04a682c2c3f90a4b8aa" + }, + { + "seq": 160, + "digest": "805371be67b8d02b8d9a32ceb4bacffdbe7a1077c2aedd22b57eafb7efd06cb7" + }, + { + "seq": 161, + "digest": "c5346e439fa61237f6007bd9629454355e0ce3932253284e9a17c3545e705e2d" + }, + { + "seq": 162, + "digest": "8b307d32a8105f9d43ce6922023b6a3bd425cc1fb8916606ad59fe849ba7cde9" + }, + { + "seq": 163, + "digest": "9291367a14e218c4ce62fe693b53f7701a773e85e7d90a84d666bb0963d358cd" + }, + { + "seq": 164, + "digest": "3a2259ca9d6612727d20dfd93ae9969c56996b2ab4d7085367a6cc47f43ac632" + }, + { + "seq": 165, + "digest": "095a3e335b265b91358e5884486e3ce605052c0a2e09d33f3f2338df1691c2fc" + }, + { + "seq": 166, + "digest": "2f7a45cf5c4c6432dad743a25004df827ff6bd66d5586a9d523401fd2ac54b58" + }, + { + "seq": 167, + "digest": "7e607629cd08a5c668b31db2de4d862f15b2bc61547e7ba133785556c3e87be6" + }, + { + "seq": 168, + "digest": "a9c038027a33576772627ffa289cb30ac2c03522bd43abecf6ebd4f7cff25e0b" + } + ], + "replay": [ + { + "seq": 1, + "digest": "e2dc73553e21d9e51f57370421432c070eceaed4a75ed84f323ae88d6b939d5f" + }, + { + "seq": 2, + "digest": "6ac905f9d618e1c34f5fdad7f3d881ee94251830798dc1fbfd68401a806e5515" + }, + { + "seq": 3, + "digest": "a3fa018c1a36d8e3439aba7a81f1508255c152bca3dbc38cda2b430104a18121" + }, + { + "seq": 4, + "digest": "84db741a32ec8f5776493d1f18bca0fe447d7e695f3d60949d20fec7c5184970" + }, + { + "seq": 5, + "digest": "64a6f16b3c4faba9f8009120d7d13443d1306af8f26576da9171ca652c74f6cf" + }, + { + "seq": 6, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 7, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 8, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 9, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 10, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 11, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 12, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 13, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 14, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 15, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 16, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 17, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 18, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 19, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 20, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 21, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 22, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 23, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 24, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 25, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 26, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 27, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 28, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 29, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 30, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 31, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 32, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 33, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 34, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 35, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 36, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 37, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 38, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 39, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 40, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 41, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 42, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 43, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 44, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 45, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 46, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 47, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 48, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 49, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 50, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 51, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 52, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 53, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 54, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 55, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 56, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 57, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 58, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 59, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 60, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 61, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 62, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 63, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 64, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 65, + "digest": "27c3232c04a6274399deba5709c34fc533c7a0d0874069dbb38fb786cca80410" + }, + { + "seq": 66, + "digest": "258a452f167bd26e28b2f3a1737b6e2115d934282635d2a822c7a605c91860dd" + }, + { + "seq": 67, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 68, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 69, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 70, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 71, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 72, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 73, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 74, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 75, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 76, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 77, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 78, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 79, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 80, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 81, + "digest": "b1d44050e44aac1ed94ab6ebb0dea35dd070036dcd6ecf313f576ea26767d9fe" + }, + { + "seq": 82, + "digest": "091c7f2448bfba5e1179a5ef8169eb2e9cf3ad869ee776bc6e82f4cf28a4ba42" + }, + { + "seq": 83, + "digest": "893f76c9811c522336ad6100904bcdab7359caa5e9183dbd5e11bc2f0d0e1843" + }, + { + "seq": 84, + "digest": "93ddce95bec43ce4b04684764c798fcd764cd0a7df56d78b48a01f09af08efbe" + }, + { + "seq": 85, + "digest": "dc4da566826a383bccd361848c4586d4fac73df95848300f5ae2a35e443e6fa2" + }, + { + "seq": 86, + "digest": "1129fd7f26688d814d3b1a9590c29d6c6c4ce13924cee4545b938b85eaf44ad8" + }, + { + "seq": 87, + "digest": "23f74364c345926d3d39df9eb557d3a9f4accb010fc75ef6e93b9c009dc18fb8" + }, + { + "seq": 88, + "digest": "9488c81df85323069e49624a52ead0d1695ed81b68be652a8ff9f07e30f8e8d6" + }, + { + "seq": 89, + "digest": "0856ec4adabbf231bcf1c1a647525f0b7267c841d6d6b5a17b6d374586f2d60b" + }, + { + "seq": 90, + "digest": "84c32c6983ccfa80c50cdb968358982223ff51a943632932bf0685a4d7b65ad0" + }, + { + "seq": 91, + "digest": "0cad09547877955f65fbaa4b5ee258bbd542230acffe05f11e674cb6ee539fca" + }, + { + "seq": 92, + "digest": "22736bb74ba98b0892b98ff78105b14ce7ccb164e37a8fd358b6c20cd34a83d4" + }, + { + "seq": 93, + "digest": "924a874ae3b873234c9184db683963cc7358a00b1e9ce8d8a1e23f9f0c51eff2" + }, + { + "seq": 94, + "digest": "469c5612212a13f5f7b8d06d42bff789a443b89d558f7df30e09edf7b46a71c9" + }, + { + "seq": 95, + "digest": "9fe6e2b902c7ab996ece9f388b1256a02dffc31ecd178cbf85918946f502896a" + }, + { + "seq": 96, + "digest": "414a52b52cb124c076bdb041b52840cf862f81e2335538e69a06a7678549b616" + }, + { + "seq": 97, + "digest": "c244ac4e31909b0ebe3bc09922c874ab71906ce72aa1e07fde3eb7250c93fe1f" + }, + { + "seq": 98, + "digest": "0cf7685989517c70ab72f5a95b81981b69b36645061804179faf2bbde846ee00" + }, + { + "seq": 99, + "digest": "0d7e4d13d74766c70042ef041b66c70c40f86178fc41ff1f737a5752eb9e8ead" + }, + { + "seq": 100, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 101, + "digest": "97286c21880a38583f7fef609f5161f1f7a2ea6ef8c78264af701f7371c5a20c" + }, + { + "seq": 102, + "digest": "2000c4f0b234cce1234f628c351c9bee336e7131c672a647f8d8cee0280dafb9" + }, + { + "seq": 103, + "digest": "33a2aebfc6a6b03fc7bacc6349178f724f28396be79ff360637efec5a4d022e4" + }, + { + "seq": 104, + "digest": "965bada94e3e652ec836ad58a19c982572674cc0e84585f1dcf508733e80ee63" + }, + { + "seq": 105, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 106, + "digest": "cf5695ab9373c64bde821bde01c82b38e7ab05c9482ae47c0657c28295018df0" + }, + { + "seq": 107, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 108, + "digest": "d7c8cfa2bb32597580c4eb2507d5305c5ce47cf332f38da481611f6249d24c4d" + }, + { + "seq": 109, + "digest": "b4b46b29c74c659383a8e61de3d48ab841b38f704b25833e297ef63ffa256066" + }, + { + "seq": 110, + "digest": "6f8376c298f8b1be71d98e7ed09055e76cf24fc27a9bdc850100c1032be2ba6a" + }, + { + "seq": 111, + "digest": "82c2f25b5061bee056ec9bb84df049dd6319300831c9b010d29e09ddeb6a96a4" + }, + { + "seq": 112, + "digest": "2a9f50f57cc8dc0a5f502a5f20ad3e6a96ad5ee9c8d754af90ee047beffd7991" + }, + { + "seq": 113, + "digest": "a84d626533723885f773c990ca626a3e1796eba12d3c9575fe2930a6c73627e5" + }, + { + "seq": 114, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 115, + "digest": "aed1655cae9150520c876fd9b6970690e76f5e7668f527aa4b00fde327065303" + }, + { + "seq": 116, + "digest": "50bb9b47edd9804ee5926045cf2430a5d9e65c9e1401a78a754c6970ddef8c0f" + }, + { + "seq": 117, + "digest": "0b79a867b846de91bba730509a77f158254381a0e7856bfa9b7bbdcd5a9ef0ca" + }, + { + "seq": 118, + "digest": "f1acc2ef36617ebf4aae9ecb0748eec97115733a7649fa99a49c150abd31903f" + }, + { + "seq": 119, + "digest": "f6fcb0d0a3fadf1c46644783cacb5541185bbe832e0918dae7686ed3977424ae" + }, + { + "seq": 120, + "digest": "2475f62fba3b27b986a8b2552710fa9f5b3169a3fff478229baa735f4b82ff2e" + }, + { + "seq": 121, + "digest": "311e8479904bcd203e3986a386e0029f2284f10b3f9a0e605e180e101c448b72" + }, + { + "seq": 122, + "digest": "41c5cd6e35bad8467ddafc558ae5445ca44ab9145621614d9c7f92abb7075422" + }, + { + "seq": 123, + "digest": "b61daeb47c9176ebfdea34c9b4622aa499a857a8d25c7692ff61dd0e88993251" + }, + { + "seq": 124, + "digest": "bf6b9971d0e12d0fc40c999573b7ba1911e76466d552dcc0b0050baf30801a6b" + }, + { + "seq": 125, + "digest": "2f860fb5b5f703c5be7ffa63323b27a935639e269c659e35b71680c25046fe01" + }, + { + "seq": 126, + "digest": "1981999076a47eee0924064f0ce51bc78ed8e2715dc0ca4071122db722b1ee7c" + }, + { + "seq": 127, + "digest": "df152bfcab10e61cc729362516068868fbcfe43f8c9aa1150f773fd661106f17" + }, + { + "seq": 128, + "digest": "e6db7c9c5f88e55bbdfc455c087616b9c592eb4efc14797582ee9add5ef15964" + }, + { + "seq": 129, + "digest": "8df24185d6a2d7fbf5db031e5037251970046d0df0c997401e854d376501451b" + }, + { + "seq": 130, + "digest": "4197690d9005d30333bc6c676b464ffd49ab309a399a4a873e40f519750e5594" + }, + { + "seq": 131, + "digest": "77a5bebec180fb84bab3ac3fbf5fbd74e052ee0b61d5da5ab05697c0b4314fc8" + }, + { + "seq": 132, + "digest": "54b5a5138704d736937975ae6a63cdc516e85c68fe9c7d6484b624140ddc00c4" + }, + { + "seq": 133, + "digest": "1fc2880c38ccc599525c6e65592067e9d0678e4a85564a65a1f0ae44f9ce8273" + }, + { + "seq": 134, + "digest": "37293e18a980f63ac186e1a1fdb2ea0058304a6ee760b8eb45d6e9a48c25f539" + }, + { + "seq": 135, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 136, + "digest": "62d2147917de6f5d122d0dd04b99f487592e1ed3f2464698a9968da3718c0c07" + }, + { + "seq": 137, + "digest": "a2df80b19491c5ea923d226b5aa834a8119aa92ea9248a66287fd310c5b9b995" + }, + { + "seq": 138, + "digest": "2896f1de2319850b7e5d9e13eb3eebf854cb3605aa58d9f65ae43cf74cb5b16b" + }, + { + "seq": 139, + "digest": "96094cfb58b32316f84012d775b01da601a271f6ff366a2613218b0c0f1ff541" + }, + { + "seq": 140, + "digest": "e937c91eb950623051ac598bfa7f98d955a4034c15a9d67d203e2cc612498c57" + }, + { + "seq": 141, + "digest": "0996376a632df35da3751d53b59bc3d7c87ad8850cd4a618fbf78368ae61ecd1" + }, + { + "seq": 142, + "digest": "bbf9b38232422eb2ac60f7eb63c31fbc8c78dfad82169ab5782b89c957bb107d" + }, + { + "seq": 143, + "digest": "8103affe397953ca9ef964149778085a3592045ed7aca4babd7b0aca088c271b" + }, + { + "seq": 144, + "digest": "3713ff52950c7d1c722424124fcadfdfbcfab6e685779156b5a2a1f42eff8858" + }, + { + "seq": 145, + "digest": "64d73bbb15722c80e3ea871758d3663e1379af2e1f2489ada9fbbf9cd6f3971c" + }, + { + "seq": 146, + "digest": "68abf2b7d0b22556d17196e6a2a72caf00f78586dfd2133a587b09d2ddde72d2" + }, + { + "seq": 147, + "digest": "4ef9eaa750be382c87d3658b023d493880051dad257677f51b1b3599d577418c" + }, + { + "seq": 148, + "digest": "f0bc2c5a2c55a9daf6174d2585905b08f7fc6f8e0f0773c13adfc860811c4178" + }, + { + "seq": 149, + "digest": "ed1ba2be16b0643ed7d3d8906d38603313c29d168e48a0a678174253e3fd3ca8" + }, + { + "seq": 150, + "digest": "5a63915716f62f25152fd39a3bbe96c12185bc1ed80625bbe5e3b9ba559bdb39" + }, + { + "seq": 151, + "digest": "4004904b161afb1cb3498ad4ffd8e0aa5451d5469f4a60263d3a9e60bc6f9a3f" + }, + { + "seq": 152, + "digest": "131c228b903e1618d7951d7ac799f4c7d1791f4f30505b2636e7b96586db5ea0" + }, + { + "seq": 153, + "digest": "24a90676a89a9ac8bc7cf026c8e5df676e25eb6ccc8ae3c99a8e900b17ef2720" + }, + { + "seq": 154, + "digest": "d6ff7e31e96bf3f1720106395d85cfd989cce910d694511c6a9aa9408df17c77" + }, + { + "seq": 155, + "digest": "c243ad9aa5e2acb50a05f309b58c106bc9e961005e7ee6ef2bf1230120344c6d" + }, + { + "seq": 156, + "digest": "5239d998874aa2876e1c7dd6271777dd308589b2a2d43497201bdbb4a77b62fd" + }, + { + "seq": 157, + "digest": "0cbe121e2dc16629aa198e400fe82df70454ccf544520790eaff4e9580c5d2da" + }, + { + "seq": 158, + "digest": "641dabb7e022ff6fff535fa45f4ecd0575e8c471665f1d8606dde54fc41be893" + }, + { + "seq": 159, + "digest": "273818b42c7a14358058326d98305e090db661fb097ff04a682c2c3f90a4b8aa" + }, + { + "seq": 160, + "digest": "805371be67b8d02b8d9a32ceb4bacffdbe7a1077c2aedd22b57eafb7efd06cb7" + }, + { + "seq": 161, + "digest": "c5346e439fa61237f6007bd9629454355e0ce3932253284e9a17c3545e705e2d" + }, + { + "seq": 162, + "digest": "8b307d32a8105f9d43ce6922023b6a3bd425cc1fb8916606ad59fe849ba7cde9" + }, + { + "seq": 163, + "digest": "9291367a14e218c4ce62fe693b53f7701a773e85e7d90a84d666bb0963d358cd" + }, + { + "seq": 164, + "digest": "3a2259ca9d6612727d20dfd93ae9969c56996b2ab4d7085367a6cc47f43ac632" + }, + { + "seq": 165, + "digest": "095a3e335b265b91358e5884486e3ce605052c0a2e09d33f3f2338df1691c2fc" + }, + { + "seq": 166, + "digest": "2f7a45cf5c4c6432dad743a25004df827ff6bd66d5586a9d523401fd2ac54b58" + }, + { + "seq": 167, + "digest": "7e607629cd08a5c668b31db2de4d862f15b2bc61547e7ba133785556c3e87be6" + }, + { + "seq": 168, + "digest": "a9c038027a33576772627ffa289cb30ac2c03522bd43abecf6ebd4f7cff25e0b" + } + ] + }, + { + "step": "terminal-close", + "acknowledgement": { + "status": "unknown", + "terminalSessionId": "retained-session:runtime-773-no-inference-1788987147360:runtime-773-no-inference-1788987147360-session", + "message": "The terminal connection closed, but only the interactive stop operation can prove process termination.", + "retryable": false + } + }, + { + "step": "cleanup-confirmed", + "environmentId": "sandbox-1fd5bc901d7a", + "gone": true + } + ], + "verification": { + "identicalReplay": true, + "contiguousSequences": true, + "runtimeCommit": "2f13fc7b", + "interface": "2.6.0", + "requestedLimits": { + "cpu": 1, + "memoryMB": 1024, + "diskGB": 1, + "maxLifetimeSeconds": 180, + "idleTimeoutSeconds": 60, + "egress": "blocked" + }, + "command": "tsx proof.ts", + "exitCode": 0 + } +} diff --git a/docs/agent-managed-compute/evidence/interactive-2026-09-09/stop-proof.mts b/docs/agent-managed-compute/evidence/interactive-2026-09-09/stop-proof.mts new file mode 100644 index 00000000..7f6a4810 --- /dev/null +++ b/docs/agent-managed-compute/evidence/interactive-2026-09-09/stop-proof.mts @@ -0,0 +1,228 @@ +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import { writeFile } from 'node:fs/promises' +import { setTimeout as delay } from 'node:timers/promises' +import { agentInteractiveSessionStopRequestDigest } from '@tangle-network/agent-interface' +import { Sandbox } from '@tangle-network/sandbox' +import { createTangleProvider } from '@tangle-network/agent-provider-tangle' +const { getApiKey, getBaseUrl, resolveProductProfile } = await import( + `${process.env.SANDBOX_CLI_SOURCE}/lib/config.ts` +) +const { fetchAuthenticatedAccount } = await import(`${process.env.SANDBOX_CLI_SOURCE}/lib/auth.ts`) +const { startRetainedInteractiveRun, reconnectRetainedInteractiveRun } = await import( + `${process.env.RUNTIME_SOURCE}/src/runtime/retained-interactive.ts` +) +const { claimRetainedInteractiveControl } = await import( + `${process.env.RUNTIME_SOURCE}/src/runtime/retained-interactive-control.ts` +) + +const name = `runtime-773-no-inference-${Date.now()}` +const admissions = [] +const evidence = { + name, + sdk: '0.38.2', + provider: '1.1.6', + modelPrompts: 0, + startedAt: new Date().toISOString(), + steps: [], +} +const profileName = resolveProductProfile('sandbox') +const apiKey = getApiKey(undefined, profileName) +const baseUrl = getBaseUrl(undefined, profileName) +assert.equal(new URL(baseUrl).origin, 'https://sandbox.tangle.tools') +const client = new Sandbox({ apiKey, baseUrl, timeoutMs: 15000 }) +const controller = new AbortController() +const timeout = setTimeout(() => controller.abort('180-second proof limit'), 180000) +let environmentId +let terminal +async function record(step, data = {}) { + evidence.steps.push({ step, ...data }) + console.log(JSON.stringify({ step })) + await writeFile('./stop-evidence.json', JSON.stringify(evidence, null, 2)) +} +const provider = createTangleProvider({ + client, + defaultBackend: 'codex', + readyTimeoutMs: 120000, + mapCreateInput: (input) => ({ + name, + backend: { type: 'codex', profile: input.profile }, + resources: { cpuCores: 1, memoryMB: 1024, diskGB: 1 }, + egressPolicy: { mode: 'blocked' }, + idempotencyKey: input.idempotencyKey, + metadata: input.metadata, + maxLifetimeSeconds: 180, + idleTimeoutSeconds: 60, + }), +}) +async function main() { + try { + const account = await fetchAuthenticatedAccount({ apiKey, baseUrl, timeoutMs: 10000 }) + assert.equal(account.email, process.env.EXPECTED_SANDBOX_EMAIL) + await record('sandbox-account-verified', {}) + const caps = await provider.capabilities() + assert.equal(caps.interactiveAgent?.start, true) + await record('provider-client-capability-confirmed') + const handle = await startRetainedInteractiveRun({ + provider, + environment: { + name, + idempotencyKey: name, + profile: { name: 'terminal-conformance-no-inference', harness: 'codex' }, + backend: 'codex', + resources: { cpu: 1, memoryMb: 1024, diskMb: 1024 }, + egress: { mode: 'blocked' }, + }, + interactiveIdempotencyKey: `${name}-session`, + cols: 80, + rows: 24, + signal: controller.signal, + onAdmission: async (admission) => { + admissions.push({ + phase: admission.phase, + ...(admission.environmentId ? { environmentId: admission.environmentId } : {}), + }) + if (admission.phase === 'interactive_environment') environmentId = admission.environmentId + await writeFile('./stop-admissions.json', JSON.stringify(admissions, null, 2)) + await record('admission', { + phase: admission.phase, + ...(environmentId ? { environmentId } : {}), + }) + }, + }) + environmentId = handle.ref.environmentId ?? handle.ref.run.environmentId + await record('interactive-started', { environmentId, ref: handle.ref }) + const control = await claimRetainedInteractiveControl({ + handle, + holderId: name, + signal: controller.signal, + }) + terminal = await handle.attach({ control, cols: 80, rows: 24 }, { signal: controller.signal }) + const frames = [] + const readerCtl = new AbortController() + const reader = (async () => { + try { + for await (const event of terminal.events({ signal: readerCtl.signal })) { + if (event.type === 'output') + frames.push({ + seq: event.seq, + digest: createHash('sha256').update(event.data).digest('hex'), + }) + } + } catch (error) { + if (!readerCtl.signal.aborted) throw error + } + })() + await delay(1000) + await terminal.resize({ cols: 100, rows: 30 }, { signal: controller.signal }) + await terminal.resize({ cols: 120, rows: 40 }, { signal: controller.signal }) + await terminal.input({ data: '/help\r' }, { signal: controller.signal }) + await delay(1500) + const detached = await terminal.detach({ signal: controller.signal }) + readerCtl.abort() + await reader + assert.equal(detached.status, 'detached') + assert.equal((await handle.status()).state, 'running') + await record('detached-process-running', { outputFrames: frames.length }) + const recovered = await reconnectRetainedInteractiveRun({ + provider, + ref: handle.ref, + signal: controller.signal, + }) + assert.ok(recovered) + const nextControl = await claimRetainedInteractiveControl({ + handle: recovered, + holderId: `${name}-reconnect`, + signal: controller.signal, + }) + terminal = await recovered.attach( + { control: nextControl, cols: 120, rows: 40 }, + { signal: controller.signal }, + ) + const replay = [] + const replayCtl = new AbortController() + const replayTimer = setTimeout(() => replayCtl.abort(), 2000) + try { + for await (const event of terminal.events({ signal: replayCtl.signal })) { + if (event.type === 'output') + replay.push({ + seq: event.seq, + digest: createHash('sha256').update(event.data).digest('hex'), + }) + } + } catch (error) { + if (!replayCtl.signal.aborted) throw error + } finally { + clearTimeout(replayTimer) + } + assert.ok(frames.length > 0) + assert.ok(replay.length > 0) + for (let i = 1; i < replay.length; i++) assert.ok(replay[i].seq > replay[i - 1].seq) + const prior = new Map(frames.map((frame) => [frame.seq, frame.digest])) + for (const frame of replay) + if (prior.has(frame.seq)) assert.equal(prior.get(frame.seq), frame.digest) + await record('reconnected-ordered-history', { + firstFrames: frames.length, + replayFrames: replay.length, + frames, + replay, + }) + const material = { operationId: `${name}:stop`, ref: recovered.ref, control: nextControl } + const acknowledgement = await recovered.stop( + { ...material, requestDigest: agentInteractiveSessionStopRequestDigest(material) }, + { signal: controller.signal }, + ) + await record('interactive-stop', { + status: acknowledgement.status, + effect: acknowledgement.effect, + operationId: acknowledgement.operationId, + requestDigest: acknowledgement.requestDigest, + }) + assert.ok(['accepted', 'replayed'].includes(acknowledgement.status)) + let status = await recovered.status({ signal: controller.signal }) + for (let i = 0; i < 20 && status.state !== 'exited'; i++) { + await delay(250) + status = await recovered.status({ signal: controller.signal }) + } + await record('interactive-status-after-stop', { + state: status.state, + ...(status.exited ? { exited: status.exited } : {}), + }) + assert.equal(status.state, 'exited') + } catch (error) { + await record('failure', { name: error.name, status: error.status ?? error.statusCode }) + if (error.cleanupHandle?.id) environmentId ??= error.cleanupHandle.id + process.exitCode = 1 + } finally { + clearTimeout(timeout) + if (!environmentId) { + try { + const matches = (await client.list({ limit: 100 })).filter((box) => box.name === name) + if (matches.length === 1) environmentId = matches[0].id + await record('cleanup-inventory', { matchingSandboxes: matches.length }) + } catch (error) { + await record('cleanup-inventory-failed', { name: error.name }) + } + } + if (environmentId) { + try { + const box = await client.get(environmentId) + if (box) await box.delete() + let gone = false + try { + gone = (await client.get(environmentId)) === null + } catch (error) { + if (error.status === 404 || error.statusCode === 404 || error.name === 'NotFoundError') + gone = true + else throw error + } + await record('cleanup-confirmed', { environmentId, gone }) + assert.ok(gone) + } catch (error) { + await record('cleanup-failed', { environmentId, name: error.name }) + process.exitCode = 1 + } + } + } +} +main() diff --git a/docs/agent-managed-compute/evidence/interactive-2026-09-09/stop-result.json b/docs/agent-managed-compute/evidence/interactive-2026-09-09/stop-result.json new file mode 100644 index 00000000..d0cd184b --- /dev/null +++ b/docs/agent-managed-compute/evidence/interactive-2026-09-09/stop-result.json @@ -0,0 +1,1664 @@ +{ + "name": "runtime-773-no-inference-1788987434437", + "sdk": "0.38.2", + "provider": "1.1.6", + "modelPrompts": 0, + "startedAt": "2026-09-09T20:57:14.437Z", + "steps": [ + { + "step": "sandbox-account-verified" + }, + { + "step": "provider-client-capability-confirmed" + }, + { + "step": "admission", + "phase": "interactive_intent" + }, + { + "step": "admission", + "phase": "interactive_environment", + "environmentId": "sandbox-ebc9ff348f2b" + }, + { + "step": "admission", + "phase": "interactive_started", + "environmentId": "sandbox-ebc9ff348f2b" + }, + { + "step": "interactive-started", + "environmentId": "sandbox-ebc9ff348f2b", + "ref": { + "run": { + "runId": "interactive-run-cd5f3af28932faa054f6bbe5f9a2e9ba450eebd67bcbd1d3aa303317146ae322", + "provider": "tangle-sandbox", + "environmentId": "sandbox-ebc9ff348f2b", + "sessionId": "retained-session:runtime-773-no-inference-1788987434437:runtime-773-no-inference-1788987434437-session", + "executionId": "retained-execution:runtime-773-no-inference-1788987434437:runtime-773-no-inference-1788987434437-session", + "requestDigest": "sha256:cd5f3af28932faa054f6bbe5f9a2e9ba450eebd67bcbd1d3aa303317146ae322" + }, + "preparationReceipt": { + "kind": "agent-execution-preparation", + "schemaVersion": 1, + "preparationId": "interactive-preparation:0cf8ae42-807d-4d97-85a6-3bd8e177fe73", + "requestDigest": "sha256:cd5f3af28932faa054f6bbe5f9a2e9ba450eebd67bcbd1d3aa303317146ae322", + "authoredProfileDigest": "sha256:670375e14fc065a0b3e2c1020829beecfd261be768ec3c2ce5f97de40a0fa41a", + "effectiveProfileDigest": "sha256:670375e14fc065a0b3e2c1020829beecfd261be768ec3c2ce5f97de40a0fa41a", + "backend": "codex", + "harness": "codex", + "harnessVersion": "sidecar-registry", + "resolvedModel": { + "requested": "openai/gpt-5.6-luna", + "resolved": "openai/gpt-5.6-luna" + }, + "workspace": { + "leaseId": "interactive:retained-session:runtime-773-no-inference-1788987434437:runtime-773-no-inference-1788987434437-session:0cf8ae42-807d-4d97-85a6-3bd8e177fe73", + "provider": "tangle-sidecar", + "identityDigest": "sha256:1d36689b00d03846c3e40420d19018a1df4234831d0137f47e14ba7f935c9dd3", + "isolation": "per-run", + "sourceSnapshotDigest": "sha256:7e0af78fe503afa8801ac87d9b5ff2c17f09f2f2281d13c1673b16ad93cd1e4c", + "sourceSnapshotPolicy": { + "kind": "provider-declared", + "name": "sidecar-interactive-workspace", + "version": 1, + "digest": "sha256:3c1690cf4d29a6098aeac0b1b5e457044dc6f928caa6780de536f1cc23344739" + }, + "preparedWorkspaceDigest": "sha256:bf37d0a15048b50e188416828e441386d3ad17d3f1db8221dee39e6212a8e547", + "profileActivationDigest": "sha256:369b26d8aee7b2a3d257cb4ecdae63f0ee4475660ad4e5c34b86d541aacfacd3" + }, + "axisResults": [], + "executionPlanDigest": "sha256:4d77c3c772a6e4e51147810e5ddf060b6f676dfd8025697db53fd10ce00a7657", + "materializer": { + "name": "@tangle-network/agent-profile-materialize", + "version": "workspace" + }, + "expiresAtMs": 1789073838615, + "digest": "sha256:97c89073d2253a22d2a912c81e3ad6b8195b20bbddd7e40d39d5e1183df434cc" + }, + "incarnationId": "0cf8ae42-807d-4d97-85a6-3bd8e177fe73", + "startedAt": "2026-09-09T20:57:18.615Z" + } + }, + { + "step": "detached-process-running", + "outputFrames": 192 + }, + { + "step": "reconnected-ordered-history", + "firstFrames": 192, + "replayFrames": 192, + "frames": [ + { + "seq": 1, + "digest": "8fc8b432b7ec069970a0bc9258399f62cb7ed8e5b161352ce1c1b644f39f8555" + }, + { + "seq": 2, + "digest": "7fca56273b39e327db125f6307a6960b2012c4e1a170e61ea91f9e0a3148f42d" + }, + { + "seq": 3, + "digest": "d34c148636c576b8c4810d46359cc6d08248c830c72e80434808424848bc7e97" + }, + { + "seq": 4, + "digest": "84db741a32ec8f5776493d1f18bca0fe447d7e695f3d60949d20fec7c5184970" + }, + { + "seq": 5, + "digest": "f47585fbe565364ea24085d0b7181189de031ce11204956041029036ff5a4f1f" + }, + { + "seq": 6, + "digest": "fb68f3c3d1ffdd439709a7cee789956657ac425f3811adee3e700473246c9b79" + }, + { + "seq": 7, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 8, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 9, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 10, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 11, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 12, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 13, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 14, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 15, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 16, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 17, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 18, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 19, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 20, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 21, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 22, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 23, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 24, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 25, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 26, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 27, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 28, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 29, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 30, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 31, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 32, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 33, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 34, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 35, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 36, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 37, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 38, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 39, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 40, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 41, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 42, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 43, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 44, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 45, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 46, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 47, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 48, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 49, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 50, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 51, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 52, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 53, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 54, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 55, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 56, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 57, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 58, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 59, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 60, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 61, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 62, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 63, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 64, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 65, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 66, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 67, + "digest": "27c3232c04a6274399deba5709c34fc533c7a0d0874069dbb38fb786cca80410" + }, + { + "seq": 68, + "digest": "7fc8c8596165b127063f7daba06653386e3888cf99a1fed02340ea632223c8f3" + }, + { + "seq": 69, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 70, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 71, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 72, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 73, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 74, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 75, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 76, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 77, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 78, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 79, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 80, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 81, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 82, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 83, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 84, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 85, + "digest": "83cddbcd29c963a0979d070f11493f3a0055e0200d64e2df138b8d77a6bc88db" + }, + { + "seq": 86, + "digest": "6a0bcf9a606c6d60151b72414478ed17fbe70bf62a0d956360de9a5d75831148" + }, + { + "seq": 87, + "digest": "40fa051322fbdd0cbe2d74e1416309b33fd906e19a5e900866525286d409ec37" + }, + { + "seq": 88, + "digest": "eaadac7da15ba5efc8f4c9f1d67044ed61117b90a9bf34b9c2146d97f2179d54" + }, + { + "seq": 89, + "digest": "893f76c9811c522336ad6100904bcdab7359caa5e9183dbd5e11bc2f0d0e1843" + }, + { + "seq": 90, + "digest": "93ddce95bec43ce4b04684764c798fcd764cd0a7df56d78b48a01f09af08efbe" + }, + { + "seq": 91, + "digest": "dc4da566826a383bccd361848c4586d4fac73df95848300f5ae2a35e443e6fa2" + }, + { + "seq": 92, + "digest": "1129fd7f26688d814d3b1a9590c29d6c6c4ce13924cee4545b938b85eaf44ad8" + }, + { + "seq": 93, + "digest": "23f74364c345926d3d39df9eb557d3a9f4accb010fc75ef6e93b9c009dc18fb8" + }, + { + "seq": 94, + "digest": "7ca09fb5cbc83d509fd952a6dfe79f5e27d80fb6b9de2b387bb8f60ac5534eae" + }, + { + "seq": 95, + "digest": "0180ed417f4ee697f43903da1fc1af9058a064ad0d32237d4c5a1c8ea5ade96a" + }, + { + "seq": 96, + "digest": "e7522e634791261b04331c33ebe8798fad050f0b6ae6a3e70a3ed28cc0b4d7b9" + }, + { + "seq": 97, + "digest": "ff61b2621450cf20d5785d04b0f8a3a4628280a54712b6d72bf931e890a0d434" + }, + { + "seq": 98, + "digest": "0cad09547877955f65fbaa4b5ee258bbd542230acffe05f11e674cb6ee539fca" + }, + { + "seq": 99, + "digest": "22736bb74ba98b0892b98ff78105b14ce7ccb164e37a8fd358b6c20cd34a83d4" + }, + { + "seq": 100, + "digest": "924a874ae3b873234c9184db683963cc7358a00b1e9ce8d8a1e23f9f0c51eff2" + }, + { + "seq": 101, + "digest": "469c5612212a13f5f7b8d06d42bff789a443b89d558f7df30e09edf7b46a71c9" + }, + { + "seq": 102, + "digest": "9fe6e2b902c7ab996ece9f388b1256a02dffc31ecd178cbf85918946f502896a" + }, + { + "seq": 103, + "digest": "589d5a3151957ff8a4377f3746087348cdce88011680fa42e34fbab9eb495656" + }, + { + "seq": 104, + "digest": "35d424c876bc5be16a84323a605f9fbc8c4cdf79f1c58975d6efc94fa49ea256" + }, + { + "seq": 105, + "digest": "c244ac4e31909b0ebe3bc09922c874ab71906ce72aa1e07fde3eb7250c93fe1f" + }, + { + "seq": 106, + "digest": "0cf7685989517c70ab72f5a95b81981b69b36645061804179faf2bbde846ee00" + }, + { + "seq": 107, + "digest": "a69c7f65a597d4c43452749c942f6cb5f16c9ecdea679451b24bac2349a97dbc" + }, + { + "seq": 108, + "digest": "0f1f644d59e2a6e462dbc90df9ef681baf3d79f41e89f0eb93e1fec9f9d70994" + }, + { + "seq": 109, + "digest": "97286c21880a38583f7fef609f5161f1f7a2ea6ef8c78264af701f7371c5a20c" + }, + { + "seq": 110, + "digest": "2000c4f0b234cce1234f628c351c9bee336e7131c672a647f8d8cee0280dafb9" + }, + { + "seq": 111, + "digest": "8aeb845896f474ffde083ff37697608bc7a531c96ca923b78b041f4cab8cc196" + }, + { + "seq": 112, + "digest": "c588bf1daea21f9fc6c65e642cf7f65ae692b2aff439b5297fcab29e371dae9c" + }, + { + "seq": 113, + "digest": "f986e91c2f2ba5e49873b59783de7988d97a8896f48b3090804dfa75ff4f1477" + }, + { + "seq": 114, + "digest": "451ad82e25ea76ec32f6a2c1bb514c44a6a08c44901e2b1a6450e42c5e021859" + }, + { + "seq": 115, + "digest": "d6baa3243d6d3be62cc15eb22cd48c01c9745770393388c4ebc20c2bc9633ef2" + }, + { + "seq": 116, + "digest": "d7c8cfa2bb32597580c4eb2507d5305c5ce47cf332f38da481611f6249d24c4d" + }, + { + "seq": 117, + "digest": "9d22e8e49d4ba5471c4a5cdac11513d543ebb938b282cca4b467c494b1b146b2" + }, + { + "seq": 118, + "digest": "5ce3bbff071b4353ec364ae3b48c9dcd8544d9121843d67294ae500f3f88b1b8" + }, + { + "seq": 119, + "digest": "e529fc567d89682e1fa2ad7d3b1ab16e50a40a514ccb5bd04576bb36dfb57573" + }, + { + "seq": 120, + "digest": "1f6e97353e985ef5e7d253a4320be3fc7b21ebb3a671fdcf9ddcd3aaf517642c" + }, + { + "seq": 121, + "digest": "eed356a532db2e25e41de6fd53f1844ae5c80acb8b6d64342a5264e03e208fbf" + }, + { + "seq": 122, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 123, + "digest": "cc81ceee5c9b6ce7cd8cea583b04a11560143686cbe38e691bc6a23a0b164570" + }, + { + "seq": 124, + "digest": "6be894f182786a691d0a14946a1198790f44164418680e67b0605d46e8c3dcef" + }, + { + "seq": 125, + "digest": "50bb9b47edd9804ee5926045cf2430a5d9e65c9e1401a78a754c6970ddef8c0f" + }, + { + "seq": 126, + "digest": "0b79a867b846de91bba730509a77f158254381a0e7856bfa9b7bbdcd5a9ef0ca" + }, + { + "seq": 127, + "digest": "315b03eb5e8e790524fc95b49d9fa03176066cd449bb9515f7da0e527110c186" + }, + { + "seq": 128, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 129, + "digest": "2475f62fba3b27b986a8b2552710fa9f5b3169a3fff478229baa735f4b82ff2e" + }, + { + "seq": 130, + "digest": "90df8742453442283db6538f27f447e19b137701309d68a6d0038dbc84d80d6f" + }, + { + "seq": 131, + "digest": "d4a57ba124f2c66aafc1f76ce206a9051e6a6a5dd03fbab7ce1de810379251a2" + }, + { + "seq": 132, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 133, + "digest": "f0683239d31eb01dcf54dd70bdc3bd18ef1dd0850a87018652e7c6850f6db578" + }, + { + "seq": 134, + "digest": "6d8543c26c45f1da195d8500eb14ccb5255eb399ca1b788aad478461344711b2" + }, + { + "seq": 135, + "digest": "7992b70ec2ac0aeec69a8a6457d4b6d82ff47ea40448f4b632df5d7b955dd643" + }, + { + "seq": 136, + "digest": "773d9a57668af9cb519a584bcf520eb11f4004dd58c19c67733593c5bdfb1735" + }, + { + "seq": 137, + "digest": "2ee730d336771cb8f011d0e65b42ba4e79b7fa40d62070a5a8bc503e18813b45" + }, + { + "seq": 138, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 139, + "digest": "d35fca00662fdf70c7357d22fdfb82eadfeda311424290e138e92c8f130353bd" + }, + { + "seq": 140, + "digest": "17a66c525fb27830810f38f3b725b3db5550181a8fcbb1daf61e49239c714e59" + }, + { + "seq": 141, + "digest": "df152bfcab10e61cc729362516068868fbcfe43f8c9aa1150f773fd661106f17" + }, + { + "seq": 142, + "digest": "e6db7c9c5f88e55bbdfc455c087616b9c592eb4efc14797582ee9add5ef15964" + }, + { + "seq": 143, + "digest": "8df24185d6a2d7fbf5db031e5037251970046d0df0c997401e854d376501451b" + }, + { + "seq": 144, + "digest": "4197690d9005d30333bc6c676b464ffd49ab309a399a4a873e40f519750e5594" + }, + { + "seq": 145, + "digest": "77a5bebec180fb84bab3ac3fbf5fbd74e052ee0b61d5da5ab05697c0b4314fc8" + }, + { + "seq": 146, + "digest": "1e1be3dcc92e72923c6ccdc416d3be6ec8c1d2b2c6fb2559186beb248000a592" + }, + { + "seq": 147, + "digest": "550d16ef7234dda0021e596c1c7ce104b4c4b9016c848c956465292dcd86bfa8" + }, + { + "seq": 148, + "digest": "39355cb4bcdf2ec2de8ecbc92ae6c0355888b0e16209dbdafc4053820449dc9f" + }, + { + "seq": 149, + "digest": "f9d7e1eb815a9a69a5a833bb8acbbaa2e72d38a1b860ebe375f71dc685bdf615" + }, + { + "seq": 150, + "digest": "5d5c6383c08b8753f15956576946c7374c33f7c79a5fecc89b42b0b525af521b" + }, + { + "seq": 151, + "digest": "d5ae5ff31af1f5f134d16eb956004742c45a16ffe1f9b3ed8bae3b1b80d56343" + }, + { + "seq": 152, + "digest": "e45a895195568a3a08282db9d069da8cb06f55f49d66e1ae2a974434c68a84d5" + }, + { + "seq": 153, + "digest": "343bb0c0377627f75780d3b6fb4853da961f5d88a514d6421ae079f2c5ac851d" + }, + { + "seq": 154, + "digest": "8253a6ea59d58cba0d9e08f488fcb8b08279f41f9c8097db178d3dd29284ecc6" + }, + { + "seq": 155, + "digest": "621b1fea529b1c979e2c7713b3030d6c38ad08408788faac37505f35a950ab7b" + }, + { + "seq": 156, + "digest": "bbf9b38232422eb2ac60f7eb63c31fbc8c78dfad82169ab5782b89c957bb107d" + }, + { + "seq": 157, + "digest": "c599f8aafa381bf99a301b0bd0895dfc6f43069f7fd3363e2dff3d5576f546ff" + }, + { + "seq": 158, + "digest": "2746a2de4a03a989fda9103d51d099d6fc97326af25a35909b9e6357ad818492" + }, + { + "seq": 159, + "digest": "3713ff52950c7d1c722424124fcadfdfbcfab6e685779156b5a2a1f42eff8858" + }, + { + "seq": 160, + "digest": "64d73bbb15722c80e3ea871758d3663e1379af2e1f2489ada9fbbf9cd6f3971c" + }, + { + "seq": 161, + "digest": "68abf2b7d0b22556d17196e6a2a72caf00f78586dfd2133a587b09d2ddde72d2" + }, + { + "seq": 162, + "digest": "6bf117e5be74862bdac257c1f8fdaa0fe615aea6e21a0921f226ca87f890d828" + }, + { + "seq": 163, + "digest": "1c1dfc92f2c797b4f0d7815e84ffb4a27d624578ab4798c947e4f5bc5fa01dfb" + }, + { + "seq": 164, + "digest": "f0bc2c5a2c55a9daf6174d2585905b08f7fc6f8e0f0773c13adfc860811c4178" + }, + { + "seq": 165, + "digest": "ed1ba2be16b0643ed7d3d8906d38603313c29d168e48a0a678174253e3fd3ca8" + }, + { + "seq": 166, + "digest": "5a63915716f62f25152fd39a3bbe96c12185bc1ed80625bbe5e3b9ba559bdb39" + }, + { + "seq": 167, + "digest": "4004904b161afb1cb3498ad4ffd8e0aa5451d5469f4a60263d3a9e60bc6f9a3f" + }, + { + "seq": 168, + "digest": "131c228b903e1618d7951d7ac799f4c7d1791f4f30505b2636e7b96586db5ea0" + }, + { + "seq": 169, + "digest": "321f2c0edf02edf1d9ced11f00db0070d3174b9949bc95b3f12a097f4fe8d31a" + }, + { + "seq": 170, + "digest": "8ee9cc021e9fea3832025122d844e622c187a5289baf29ffd7e4d64b25b51022" + }, + { + "seq": 171, + "digest": "c5128c6dc1b117762533818e56091b25139fffe86123b1584d845e9f2b0d6498" + }, + { + "seq": 172, + "digest": "c5759d30a8195876b846c71fe423852c136f90101e67e1f174322538e8923d17" + }, + { + "seq": 173, + "digest": "6b2d3939a0667aab66f3f5dab3162f36a39a55f9d812bd180da59620164923e4" + }, + { + "seq": 174, + "digest": "e18685891975b1f611d8d8cdb76085ead572a9f3f9bb2855dfe84a33b269ad3b" + }, + { + "seq": 175, + "digest": "8751a803675311e7c810c756be7fe404ad91afddba846d723d09147b230e803a" + }, + { + "seq": 176, + "digest": "070bcb1007edbf2a40f2bea130e6439674b53d146918329a018fb39dd5c0f8d0" + }, + { + "seq": 177, + "digest": "d95caaf9bdf20018ecfb0d77b56a9af77b8ec4934d9a727cadcdb6f0a439247e" + }, + { + "seq": 178, + "digest": "64196685b5c5340b159a3b491bbc2e92d04b109374419f859d818ab1eb093099" + }, + { + "seq": 179, + "digest": "7433db8200ba38393f1165afb43394f5032e438e276702b61199e71639bc8dd7" + }, + { + "seq": 180, + "digest": "bf44a10fb07252c275797421c121d8c449cc278ae78d8498d50261df6a190e4b" + }, + { + "seq": 181, + "digest": "c5346e439fa61237f6007bd9629454355e0ce3932253284e9a17c3545e705e2d" + }, + { + "seq": 182, + "digest": "8b307d32a8105f9d43ce6922023b6a3bd425cc1fb8916606ad59fe849ba7cde9" + }, + { + "seq": 183, + "digest": "ff738a2d877161d9ae89a6cad26db9397a7b9a0b79efab3195d09838aa23690b" + }, + { + "seq": 184, + "digest": "135ed63f4ed8742bc694373154e40bcb7d7ed539da2175b16c19ad3d4484107e" + }, + { + "seq": 185, + "digest": "5d98159a124bc25128209842f08d6ebae21f222bbc3bd51f525c6055e3e03670" + }, + { + "seq": 186, + "digest": "186a84229328e2b8b174549025a418f9d8af57a24c9c152b15a1a324aa0f9232" + }, + { + "seq": 187, + "digest": "d7699f68ea89bdeb0e0ff04cf655288734a2257c664b49ffde3440d5c3241aec" + }, + { + "seq": 188, + "digest": "51a481d3d585e0441dcaf0d9237a2313fb7beab8e56888fe119c17e1d4370e96" + }, + { + "seq": 189, + "digest": "4d825ab75bae576f7390a039347701d19c233464e9de5ff6a1bd26c0357dfbea" + }, + { + "seq": 190, + "digest": "90d8d23d0fb65d8d91cfa5b7e5a84a4f5c483befd0b09720f28fefcfd5a83481" + }, + { + "seq": 191, + "digest": "ca1945ff4445fb973eeb4d46bf61f6b3ff9c0072cec3f194bd694b3a2d5a7a15" + }, + { + "seq": 192, + "digest": "e4bd24987be41a521ca05373a2e4ab4525c4387a30bc435c472aee27815e8e31" + } + ], + "replay": [ + { + "seq": 1, + "digest": "8fc8b432b7ec069970a0bc9258399f62cb7ed8e5b161352ce1c1b644f39f8555" + }, + { + "seq": 2, + "digest": "7fca56273b39e327db125f6307a6960b2012c4e1a170e61ea91f9e0a3148f42d" + }, + { + "seq": 3, + "digest": "d34c148636c576b8c4810d46359cc6d08248c830c72e80434808424848bc7e97" + }, + { + "seq": 4, + "digest": "84db741a32ec8f5776493d1f18bca0fe447d7e695f3d60949d20fec7c5184970" + }, + { + "seq": 5, + "digest": "f47585fbe565364ea24085d0b7181189de031ce11204956041029036ff5a4f1f" + }, + { + "seq": 6, + "digest": "fb68f3c3d1ffdd439709a7cee789956657ac425f3811adee3e700473246c9b79" + }, + { + "seq": 7, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 8, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 9, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 10, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 11, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 12, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 13, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 14, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 15, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 16, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 17, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 18, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 19, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 20, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 21, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 22, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 23, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 24, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 25, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 26, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 27, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 28, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 29, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 30, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 31, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 32, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 33, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 34, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 35, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 36, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 37, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 38, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 39, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 40, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 41, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 42, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 43, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 44, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 45, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 46, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 47, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 48, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 49, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 50, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 51, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 52, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 53, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 54, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 55, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 56, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 57, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 58, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 59, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 60, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 61, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 62, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 63, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 64, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 65, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 66, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 67, + "digest": "27c3232c04a6274399deba5709c34fc533c7a0d0874069dbb38fb786cca80410" + }, + { + "seq": 68, + "digest": "7fc8c8596165b127063f7daba06653386e3888cf99a1fed02340ea632223c8f3" + }, + { + "seq": 69, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 70, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 71, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 72, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 73, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 74, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 75, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 76, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 77, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 78, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 79, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 80, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 81, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 82, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 83, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 84, + "digest": "f026d781d2dbd20d1862d6fd59b8dd8a194b4cc139a5ee5817f059e0d9b1ac5e" + }, + { + "seq": 85, + "digest": "83cddbcd29c963a0979d070f11493f3a0055e0200d64e2df138b8d77a6bc88db" + }, + { + "seq": 86, + "digest": "6a0bcf9a606c6d60151b72414478ed17fbe70bf62a0d956360de9a5d75831148" + }, + { + "seq": 87, + "digest": "40fa051322fbdd0cbe2d74e1416309b33fd906e19a5e900866525286d409ec37" + }, + { + "seq": 88, + "digest": "eaadac7da15ba5efc8f4c9f1d67044ed61117b90a9bf34b9c2146d97f2179d54" + }, + { + "seq": 89, + "digest": "893f76c9811c522336ad6100904bcdab7359caa5e9183dbd5e11bc2f0d0e1843" + }, + { + "seq": 90, + "digest": "93ddce95bec43ce4b04684764c798fcd764cd0a7df56d78b48a01f09af08efbe" + }, + { + "seq": 91, + "digest": "dc4da566826a383bccd361848c4586d4fac73df95848300f5ae2a35e443e6fa2" + }, + { + "seq": 92, + "digest": "1129fd7f26688d814d3b1a9590c29d6c6c4ce13924cee4545b938b85eaf44ad8" + }, + { + "seq": 93, + "digest": "23f74364c345926d3d39df9eb557d3a9f4accb010fc75ef6e93b9c009dc18fb8" + }, + { + "seq": 94, + "digest": "7ca09fb5cbc83d509fd952a6dfe79f5e27d80fb6b9de2b387bb8f60ac5534eae" + }, + { + "seq": 95, + "digest": "0180ed417f4ee697f43903da1fc1af9058a064ad0d32237d4c5a1c8ea5ade96a" + }, + { + "seq": 96, + "digest": "e7522e634791261b04331c33ebe8798fad050f0b6ae6a3e70a3ed28cc0b4d7b9" + }, + { + "seq": 97, + "digest": "ff61b2621450cf20d5785d04b0f8a3a4628280a54712b6d72bf931e890a0d434" + }, + { + "seq": 98, + "digest": "0cad09547877955f65fbaa4b5ee258bbd542230acffe05f11e674cb6ee539fca" + }, + { + "seq": 99, + "digest": "22736bb74ba98b0892b98ff78105b14ce7ccb164e37a8fd358b6c20cd34a83d4" + }, + { + "seq": 100, + "digest": "924a874ae3b873234c9184db683963cc7358a00b1e9ce8d8a1e23f9f0c51eff2" + }, + { + "seq": 101, + "digest": "469c5612212a13f5f7b8d06d42bff789a443b89d558f7df30e09edf7b46a71c9" + }, + { + "seq": 102, + "digest": "9fe6e2b902c7ab996ece9f388b1256a02dffc31ecd178cbf85918946f502896a" + }, + { + "seq": 103, + "digest": "589d5a3151957ff8a4377f3746087348cdce88011680fa42e34fbab9eb495656" + }, + { + "seq": 104, + "digest": "35d424c876bc5be16a84323a605f9fbc8c4cdf79f1c58975d6efc94fa49ea256" + }, + { + "seq": 105, + "digest": "c244ac4e31909b0ebe3bc09922c874ab71906ce72aa1e07fde3eb7250c93fe1f" + }, + { + "seq": 106, + "digest": "0cf7685989517c70ab72f5a95b81981b69b36645061804179faf2bbde846ee00" + }, + { + "seq": 107, + "digest": "a69c7f65a597d4c43452749c942f6cb5f16c9ecdea679451b24bac2349a97dbc" + }, + { + "seq": 108, + "digest": "0f1f644d59e2a6e462dbc90df9ef681baf3d79f41e89f0eb93e1fec9f9d70994" + }, + { + "seq": 109, + "digest": "97286c21880a38583f7fef609f5161f1f7a2ea6ef8c78264af701f7371c5a20c" + }, + { + "seq": 110, + "digest": "2000c4f0b234cce1234f628c351c9bee336e7131c672a647f8d8cee0280dafb9" + }, + { + "seq": 111, + "digest": "8aeb845896f474ffde083ff37697608bc7a531c96ca923b78b041f4cab8cc196" + }, + { + "seq": 112, + "digest": "c588bf1daea21f9fc6c65e642cf7f65ae692b2aff439b5297fcab29e371dae9c" + }, + { + "seq": 113, + "digest": "f986e91c2f2ba5e49873b59783de7988d97a8896f48b3090804dfa75ff4f1477" + }, + { + "seq": 114, + "digest": "451ad82e25ea76ec32f6a2c1bb514c44a6a08c44901e2b1a6450e42c5e021859" + }, + { + "seq": 115, + "digest": "d6baa3243d6d3be62cc15eb22cd48c01c9745770393388c4ebc20c2bc9633ef2" + }, + { + "seq": 116, + "digest": "d7c8cfa2bb32597580c4eb2507d5305c5ce47cf332f38da481611f6249d24c4d" + }, + { + "seq": 117, + "digest": "9d22e8e49d4ba5471c4a5cdac11513d543ebb938b282cca4b467c494b1b146b2" + }, + { + "seq": 118, + "digest": "5ce3bbff071b4353ec364ae3b48c9dcd8544d9121843d67294ae500f3f88b1b8" + }, + { + "seq": 119, + "digest": "e529fc567d89682e1fa2ad7d3b1ab16e50a40a514ccb5bd04576bb36dfb57573" + }, + { + "seq": 120, + "digest": "1f6e97353e985ef5e7d253a4320be3fc7b21ebb3a671fdcf9ddcd3aaf517642c" + }, + { + "seq": 121, + "digest": "eed356a532db2e25e41de6fd53f1844ae5c80acb8b6d64342a5264e03e208fbf" + }, + { + "seq": 122, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 123, + "digest": "cc81ceee5c9b6ce7cd8cea583b04a11560143686cbe38e691bc6a23a0b164570" + }, + { + "seq": 124, + "digest": "6be894f182786a691d0a14946a1198790f44164418680e67b0605d46e8c3dcef" + }, + { + "seq": 125, + "digest": "50bb9b47edd9804ee5926045cf2430a5d9e65c9e1401a78a754c6970ddef8c0f" + }, + { + "seq": 126, + "digest": "0b79a867b846de91bba730509a77f158254381a0e7856bfa9b7bbdcd5a9ef0ca" + }, + { + "seq": 127, + "digest": "315b03eb5e8e790524fc95b49d9fa03176066cd449bb9515f7da0e527110c186" + }, + { + "seq": 128, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 129, + "digest": "2475f62fba3b27b986a8b2552710fa9f5b3169a3fff478229baa735f4b82ff2e" + }, + { + "seq": 130, + "digest": "90df8742453442283db6538f27f447e19b137701309d68a6d0038dbc84d80d6f" + }, + { + "seq": 131, + "digest": "d4a57ba124f2c66aafc1f76ce206a9051e6a6a5dd03fbab7ce1de810379251a2" + }, + { + "seq": 132, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 133, + "digest": "f0683239d31eb01dcf54dd70bdc3bd18ef1dd0850a87018652e7c6850f6db578" + }, + { + "seq": 134, + "digest": "6d8543c26c45f1da195d8500eb14ccb5255eb399ca1b788aad478461344711b2" + }, + { + "seq": 135, + "digest": "7992b70ec2ac0aeec69a8a6457d4b6d82ff47ea40448f4b632df5d7b955dd643" + }, + { + "seq": 136, + "digest": "773d9a57668af9cb519a584bcf520eb11f4004dd58c19c67733593c5bdfb1735" + }, + { + "seq": 137, + "digest": "2ee730d336771cb8f011d0e65b42ba4e79b7fa40d62070a5a8bc503e18813b45" + }, + { + "seq": 138, + "digest": "4ec4e904b2aa77d61b527d4f83b9e2530196080ac0d5aab786db42d7c4dfceb1" + }, + { + "seq": 139, + "digest": "d35fca00662fdf70c7357d22fdfb82eadfeda311424290e138e92c8f130353bd" + }, + { + "seq": 140, + "digest": "17a66c525fb27830810f38f3b725b3db5550181a8fcbb1daf61e49239c714e59" + }, + { + "seq": 141, + "digest": "df152bfcab10e61cc729362516068868fbcfe43f8c9aa1150f773fd661106f17" + }, + { + "seq": 142, + "digest": "e6db7c9c5f88e55bbdfc455c087616b9c592eb4efc14797582ee9add5ef15964" + }, + { + "seq": 143, + "digest": "8df24185d6a2d7fbf5db031e5037251970046d0df0c997401e854d376501451b" + }, + { + "seq": 144, + "digest": "4197690d9005d30333bc6c676b464ffd49ab309a399a4a873e40f519750e5594" + }, + { + "seq": 145, + "digest": "77a5bebec180fb84bab3ac3fbf5fbd74e052ee0b61d5da5ab05697c0b4314fc8" + }, + { + "seq": 146, + "digest": "1e1be3dcc92e72923c6ccdc416d3be6ec8c1d2b2c6fb2559186beb248000a592" + }, + { + "seq": 147, + "digest": "550d16ef7234dda0021e596c1c7ce104b4c4b9016c848c956465292dcd86bfa8" + }, + { + "seq": 148, + "digest": "39355cb4bcdf2ec2de8ecbc92ae6c0355888b0e16209dbdafc4053820449dc9f" + }, + { + "seq": 149, + "digest": "f9d7e1eb815a9a69a5a833bb8acbbaa2e72d38a1b860ebe375f71dc685bdf615" + }, + { + "seq": 150, + "digest": "5d5c6383c08b8753f15956576946c7374c33f7c79a5fecc89b42b0b525af521b" + }, + { + "seq": 151, + "digest": "d5ae5ff31af1f5f134d16eb956004742c45a16ffe1f9b3ed8bae3b1b80d56343" + }, + { + "seq": 152, + "digest": "e45a895195568a3a08282db9d069da8cb06f55f49d66e1ae2a974434c68a84d5" + }, + { + "seq": 153, + "digest": "343bb0c0377627f75780d3b6fb4853da961f5d88a514d6421ae079f2c5ac851d" + }, + { + "seq": 154, + "digest": "8253a6ea59d58cba0d9e08f488fcb8b08279f41f9c8097db178d3dd29284ecc6" + }, + { + "seq": 155, + "digest": "621b1fea529b1c979e2c7713b3030d6c38ad08408788faac37505f35a950ab7b" + }, + { + "seq": 156, + "digest": "bbf9b38232422eb2ac60f7eb63c31fbc8c78dfad82169ab5782b89c957bb107d" + }, + { + "seq": 157, + "digest": "c599f8aafa381bf99a301b0bd0895dfc6f43069f7fd3363e2dff3d5576f546ff" + }, + { + "seq": 158, + "digest": "2746a2de4a03a989fda9103d51d099d6fc97326af25a35909b9e6357ad818492" + }, + { + "seq": 159, + "digest": "3713ff52950c7d1c722424124fcadfdfbcfab6e685779156b5a2a1f42eff8858" + }, + { + "seq": 160, + "digest": "64d73bbb15722c80e3ea871758d3663e1379af2e1f2489ada9fbbf9cd6f3971c" + }, + { + "seq": 161, + "digest": "68abf2b7d0b22556d17196e6a2a72caf00f78586dfd2133a587b09d2ddde72d2" + }, + { + "seq": 162, + "digest": "6bf117e5be74862bdac257c1f8fdaa0fe615aea6e21a0921f226ca87f890d828" + }, + { + "seq": 163, + "digest": "1c1dfc92f2c797b4f0d7815e84ffb4a27d624578ab4798c947e4f5bc5fa01dfb" + }, + { + "seq": 164, + "digest": "f0bc2c5a2c55a9daf6174d2585905b08f7fc6f8e0f0773c13adfc860811c4178" + }, + { + "seq": 165, + "digest": "ed1ba2be16b0643ed7d3d8906d38603313c29d168e48a0a678174253e3fd3ca8" + }, + { + "seq": 166, + "digest": "5a63915716f62f25152fd39a3bbe96c12185bc1ed80625bbe5e3b9ba559bdb39" + }, + { + "seq": 167, + "digest": "4004904b161afb1cb3498ad4ffd8e0aa5451d5469f4a60263d3a9e60bc6f9a3f" + }, + { + "seq": 168, + "digest": "131c228b903e1618d7951d7ac799f4c7d1791f4f30505b2636e7b96586db5ea0" + }, + { + "seq": 169, + "digest": "321f2c0edf02edf1d9ced11f00db0070d3174b9949bc95b3f12a097f4fe8d31a" + }, + { + "seq": 170, + "digest": "8ee9cc021e9fea3832025122d844e622c187a5289baf29ffd7e4d64b25b51022" + }, + { + "seq": 171, + "digest": "c5128c6dc1b117762533818e56091b25139fffe86123b1584d845e9f2b0d6498" + }, + { + "seq": 172, + "digest": "c5759d30a8195876b846c71fe423852c136f90101e67e1f174322538e8923d17" + }, + { + "seq": 173, + "digest": "6b2d3939a0667aab66f3f5dab3162f36a39a55f9d812bd180da59620164923e4" + }, + { + "seq": 174, + "digest": "e18685891975b1f611d8d8cdb76085ead572a9f3f9bb2855dfe84a33b269ad3b" + }, + { + "seq": 175, + "digest": "8751a803675311e7c810c756be7fe404ad91afddba846d723d09147b230e803a" + }, + { + "seq": 176, + "digest": "070bcb1007edbf2a40f2bea130e6439674b53d146918329a018fb39dd5c0f8d0" + }, + { + "seq": 177, + "digest": "d95caaf9bdf20018ecfb0d77b56a9af77b8ec4934d9a727cadcdb6f0a439247e" + }, + { + "seq": 178, + "digest": "64196685b5c5340b159a3b491bbc2e92d04b109374419f859d818ab1eb093099" + }, + { + "seq": 179, + "digest": "7433db8200ba38393f1165afb43394f5032e438e276702b61199e71639bc8dd7" + }, + { + "seq": 180, + "digest": "bf44a10fb07252c275797421c121d8c449cc278ae78d8498d50261df6a190e4b" + }, + { + "seq": 181, + "digest": "c5346e439fa61237f6007bd9629454355e0ce3932253284e9a17c3545e705e2d" + }, + { + "seq": 182, + "digest": "8b307d32a8105f9d43ce6922023b6a3bd425cc1fb8916606ad59fe849ba7cde9" + }, + { + "seq": 183, + "digest": "ff738a2d877161d9ae89a6cad26db9397a7b9a0b79efab3195d09838aa23690b" + }, + { + "seq": 184, + "digest": "135ed63f4ed8742bc694373154e40bcb7d7ed539da2175b16c19ad3d4484107e" + }, + { + "seq": 185, + "digest": "5d98159a124bc25128209842f08d6ebae21f222bbc3bd51f525c6055e3e03670" + }, + { + "seq": 186, + "digest": "186a84229328e2b8b174549025a418f9d8af57a24c9c152b15a1a324aa0f9232" + }, + { + "seq": 187, + "digest": "d7699f68ea89bdeb0e0ff04cf655288734a2257c664b49ffde3440d5c3241aec" + }, + { + "seq": 188, + "digest": "51a481d3d585e0441dcaf0d9237a2313fb7beab8e56888fe119c17e1d4370e96" + }, + { + "seq": 189, + "digest": "4d825ab75bae576f7390a039347701d19c233464e9de5ff6a1bd26c0357dfbea" + }, + { + "seq": 190, + "digest": "90d8d23d0fb65d8d91cfa5b7e5a84a4f5c483befd0b09720f28fefcfd5a83481" + }, + { + "seq": 191, + "digest": "ca1945ff4445fb973eeb4d46bf61f6b3ff9c0072cec3f194bd694b3a2d5a7a15" + }, + { + "seq": 192, + "digest": "e4bd24987be41a521ca05373a2e4ab4525c4387a30bc435c472aee27815e8e31" + } + ] + }, + { + "step": "interactive-stop", + "status": "accepted", + "effect": "stopped", + "operationId": "runtime-773-no-inference-1788987434437:stop", + "requestDigest": "sha256:30c3f4746953a1f447bb24af96ad67ccda184f09791c0cd931c509b48df1f683" + }, + { + "step": "interactive-status-after-stop", + "state": "exited" + }, + { + "step": "cleanup-confirmed", + "environmentId": "sandbox-ebc9ff348f2b", + "gone": true + } + ], + "verification": { + "identicalReplay": true, + "contiguousSequences": true, + "runtimeCommit": "2f13fc7b", + "interface": "2.6.0", + "requestedLimits": { + "cpu": 1, + "memoryMB": 1024, + "diskGB": 1, + "maxLifetimeSeconds": 180, + "idleTimeoutSeconds": 60, + "egress": "blocked" + }, + "command": "tsx stop-proof.ts", + "exitCode": 0 + } +} diff --git a/docs/agent-managed-compute/reliability.md b/docs/agent-managed-compute/reliability.md index a96b456f..17a29dcc 100644 --- a/docs/agent-managed-compute/reliability.md +++ b/docs/agent-managed-compute/reliability.md @@ -277,3 +277,38 @@ Deleting a run must either delete referenced private blobs or record why shared - Do not use one shared mutable checkout as the default parallel workspace. - Do not make every provider pretend it supports live steering or sessions. - Do not merge agent knowledge state into coordinator state. + +## Caller-named resource accounting + +`Budget.resources` declares caller-owned names, explicit units, and non-negative safe-integer limits. +`Spend.resources` reports safe-integer amounts and explicit completeness flags in those same units. +Choose units fine enough for the measurement, such as GPU-milliseconds or bytes. +Fractional values and values above `Number.MAX_SAFE_INTEGER` are rejected. +Default worker partitions round each resource allocation down to an integer. +A resource usage event reports an increment; a terminal spend reports the total for the same invocation. +The runtime avoids counting streamed and terminal measurements twice. +Conflicting totals retain the larger subtotal and mark completeness unknown. +An aggregate above `Number.MAX_SAFE_INTEGER` retains that value as a lower bound and marks completeness unknown. +Original child receipts remain in the journal. +Overflow closes enforced admission and settles the affected reservation. + +The existing budget pool reserves standard and named channels together before starting a child. +Each child must declare every resource enforced by its parent, with matching units. +`supervise` rejects incompatible `perWorker` resource declarations before invoking its driver. +Known settlement commits measured usage and refunds the unused allocation. +An overrun remains recorded and fails settlement. +Missing or unknown enforced measurements close admission for that dimension. +An omitted measurement becomes unknown, including when an executor terminates or cancellation interrupts reporting. +Only a proven refusal before execution can refund an unmeasured allocation as known zero. + +Names, units, amounts, and unknown flags survive journal aggregation and retained execution recovery. +Unknown recovery evidence cannot grant fresh usable capacity. +These are accounting limits: callers must supply trustworthy measurements from their executors. +Standard backends cannot invent measurements for caller-defined resources. +Custom executor receipts or explicit metering must supply those measurements, including the driver's own work. +A caller-owned `ToolLoopChat` can return `resources` with each result, including explicit known-zero measurements. +GPU usage and `boxMinutes` do not convert into named resources automatically. +Estimated box lifetime remains separate evidence and does not become a measured resource receipt. + +See the offline [example](../../examples/supervise/named-resources.ts). +The concurrent conservation and durable replay proof is in `tests/kernel/named-resource-budgets.test.ts`. diff --git a/docs/agent-managed-compute/validation.md b/docs/agent-managed-compute/validation.md index c8da4486..ee6a998d 100644 --- a/docs/agent-managed-compute/validation.md +++ b/docs/agent-managed-compute/validation.md @@ -216,3 +216,10 @@ Ship the distributed path only when: A failed or underpowered benchmark remains open work. It is not evidence that agent-managed compute is ineffective. + +## Retained interactive Sandbox evidence + +The [2026-09-09 live records](./evidence/interactive-2026-09-09/README.md) cover two bounded Codex terminal runs without model prompts. +Exact-reference reconnect replayed all 168 and 192 captured frames, respectively. +The second run received an accepted stop acknowledgement and then observed `exited` before verified sandbox deletion. +Worker native-child binding, live coordinator restart, environment expiry, and local Bridge conformance remain outside this proof. diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index f47b0990..3e79d6e2 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.208.1` and `@tangle-network/agent-eval@0.179.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.209.0` and `@tangle-network/agent-eval@0.180.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface @@ -416,7 +416,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 167 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 933 exports. +Import from `@tangle-network/agent-runtime/kernel` — 935 exports. | Symbol | Kind | Summary | |---|---|---| @@ -939,6 +939,8 @@ Import from `@tangle-network/agent-runtime/kernel` — 933 exports. | `ResolvedMcpServerLaunch` | interface | The spawn-ready strings for one stdio MCP server: profile config values | | `ResolvedSupervisorProfile` | interface | The exact profile fields consumed by supervisor materialization. | | `ResolvedWorkerSpawnRetry` | interface | A policy with every bound decided — what {@link retryPreSpawnRefusals} acts on. Produced by | +| `ResourceLimit` | interface | Caller-defined resource ceiling in non-negative safe-integer units, consistent throughout a tree. | +| `ResourceSpend` | interface | Non-negative safe-integer subtotal. False means unknown, even when amount is zero. | | `ResultBlobStore` | interface | Content-addressed result blobs (the `outRef` → artifact map) backing the replay | | `ResumedKeyState` | interface | What the journal proves about one keyed assignment at resume time. | | `ResumedWork` | interface | The committed work a resumed run inherits from its journal. `settled` is the replayed | diff --git a/docs/api/runtime.md b/docs/api/runtime.md index 260d8a84..e1e7cc86 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -6338,7 +6338,7 @@ readonly `AnalystFinding`[] ###### budget -`Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +`Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> ###### Returns @@ -8277,7 +8277,7 @@ policy belong to the exact executable `AgentProfile` consumed by `streamAgentTur > `optional` **complete?**: (`body`, `request?`) => `Promise`\<`unknown`\> -Injectable OpenAI-compatible transport for offline execution. +Injectable OpenAI-compatible transport. Optional usage.resources carries measured turn totals. ###### Parameters @@ -12105,6 +12105,10 @@ Opaque, single-use reservation handle returned by `reserve` and consumed by > `readonly` **reserved**: `object` +###### resources? + +> `readonly` `optional` **resources?**: `Readonly`\<`Record`\<`string`, [`ResourceLimit`](#resourcelimit)\>\> + ###### tokens > `readonly` **tokens**: `number` @@ -12158,7 +12162,7 @@ while the public readout remains explicitly unknown. > **reserve**(`b`): \{ `ok`: `true`; `ticket`: [`ReservationTicket`](#reservationticket); \} \| \{ `ok`: `false`; `reason`: [`ReservationRejection`](#reservationrejection); \} Atomically reserve a child's full ceiling from the free balance. Fails closed -({ ok: false }) when the pool can't cover tokens, usd, or iterations — the +({ ok: false }) when the pool can't cover standard or named channels — the caller inspects `ok` before `ticket`. ###### Parameters @@ -12223,14 +12227,14 @@ The current readout, reflecting all outstanding reservations. ##### observe() -> **observe**(`spend`): `void` +> **observe**(`spend`, `options?`): `void` Record OBSERVED spend that did NOT go through reserve/reconcile — the driver's OWN inference (its chat turns), which is real compute but not a spawned child. A direct `free → committed` debit, so `total ≡ free + reserved + committed` is preserved: equal-k counts the driver's tokens and the in-loop budget guard (`readout().tokensLeft`) sees them. `free` may go negative -when a run overspends — that is honest (the readout then signals exhaustion). It never throws: -the spend already happened, so accounting records reality; the in-loop guard prevents MORE. +when a run overspends. Unknown enforced resource usage is recorded before throwing. +Partial increments defer completeness checks until the invocation reports its terminal spend. The DURABLE record is the journal's `metered` event (written by `Scope.meter`); this debit only makes the live `readout()` reflect driver inference for the in-loop guard. @@ -12240,6 +12244,12 @@ only makes the live `readout()` reflect driver inference for the in-loop guard. [`Spend`](#spend) +###### options? + +###### partial? + +`boolean` + ###### Returns `void` @@ -13849,7 +13859,7 @@ What a finalizer gets to decide with. `delivered` is the ONLY output material; ` ##### budget -> `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +> `readonly` **budget**: `Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> *** @@ -15924,6 +15934,10 @@ False when observed `tokens` is only a known subtotal, not a complete total — False when observed dollar spend is only a known subtotal, not a complete total. +##### resources? + +> `readonly` `optional` **resources?**: `Readonly`\<`Record`\<`string`, [`ResourceSpend`](#resourcespend)\>\> + ##### pendingMessages > `readonly` **pendingMessages**: `number` @@ -16051,6 +16065,10 @@ The scope-side facts about a child, independent of whether its executor cooperat > `readonly` `optional` **usdKnown?**: `boolean` +##### resources? + +> `readonly` `optional` **resources?**: `Readonly`\<`Record`\<`string`, [`ResourceSpend`](#resourcespend)\>\> + *** ### PromptHandle @@ -21095,6 +21113,43 @@ inspects `succeeded` before `value` (no silent fallback). *** +### ResourceLimit + +Caller-defined resource ceiling in non-negative safe-integer units, consistent throughout a tree. + +#### Properties + +##### unit + +> `readonly` **unit**: `string` + +##### limit + +> `readonly` **limit**: `number` + +*** + +### ResourceSpend + +Non-negative safe-integer subtotal. False means unknown, even when amount is zero. +Select sufficiently fine units, such as GPU-milliseconds. + +#### Properties + +##### unit + +> `readonly` **unit**: `string` + +##### amount + +> `readonly` **amount**: `number` + +##### known + +> `readonly` **known**: `boolean` + +*** + ### Budget A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. @@ -21117,6 +21172,10 @@ A budget envelope on a spawn or the root. All ceilings; the pool reserves agains > `readonly` `optional` **deadlineMs?**: `number` +##### resources? + +> `readonly` `optional` **resources?**: `Readonly`\<`Record`\<`string`, [`ResourceLimit`](#resourcelimit)\>\> + *** ### Spend @@ -21130,6 +21189,10 @@ holds the reason. #### Properties +##### resources? + +> `optional` **resources?**: `Readonly`\<`Record`\<`string`, [`ResourceSpend`](#resourcespend)\>\> + ##### iterations > **iterations**: `number` @@ -21389,7 +21452,7 @@ The live tree — reads the in-memory nursery, not the journal. ##### budget -> `readonly` **budget**: `Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +> `readonly` **budget**: `Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> Conserved-pool readouts (post-reservation). @@ -22532,7 +22595,7 @@ Default impl returns false for every settlement (flat — never widens). ###### budget -`Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +`Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> ###### Returns @@ -26761,7 +26824,7 @@ What the supervisor AUTHORS per sub-task: one complete canonical profile whose n ### BudgetReadout -> **BudgetReadout** = `Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +> **BudgetReadout** = `Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> Post-reservation pool readout — the shape `Scope.budget` exposes. `tokensLeft`, `usdLeft`, and `reservedTokens` reflect committed-but-unsettled reservations; @@ -27356,7 +27419,7 @@ How a token count was obtained. ### UsageEvent -> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; `freshInput?`: `number`; `cacheRead?`: `number`; `cacheWrite?`: `number`; `cacheBreakdownKnown?`: `false`; `provenance?`: [`TokenUsageProvenance`](#tokenusageprovenance); \} \| \{ `kind`: `"cost"`; `usdKnown`: `true`; `usd`: `number`; `provenance`: `"provider-receipt"` \| `"billing-receipt"`; \} \| \{ `kind`: `"cost"`; `usdKnown`: `false`; `usd`: `number`; `usdEstimated?`: `number`; `provenance`: `"catalog-estimate"` \| `"uncaptured"`; \} \| \{ `kind`: `"progress"`; `progress`: [`ExecutorProgressEvent`](#executorprogressevent); \} \| \{ `kind`: `"iteration"`; \} +> **UsageEvent** = \{ `kind`: `"tokens"`; `tokensKnown?`: `false`; `input`: `number`; `output`: `number`; `freshInput?`: `number`; `cacheRead?`: `number`; `cacheWrite?`: `number`; `cacheBreakdownKnown?`: `false`; `provenance?`: [`TokenUsageProvenance`](#tokenusageprovenance); \} \| \{ `kind`: `"cost"`; `usdKnown`: `true`; `usd`: `number`; `provenance`: `"provider-receipt"` \| `"billing-receipt"`; \} \| \{ `kind`: `"cost"`; `usdKnown`: `false`; `usd`: `number`; `usdEstimated?`: `number`; `provenance`: `"catalog-estimate"` \| `"uncaptured"`; \} \| \{ `kind`: `"progress"`; `progress`: [`ExecutorProgressEvent`](#executorprogressevent); \} \| \{ `kind`: `"resource"`; `name`: `string`; `unit`: `string`; `amount`: `number`; `known`: `boolean`; \} \| \{ `kind`: `"iteration"`; \} #### Union Members @@ -27508,6 +27571,12 @@ Observed output, not accounting. Meters ignore it; the turn projection publishes ##### Type Literal +\{ `kind`: `"resource"`; `name`: `string`; `unit`: `string`; `amount`: `number`; `known`: `boolean`; \} + +*** + +##### Type Literal + \{ `kind`: `"iteration"`; \} *** @@ -28598,7 +28667,7 @@ Durable provider identity evidence, independent from the planned materialization ### SpendChannel -> **SpendChannel** = `"tokens"` \| `"usd"` +> **SpendChannel** = `"tokens"` \| `"usd"` \| `` `resource:${string}` `` The accounting channels a usage gap leaves incomplete. @@ -29008,7 +29077,7 @@ Provider-neutral conversation record accepted by a tool-loop brain. ### ToolLoopChat -> **ToolLoopChat** = (`messages`, `tools`, `context?`) => `Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; `reasoning?`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; `model?`: `string`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `transportAttempts?`: `number`; \}\> +> **ToolLoopChat** = (`messages`, `tools`, `context?`) => `Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; `reasoning?`: `number`; \}; `resources?`: [`Spend`](#spend)\[`"resources"`\]; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; `model?`: `string`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `transportAttempts?`: `number`; \}\> One inference turn over the running conversation + the tool specs → the model's text, any tool calls, and token usage. The seam every brain satisfies. @@ -29029,7 +29098,7 @@ One inference turn over the running conversation + the tool specs → the model' #### Returns -`Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; `reasoning?`: `number`; \}; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; `model?`: `string`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `transportAttempts?`: `number`; \}\> +`Promise`\<\{ `content?`: `string` \| `null`; `toolCalls`: [`ToolLoopToolCall`](#toollooptoolcall)[]; `usage?`: \{ `input`: `number`; `output`: `number`; `reasoning?`: `number`; \}; `resources?`: [`Spend`](#spend)\[`"resources"`\]; `costUsd?`: `number`; `costProvenance?`: `"provider-receipt"` \| `"billing-receipt"` \| `"catalog-estimate"`; `usageUnknown?`: `true`; `model?`: `string`; `promptCache?`: `Readonly`\<`Record`\<`string`, `number` \| `string`\>\>; `transportAttempts?`: `number`; \}\> *** @@ -33658,7 +33727,7 @@ readonly [`FinalizerSettled`](#finalizersettled)[] ###### budget -`Readonly`\<\{ `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> +`Readonly`\<\{ `resources?`: `Readonly`\<`Record`\<`string`, \{ `unit`: `string`; `limit`: `number`; `remaining`: `number`; `reserved`: `number`; `committed`: `number`; `known`: `boolean`; \}\>\>; `tokensLeft`: `number`; `tokensKnown`: `boolean`; `cacheBreakdownKnown`: `boolean`; `usdLeft`: `number`; `usdCapped`: `boolean`; `usdKnown`: `boolean`; `iterationsLeft`: `number`; `deadlineMs`: `number`; `reservedTokens`: `number`; \}\> #### Returns diff --git a/docs/canonical-api.md b/docs/canonical-api.md index 7e9d7763..2a1d17af 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,9 +4,9 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.208.1.** +> **Version 0.209.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. -> `agent-eval` must satisfy `>=0.179.0 <0.180.0`. +> `agent-eval` must satisfy `>=0.180.0 <0.181.0`. > `sandbox` must satisfy `>=0.36.4 <0.39.0`. > Portable profile and tool-part types come from `@tangle-network/agent-interface` `^2.6.0`. > diff --git a/examples/supervise/README.md b/examples/supervise/README.md index 38ae4b97..b5cd77bc 100644 --- a/examples/supervise/README.md +++ b/examples/supervise/README.md @@ -44,3 +44,45 @@ Three settings are worth knowing. Multi-agent orchestration usually becomes glue code: spawn, track, collect, and guess when the work is done. This is one call with defaults, so you write a goal and a profile instead of a framework. "Done" is your check against a worker's output, so a worker cannot claim success, and a failure reports the real reason and the spend. + +## Caller-named resource limits + +Run the offline [resource accounting example](./named-resources.ts): + +```bash +pnpm tsx --tsconfig tsconfig.examples.json examples/supervise/named-resources.ts +``` + +`Budget.resources` pairs each caller-owned name with a unit and limit. +Executors report matching `Spend.resources` totals or incremental resource usage events. +Every child must declare all dimensions enforced by its parent. +Reservations include all standard and named channels atomically. +Known unused allocations return to the pool. +Missing or unknown enforced measurements block further admission and remain unknown after restart. +The runtime trusts executor measurements; it does not measure accelerator use or network traffic itself. + +A model-driven root also needs a measured receipt for every enforced dimension. +A custom child executor alone cannot supply the root's measurements. +The existing `router.complete` transport can return them as `usage.resources`: + +```ts +return { + ...completion, + usage: { + ...completion.usage, + resources: { + compute: { unit: 'millisecond', amount: measuredComputeMs, known: true }, + transfer: { unit: 'byte', amount: measuredTransferBytes, known: true }, + }, + }, +} +``` + +The transport must forward the Runtime-authored request without changing its model, profile, tools, or settings. +It must derive those amounts from trustworthy measurements for that completion. +An explicit measured zero is valid; an omitted dimension remains unknown and blocks admission. +Buffered and streamed Router responses use the same receipt validation. +After a transport retry, the final measured subtotal remains available, but completeness becomes unknown because earlier attempts lack receipts. +Inline Router executors sum turn receipts and preserve unknown measurements across turns. +Custom tool-loop brains can return the same map as `resources` on their existing response. +The public `supervise` regression is `tests/kernel/named-resource-driver.test.ts`. diff --git a/examples/supervise/named-resources.ts b/examples/supervise/named-resources.ts new file mode 100644 index 00000000..ef749bfa --- /dev/null +++ b/examples/supervise/named-resources.ts @@ -0,0 +1,36 @@ +import { + type Budget, + createBudgetPool, + spendFromUsageEvents, +} from '@tangle-network/agent-runtime/kernel' + +// The executor supplies measured increments in caller-defined safe-integer units. +// Select fine units instead of reporting fractional seconds. +const budget: Budget = { + maxTokens: 100, + maxIterations: 2, + resources: { + accelerator: { unit: 'GPU-millisecond', limit: 10_000 }, + transfer: { unit: 'byte', limit: 1_000 }, + }, +} +const pool = createBudgetPool(budget, 0) +const reservation = pool.reserve({ + ...budget, + resources: { + accelerator: { unit: 'GPU-millisecond', limit: 6_000 }, + transfer: { unit: 'byte', limit: 600 }, + }, +}) +if (!reservation.ok) throw new Error(reservation.reason) + +pool.reconcile( + reservation.ticket, + spendFromUsageEvents([ + { kind: 'resource', name: 'accelerator', unit: 'GPU-millisecond', amount: 2_000, known: true }, + { kind: 'resource', name: 'transfer', unit: 'byte', amount: 400, known: true }, + ]), +) +console.log(pool.readout().resources) +// accelerator: remaining 8_000, committed 2_000; transfer: remaining 600, committed 400. +// Scope performs this reservation and reconciliation around each child automatically. diff --git a/package.json b/package.json index 42503da7..dae7bddb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.208.1", + "version": "0.209.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { @@ -131,7 +131,7 @@ "@arethetypeswrong/cli": "catalog:", "@biomejs/biome": "^2.5.11", "@modelcontextprotocol/sdk": "catalog:", - "@tangle-network/agent-eval": "0.179.0", + "@tangle-network/agent-eval": "0.180.0", "@tangle-network/agent-interface": "catalog:", "@tangle-network/sandbox": "catalog:", "@types/node": "catalog:", @@ -155,7 +155,7 @@ "license": "MIT", "packageManager": "pnpm@11.17.0", "peerDependencies": { - "@tangle-network/agent-eval": ">=0.179.0 <0.180.0", + "@tangle-network/agent-eval": ">=0.180.0 <0.181.0", "@tangle-network/agent-interface": "^2.6.0", "@tangle-network/sandbox": ">=0.36.4 <0.39.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5089227d..31ac6df9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,14 +16,14 @@ catalogs: specifier: '>=0.9.6 <0.10.0' version: 0.9.6 '@tangle-network/agent-eval': - specifier: '>=0.179.0 <0.180.0' - version: 0.179.0 + specifier: '>=0.180.0 <0.181.0' + version: 0.180.0 '@tangle-network/agent-interface': specifier: ^2.6.0 version: 2.6.0 '@tangle-network/agent-knowledge': - specifier: ^15.0.2 - version: 15.0.2 + specifier: ^15.0.3 + version: 15.0.3 '@tangle-network/agent-profile-materialize': specifier: '>=0.19.0 <0.20.0' version: 0.19.0 @@ -58,7 +58,7 @@ importers: version: 0.9.6(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 15.0.2(@tangle-network/agent-eval@0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0) + version: 15.0.3(@tangle-network/agent-eval@0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0) '@tangle-network/agent-profile-materialize': specifier: 'catalog:' version: 0.19.0(@tangle-network/agent-interface@2.6.0) @@ -79,8 +79,8 @@ importers: specifier: 'catalog:' version: 1.30.0(supports-color@10.2.2)(zod@4.5.4) '@tangle-network/agent-eval': - specifier: 0.179.0 - version: 0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) + specifier: 0.180.0 + version: 0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': specifier: 'catalog:' version: 2.6.0 @@ -134,13 +134,13 @@ importers: dependencies: '@tangle-network/agent-eval': specifier: 'catalog:' - version: 0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) + version: 0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': specifier: 'catalog:' version: 2.6.0 '@tangle-network/agent-knowledge': specifier: 'catalog:' - version: 15.0.2(@tangle-network/agent-eval@0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0) + version: 15.0.3(@tangle-network/agent-eval@0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0) '@tangle-network/agent-runtime': specifier: workspace:^ version: link:.. @@ -1297,20 +1297,20 @@ packages: '@modelcontextprotocol/sdk': optional: true - '@tangle-network/agent-eval@0.179.0': - resolution: {integrity: sha512-rC3uAya4nxH0+8IISJrh8i6N4+wQbhRZiNnX291ChQBiSqYQG85JZ+yLYrgHQQt3ttJyIRtaOnfi8KUw8Vc4xQ==} + '@tangle-network/agent-eval@0.180.0': + resolution: {integrity: sha512-sQFExEf/3eaVM0/PbNWsFJ4cSWRgQayx6VB0nR6yy1Av9XNzQ7RuVMllOmJYz3anoJ4xHjiMpbrFlRnu4MM5VQ==} engines: {node: '>=20.19.0'} hasBin: true '@tangle-network/agent-interface@2.6.0': resolution: {integrity: sha512-tEByATif9oM5EEQB94Fytjt1Jtd70KxbWudKpwMQYyqXw2dcVFh2JTkeE8rIQP2F8KDLj9n5VHFkwg/ikbr7dg==} - '@tangle-network/agent-knowledge@15.0.2': - resolution: {integrity: sha512-8BJ4udy14I/U3qVUXrc3EI9vIpmxMkWLIv3Y8PVYVPa/ASjtGz2E42PtqhW2lhCbkCfY3Ar0D8ZcLFW49em93g==} + '@tangle-network/agent-knowledge@15.0.3': + resolution: {integrity: sha512-a9tmJXloklXHRTFVBPLmJGvgJFTCVfLp/GCo/zY31pA0g6/3f7jdGoQPjuhdNhbkJixxJCaCy4whevx9uyO86Q==} engines: {node: '>=20.19.0'} hasBin: true peerDependencies: - '@tangle-network/agent-eval': '>=0.174.0 <0.180.0' + '@tangle-network/agent-eval': '>=0.174.0 <0.181.0' '@tangle-network/agent-interface': ^2.0.0 '@tangle-network/agent-profile-materialize@0.19.0': @@ -3567,7 +3567,7 @@ snapshots: optionalDependencies: '@modelcontextprotocol/sdk': 1.30.0(supports-color@10.2.2)(zod@4.5.4) - '@tangle-network/agent-eval@0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))': + '@tangle-network/agent-eval@0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4))': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.5.4) '@hono/node-server': 2.1.1(hono@4.13.7) @@ -3587,9 +3587,9 @@ snapshots: spdx-expression-parse: 5.0.0 zod: 4.5.4 - '@tangle-network/agent-knowledge@15.0.2(@tangle-network/agent-eval@0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0)': + '@tangle-network/agent-knowledge@15.0.3(@tangle-network/agent-eval@0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)))(@tangle-network/agent-interface@2.6.0)': dependencies: - '@tangle-network/agent-eval': 0.179.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) + '@tangle-network/agent-eval': 0.180.0(@modelcontextprotocol/sdk@1.30.0(supports-color@10.2.2)(zod@4.5.4)) '@tangle-network/agent-interface': 2.6.0 '@types/proper-lockfile': 4.1.4 proper-lockfile: 4.1.2 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 3a1bd282..b15a6b75 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,8 @@ packages: - bench +strictPeerDependencies: true + minimumReleaseAge: 4320 minimumReleaseAgeExclude: - '@tangle-network/*' @@ -21,9 +23,9 @@ catalog: '@modelcontextprotocol/sdk': 1.30.0 '@tangle-network/agent-core': '>=0.9.6 <0.10.0' '@types/node': 26.4.0 - '@tangle-network/agent-eval': '>=0.179.0 <0.180.0' + '@tangle-network/agent-eval': '>=0.180.0 <0.181.0' '@tangle-network/agent-interface': ^2.6.0 - '@tangle-network/agent-knowledge': ^15.0.2 + '@tangle-network/agent-knowledge': ^15.0.3 '@tangle-network/agent-profile-materialize': '>=0.19.0 <0.20.0' '@tangle-network/agent-trace-contract': ^1.0.2 '@tangle-network/sandbox': '>=0.36.4 <0.39.0' diff --git a/release/cohort.json b/release/cohort.json index 36817e5b..c5c44aa5 100644 --- a/release/cohort.json +++ b/release/cohort.json @@ -10,14 +10,14 @@ "agentEval": { "name": "@tangle-network/agent-eval", "repository": "tangle-network/agent-eval", - "version": "0.179.0", - "ref": "c7718399a52734a0104793370416b467e52d69b0" + "version": "0.180.0", + "ref": "cde7a8841e2b49ede46bc0f155d05362f4b02427" }, "agentKnowledge": { "name": "@tangle-network/agent-knowledge", "repository": "tangle-network/agent-knowledge", - "version": "15.0.2", - "ref": "1b1408efbf4a652ff9824e7b805f9d55d312fb9d" + "version": "15.0.3", + "ref": "9dd2198cf92bb59d2d9513461640c4e99113a6f3" } } } diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index 2adcfcdd..0ead2a0f 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -1595,7 +1595,19 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin const maxTokens = field('maxTokens') const maxUsd = field('maxUsd') const deadlineMs = field('deadlineMs') + const rawResources = o.resources + if ( + rawResources !== undefined && + (!rawResources || typeof rawResources !== 'object' || Array.isArray(rawResources)) + ) { + throw new Error('coordination tools: "budget.resources" must be a resource map') + } + const resources = + rawResources === undefined ? base.resources : { ...base.resources, ...rawResources } const merged: Budget = { + ...(resources === undefined + ? {} + : { resources: resources as NonNullable }), maxIterations: maxIterations ?? base.maxIterations, maxTokens: maxTokens ?? base.maxTokens, ...((maxUsd ?? base.maxUsd) === undefined ? {} : { maxUsd: maxUsd ?? base.maxUsd }), @@ -2816,6 +2828,17 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin maxTokens: { type: 'number', minimum: 0 }, maxUsd: { type: 'number', minimum: 0 }, deadlineMs: { type: 'number', minimum: 0 }, + resources: { + type: 'object', + additionalProperties: { + type: 'object', + properties: { + unit: { type: 'string', minLength: 1 }, + limit: { type: 'integer', minimum: 0, maximum: Number.MAX_SAFE_INTEGER }, + }, + required: ['unit', 'limit'], + }, + }, }, }, }, diff --git a/src/runtime/index.ts b/src/runtime/index.ts index 64295de7..b4f9d036 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -1084,6 +1084,8 @@ export type { ProfileMaterializationReceipt, ProviderModelAttemptEvidence, ProviderModelExecutionEvidence, + ResourceLimit, + ResourceSpend, ResultBlobStore, ResumedKeyState, ResumedWork, diff --git a/src/runtime/personify/persona.ts b/src/runtime/personify/persona.ts index 5fa31a80..24afeda6 100644 --- a/src/runtime/personify/persona.ts +++ b/src/runtime/personify/persona.ts @@ -196,6 +196,16 @@ function shapeName( function resolveShapeBudget(root: Budget, over?: Partial): ShapeBudget { const fanout = over?.fanout ?? defaultFanout const perChild: Budget = over?.perChild ?? { + ...(root.resources === undefined + ? {} + : { + resources: Object.fromEntries( + Object.entries(root.resources).map(([name, resource]) => [ + name, + { ...resource, limit: Math.floor(resource.limit / fanout) }, + ]), + ), + }), maxIterations: Math.max(1, Math.floor(root.maxIterations / fanout)), maxTokens: Math.max(1, Math.floor(root.maxTokens / fanout)), ...(root.maxUsd !== undefined ? { maxUsd: root.maxUsd / fanout } : {}), diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index a21b69fc..1a12838d 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -20,6 +20,8 @@ import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' import type { ReasoningEffort } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' import { type RouterRetryPolicy, resolveRouterRetryPolicy } from './router-retry-policy' +import { addResourceSpend } from './supervise/resources' +import type { Spend } from './supervise/types' import { runBrainLoop, type ToolLoopChat } from './tool-loop' /** @@ -31,7 +33,7 @@ import { runBrainLoop, type ToolLoopChat } from './tool-loop' export interface RouterTransportConfig { routerBaseUrl: string routerKey: string - /** Injectable OpenAI-compatible transport for offline execution. */ + /** Injectable OpenAI-compatible transport. Optional usage.resources carries measured turn totals. */ complete?: ( body: Record, request?: { @@ -110,6 +112,8 @@ export interface RouterChatResult { reasoning?: string /** REAL usage, or undefined when the provider reported none. */ usage?: { input: number; output: number; reasoning?: number } + /** Explicit per-turn resource measurements supplied by the transport. */ + resources?: Spend['resources'] /** Local catalog estimate derived from usage; never a provider billing receipt. */ costUsd?: number /** Present with `costUsd` so consumers cannot mistake a catalog estimate for billed spend. */ @@ -227,7 +231,11 @@ function parseChatResult( }> usage?: RawUsage } - const { usage, costUsd, costProvenance, billedCostUsd, cache } = meterTurn(data.usage, model) + const { usage, resources, costUsd, costProvenance, billedCostUsd, cache } = meterTurn( + data.usage, + model, + transportAttempts, + ) const msg = data.choices?.[0]?.message if (!msg) throw new ValidationError('router completion: no choices[0].message') const { content, reasoning } = splitReasoning( @@ -240,6 +248,7 @@ function parseChatResult( ...(reportedModel(data.model) ? { model: reportedModel(data.model) } : {}), ...(reasoning ? { reasoning } : {}), ...(usage ? { usage } : {}), + ...(resources ? { resources } : {}), ...(costUsd !== undefined ? { costUsd } : {}), ...(costProvenance ? { costProvenance } : {}), ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), @@ -287,6 +296,8 @@ export interface RouterChatToolsResult { content: string | null toolCalls: RouterToolCall[] usage?: { input: number; output: number; reasoning?: number } + /** Explicit per-turn resource measurements supplied by the transport. */ + resources?: Spend['resources'] costUsd?: number /** Present with `costUsd` so consumers cannot mistake a catalog estimate for billed spend. */ costProvenance?: 'catalog-estimate' @@ -400,13 +411,18 @@ export async function routerChatWithTools( name: tc.function?.name ?? '', arguments: tc.function?.arguments ?? '{}', })) - const { usage, costUsd, costProvenance, billedCostUsd, cache } = meterTurn(data.usage, cfg.model) + const { usage, resources, costUsd, costProvenance, billedCostUsd, cache } = meterTurn( + data.usage, + cfg.model, + transportAttempts, + ) return { content: msg?.content ?? null, toolCalls, transportAttempts, ...(reportedModel(data.model) ? { model: reportedModel(data.model) } : {}), ...(usage ? { usage } : {}), + ...(resources ? { resources } : {}), ...(costUsd !== undefined ? { costUsd } : {}), ...(costProvenance ? { costProvenance } : {}), ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), @@ -477,13 +493,23 @@ function providerRequestExtras( function meterTurn( raw: RawUsage | undefined, model: string, + transportAttempts: number, ): { usage?: { input: number; output: number; reasoning?: number } + /** Explicit per-turn resource measurements supplied by the transport. */ + resources?: Spend['resources'] costUsd?: number costProvenance?: 'catalog-estimate' billedCostUsd?: number cache?: PromptCacheUsage } { + let resources = addResourceSpend(raw?.resources).resources + if (transportAttempts > 1 && resources !== undefined) { + // The final receipt cannot prove that failed transport attempts consumed no resources. + resources = Object.fromEntries( + Object.entries(resources).map(([name, value]) => [name, { ...value, known: false }]), + ) + } const reasoning = providerReasoningTokens(raw) const usage = raw && typeof raw.prompt_tokens === 'number' && typeof raw.completion_tokens === 'number' @@ -497,6 +523,7 @@ function meterTurn( const billedCostUsd = providerBilledCost(raw) if (!usage) { return { + ...(resources ? { resources } : {}), ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), ...(cache ? { cache } : {}), } @@ -514,6 +541,7 @@ function meterTurn( : localEstimate return { usage, + ...(resources ? { resources } : {}), ...(costUsd !== undefined ? { costUsd, costProvenance: 'catalog-estimate' as const } : {}), ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), ...(cache ? { cache } : {}), @@ -553,6 +581,7 @@ export interface PromptCacheUsage { } interface RawUsage { + resources?: Spend['resources'] prompt_tokens?: number completion_tokens?: number cost?: number @@ -779,7 +808,11 @@ export async function streamRouterChatWithTools( name: call.name ?? '', arguments: call.arguments || '{}', })) - const { usage, costUsd, costProvenance, billedCostUsd, cache } = meterTurn(rawUsage, cfg.model) + const { usage, resources, costUsd, costProvenance, billedCostUsd, cache } = meterTurn( + rawUsage, + cfg.model, + transportAttempts, + ) return { // `null` only when NO content field was ever sent — the buffered path's `msg?.content ?? null`. content: sawContent ? split.content : null, @@ -789,6 +822,7 @@ export async function streamRouterChatWithTools( ...(split.reasoning ? { reasoning: split.reasoning } : {}), ...(finishReason !== undefined ? { finishReason } : {}), ...(usage ? { usage } : {}), + ...(resources ? { resources } : {}), ...(costUsd !== undefined ? { costUsd } : {}), ...(costProvenance ? { costProvenance } : {}), ...(billedCostUsd !== undefined ? { billedCostUsd } : {}), @@ -1207,6 +1241,7 @@ export function routerBrain( content: result.content, toolCalls: result.toolCalls, ...(result.usage !== undefined ? { usage: result.usage } : {}), + ...addResourceSpend(result.resources), ...(result.usageUnknown === true ? { usageUnknown: true as const } : {}), ...(result.model !== undefined ? { model: result.model } : {}), ...(result.cache !== undefined ? { promptCache: Object.freeze({ ...result.cache }) } : {}), diff --git a/src/runtime/supervise/budget.ts b/src/runtime/supervise/budget.ts index dafc56ac..cf6dd59e 100644 --- a/src/runtime/supervise/budget.ts +++ b/src/runtime/supervise/budget.ts @@ -1,8 +1,8 @@ /** * * The conserved budget reservation pool — the invariant the whole instrument - * rests on (critique M5/B3). One root `Budget` becomes a conserved pool of three - * quantities (tokens, usd, iterations) plus an absolute deadline. Children reserve + * rests on (critique M5/B3). One root `Budget` becomes a conserved pool of standard + * quantities (tokens, usd, iterations) and caller-named resources plus an absolute deadline. Children reserve * atomically at spawn and reconcile at settle: * * total ≡ free + reserved + committed (for every known quantity) @@ -26,16 +26,11 @@ * it trusts a reported `input`: the token channel is an accounting unit, not a trust boundary * against a provider that misreports its own usage. * - * THE POOL RESERVES AGAINST THREE QUANTITIES AND NO MORE. `Spend.boxMinutes` — platform box wall - * time — is reported and summed, and it is never reserved, never committed, and never refunded. - * The reason is the conservation law above: a reservation is only meaningful against a ceiling a - * caller declared and a measurement the pool can trust, and box minutes have neither. `Budget` - * declares no box ceiling, and the number the sandbox executor reports is DERIVED from the box - * lifetime it watched (`boxMinutesProvenance: 'estimated'`), not a platform receipt. Reserving - * against an estimate would let a derived number refuse real work, which is the same defect as a - * gate that cannot fire, inverted. When the platform reports minutes itself - * (`boxMinutesProvenance: 'observed'`) and a `Budget` gains a box ceiling, the channel becomes a - * conserved quantity; until both hold, it is evidence only. + * Named resources require explicit units, ceilings, and complete executor measurements. + * Missing or unknown enforced measurements close admission for that resource. + * The pool trusts those receipts; it does not independently meter external systems. + * `Spend.boxMinutes` remains evidence only and has no implicit resource ceiling. + * Callers must not present estimated box lifetime as a measured resource receipt. * * Pure and deterministic: the run's start instant is supplied, there is no I/O, and no * wall-clock or RNG read. A `reserve`/`reconcile` ticket is single-use (fail-loud on double or @@ -63,6 +58,7 @@ import { ValidationError } from '../../errors' import type { LoopTokenUsage } from '../types' import { addTokenUsage, chargedTokens, hasCompleteCacheBreakdown, zeroTokenUsage } from '../util' +import { addResourceSpend, assertResources, withBudgetResources } from './resources' import type { Budget, Spend, TokenUsageProvenance, UsageEvent } from './types' export type { Budget, Spend, UsageEvent } @@ -72,6 +68,7 @@ export type { Budget, Spend, UsageEvent } export interface ReservationTicket { readonly id: number readonly reserved: { + readonly resources?: Budget['resources'] readonly tokens: number readonly usd: number readonly iterations: number @@ -94,6 +91,19 @@ export interface ReservationTicket { * `usdCapped` distinguishes a real `usdLeft <= 0` exhaustion from an uncapped pool (which always * reads `usdLeft: 0`) — the in-loop guard needs it to bound a usd-capped driver. */ export type BudgetReadout = Readonly<{ + resources?: Readonly< + Record< + string, + { + unit: string + limit: number + remaining: number + reserved: number + committed: number + known: boolean + } + > + > tokensLeft: number /** * False once the pool has recorded work whose token count was UNREPORTED (a `Spend` with @@ -147,6 +157,7 @@ export function assertValidBudget(budget: Budget, label = 'budget'): void { throw new Error(`${label}.${field} must be a non-negative finite number`) } } + assertResources(budget.resources, 'limit', `${label}.resources`) safeInteger(budget.maxIterations, 'maxIterations') safeInteger(budget.maxTokens, 'maxTokens') if (budget.maxUsd !== undefined) finiteNonNegative(budget.maxUsd, 'maxUsd') @@ -154,6 +165,7 @@ export function assertValidBudget(budget: Budget, label = 'budget'): void { } export function assertValidSpend(spend: Spend, label: string): void { + assertResources(spend.resources, 'amount', `${label}.resources`) if (!Number.isSafeInteger(spend.iterations) || spend.iterations < 0) { throw new Error(`${label}.iterations must be a non-negative safe integer`) } @@ -223,7 +235,7 @@ export function assertValidSpend(spend: Spend, label: string): void { export interface BudgetPool { /** * Atomically reserve a child's full ceiling from the free balance. Fails closed - * ({ ok: false }) when the pool can't cover tokens, usd, or iterations — the + * ({ ok: false }) when the pool can't cover standard or named channels — the * caller inspects `ok` before `ticket`. */ reserve( @@ -246,12 +258,12 @@ export interface BudgetPool { * (its chat turns), which is real compute but not a spawned child. A direct `free → committed` * debit, so `total ≡ free + reserved + committed` is preserved: equal-k counts the driver's * tokens and the in-loop budget guard (`readout().tokensLeft`) sees them. `free` may go negative - * when a run overspends — that is honest (the readout then signals exhaustion). It never throws: - * the spend already happened, so accounting records reality; the in-loop guard prevents MORE. + * when a run overspends. Unknown enforced resource usage is recorded before throwing. + * Partial increments defer completeness checks until the invocation reports its terminal spend. * The DURABLE record is the journal's `metered` event (written by `Scope.meter`); this debit * only makes the live `readout()` reflect driver inference for the in-loop guard. */ - observe(spend: Spend): void + observe(spend: Spend, options?: { partial?: boolean }): void /** Fail loud if any reservation is still open — the conserved-pool leak detector. Called at the * supervisor's join barrier: once every child has settled, no ticket may remain (a leaked * reservation would silently break `total ≡ free + reserved + committed`). */ @@ -269,6 +281,7 @@ export function spendFromUsageEvents(events: UsageEvent[]): Spend { /** Running accounting totals folded from `UsageEvent`s. */ export interface UsageTotals { + resources?: Spend['resources'] tokens: LoopTokenUsage tokensKnown: boolean /** Which evidence the counted tokens came from. `undefined` until the first `tokens` event. */ @@ -297,6 +310,13 @@ export function newUsageTotals(): UsageTotals { * `iteration` advances the iteration count. */ export function meterUsageEvent(totals: UsageTotals, ev: UsageEvent): void { + if (ev.kind === 'resource') { + totals.resources = addResourceSpend( + totals.resources, + Object.fromEntries([[ev.name, { unit: ev.unit, amount: ev.amount, known: ev.known }]]), + ).resources + return + } if (ev.kind === 'tokens') { addTokenUsage(totals.tokens, ev) if (ev.tokensKnown === false) totals.tokensKnown = false @@ -322,6 +342,7 @@ export function meterUsageEvent(totals: UsageTotals, ev: UsageEvent): void { * read wall-clock. */ export function spendFromUsageTotals(totals: UsageTotals): Spend { return { + ...addResourceSpend(totals.resources), iterations: totals.iterations, tokens: totals.tokens, ...(totals.tokensKnown ? {} : { tokensKnown: false }), @@ -394,6 +415,69 @@ export function createBudgetPool( const absoluteDeadlineMs = root.deadlineMs === undefined ? 0 : runStartedAtMs + root.deadlineMs + const resources = new Map< + string, + { + unit: string + limit: number + remaining: number + reserved: number + committed: number + known: boolean + } + >( + Object.entries(root.resources ?? {}).map( + ([name, value]) => + [ + name, + { ...value, remaining: value.limit, reserved: 0, committed: 0, known: true }, + ] as const, + ), + ) + + function validateResourceUnits(spend: Spend): void { + for (const [name, value] of Object.entries(spend.resources ?? {})) { + const limit = resources.get(name) + if (limit && limit.unit !== value.unit) throw new Error(`resource ${name}: unit mismatch`) + } + } + + function commitResources( + spend: Spend, + reserved: Budget['resources'] = {}, + requireAll = true, + ): string | undefined { + let violation: string | undefined + for (const [name, state] of resources) { + const value = spend.resources?.[name] + if (!requireAll && value === undefined) continue + const allocation = reserved?.[name]?.limit ?? 0 + const amount = value?.amount ?? 0 + state.reserved -= allocation + const committed = state.committed + amount + const overflow = !Number.isSafeInteger(committed) + state.committed = Math.min(committed, Number.MAX_SAFE_INTEGER) + // Subtract reserved capacity first so every intermediate balance stays a safe integer. + state.remaining = state.limit - state.reserved - state.committed + if (value?.known !== true || overflow) { + state.known = false + // The remaining balance cannot fund another invocation without a complete receipt. + // Consume free capacity conservatively; later known refunds do not reopen admission. + const retained = Math.max(0, state.remaining) + state.committed += retained + state.remaining -= retained + violation ??= overflow + ? `resource ${name}: amount overflow under an enforced limit` + : `resource ${name}: unknown usage under an enforced limit` + } else if (reserved?.[name] && amount > allocation) { + violation ??= `resource ${name}: spent ${amount} > reserved ${allocation}` + } else if (state.remaining < 0) { + violation ??= `resource ${name}: exceeded root limit ${state.limit}` + } + } + return violation + } + let nextTicketId = 0 const open = new Set() @@ -401,6 +485,17 @@ export function createBudgetPool( b: Budget, ): { ok: true; ticket: ReservationTicket } | { ok: false; reason: ReservationRejection } { assertValidBudget(b, 'reservation budget') + for (const [name, state] of resources) { + const wanted = b.resources?.[name] + if (!wanted) throw new ValidationError(`resource ${name}: child must declare its limit`) + if (wanted.unit !== state.unit) throw new ValidationError(`resource ${name}: unit mismatch`) + if (!state.known || wanted.limit > state.remaining) + return { ok: false, reason: 'budget-exhausted' } + } + for (const name of Object.keys(b.resources ?? {})) { + if (!resources.has(name)) + throw new ValidationError(`resource ${name}: root must declare its limit`) + } const wantTokens = b.maxTokens const wantUsd = b.maxUsd ?? 0 const wantIterations = b.maxIterations @@ -417,6 +512,11 @@ export function createBudgetPool( if (wantUsd > 0 && !usdCapped) return { ok: false, reason: 'usd-unbudgeted' } if (wantUsd > freeUsd) return { ok: false, reason: 'budget-exhausted' } + for (const [name, state] of resources) { + const amount = b.resources![name]!.limit + state.remaining -= amount + state.reserved += amount + } freeTokens -= wantTokens reservedTokens += wantTokens freeIterations -= wantIterations @@ -433,6 +533,13 @@ export function createBudgetPool( ticket: { id, reserved: { + ...(b.resources === undefined + ? {} + : { + resources: Object.fromEntries( + Object.entries(b.resources).map(([name, value]) => [name, { ...value }]), + ), + }), tokens: wantTokens, usd: wantUsd, iterations: wantIterations, @@ -447,6 +554,7 @@ export function createBudgetPool( throw new Error(`budget pool: reconcile of unknown or already-settled ticket ${ticket.id}`) } assertValidSpend(spent, `budget pool ticket ${ticket.id} spend`) + validateResourceUnits(spent) const { tokens: rTokens, usd: rUsd, iterations: rIterations } = ticket.reserved const unknownUnderCap = usdCapped && spent.usdKnown === false const spentTokens = chargedTokens(spent.tokens) @@ -520,11 +628,14 @@ export function createBudgetPool( committedUsd += spent.usd } + const resourceViolation = commitResources(spent, ticket.reserved.resources) + violation ??= resourceViolation if (violation !== undefined) throw new Error(`budget pool: ${violation}`) } - function observe(spend: Spend): void { + function observe(spend: Spend, options: { partial?: boolean } = {}): void { assertValidSpend(spend, 'observed spend') + validateResourceUnits(spend) // Unknown dollars under a dollar cap are REFUSED before any balance mutates: unlike a // reconciled child (whose reservation must settle), an observation has no ticket to strand, // so the honest reading is a fail-loud refusal the caller surfaces — `driver-failed` carrying @@ -554,10 +665,19 @@ export function createBudgetPool( committedIterations += spend.iterations committedUsd += spend.usd if (usdCapped) freeUsd -= spend.usd + const violation = commitResources(spend, {}, !options.partial) + if (violation) throw new ValidationError(`budget pool: ${violation}`) } function readout(): BudgetReadout { return { + ...(resources.size + ? { + resources: Object.fromEntries( + [...resources].map(([name, value]) => [name, { ...value }]), + ), + } + : {}), tokensLeft: freeTokens, tokensKnown: !tokensTainted, cacheBreakdownKnown: !cacheBreakdownTainted, @@ -586,6 +706,8 @@ export function createBudgetPool( // refusing to record it would be the zero-cost restart the pool must never allow. if (restore.committed !== undefined) { assertValidSpend(restore.committed, 'budget restore committed') + validateResourceUnits(restore.committed) + commitResources(restore.committed) const committed = restore.committed if (committed.tokensKnown === false) tokensTainted = true if (!hasCompleteCacheBreakdown(committed.tokens)) cacheBreakdownTainted = true @@ -612,6 +734,11 @@ export function createBudgetPool( // The child ran (or may have run) without a settle record: its telemetry is unknown, so the // pool charges the full declared ceiling and reports the balance as a ceiling, never a // measurement. + const unknownResources = withBudgetResources( + { iterations: 0, tokens: zeroTokenUsage(), usd: 0, ms: 0 }, + root, + ) + commitResources(unknownResources) tokensTainted = true usdMeasured = false freeTokens -= uncertain.maxTokens diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index e7350561..01a14924 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -69,6 +69,7 @@ import { import { createInbox, type Inbox } from './inbox' import { providerAttemptEvidence } from './materialization' import { isTerminalNodeStatus } from './node-status' +import { addResourceSpend, withBudgetResources } from './resources' import { applyRunCancellation } from './run-cancellation' import { claimWorkerSteerDelivery, @@ -318,7 +319,13 @@ function poolStarved(scope: Scope, perWorker: Budget): boolean { const usdStarved = b.usdCapped && (b.usdLeft <= 0 || (perWorker.maxUsd !== undefined && b.usdLeft < perWorker.maxUsd)) - return tokenStarved || iterationStarved || usdStarved + const resourceStarved = Object.entries(b.resources ?? {}).some( + ([name, value]) => + !value.known || + value.remaining <= 0 || + value.remaining < (perWorker.resources?.[name]?.limit ?? 0), + ) + return tokenStarved || iterationStarved || usdStarved || resourceStarved } /** The absolute wall-clock deadline (when the root set one) has passed. */ @@ -1040,7 +1047,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { opts.onProviderModel?.(undefined) await meterRuntimeOwnedProviderAttempt( scope, - unmeteredSpend(0), + withBudgetResources(unmeteredSpend(0), scope.budget), providerAttemptEvidence(undefined), { driver: opts.name, @@ -1079,6 +1086,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { res.costProvenance === 'provider-receipt' || res.costProvenance === 'billing-receipt' const cacheUsage = promptCacheTokenClasses(res.usage?.input, res.promptCache) const turnSpend: Spend = { + ...addResourceSpend(res.resources), iterations: 0, tokens: { input: res.usage?.input ?? 0, @@ -1092,7 +1100,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { } await meterRuntimeOwnedProviderAttempt( scope, - turnSpend, + withBudgetResources(turnSpend, scope.budget), providerAttemptEvidence(res.model), { driver: opts.name, diff --git a/src/runtime/supervise/driver-executor.ts b/src/runtime/supervise/driver-executor.ts index 8da98890..17e45942 100644 --- a/src/runtime/supervise/driver-executor.ts +++ b/src/runtime/supervise/driver-executor.ts @@ -46,6 +46,7 @@ import { attestRuntimeOwnedDeferredExecutor, runtimeOwnedScopeOwnerRuntime, } from './materialization' +import { addResourceSpend } from './resources' import { RetainedExecutionPendingError, retainedExecutorContext } from './retained-executor' import { finalizeScopeOwnerMaterialization, @@ -483,7 +484,8 @@ function isNonZeroSpend(s: Spend): boolean { s.usd > 0 || s.ms > 0 || s.tokensKnown === false || - s.usdKnown === false + s.usdKnown === false || + Object.keys(s.resources ?? {}).length > 0 ) } @@ -584,6 +586,25 @@ function unreportedSpend(total: Spend, prior: Spend): Spend { } return { ...total, + ...addResourceSpend( + total.resources === undefined + ? undefined + : Object.fromEntries( + Object.entries(total.resources).map(([name, value]) => { + const previous = prior.resources?.[name] + if (previous && previous.unit !== value.unit) + throw new ValidationError(`resource ${name}: unit mismatch`) + return [ + name, + { + ...value, + amount: Math.max(0, value.amount - (previous?.amount ?? 0)), + known: value.known && (previous?.known ?? true), + }, + ] + }), + ), + ), iterations: Math.max(0, total.iterations - prior.iterations), tokens, usd: Math.max(0, total.usd - prior.usd), diff --git a/src/runtime/supervise/driver-retry.ts b/src/runtime/supervise/driver-retry.ts index 641d4415..1deeecf0 100644 --- a/src/runtime/supervise/driver-retry.ts +++ b/src/runtime/supervise/driver-retry.ts @@ -294,6 +294,12 @@ export function budgetStop( // retry could only reproduce the same refusal. The pool's own state proves this, independently of // how the refusal was thrown. if (budget.usdCapped && budget.usdKnown === false) return 'budget-exhausted' + if ( + Object.values(budget.resources ?? {}).some( + (resource) => !resource.known || resource.remaining <= 0, + ) + ) + return 'budget-exhausted' return undefined } diff --git a/src/runtime/supervise/progress.ts b/src/runtime/supervise/progress.ts index a405aabc..8c045ffa 100644 --- a/src/runtime/supervise/progress.ts +++ b/src/runtime/supervise/progress.ts @@ -23,7 +23,8 @@ */ import { isLiveNodeStatus } from './node-status' -import type { NodeStatus } from './types' +import { addResourceSpend } from './resources' +import type { NodeStatus, Spend } from './types' /** How long a worker may produce no metered activity before a `progress()` read calls it stalled. * Deliberately generous: a coding harness routinely spends minutes inside one tool call, and a @@ -91,6 +92,7 @@ export interface WorkerProgress { readonly usd: number /** False when observed dollar spend is only a known subtotal, not a complete total. */ readonly usdKnown?: boolean + readonly resources?: Spend['resources'] /** Steers delivered but not yet read by the worker. */ readonly pendingMessages: number /** Newest-last window of tool/turn activity; empty when the executor exposes none. */ @@ -152,6 +154,7 @@ export interface ScopeProgressInput { readonly tokensKnown?: boolean readonly usd: number readonly usdKnown?: boolean + readonly resources?: Spend['resources'] } /** Fold the scope-derived facts and the executor's optional enrichment into one read. Pure: the @@ -172,6 +175,7 @@ export function readWorkerProgress( const idleMs = Math.max(0, now - lastActivityAt) const note = executor?.note return { + ...addResourceSpend(scope.resources), id: scope.id, status: scope.status, live, diff --git a/src/runtime/supervise/recover-executors.ts b/src/runtime/supervise/recover-executors.ts index 517fc49e..64903516 100644 --- a/src/runtime/supervise/recover-executors.ts +++ b/src/runtime/supervise/recover-executors.ts @@ -15,6 +15,7 @@ import { RuntimeRunStateError } from '../../errors' import { addSpend, zeroSpend } from '../util' import { assertValidSpend, type BudgetPoolRestore, createBudgetPool } from './budget' import { executorFailureReason } from './executor-outcome' +import { addResourceSpend, withBudgetResources } from './resources' import { prepareRetainedExecutor, type RetainedChildRecovery } from './retained-executor' import type { ScopeArgs } from './scope' import { detachedSnapshot } from './snapshot' @@ -289,6 +290,8 @@ export function sumSpendFromEvents(events: SpawnEvent[]): { (rootBudget?.maxUsd ?? 0) - totals.childWork.usd - totals.driverInference.usd, ) for (const budget of uncertainSpawnBudgets(events)) { + const unknown = withBudgetResources(zeroSpend(), budget) + Object.assign(totals.childWork, addResourceSpend(totals.childWork.resources, unknown.resources)) totals.childWork.iterations += budget.maxIterations // The numeric value is the charged upper bound, not a fabricated measurement. The false flag // makes that distinction machine-readable in every report. @@ -306,12 +309,28 @@ export function sumMeasuredSpendFromEvents(events: SpawnEvent[]): { childWork: Spend driverInference: Spend } { + const budgets = new Map( + events.flatMap((event) => + event.kind === 'spawned' ? [[event.id, event.budget] as const] : [], + ), + ) let childWork = zeroSpend() let driverInference = zeroSpend() + const owners = new Map() for (const ev of events) { - if (ev.kind === 'settled' || (ev.kind === 'cancelled' && ev.spent !== undefined)) - childWork = addSpend(childWork, ev.spent!) - else if (ev.kind === 'metered') driverInference = addSpend(driverInference, ev.spend) + if (ev.kind === 'settled' || ev.kind === 'cancelled') + childWork = addSpend( + childWork, + withBudgetResources( + ev.spent ?? { ...zeroSpend(), tokensKnown: false, usdKnown: false }, + budgets.get(ev.id) ?? {}, + ), + ) + else if (ev.kind === 'metered') + owners.set(ev.id, addSpend(owners.get(ev.id) ?? zeroSpend(), ev.spend)) + } + for (const [id, spend] of owners) { + driverInference = addSpend(driverInference, withBudgetResources(spend, budgets.get(id) ?? {})) } return { childWork, driverInference } } @@ -329,14 +348,20 @@ export async function prepareScopeResume( const recovering = new Set(prepared.recoveries.map((item) => item.spawned.id)) // A restored manager's full reservation already covers its previous inference. // Its executor reconciles total spend and publishes only the unrecorded meter delta. - const measured = sumMeasuredSpendFromEvents( - prior.filter((event) => event.kind !== 'metered' || !recovering.has(event.id)), + const measuredEvents = prior.filter( + (event) => event.kind !== 'metered' || !recovering.has(event.id), + ) + const measured = sumMeasuredSpendFromEvents(measuredEvents) + const hasCommittedEvidence = measuredEvents.some( + (event) => event.kind === 'metered' || event.kind === 'settled' || event.kind === 'cancelled', ) const settled = await replaySpawnTree(opts.journal, opts.blobs, opts.runId) signal.throwIfAborted() return { poolRestore: { - committed: addSpend(measured.childWork, measured.driverInference), + ...(hasCommittedEvidence + ? { committed: addSpend(measured.childWork, measured.driverInference) } + : {}), uncertainReservations: uncertainSpawnBudgets(prior, recovering), }, resumeFrom: { diff --git a/src/runtime/supervise/resources.ts b/src/runtime/supervise/resources.ts new file mode 100644 index 00000000..5e86ab0c --- /dev/null +++ b/src/runtime/supervise/resources.ts @@ -0,0 +1,93 @@ +import type { Budget, ResourceSpend, Spend } from './types' + +/** Validate caller-owned names and units without assigning domain meaning to them. */ +export function assertResources( + resources: Budget['resources'] | Spend['resources'], + kind: 'limit' | 'amount', + label: string, +): void { + if (resources === undefined) return + if (resources === null || typeof resources !== 'object' || Array.isArray(resources)) { + throw new Error(`${label} must be a resource map`) + } + for (const [name, resource] of Object.entries(resources)) { + if ( + !name.trim() || + !resource || + typeof resource !== 'object' || + typeof resource.unit !== 'string' || + !resource.unit.trim() + ) { + throw new Error(`${label}: resource names and units must be non-empty strings`) + } + const value = + kind === 'limit' + ? 'limit' in resource + ? resource.limit + : undefined + : 'amount' in resource + ? resource.amount + : undefined + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`${label}.${name}.${kind} must be a non-negative safe integer`) + } + if (kind === 'amount' && (!('known' in resource) || typeof resource.known !== 'boolean')) { + throw new Error(`${label}.${name}.known must be a boolean`) + } + } +} + +/** Missing contributors are not usage claims; enforced omissions are marked before aggregation. + * Overflow retains a safe lower bound and marks it unknown, preserving readable component receipts. */ +export function addResourceSpend(...maps: ReadonlyArray): { + resources?: Record +} { + const totals = new Map() + for (const resources of maps) { + assertResources(resources, 'amount', 'resource spend') + for (const [name, resource] of Object.entries(resources ?? {})) { + const prior = totals.get(name) + if (prior && prior.unit !== resource.unit) throw new Error(`resource ${name}: unit mismatch`) + const amount = (prior?.amount ?? 0) + resource.amount + totals.set(name, { + unit: resource.unit, + amount: Math.min(amount, Number.MAX_SAFE_INTEGER), + known: (prior?.known ?? true) && resource.known && Number.isSafeInteger(amount), + }) + } + } + return totals.size ? { resources: Object.fromEntries(totals) } : {} +} + +/** A terminal omission under an enforced ceiling is unknown, never measured zero. */ +export function withBudgetResources( + spend: Spend, + budget: Pick, + notStarted = false, +): Spend { + const resources = new Map(Object.entries(addResourceSpend(spend.resources).resources ?? {})) + for (const [name, limit] of Object.entries(budget.resources ?? {})) { + const observed = resources.get(name) + if (observed && observed.unit !== limit.unit) throw new Error(`resource ${name}: unit mismatch`) + if (!observed) resources.set(name, { unit: limit.unit, amount: 0, known: notStarted }) + } + return resources.size ? { ...spend, resources: Object.fromEntries(resources) } : spend +} + +/** Stream increments and a terminal total describe the same work; never sum them twice. */ +export function resourceTelemetry(streamed: Spend, terminal: Spend): Pick { + const resources = new Map(Object.entries(addResourceSpend(terminal.resources).resources ?? {})) + for (const [name, value] of Object.entries(streamed.resources ?? {})) { + const other = resources.get(name) + if (other && other.unit !== value.unit) throw new Error(`resource ${name}: unit mismatch`) + resources.set(name, { + ...value, + amount: Math.max(value.amount, other?.amount ?? 0), + known: + value.known && + (other?.known ?? true) && + (other === undefined || other.amount === value.amount), + }) + } + return addResourceSpend(Object.fromEntries(resources)) +} diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index 202ff822..0655d47a 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -1,4 +1,5 @@ import { selectProviderPlacement } from '../provider-placement' +import { addResourceSpend } from './resources' /** * * The leaf runtime — the built-in `Executor` IMPLEMENTATIONS behind the ONE @@ -817,6 +818,7 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { ) if (r.model !== undefined) recordRuntimeOwnedProviderModel(executor, r.model) const spent: Spend = { + ...addResourceSpend(r.resources), iterations: 1, tokens: r.usage ? cloneTokenUsage({ @@ -1031,6 +1033,7 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = let billedUsd = 0 let usdKnown = true let turns = 0 + let resources: Spend['resources'] let transportAttempts = 0 let observedModel: string | undefined let reasoningTokens = 0 @@ -1114,6 +1117,14 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = !controller.signal.aborted if (interruptAbort) { turns += 1 + resources = addResourceSpend( + Object.fromEntries( + Object.entries(resources ?? {}).map(([name, value]) => [ + name, + { ...value, known: false }, + ]), + ), + ).resources transportAttempts += routerTransportAttemptsFromError(e) ?? 1 tokensKnown = false usdKnown = false @@ -1125,6 +1136,22 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = cleanup() // The inference completed — count the turn and merge its terminal receipt. turns += 1 + const priorResources = resources + resources = addResourceSpend(resources, res.resources).resources + resources = addResourceSpend( + Object.fromEntries( + Object.entries(resources ?? {}).map(([name, value]) => [ + name, + { + ...value, + known: + value.known && + res.resources?.[name] !== undefined && + (turns === 1 || priorResources?.[name] !== undefined), + }, + ]), + ), + ).resources transportAttempts += res.transportAttempts if (res.model !== undefined) recordRuntimeOwnedProviderModel(executor, res.model) assertObservedRouterModel(res.model, model, 'routerToolsInlineExecutor') @@ -1237,6 +1264,7 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = const priced = isModelPriced(model) const estimatedUsd = priced ? estimateCost(tokens.input, tokens.output, model) : undefined const spent: Spend = { + ...addResourceSpend(resources), iterations: turns, tokens, ...(tokensKnown ? {} : { tokensKnown: false }), diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 756c184b..24fc050f 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -80,6 +80,7 @@ import { type WorkerProgress, } from './progress' import { prepareScopeResume } from './recover-executors' +import { addResourceSpend, resourceTelemetry, withBudgetResources } from './resources' import { type RetainedChildRecovery, RetainedExecutionPendingError, @@ -705,7 +706,7 @@ export function createScope(args: ScopeArgs): Scope { spec = prepared.spec identity = prepared.identity if (opts.key !== undefined && !isCompleteIdentity(identity)) { - args.pool.reconcile(reservation.ticket, zeroSpend()) + args.pool.reconcile(reservation.ticket, withBudgetResources(zeroSpend(), opts.budget, true)) permit.release() return { ok: false, reason: 'invalid-identity' } } @@ -718,7 +719,7 @@ export function createScope(args: ScopeArgs): Scope { if (!outcome.succeeded) throw new ValidationError(`scope.spawn: ${outcome.error}`) resolved = outcome } catch (error) { - args.pool.reconcile(reservation.ticket, zeroSpend()) + args.pool.reconcile(reservation.ticket, withBudgetResources(zeroSpend(), opts.budget, true)) permit.release() throw error } @@ -1291,7 +1292,7 @@ export function createScope(args: ScopeArgs): Scope { permit.release() clearChildDeadline?.() if (cascadeAbort) args.signal.removeEventListener('abort', cascadeAbort) - args.pool.reconcile(reservation.ticket, zeroSpend()) + args.pool.reconcile(reservation.ticket, withBudgetResources(zeroSpend(), opts.budget, true)) throw err } } @@ -1568,6 +1569,7 @@ export function createScope(args: ScopeArgs): Scope { steerable: child.deliver !== undefined && !child.delivered, startedAt: child.startedAt, lastActivityAt: child.lastActivityAt, + ...addResourceSpend(child.spent.resources), turns: child.spent.iterations, tokens: child.spent.tokens, ...(child.spent.tokensKnown === false ? { tokensKnown: false } : {}), @@ -1640,6 +1642,8 @@ export function createScope(args: ScopeArgs): Scope { if (args.signal.aborted) { throw new ValidationError('scope.meter: cannot record new driver work after scope abort') } + const partial = providerModel !== undefined || accountingOnly + if (!partial) spend = withBudgetResources(spend, args.pool.readout()) const seq = meterSeq++ // Debit the driver's own inference against the shared conserved pool (free → committed), so // equal-k counts it live and `budget.tokensLeft` reflects it for the in-loop guard. @@ -1648,7 +1652,7 @@ export function createScope(args: ScopeArgs): Scope { // returning the refusal; otherwise the terminal result would falsely report that cost as $0. let observeError: unknown try { - args.pool.observe(spend) + args.pool.observe(spend, { partial }) } catch (error) { observeError = error } @@ -2562,6 +2566,24 @@ async function runChild( // A refused pre-execution path (including an unmetered executor) reconciles zero and refunds // its whole reservation. Every path that actually executes reports measured or unknown spend. try { + try { + spend = withBudgetResources(spend, opts.budget, !started) + live.spent = withBudgetResources( + live.spent, + opts.budget, + !started || executor.accounting?.() !== undefined, + ) + } catch (error) { + // Invalid resource receipts cannot leave a ticket open or authorize a refund. + spend = withBudgetResources({ ...spend, resources: undefined }, opts.budget) + live.spent = withBudgetResources({ ...live.spent, resources: undefined }, opts.budget) + try { + pool.reconcile(ticket, spend) + } catch { + // Unknown enforced usage closes the ticket before reporting its violation. + } + throw error + } pool.reconcile(ticket, spend) return undefined } catch (error) { @@ -2773,7 +2795,21 @@ async function runChild( // lower bound, but never reinterpret the unreported remainder as zero under either root // ceiling. A recursive executor's explicit accounting remains authoritative on its throw // path; a persistence/teardown failure after a terminal artifact does too. - live.spent = { ...live.spent, tokensKnown: false, usdKnown: false } + live.spent = { + ...live.spent, + tokensKnown: false, + usdKnown: false, + ...(live.spent.resources === undefined + ? {} + : { + resources: Object.fromEntries( + Object.entries(live.spent.resources).map(([name, value]) => [ + name, + { ...value, known: false }, + ]), + ), + }), + } } const reconcileError = reconcileOnce(accounting?.reservation ?? live.spent) // A crashed driver child still re-homes the partial inference it durably metered. @@ -3014,6 +3050,7 @@ async function foldStream( const ev = next.value meterUsageEvent(totals, ev) await onProgress?.({ + ...addResourceSpend(totals.resources), iterations: totals.iterations, tokens: cloneTokenUsage(totals.tokens), ...(totals.tokensKnown ? {} : { tokensKnown: false }), @@ -3042,6 +3079,7 @@ function preserveUnknownTelemetry(streamed: Spend, terminal: Spend): Spend { const terminalTokens = cloneTokenUsage(terminal.tokens) return { ...streamed, + ...resourceTelemetry(streamed, terminal), tokens: { ...streamed.tokens, ...(streamed.tokens.freshInput === undefined && terminalTokens.freshInput !== undefined diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index f4ccf347..08ac08ef 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -92,6 +92,7 @@ import { type SupervisorSpanRecorder, } from './otel-spans' import type { PeerMailLimits } from './peer-mail' +import { addResourceSpend, resourceTelemetry, withBudgetResources } from './resources' import { registerRetainedExecutorPreparation, retainedExecutorSeamKey } from './retained-executor' import { consumeScopeRetainedOwnerResult, @@ -802,6 +803,9 @@ function driveHarnessFromBackend( (initialBudget.tokensLeft <= 0 || initialBudget.iterationsLeft <= 0 || (initialBudget.usdCapped && initialBudget.usdLeft <= 0) || + Object.values(initialBudget.resources ?? {}).some( + (resource) => !resource.known || resource.remaining <= 0, + ) || (initialBudget.deadlineMs > 0 && now() >= initialBudget.deadlineMs)) ) { throw new ValidationError('driveHarnessFromBackend: supervisor budget exhausted') @@ -923,6 +927,25 @@ function driveHarnessFromBackend( ? await scopeRetainedOwnerPriorSpend(scope) : zeroSpend() const ownerDelta = (total: Spend): Spend => ({ + ...addResourceSpend( + total.resources === undefined + ? undefined + : Object.fromEntries( + Object.entries(total.resources).map(([name, value]) => { + const prior = committedOwnerSpend.resources?.[name] + if (prior && prior.unit !== value.unit) + throw new ValidationError(`resource ${name}: unit mismatch`) + return [ + name, + { + ...value, + amount: Math.max(0, value.amount - (prior?.amount ?? 0)), + known: value.known && (prior?.known ?? true), + }, + ] + }), + ), + ), iterations: Math.max(0, total.iterations - committedOwnerSpend.iterations), tokens: { input: Math.max(0, total.tokens.input - committedOwnerSpend.tokens.input), @@ -1017,6 +1040,9 @@ function driveHarnessFromBackend( if ( budget.tokensLeft <= 0 || (budget.usdCapped && budget.usdLeft <= 0) || + Object.values(budget.resources ?? {}).some( + (resource) => !resource.known || resource.remaining <= 0, + ) || (budget.deadlineMs > 0 && now() >= budget.deadlineMs) ) { throw new ValidationError('driveHarnessFromBackend: supervisor budget exhausted') @@ -1082,7 +1108,10 @@ function driveHarnessFromBackend( ownerMaterializationPublished = true } await meterPending() - const total = { ...result.spent, iterations: 0 } + const total = withBudgetResources( + { ...result.spent, ...resourceTelemetry(observedOwnerSpend, result.spent), iterations: 0 }, + scope.budget, + ) const delta = ownerDelta(total) await meterRuntimeOwnedAccounting(scope, delta, { role: 'driver', runtime: executor.runtime }) committedOwnerSpend = addRetainedSpend(committedOwnerSpend, delta) @@ -1151,16 +1180,38 @@ function driveHarnessFromBackend( } await meterPending() const artifact = executor.resultArtifact() - terminalAccountingCaptured = true + const terminalResources = withBudgetResources( + { ...artifact.spent, ...resourceTelemetry(observedOwnerSpend, artifact.spent) }, + scope.budget, + ).resources + const resourceCorrection = addResourceSpend( + terminalResources === undefined + ? undefined + : Object.fromEntries( + Object.entries(terminalResources).map(([name, value]) => [ + name, + { + ...value, + amount: Math.max( + 0, + value.amount - (observedOwnerSpend.resources?.[name]?.amount ?? 0), + ), + }, + ]), + ), + ) // A stream carries increments, while its terminal artifact says whether either accounting // channel was omitted. Preserve unknowns in the shared pool instead of treating them as 0. if ( !retainedOwner && - (artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false) + (artifact.spent.tokensKnown === false || + artifact.spent.usdKnown === false || + resourceCorrection.resources !== undefined) ) { await meterRuntimeOwnedAccounting( scope, { + ...resourceCorrection, iterations: 0, tokens: { input: 0, output: 0 }, ...(artifact.spent.tokensKnown === false ? { tokensKnown: false } : {}), @@ -1171,15 +1222,16 @@ function driveHarnessFromBackend( { role: 'driver', runtime: executor.runtime, telemetry: 'unknown' }, ) } + terminalAccountingCaptured = true } else { const artifact = await run - terminalAccountingCaptured = true await meterRuntimeOwnedProviderAttempt( scope, - { ...artifact.spent, iterations: 0 }, + withBudgetResources({ ...artifact.spent, iterations: 0 }, scope.budget), providerEvidenceForNextMeter(), { role: 'driver', runtime: executor.runtime }, ) + terminalAccountingCaptured = true } if (pending !== undefined && !ownerMaterializationPublished) { const acknowledged = runtimeOwnedExecutorMaterialization(executor) @@ -1245,7 +1297,11 @@ function driveHarnessFromBackend( evidence?.attempts[meteredProviderAttempts]?.providerDispatch === 'not_started' await meterRuntimeOwnedProviderAttempt( scope, - providerDispatchDidNotStart ? zeroSpend() : unmeteredSpend(0), + withBudgetResources( + providerDispatchDidNotStart ? zeroSpend() : unmeteredSpend(0), + scope.budget, + providerDispatchDidNotStart, + ), providerEvidenceForNextMeter(), { role: 'driver', @@ -1259,11 +1315,24 @@ function driveHarnessFromBackend( if (meteredProviderAttempts === 0) { await meterRuntimeOwnedProviderAttempt( scope, - unmeteredSpend(0), + withBudgetResources(unmeteredSpend(0), scope.budget), providerEvidenceForNextMeter(), { role: 'driver', runtime: executor.runtime, telemetry: 'unknown-after-failure' }, ) } + // Increments cannot prove the final resource total after an interrupted invocation. + const attempts = runtimeOwnedExecutorProviderEvidence(executor)?.attempts + const provenNotStarted = + attempts !== undefined && + attempts.length > 0 && + attempts.every((attempt) => attempt.providerDispatch === 'not_started') + if (scope.budget.resources !== undefined && !provenNotStarted) { + await meterRuntimeOwnedAccounting( + scope, + withBudgetResources(zeroSpend(), scope.budget), + { role: 'driver', runtime: executor.runtime, telemetry: 'unknown-after-failure' }, + ) + } } catch (error) { // The budget pool intentionally throws after durably recording unknown capped usage and // closing that capacity. Only replace the original failure if the marker did not land. @@ -2257,6 +2326,22 @@ function assertPerWorkerWithinPool(perWorker: Budget, pool: Budget): void { child > total ? `supervise perWorker.${field} (${child}) exceeds budget.${field} (${total})` : null + for (const [name, resource] of Object.entries(pool.resources ?? {})) { + const child = perWorker.resources?.[name] + if (!child) + throw new ValidationError( + `supervise perWorker resource ${name}: child must declare its limit`, + ) + if (child.unit !== resource.unit) + throw new ValidationError(`supervise perWorker resource ${name}: unit mismatch`) + const problem = over(child.limit, resource.limit, `resources.${name}.limit`) + if (problem) throw new ValidationError(problem) + } + for (const name of Object.keys(perWorker.resources ?? {})) { + if (!Object.hasOwn(pool.resources ?? {}, name)) { + throw new ValidationError(`supervise perWorker resource ${name}: root must declare its limit`) + } + } const problems = [ over(perWorker.maxTokens, pool.maxTokens, 'maxTokens'), over(perWorker.maxIterations, pool.maxIterations, 'maxIterations'), @@ -2275,6 +2360,16 @@ function assertPerWorkerWithinPool(perWorker: Budget, pool: Budget): void { function defaultPerWorker(budget: Budget): Budget { return { + ...(budget.resources === undefined + ? {} + : { + resources: Object.fromEntries( + Object.entries(budget.resources).map(([name, value]) => [ + name, + { unit: value.unit, limit: Math.floor(value.limit / 4) }, + ]), + ), + }), maxIterations: Math.max(1, Math.floor(budget.maxIterations / 4)), maxTokens: Math.max(1, Math.floor(budget.maxTokens / 4)), ...(budget.maxUsd !== undefined ? { maxUsd: budget.maxUsd / 4 } : {}), diff --git a/src/runtime/supervise/supervisor.ts b/src/runtime/supervise/supervisor.ts index 18bcba52..4f75ee53 100644 --- a/src/runtime/supervise/supervisor.ts +++ b/src/runtime/supervise/supervisor.ts @@ -58,6 +58,7 @@ import { prepareScopeResume, type ScopeResumeState, sumSpendFromEvents } from '. export { maxSeqOf, sumMeasuredSpendFromEvents, uncertainSpawnBudgets } from './recover-executors' +import { withBudgetResources } from './resources' import { createScope, finalizeScopeOwnerMaterialization, startScopeRecoveries } from './scope' import { detachedSnapshot } from './snapshot' import type { @@ -1214,10 +1215,13 @@ async function terminalAccounting( * result this list rides on. */ function spendGapsFromEvents(events: SpawnEvent[]): SpendGap[] { const labels = new Map() + const budgets = new Map() const terminal = new Set() for (const ev of events) { - if (ev.kind === 'spawned') labels.set(ev.id, ev.label) - else if (closesCursorSlot(ev)) terminal.add(ev.id) + if (ev.kind === 'spawned') { + labels.set(ev.id, ev.label) + budgets.set(ev.id, ev.budget) + } else if (closesCursorSlot(ev)) terminal.add(ev.id) } const gaps = new Map< string, @@ -1236,11 +1240,25 @@ function spendGapsFromEvents(events: SpawnEvent[]): SpendGap[] { } for (const ev of events) { if (ev.kind === 'spawned' && ev.parent !== undefined && !terminal.has(ev.id)) { - record(ev.id, 'never-settled', ['tokens', 'usd']) + record(ev.id, 'never-settled', [ + 'tokens', + 'usd', + ...Object.keys(ev.budget.resources ?? {}).map((name): SpendChannel => `resource:${name}`), + ]) } else if (ev.kind === 'cancelled' && ev.spent === undefined) { - record(ev.id, 'unreported', ['tokens', 'usd']) + record(ev.id, 'unreported', [ + 'tokens', + 'usd', + ...Object.keys(budgets.get(ev.id)?.resources ?? {}).map( + (name): SpendChannel => `resource:${name}`, + ), + ]) } else if (ev.kind === 'settled' || (ev.kind === 'cancelled' && ev.spent !== undefined)) { - record(ev.id, 'unreported', unknownChannels(ev.spent!)) + record( + ev.id, + 'unreported', + unknownChannels(withBudgetResources(ev.spent!, budgets.get(ev.id) ?? {})), + ) } else if (ev.kind === 'metered') { record(ev.id, 'unreported', unknownChannels(ev.spend)) } @@ -1261,6 +1279,9 @@ function unknownChannels(spend: Spend): SpendChannel[] { const channels: SpendChannel[] = [] if (spend.tokensKnown === false) channels.push('tokens') if (spend.usdKnown === false) channels.push('usd') + for (const [name, resource] of Object.entries(spend.resources ?? {})) { + if (!resource.known) channels.push(`resource:${name}`) + } return channels } @@ -1275,6 +1296,7 @@ function isNonEmptySpend(s: Spend): boolean { s.usd > 0 || s.ms > 0 || s.tokensKnown === false || - s.usdKnown === false + s.usdKnown === false || + Object.values(s.resources ?? {}).some((resource) => resource.amount > 0 || !resource.known) ) } diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 9e96195c..9833c2fa 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -485,6 +485,7 @@ export type UsageEvent = kind: 'progress' progress: ExecutorProgressEvent } + | { kind: 'resource'; name: string; unit: string; amount: number; known: boolean } | { kind: 'iteration' } /** The runtime tag of a `Executor` impl. Open by intent: custom runtimes use their own string name. @@ -704,12 +705,27 @@ export interface ExecutorRegistry { // ── Budget — the conserved reservation pool ─────────────────────────────────── +/** Caller-defined resource ceiling in non-negative safe-integer units, consistent throughout a tree. */ +export interface ResourceLimit { + readonly unit: string + readonly limit: number +} + +/** Non-negative safe-integer subtotal. False means unknown, even when amount is zero. + * Select sufficiently fine units, such as GPU-milliseconds. */ +export interface ResourceSpend { + readonly unit: string + readonly amount: number + readonly known: boolean +} + /** A budget envelope on a spawn or the root. All ceilings; the pool reserves against them. */ export interface Budget { readonly maxIterations: number readonly maxTokens: number readonly maxUsd?: number readonly deadlineMs?: number + readonly resources?: Readonly> } /** @@ -721,6 +737,7 @@ export interface Budget { * holds the reason. */ export interface Spend { + resources?: Readonly> iterations: number tokens: LoopTokenUsage /** Token accounting is known unless explicitly false. A false value marks work that HAPPENED with @@ -1046,22 +1063,7 @@ export interface Scope { /** The live tree — reads the in-memory nursery, not the journal. */ readonly view: TreeView /** Conserved-pool readouts (post-reservation). */ - readonly budget: Readonly<{ - tokensLeft: number - /** `false` once a turn settled without reporting its tokens: `tokensLeft` is then a ceiling, - * not a measurement. */ - tokensKnown: boolean - /** `false` once a charged spend arrived without a readable prompt-cache split. That spend was - * charged at its rolled-up prompt total, which counts a cached prefix again on every turn that - * reads it, so `tokensLeft` is an upper bound on newly-presented work. */ - cacheBreakdownKnown: boolean - usdLeft: number - usdCapped: boolean - usdKnown: boolean - iterationsLeft: number - deadlineMs: number - reservedTokens: number - }> + readonly budget: import('./budget').BudgetReadout /** One tree-wide view of simultaneous spawned work. Every nested scope reads the same counter; * the root agent itself is not a spawned worker. `freeSlots` is `null` when no limit is set. * `unconfirmed` NAMES the settled children whose executor teardown was never acknowledged — @@ -1608,7 +1610,7 @@ export interface NoWinnerError { } /** The accounting channels a usage gap leaves incomplete. */ -export type SpendChannel = 'tokens' | 'usd' +export type SpendChannel = 'tokens' | 'usd' | `resource:${string}` /** * One journaled node whose usage accounting is incomplete — the named gap behind a `false` diff --git a/src/runtime/tool-loop.ts b/src/runtime/tool-loop.ts index 0b95645a..b6f2fd7d 100644 --- a/src/runtime/tool-loop.ts +++ b/src/runtime/tool-loop.ts @@ -11,6 +11,7 @@ import { ValidationError } from '../errors' import type { ToolSpec } from './router-client' +import type { Spend } from './supervise/types' /** Provider-neutral conversation record accepted by a tool-loop brain. */ export type ToolLoopMessageRecord = Record @@ -41,6 +42,8 @@ export type ToolLoopChat = ( content?: string | null toolCalls: ToolLoopToolCall[] usage?: { input: number; output: number; reasoning?: number } + /** Caller-measured resource totals for this turn; omission makes enforced dimensions unknown. */ + resources?: Spend['resources'] /** Dollar value reported for the turn. It is not billed spend unless provenance says so. */ costUsd?: number costProvenance?: 'provider-receipt' | 'billing-receipt' | 'catalog-estimate' diff --git a/src/runtime/util.ts b/src/runtime/util.ts index 4ac47d22..eaafaa04 100644 --- a/src/runtime/util.ts +++ b/src/runtime/util.ts @@ -7,6 +7,7 @@ */ import type { SandboxInstance } from '@tangle-network/sandbox' +import { addResourceSpend } from './supervise/resources' import type { Spend, TokenUsageProvenance } from './supervise/types' import type { LoopTokenUsage } from './types' @@ -197,6 +198,7 @@ export function unmeteredSpend(ms: number): Spend { /** Copy a conserved spend without dropping a completeness marker or the catalog-priced part. */ export function cloneSpend(spend: Spend): Spend { return { + ...addResourceSpend(spend.resources), iterations: spend.iterations, tokens: cloneTokenUsage(spend.tokens), ...(spend.tokensKnown === false ? { tokensKnown: false } : {}), @@ -217,6 +219,7 @@ export function addSpend(a: Spend, b: Spend): Spend { const tokens = cloneTokenUsage(a.tokens) addTokenUsage(tokens, b.tokens) return { + ...addResourceSpend(a.resources, b.resources), iterations: a.iterations + b.iterations, tokens, ...(a.tokensKnown === false || b.tokensKnown === false ? { tokensKnown: false } : {}), diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 1c602d6a..2cd16aea 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:f666113f8a62b9055193ab5b121c409c59790ef7e28fd57ab4b17b507e68c15a", + "digest": "sha256:ec41f15a91c96bfd97b7b5f2f7439e7a5c06c2311ba59e7cbb863a0feedbd487", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.208.1" + "runtimeVersion": "0.209.0" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:d726719e07878264cd67cbad7e11df3e93e947245d83864f070013f3cb0e6e88", - "runId": "agent-runtime-0.208.1-proposal-fixture", + "recordDigest": "sha256:1ac20f16209d080a7ec021d1eab0d35bc7544689e2458a031c920a71ccc7fe69", + "runId": "agent-runtime-0.209.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.208.1-proposal-fixture" + "runId": "agent-runtime-0.209.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 0b2b0a46..99cd1bce 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:ef6030831b942d3552d6e374808691731c87268f53632145a3aea4a36675a06c", + "digest": "sha256:2f19687492c33beb201a63d1a33c9cb4fb4657d9a45de02899565ba51d57532d", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.208.1" + "runtimeVersion": "0.209.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:4731856ecdc511cb3818a13dcbc5611013c8e95d236e8e0bd68fbb3e1a57bbe8", + "recordDigest": "sha256:b2382dafc570b8b7dbc9b86132644018380f31951d43c3889dd90917986a33e9", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/kernel/named-resource-aggregation.test.ts b/tests/kernel/named-resource-aggregation.test.ts new file mode 100644 index 00000000..54688147 --- /dev/null +++ b/tests/kernel/named-resource-aggregation.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from 'vitest' +import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { createBudgetPool } from '../../src/runtime/supervise/budget' +import { budgetStop } from '../../src/runtime/supervise/driver-retry' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createScope } from '../../src/runtime/supervise/scope' +import type { + Agent, + AgentSpec, + Executor, + Spend, + UsageEvent, +} from '../../src/runtime/supervise/types' +import { testAgentProfile } from './test-agent-profile' + +const resources = { compute: { unit: 'GPU-second', limit: 200 } } + +async function runStream(terminal: Spend['resources'], interrupted = false, increments = [3, 2]) { + const budget = { maxIterations: 3, maxTokens: 100, resources } + const pool = createBudgetPool(budget, 0) + const journal = new InMemorySpawnJournal() + await journal.beginTree('resources', new Date(0).toISOString()) + const scope = createScope({ + parentId: 'resources', + root: 'resources', + pool, + journal, + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + seams: {}, + depth: 0, + signal: new AbortController().signal, + now: () => 0, + }) + const executor: Executor = { + runtime: 'router', + execute: async function* (): AsyncIterable { + yield { kind: 'tokens', input: 1, output: 1 } + for (const amount of increments) { + yield { kind: 'resource', name: 'compute', unit: 'GPU-second', amount, known: true } + } + if (interrupted) throw new Error('connection interrupted') + yield { kind: 'iteration' } + }, + resultArtifact: () => ({ + outRef: 'result', + out: 'done', + spent: { + iterations: 1, + tokens: { input: 1, output: 1 }, + usd: 0, + ms: 0, + ...(terminal === undefined ? {} : { resources: terminal }), + }, + }), + teardown: async () => ({ destroyed: true }), + } + const spec: AgentSpec = { profile: testAgentProfile('resource-worker'), harness: null, executor } + const agent: Agent & { executorSpec: AgentSpec } = { + name: 'resource-worker', + act: async () => 'done', + executorSpec: spec, + } + const spawned = scope.spawn(agent, 'work', { label: 'resource-worker', budget }) + expect(spawned.ok).toBe(true) + const settled = await scope.next() + return { settled, pool, scope, id: spawned.ok ? spawned.handle.id : '' } +} + +describe('named resource aggregation at execution boundaries', () => { + it('reconciles 100 streamed increments exactly against the terminal receipt', async () => { + const { settled, pool } = await runStream( + { + compute: { unit: 'GPU-second', amount: 100, known: true }, + }, + false, + Array.from({ length: 100 }, () => 1), + ) + expect(settled?.kind).toBe('done') + expect(settled?.spent.resources?.compute).toEqual({ + unit: 'GPU-second', + amount: 100, + known: true, + }) + expect(pool.readout().resources?.compute).toMatchObject({ + committed: 100, + remaining: 100, + known: true, + }) + }) + + it('closes an overflowing stream as unknown and retains its safe lower bound', async () => { + const { settled, pool, scope, id } = await runStream(undefined, false, [ + Number.MAX_SAFE_INTEGER, + 1, + ]) + expect(settled?.kind).toBe('down') + expect(scope.view.nodes.find((node) => node.id === id)?.spent.resources?.compute).toEqual({ + unit: 'GPU-second', + amount: Number.MAX_SAFE_INTEGER, + known: false, + }) + pool.assertNoOpenTickets() + expect(pool.readout().resources?.compute.known).toBe(false) + }) + + it('charges streamed increments once when the terminal receipt repeats the total', async () => { + const { settled, pool, scope, id } = await runStream({ + compute: { unit: 'GPU-second', amount: 5, known: true }, + }) + expect(settled?.spent.resources).toEqual({ + compute: { unit: 'GPU-second', amount: 5, known: true }, + }) + expect(pool.readout().resources?.compute).toMatchObject({ + committed: 5, + remaining: 195, + known: true, + }) + expect(scope.progress(id)?.resources).toEqual(settled?.spent.resources) + }) + + it('preserves an unknown terminal total despite measured stream increments', async () => { + const { settled, pool, scope, id } = await runStream({ + compute: { unit: 'GPU-second', amount: 5, known: false }, + }) + expect(settled?.kind).toBe('down') + expect(scope.view.nodes.find((node) => node.id === id)?.spent.resources?.compute).toEqual({ + unit: 'GPU-second', + amount: 5, + known: false, + }) + expect(pool.readout().resources?.compute).toMatchObject({ remaining: 0, known: false }) + expect(budgetStop(pool.readout(), 0)).toBe('budget-exhausted') + }) + + it('accepts complete streamed resource telemetry when the terminal artifact omits the map', async () => { + const { settled, pool } = await runStream(undefined) + expect(settled?.spent.resources?.compute).toEqual({ + unit: 'GPU-second', + amount: 5, + known: true, + }) + expect(pool.readout().resources?.compute).toMatchObject({ + committed: 5, + remaining: 195, + known: true, + }) + }) + it.each([ + { unit: 'minute', amount: 5, known: true }, + { unit: 'GPU-second', amount: -1, known: true }, + { unit: 'GPU-second', amount: 0.1, known: true }, + { unit: 'GPU-second', amount: Number.MAX_SAFE_INTEGER + 1, known: true }, + { unit: 'GPU-second', amount: Number.NaN, known: true }, + ])( + 'closes reservations and preserves unknown usage for malformed receipts: %j', + async (receipt) => { + const { settled, pool, scope, id } = await runStream({ compute: receipt }) + expect(settled?.kind).toBe('down') + pool.assertNoOpenTickets() + expect(pool.readout().resources?.compute.known).toBe(false) + expect(scope.view.nodes.find((node) => node.id === id)?.spent.resources?.compute.known).toBe( + false, + ) + expect(budgetStop(pool.readout(), 0)).toBe('budget-exhausted') + }, + ) + + it('retains measured increments as an unknown lower bound when execution interrupts', async () => { + const { settled, pool, scope, id } = await runStream(undefined, true) + expect(settled?.kind).toBe('down') + pool.assertNoOpenTickets() + expect(scope.view.nodes.find((node) => node.id === id)?.spent.resources?.compute).toEqual({ + unit: 'GPU-second', + amount: 5, + known: false, + }) + expect(budgetStop(pool.readout(), 0)).toBe('budget-exhausted') + }) + it('retains the larger conflicting total and refuses to label inconsistent receipts known', async () => { + const { settled, pool, scope, id } = await runStream({ + compute: { unit: 'GPU-second', amount: 10, known: true }, + }) + expect(settled?.kind).toBe('down') + expect(scope.view.nodes.find((node) => node.id === id)?.spent.resources?.compute).toEqual({ + unit: 'GPU-second', + amount: 10, + known: false, + }) + pool.assertNoOpenTickets() + expect(budgetStop(pool.readout(), 0)).toBe('budget-exhausted') + }) +}) diff --git a/tests/kernel/named-resource-budgets.test.ts b/tests/kernel/named-resource-budgets.test.ts new file mode 100644 index 00000000..17058b7b --- /dev/null +++ b/tests/kernel/named-resource-budgets.test.ts @@ -0,0 +1,501 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { + FileResultBlobStore, + FileSpawnJournal, + materializeTreeView, + replaySpawnTree, +} from '../../src/durable/spawn-journal' +import { createBudgetPool, spendFromUsageEvents } from '../../src/runtime/supervise/budget' +import { prepareScopeResume } from '../../src/runtime/supervise/recover-executors' +import { withBudgetResources } from '../../src/runtime/supervise/resources' +import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { superviseWithTestBrain } from '../../src/runtime/supervise/supervise' +import { createSupervisor } from '../../src/runtime/supervise/supervisor' +import type { Agent, Budget, Executor, Spend, UsageEvent } from '../../src/runtime/supervise/types' +import { addSpend, cloneSpend, zeroSpend } from '../../src/runtime/util' +import { testAgentProfile } from './test-agent-profile' + +const budget = (limit = 10): Budget => ({ + maxTokens: limit * 10, + maxIterations: limit, + resources: { gpu: { unit: 'seconds', limit }, io: { unit: 'bytes', limit: limit * 10 } }, +}) +const measured = (gpu = 2, io = 20): Spend => ({ + ...zeroSpend(), + resources: { + gpu: { unit: 'seconds', amount: gpu, known: true }, + io: { unit: 'bytes', amount: io, known: true }, + }, +}) + +function conserved(pool: ReturnType) { + for (const value of Object.values(pool.readout().resources ?? {})) { + expect(value.remaining + value.reserved + value.committed).toBe(value.limit) + } +} + +describe('caller-named resource conservation', () => { + it('conserves repeated integer allocations, refunds, and restored receipts exactly', () => { + const pool = createBudgetPool(budget(200), 0) + const tickets = Array.from({ length: 100 }, () => { + const reservation = pool.reserve(budget(2)) + if (!reservation.ok) throw new Error('reservation failed') + return reservation.ticket + }) + expect(pool.reserve(budget(1)).ok).toBe(false) + let committed = zeroSpend() + for (const ticket of tickets) { + const receipt = measured(1, 10) + pool.reconcile(ticket, receipt) + committed = addSpend(committed, receipt) + } + pool.assertNoOpenTickets() + expect(pool.readout().resources?.gpu).toMatchObject({ + remaining: 100, + committed: 100, + reserved: 0, + known: true, + }) + conserved(pool) + const restored = createBudgetPool(budget(200), 0, { committed }) + expect(restored.readout().resources).toEqual(pool.readout().resources) + const final = restored.reserve(budget(100)) + if (!final.ok) throw new Error('reservation failed') + restored.reconcile(final.ticket, measured(100, 1_000)) + expect(restored.readout().resources?.gpu.remaining).toBe(0) + conserved(restored) + }) + + it('saturates overflowing totals as unknown without leaking a reservation', () => { + const maximum = Number.MAX_SAFE_INTEGER + const huge: Budget = { + maxTokens: 10, + maxIterations: 10, + resources: { gpu: { unit: 'seconds', limit: maximum } }, + } + const pool = createBudgetPool(huge, 0) + const receipt = (amount: number): Spend => ({ + ...zeroSpend(), + resources: { gpu: { unit: 'seconds', amount, known: true } }, + }) + const first = pool.reserve({ + ...huge, + resources: { gpu: { unit: 'seconds', limit: maximum - 1 } }, + }) + const second = pool.reserve({ + maxTokens: 0, + maxIterations: 0, + resources: { gpu: { unit: 'seconds', limit: 1 } }, + }) + if (!first.ok || !second.ok) throw new Error('reservation failed') + pool.reconcile(first.ticket, receipt(maximum - 1)) + expect(() => pool.reconcile(second.ticket, receipt(2))).toThrow('overflow') + pool.assertNoOpenTickets() + expect(pool.readout().resources?.gpu).toMatchObject({ + remaining: 0, + reserved: 0, + committed: maximum, + known: false, + }) + expect( + pool.reserve({ + maxTokens: 0, + maxIterations: 0, + resources: { gpu: { unit: 'seconds', limit: 0 } }, + }).ok, + ).toBe(false) + const total = addSpend(receipt(maximum - 1), receipt(2)) + expect(total.resources?.gpu).toEqual({ unit: 'seconds', amount: maximum, known: false }) + expect(createBudgetPool(huge, 0, { committed: total }).readout().resources?.gpu.known).toBe( + false, + ) + conserved(pool) + }) + + it('rounds default named worker partitions down before starting the driver', async () => { + let calls = 0 + await superviseWithTestBrain(testAgentProfile('root', { harness: 'cli-base' }), 'work', { + budget: budget(10), + brain: async () => { + calls += 1 + return { + content: 'done', + toolCalls: [], + usage: { input: 1, output: 1 }, + costUsd: 0, + costProvenance: 'provider-receipt', + } + }, + makeWorkerAgent: () => { + throw new Error('worker must not run') + }, + }) + expect(calls).toBe(1) + }) + + it.each([ + [undefined, 'must declare'], + [{ gpu: { unit: 'minutes', limit: 1 }, io: { unit: 'bytes', limit: 1 } }, 'unit mismatch'], + [{ gpu: { unit: 'seconds', limit: 11 }, io: { unit: 'bytes', limit: 1 } }, 'exceeds'], + [ + { + gpu: { unit: 'seconds', limit: 1 }, + io: { unit: 'bytes', limit: 1 }, + extra: { unit: 'items', limit: 1 }, + }, + 'root must declare', + ], + ] as const)( + 'rejects incompatible worker dimensions before invoking the driver: %j', + (resources, message) => { + let calls = 0 + expect(() => + superviseWithTestBrain(testAgentProfile('root', { harness: 'cli-base' }), 'work', { + budget: budget(), + perWorker: { maxTokens: 1, maxIterations: 1, resources }, + brain: async () => { + calls += 1 + throw new Error('driver must not run') + }, + makeWorkerAgent: () => { + throw new Error('worker must not run') + }, + }), + ).toThrow(message) + expect(calls).toBe(0) + }, + ) + + it.each([false, true])( + 'retains exact component receipts through durable replay (overflow=%s)', + async (overflow) => { + const dir = await mkdtemp(join(tmpdir(), 'named-resource-overflow-')) + try { + const maximum = Number.MAX_SAFE_INTEGER + const allocations = overflow + ? [ + [maximum - 1, maximum - 1], + [2, 1], + ] + : Array.from({ length: 100 }, () => [1, 2]) + const total = overflow ? maximum : 100 + const journal = new FileSpawnJournal(join(dir, 'journal.jsonl')) + const blobs = new FileResultBlobStore(join(dir, 'blobs')) + const root: Budget = { + maxTokens: 200, + maxIterations: 200, + resources: { gpu: { unit: 'milliseconds', limit: overflow ? maximum : 200 } }, + } + let release!: () => void + const barrier = new Promise((resolve) => { + release = resolve + }) + const leaf = (amount: number): Agent => ({ + name: `leaf-${amount}`, + act: async () => 'done', + executorSpec: { + profile: testAgentProfile(`leaf-${amount}`), + harness: null, + executor: { + runtime: 'named-resource-test', + async execute() { + await barrier + return this.resultArtifact() + }, + resultArtifact: () => ({ + outRef: 'done', + out: 'done', + spent: { + ...zeroSpend(), + resources: { gpu: { unit: 'milliseconds', amount, known: true } }, + }, + }), + teardown: async () => ({ destroyed: true }), + }, + }, + }) + const result = await createSupervisor().run( + { + name: 'root', + async act(task, scope) { + for (const [amount, limit] of allocations) { + expect( + scope.spawn(leaf(amount), task, { + label: `leaf-${amount}`, + budget: { + maxTokens: 1, + maxIterations: 1, + resources: { gpu: { unit: 'milliseconds', limit } }, + }, + }).ok, + ).toBe(true) + } + release() + for (let index = 0; index < allocations.length; index += 1) await scope.next() + expect(scope.budget.resources?.gpu.known).toBe(!overflow) + return 'done' + }, + }, + 'task', + { budget: root, runId: 'overflow', journal, blobs, executors: createExecutorRegistry() }, + ) + expect(result.spentTotal.resources?.gpu).toEqual({ + unit: 'milliseconds', + amount: total, + known: !overflow, + }) + const events = await new FileSpawnJournal(join(dir, 'journal.jsonl')).loadTree('overflow') + if (!events) throw new Error('journal absent') + const receipts = events.flatMap((event) => + event.kind === 'settled' || event.kind === 'cancelled' + ? [event.spent?.resources?.gpu] + : [], + ) + expect(receipts).toHaveLength(allocations.length) + expect(receipts).toEqual( + expect.arrayContaining( + allocations.map(([amount]) => ({ unit: 'milliseconds', amount, known: true })), + ), + ) + const restored = await prepareScopeResume( + { runId: 'overflow', journal, blobs }, + events, + new AbortController().signal, + () => Date.now(), + ) + expect( + createBudgetPool(root, 0, restored.poolRestore).readout().resources?.gpu, + ).toMatchObject({ known: !overflow, remaining: overflow ? 0 : 100 }) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }, + ) + + it('reserves all channels atomically and refunds only known unused capacity', () => { + const pool = createBudgetPool(budget(), 0) + const first = pool.reserve(budget(6)) + expect(first.ok).toBe(true) + const before = pool.readout() + expect(pool.reserve(budget(5)).ok).toBe(false) + expect(pool.readout()).toEqual(before) + expect(pool.reserve({ ...budget(1), maxTokens: 101 }).ok).toBe(false) + expect(pool.readout()).toEqual(before) + if (!first.ok) throw new Error('reservation failed') + pool.reconcile(first.ticket, measured()) + expect(pool.readout().resources?.gpu).toMatchObject({ + remaining: 8, + reserved: 0, + committed: 2, + known: true, + }) + conserved(pool) + }) + + it('rejects malformed units, negative/nonfinite values, and unbudgeted dimensions before mutation', () => { + for (const limit of [ + -1, + 0.1, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ]) { + expect(() => + createBudgetPool({ ...budget(), resources: { gpu: { unit: 'seconds', limit } } }, 0), + ).toThrow('safe integer') + } + const pool = createBudgetPool(budget(), 0) + const before = pool.readout() + expect(() => + pool.reserve({ + ...budget(), + resources: { gpu: { unit: 'minutes', limit: 1 }, io: { unit: 'bytes', limit: 1 } }, + }), + ).toThrow('unit mismatch') + expect(() => pool.reserve({ maxTokens: 1, maxIterations: 1 })).toThrow('must declare') + expect(() => + createBudgetPool({ maxTokens: 1, maxIterations: 1 }, 0).reserve(budget(1)), + ).toThrow('root must declare') + expect(pool.readout()).toEqual(before) + for (const amount of [ + -1, + 0.1, + Number.NaN, + Number.POSITIVE_INFINITY, + Number.MAX_SAFE_INTEGER + 1, + ]) { + expect(() => + spendFromUsageEvents([ + { kind: 'resource', name: 'gpu', unit: 'seconds', amount, known: true }, + ]), + ).toThrow() + } + expect(() => + addSpend(measured(), { + ...zeroSpend(), + resources: { gpu: { unit: 'minutes', amount: 1, known: true } }, + }), + ).toThrow('unit mismatch') + }) + + it('records overrun before refusing and preserves unknowns through restart', () => { + const pool = createBudgetPool(budget(), 0) + const reservation = pool.reserve(budget(4)) + if (!reservation.ok) throw new Error('reservation failed') + expect(() => pool.reconcile(reservation.ticket, measured(5, 20))).toThrow( + 'spent 5 > reserved 4', + ) + expect(pool.readout().resources?.gpu).toMatchObject({ remaining: 5, committed: 5, reserved: 0 }) + pool.assertNoOpenTickets() + conserved(pool) + + const unknown = { + ...zeroSpend(), + resources: { + gpu: { unit: 'seconds', amount: 1, known: false }, + io: { unit: 'bytes', amount: 2, known: true }, + }, + } + const next = pool.reserve(budget(2)) + if (!next.ok) throw new Error('reservation failed') + expect(() => pool.reconcile(next.ticket, unknown)).toThrow('unknown usage') + expect(pool.reserve(budget(0)).ok).toBe(false) + conserved(pool) + const restored = createBudgetPool(budget(), 0, { + committed: addSpend(measured(5, 20), unknown), + }) + expect(restored.readout().resources?.gpu).toMatchObject({ remaining: 0, known: false }) + expect(restored.reserve(budget(0)).ok).toBe(false) + conserved(restored) + }) + + it('treats explicit restored spend with omitted dimensions as unknown, including zero amounts', () => { + const restored = createBudgetPool(budget(), 0, { committed: zeroSpend() }) + expect(restored.readout().resources?.gpu.known).toBe(false) + expect(restored.reserve(budget(1)).ok).toBe(false) + conserved(restored) + expect(createBudgetPool(budget(), 0, {}).reserve(budget(1)).ok).toBe(true) + }) + + it('distinguishes a complete omission from a partial increment and a proven predispatch refusal', () => { + const pool = createBudgetPool(budget(), 0) + pool.observe(zeroSpend(), { partial: true }) + expect(pool.readout().resources?.gpu.known).toBe(true) + expect(() => pool.observe(zeroSpend())).toThrow('unknown usage') + expect(pool.readout().resources?.gpu.known).toBe(false) + const fresh = createBudgetPool(budget(), 0) + const reservation = fresh.reserve(budget(4)) + if (!reservation.ok) throw new Error('reservation failed') + fresh.reconcile(reservation.ticket, withBudgetResources(zeroSpend(), budget(4), true)) + expect(fresh.readout().resources?.gpu.remaining).toBe(10) + const missing = fresh.reserve(budget(4)) + if (!missing.ok) throw new Error('reservation failed') + expect(() => fresh.reconcile(missing.ticket, zeroSpend())).toThrow('unknown usage') + fresh.assertNoOpenTickets() + conserved(fresh) + }) + + it('retains two concurrent children, their names/units/unknowns, and admission refusal in durable replay', async () => { + const dir = await mkdtemp(join(tmpdir(), 'named-resource-budget-')) + try { + const journal = new FileSpawnJournal(join(dir, 'journal.jsonl')) + const blobs = new FileResultBlobStore(join(dir, 'blobs')) + let entered = 0 + let release!: () => void + const barrier = new Promise((resolve) => { + release = resolve + }) + const leaf = (name: string, unknown: boolean): Agent => { + const spent = measured() + if (unknown) + spent.resources = { + ...spent.resources, + gpu: { unit: 'seconds', amount: 1, known: false }, + } + const executor: Executor = { + runtime: 'named-resource-test', + execute() { + return (async function* (): AsyncGenerator { + entered += 1 + if (entered === 2) release() + await barrier + for (const [resourceName, value] of Object.entries(spent.resources ?? {})) { + yield { kind: 'resource', name: resourceName, ...value } + } + })() + }, + resultArtifact: () => ({ outRef: name, out: name, spent }), + teardown: async () => ({ destroyed: true }), + } + return { + name, + act: async () => name, + executorSpec: { profile: testAgentProfile(name), harness: null, executor }, + } + } + const runId = 'named-resources' + const result = await createSupervisor().run( + { + name: 'root', + async act(task, scope) { + expect( + scope.spawn(leaf('known', false), task, { label: 'known', budget: budget(4) }).ok, + ).toBe(true) + expect( + scope.spawn(leaf('unknown', true), task, { label: 'unknown', budget: budget(4) }).ok, + ).toBe(true) + expect(scope.budget.resources?.gpu.reserved).toBe(8) + await scope.next() + await scope.next() + expect(scope.budget.resources?.gpu.known).toBe(false) + expect( + scope.spawn(leaf('denied', false), task, { label: 'denied', budget: budget(1) }).ok, + ).toBe(false) + return 'completed' + }, + }, + 'task', + { budget: budget(), runId, journal, blobs, executors: createExecutorRegistry() }, + ) + expect(entered).toBe(2) + expect(result.spentTotal.resources).toEqual({ + gpu: { unit: 'seconds', amount: 3, known: false }, + io: { unit: 'bytes', amount: 40, known: true }, + }) + const reopened = new FileSpawnJournal(join(dir, 'journal.jsonl')) + const replayed = await replaySpawnTree( + reopened, + new FileResultBlobStore(join(dir, 'blobs')), + runId, + ) + expect(replayed).toHaveLength(2) + expect(replayed.map((row) => row.kind).sort()).toEqual(['done', 'down']) + const events = await reopened.loadTree(runId) + if (!events) throw new Error('journal absent') + expect( + materializeTreeView(events) + .nodes.filter((node) => node.parent !== undefined) + .map((node) => node.spent.resources?.gpu.known) + .sort(), + ).toEqual([false, true]) + const restored = await prepareScopeResume( + { runId, journal: reopened, blobs }, + events, + new AbortController().signal, + () => Date.now(), + ) + const pool = createBudgetPool(budget(), 0, restored.poolRestore) + expect(pool.readout().resources?.gpu.known).toBe(false) + expect(pool.reserve(budget(0)).ok).toBe(false) + conserved(pool) + const copy = cloneSpend(result.spentTotal) + expect(copy.resources).toEqual(result.spentTotal.resources) + expect(copy.resources).not.toBe(result.spentTotal.resources) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) +}) diff --git a/tests/kernel/named-resource-driver.test.ts b/tests/kernel/named-resource-driver.test.ts new file mode 100644 index 00000000..d1395829 --- /dev/null +++ b/tests/kernel/named-resource-driver.test.ts @@ -0,0 +1,141 @@ +import { expect, it } from 'vitest' +import { InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { supervise } from '../../src/runtime/supervise/supervise' +import type { Agent, Spend } from '../../src/runtime/supervise/types' +import type { ToolLoopChat } from '../../src/runtime/tool-loop' +import { runtimeToolDeclarations, testAgentProfile } from './test-agent-profile' + +it.each(['reported', 'missing', 'malformed', 'retried'])( + 'meters explicit root resource receipts before tool calls: receipt=%s', + async (mode) => { + const reported = mode === 'reported' + const resources = { + compute: { unit: 'millisecond', limit: 100 }, + transfer: { unit: 'byte', limit: 100 }, + } + const receipt: Spend['resources'] = { + compute: { unit: 'millisecond', amount: 2, known: true }, + transfer: { unit: 'byte', amount: 0, known: true }, + } + const journal = new InMemorySpawnJournal() + let entered = 0 + let calls = 0 + let transportCalls = 0 + const brain: ToolLoopChat = async () => ({ + toolCalls: + calls++ === 0 + ? [ + { + id: 'spawn', + name: 'spawn_worker', + arguments: JSON.stringify({ profile: testAgentProfile('leaf'), task: 'work' }), + }, + { id: 'await', name: 'await_event', arguments: '{}' }, + ] + : [], + usage: { input: 1, output: 1 }, + ...(reported || mode === 'retried' ? { resources: receipt } : {}), + }) + const leaf: Agent = { + name: 'leaf', + act: async () => 'unused', + executorSpec: { + profile: testAgentProfile('leaf'), + harness: null, + executor: { + runtime: 'resource-receipt', + execute: async () => { + entered++ + return { + outRef: 'leaf', + out: 'done', + spent: { + iterations: 1, + tokens: { input: 1, output: 1 }, + usd: 0, + ms: 0, + resources: receipt, + }, + } + }, + teardown: async () => ({ destroyed: true }), + }, + }, + } + const result = await supervise( + testAgentProfile('root', { + harness: 'cli-base', + model: { + metadata: { retry: { maxAttempts: 2, initialBackoffMs: 0, maxBackoffMs: 0, jitter: 0 } }, + }, + tools: runtimeToolDeclarations('spawn_worker', 'await_event'), + }), + 'work', + { + router: { + routerBaseUrl: 'http://unused.invalid/v1', + routerKey: 'offline', + complete: async (body) => { + if (mode === 'retried' && transportCalls++ === 0) + throw new TypeError('fetch failed after dispatch') + expect(body.model).toBe('offline-test-model') + const result = await brain([], []) + return { + model: body.model, + choices: [ + { + message: { + content: result.content, + tool_calls: result.toolCalls.map((call) => ({ + id: call.id, + type: 'function', + function: { name: call.name, arguments: call.arguments }, + })), + }, + }, + ], + usage: { + prompt_tokens: result.usage?.input, + completion_tokens: result.usage?.output, + ...(mode === 'malformed' + ? { resources: { compute: { unit: 'millisecond', amount: -1, known: true } } } + : result.resources + ? { resources: result.resources } + : {}), + }, + } + }, + }, + journal, + budget: { maxTokens: 100, maxIterations: 10, resources }, + perWorker: { + maxTokens: 10, + maxIterations: 1, + resources: { + compute: { unit: 'millisecond', limit: 10 }, + transfer: { unit: 'byte', limit: 10 }, + }, + }, + makeWorkerAgent: () => leaf, + }, + ) + expect(entered).toBe(reported ? 1 : 0) + expect(result.spentTotal.resources?.compute.known).toBe(reported) + expect(result.spentTotal.resources?.transfer).toEqual({ + unit: 'byte', + amount: 0, + known: reported, + }) + const events = await journal.loadTree('supervise') + const metered = events?.filter((event) => event.kind === 'metered') ?? [] + expect(metered.length).toBeGreaterThan(0) + for (const event of metered) { + if (event.kind !== 'metered') continue + expect(event.spend.resources?.compute).toEqual({ + unit: 'millisecond', + amount: reported || mode === 'retried' ? 2 : 0, + known: reported, + }) + } + }, +) diff --git a/tests/kernel/named-resource-transport.test.ts b/tests/kernel/named-resource-transport.test.ts new file mode 100644 index 00000000..083ae0ba --- /dev/null +++ b/tests/kernel/named-resource-transport.test.ts @@ -0,0 +1,182 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { + routerChatWithTools, + routerChatWithUsage, + streamRouterChatWithTools, +} from '../../src/runtime/router-client' +import { + routerInlineExecutor, + routerToolsInlineExecutor, +} from '../../src/runtime/supervise/runtime' +import { testAgentProfile } from './test-agent-profile' + +const resources = { + compute: { unit: 'millisecond', amount: 2, known: true }, + transfer: { unit: 'byte', amount: 0, known: true }, +} +const config = { + routerBaseUrl: 'http://unused.invalid/v1', + routerKey: 'offline', + model: 'offline-test-model', +} +afterEach(() => vi.unstubAllGlobals()) + +it('preserves resource-only scalar receipts and rejects malformed measurements', async () => { + const complete = async () => ({ + choices: [{ message: { content: 'done' } }], + usage: { resources }, + }) + expect((await routerChatWithUsage({ ...config, complete }, [])).resources).toEqual(resources) + await expect( + routerChatWithUsage( + { + ...config, + complete: async () => ({ + choices: [{ message: { content: 'done' } }], + usage: { resources: { compute: { unit: 'millisecond', amount: -1, known: true } } }, + }), + }, + [], + ), + ).rejects.toThrow('resource spend') +}) + +it('normalizes the same receipt from a terminal SSE usage frame', async () => { + const frame = JSON.stringify({ + choices: [{ delta: { content: 'done' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1, resources }, + }) + vi.stubGlobal( + 'fetch', + async () => + new Response(`data: ${frame}\n\ndata: [DONE]\n\n`, { + headers: { 'content-type': 'text/event-stream' }, + }), + ) + expect((await streamRouterChatWithTools(config, [], [])).resources).toEqual(resources) +}) + +it('preserves scalar executor resource totals', async () => { + const executor = routerInlineExecutor( + { profile: testAgentProfile('leaf', { harness: 'cli-base' }), harness: null }, + { + signal: new AbortController().signal, + seams: { + router: { + ...config, + complete: async () => ({ + model: config.model, + choices: [{ message: { content: 'done' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, resources }, + }), + }, + }, + }, + ) + const result = await executor.execute('task', new AbortController().signal) + if (!('spent' in result)) throw new Error('expected terminal executor result') + expect(result.spent.resources).toEqual(resources) + await executor.teardown() +}) + +it.each(['complete', 'missing-first', 'missing-last'])( + 'sums inline tool turns and preserves omissions: complete=%s', + async (mode) => { + const completeReceipt = mode === 'complete' + let turn = 0 + const executor = routerToolsInlineExecutor( + { + profile: testAgentProfile('leaf', { + harness: 'cli-base', + tools: { tick: true }, + model: { metadata: { maxTurns: 2 } }, + }), + harness: null, + }, + { + signal: new AbortController().signal, + seams: { + 'router-tools': { + ...config, + tools: [ + { type: 'function', function: { name: 'tick', parameters: { type: 'object' } } }, + ], + executeToolCall: async () => 'done', + complete: async () => { + const first = turn++ === 0 + return { + model: config.model, + choices: [ + { + message: { + content: 'done', + tool_calls: first + ? [{ id: 'tick', function: { name: 'tick', arguments: '{}' } }] + : [], + }, + }, + ], + usage: { + prompt_tokens: 1, + completion_tokens: 1, + ...(completeReceipt || (mode === 'missing-first' ? !first : first) + ? { resources } + : {}), + }, + } + }, + }, + }, + }, + ) + const result = await executor.execute('task', new AbortController().signal) + if (!('spent' in result)) throw new Error('expected terminal executor result') + expect(turn).toBe(2) + expect(result.spent.resources?.compute).toEqual({ + unit: 'millisecond', + amount: completeReceipt ? 4 : 2, + known: completeReceipt, + }) + await executor.teardown() + }, +) + +it.each(['scalar', 'tools', 'stream'] as const)( + 'marks retried %s resource totals unknown while preserving the measured subtotal', + async (mode) => { + let attempts = 0 + const receipt = { + model: config.model, + choices: [{ message: { content: 'done' } }], + usage: { prompt_tokens: 1, completion_tokens: 1, resources }, + } + const retry = { maxAttempts: 2, initialBackoffMs: 0, maxBackoffMs: 0, jitter: 0 } + const complete = async () => { + attempts += 1 + if (attempts === 1) throw new TypeError('fetch failed after dispatch') + return receipt + } + vi.stubGlobal('fetch', async () => { + attempts += 1 + if (attempts === 1) throw new TypeError('fetch failed after dispatch') + const frame = { ...receipt, choices: [{ delta: { content: 'done' }, finish_reason: 'stop' }] } + return new Response(`data: ${JSON.stringify(frame)}\n\ndata: [DONE]\n\n`, { + headers: { 'content-type': 'text/event-stream' }, + }) + }) + const result = + mode === 'scalar' + ? await routerChatWithUsage({ ...config, retry, complete }, []) + : mode === 'tools' + ? await routerChatWithTools({ ...config, retry, complete }, [], []) + : await streamRouterChatWithTools({ ...config, retry }, [], []) + expect(attempts).toBe(2) + expect(result.transportAttempts).toBe(2) + expect(result.resources).toEqual({ + compute: { ...resources.compute, known: false }, + transfer: { ...resources.transfer, known: false }, + }) + expect(resources.compute.known).toBe(true) + expect(resources.transfer.known).toBe(true) + }, +) diff --git a/tests/kernel/nested-retained-process-recovery.test.ts b/tests/kernel/nested-retained-process-recovery.test.ts index 6a0d634f..bacbe120 100644 --- a/tests/kernel/nested-retained-process-recovery.test.ts +++ b/tests/kernel/nested-retained-process-recovery.test.ts @@ -23,254 +23,282 @@ import type { Agent, SpawnJournal } from '../../src/runtime/supervise/types' import { durableRetainedProvider } from '../helpers/durable-retained-provider' import { testAgentProfile } from './test-agent-profile' -it('restores a nested manager before its retained child needs manager coordination', async () => { - const directory = await mkdtemp(join(tmpdir(), 'nested-process-recovery-')) - const runDirectory = join(directory, 'run') - const context = createFileRunContext(runDirectory, { withDriver: true }) - let releaseChild!: () => void - const parentOpened = new Promise((resolve) => { - releaseChild = resolve - }) - let releaseRoot!: () => void - const rootInspected = new Promise((resolve) => { - releaseRoot = resolve - }) - let freshFactories = 0 - let recovering = false - let parentActed = false - let creates = 0 - let interrupted = false - const provider = (): AgentEnvironmentProvider => { - const base = durableRetainedProvider(join(directory, 'provider.json')) - const wrap = (environment: AgentEnvironment): AgentEnvironment => ({ - ...environment, - session: (id, options) => { - const session = environment.session!(id, options) - return { - ...session, - async *events(eventOptions) { - // The retained child cannot make progress until its manager is running again. - // A terminal-await recovery preflight blocks the only code that can open this gate. - if (recovering) { - const signal = eventOptions?.signal - let onAbort: (() => void) | undefined - try { - await Promise.race([ - Promise.all([parentOpened, rootInspected]), - new Promise((_resolve, reject) => { - onAbort = () => reject(signal?.reason ?? new Error('child observation aborted')) - if (signal?.aborted) onAbort() - else signal?.addEventListener('abort', onAbort, { once: true }) - }), - ]) - } finally { - if (onAbort) signal?.removeEventListener('abort', onAbort) +it.each([ + { known: true, managerTokens: 5 }, + { known: false, managerTokens: 5 }, + { known: true, managerTokens: 0 }, + { known: false, managerTokens: 0 }, +])( + 'restores resource accounting (known=$known, managerTokens=$managerTokens)', + async ({ known, managerTokens }) => { + const directory = await mkdtemp(join(tmpdir(), 'nested-process-recovery-')) + const runDirectory = join(directory, 'run') + const context = createFileRunContext(runDirectory, { withDriver: true }) + let releaseChild!: () => void + const parentOpened = new Promise((resolve) => { + releaseChild = resolve + }) + let releaseRoot!: () => void + const rootInspected = new Promise((resolve) => { + releaseRoot = resolve + }) + let freshFactories = 0 + let recovering = false + let parentActed = false + let creates = 0 + let interrupted = false + const provider = (): AgentEnvironmentProvider => { + const base = durableRetainedProvider(join(directory, 'provider.json')) + const wrap = (environment: AgentEnvironment): AgentEnvironment => ({ + ...environment, + session: (id, options) => { + const session = environment.session!(id, options) + return { + ...session, + async *events(eventOptions) { + // The retained child cannot make progress until its manager is running again. + // A terminal-await recovery preflight blocks the only code that can open this gate. + if (recovering) { + const signal = eventOptions?.signal + let onAbort: (() => void) | undefined + try { + await Promise.race([ + Promise.all([parentOpened, rootInspected]), + new Promise((_resolve, reject) => { + onAbort = () => + reject(signal?.reason ?? new Error('child observation aborted')) + if (signal?.aborted) onAbort() + else signal?.addEventListener('abort', onAbort, { once: true }) + }), + ]) + } finally { + if (onAbort) signal?.removeEventListener('abort', onAbort) + } } + yield* session.events(eventOptions) + }, + result: async () => ({ + ...(await session.result()), + usage: { inputTokens: 3, outputTokens: 2 }, + }), + } + }, + }) + return { + ...base, + create: async (input) => { + creates++ + return wrap(await base.create(input)) + }, + get: async (id) => { + const environment = await base.get!(id) + return environment ? wrap(environment) : null + }, + } + } + const profile = testAgentProfile('retained-child') + const worker = (): Agent => + Object.assign( + { name: 'retained-child', act: async () => 'unused' }, + { + executorSpec: { + profile, + harness: null, + executorFactory: providerAsExecutor(provider(), { destroyOnSettle: false }), + }, + }, + ) + const managerProfile = testAgentProfile('manager') + const manager = (journal: SpawnJournal): Agent => + driverChild( + managerProfile, + { + name: 'manager', + async act(_task, scope) { + if (recovering) { + parentActed = true + expect(scope.view.inFlight).toBe(1) + expect(scope.budget.reservedTokens).toBe(10) + expect(scope.budget.tokensLeft).toBe(20 - managerTokens) + releaseChild() + } else { + await meterRuntimeOwnedAccounting(scope, { + iterations: 0, + tokens: { input: managerTokens, output: 0 }, + resources: { compute: { unit: 'GPU-second', amount: 7, known } }, + usd: 0, + ms: 0, + }) + const child = scope.spawn(worker(), 'child task', { + key: 'child', + budget: { maxIterations: 1, maxTokens: 10 }, + }) + expect(child.ok).toBe(true) + } + const settled = await scope.next() + if (!recovering) + throw new RetainedExecutionPendingError(new Error('manager observation interrupted')) + expect(settled?.handle.id).toBe('root:s0:s0') + return { + finalizedBy: 'manager', + child: settled?.kind === 'done' ? settled.out : undefined, } - yield* session.events(eventOptions) }, - result: async () => ({ - ...(await session.result()), - usage: { inputTokens: 3, outputTokens: 2 }, - }), - } - }, - }) - return { - ...base, - create: async (input) => { - creates++ - return wrap(await base.create(input)) - }, - get: async (id) => { - const environment = await base.get!(id) - return environment ? wrap(environment) : null + }, + journal, + undefined, + undefined, + providerAsExecutor(provider(), { destroyOnSettle: false }), + ) + const common = { + runId: 'root', + budget: { maxIterations: 3, maxTokens: 30 }, + maxLiveWorkers: 2, + rootIdentity: { + profileDigest: canonicalCandidateDigest({ name: 'root' }), + taskDigest: canonicalCandidateDigest('task'), }, } - } - const profile = testAgentProfile('retained-child') - const worker = (): Agent => - Object.assign( - { name: 'retained-child', act: async () => 'unused' }, - { - executorSpec: { - profile, - harness: null, - executorFactory: providerAsExecutor(provider(), { destroyOnSettle: false }), - }, + const journal: SpawnJournal = { + beginTree: context.journal.beginTree.bind(context.journal), + loadTree: context.journal.loadTree.bind(context.journal), + appendEvent: async (root, event) => { + await context.journal.appendEvent(root, event) + if ( + !interrupted && + event.kind === 'execution-admitted' && + event.admission.phase === 'dispatched' + ) { + interrupted = true + throw new Error('lost local dispatch acknowledgement') + } }, - ) - const managerProfile = testAgentProfile('manager') - const manager = (journal: SpawnJournal): Agent => - driverChild( - managerProfile, - { - name: 'manager', - async act(_task, scope) { - if (recovering) { - parentActed = true - expect(scope.view.inFlight).toBe(1) - expect(scope.budget.reservedTokens).toBe(10) - expect(scope.budget.tokensLeft).toBe(15) - releaseChild() - } else { - await meterRuntimeOwnedAccounting(scope, { - iterations: 0, - tokens: { input: 5, output: 0 }, - usd: 0, - ms: 0, - }) - const child = scope.spawn(worker(), 'child task', { - key: 'child', - budget: { maxIterations: 1, maxTokens: 10 }, + } + const abort = new AbortController() + let cutoff: ReturnType | undefined + try { + await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + const child = scope.spawn(manager(journal), 'manager task', { + key: 'manager', + budget: { maxIterations: 3, maxTokens: 30 }, }) expect(child.ok).toBe(true) - } - const settled = await scope.next() - if (!recovering) - throw new RetainedExecutionPendingError(new Error('manager observation interrupted')) - expect(settled?.handle.id).toBe('root:s0:s0') - return { - finalizedBy: 'manager', - child: settled?.kind === 'done' ? settled.out : undefined, - } + await scope.next() + }, }, - }, - journal, - undefined, - undefined, - providerAsExecutor(provider(), { destroyOnSettle: false }), - ) - const common = { - runId: 'root', - budget: { maxIterations: 3, maxTokens: 30 }, - maxLiveWorkers: 2, - rootIdentity: { - profileDigest: canonicalCandidateDigest({ name: 'root' }), - taskDigest: canonicalCandidateDigest('task'), - }, - } - const journal: SpawnJournal = { - beginTree: context.journal.beginTree.bind(context.journal), - loadTree: context.journal.loadTree.bind(context.journal), - appendEvent: async (root, event) => { - await context.journal.appendEvent(root, event) - if ( - !interrupted && - event.kind === 'execution-admitted' && - event.admission.phase === 'dispatched' - ) { - interrupted = true - throw new Error('lost local dispatch acknowledgement') - } - }, - } - const abort = new AbortController() - let cutoff: ReturnType | undefined - try { - await createSupervisor().run( - { - name: 'root', - async act(_task, scope) { - const child = scope.spawn(manager(journal), 'manager task', { - key: 'manager', - budget: { maxIterations: 3, maxTokens: 30 }, - }) - expect(child.ok).toBe(true) - await scope.next() + 'task', + { ...context, ...common, journal }, + ) + expect(interrupted).toBe(true) + expect(creates).toBe(1) + const interruptedParent = (await context.journal.loadTree('root')) ?? [] + expect( + interruptedParent + .filter((event) => event.kind === 'metered') + .reduce( + (sum, event) => + sum + + (event.kind === 'metered' ? event.spend.tokens.input + event.spend.tokens.output : 0), + 0, + ), + ).toBe(managerTokens) + recovering = true + cutoff = setTimeout( + () => abort.abort(new Error('recovery blocked parent coordination')), + 2_000, + ) + const restarted = createFileRunContext(runDirectory, { withDriver: true }) + const recoveryFactory = registerRetainedExecutorPreparation( + providerAsExecutor(provider(), { destroyOnSettle: false }), + ({ spawned }) => + spawned.ownedTreeRoot === undefined + ? undefined + : { + spec: ( + manager(restarted.journal) as Agent & { + executorSpec: import('../../src/runtime/supervise/types').AgentSpec + } + ).executorSpec, + factory: driverExecutorFactory, + }, + ) + const result = await createSupervisor().run( + { + name: 'root', + async act(_task, scope) { + expect(scope.view.inFlight).toBe(1) + expect(scope.budget.reservedTokens).toBe(30) + expect(scope.budget.tokensLeft).toBe(0) + const fresh = scope.spawn( + () => { + freshFactories++ + return worker() + }, + 'fresh task', + { + key: 'fresh', + budget: { maxIterations: 1, maxTokens: 1 }, + }, + ) + expect(fresh).toEqual({ ok: false, reason: 'max-live-workers' }) + expect(freshFactories).toBe(0) + releaseRoot() + const settled = await scope.next() + expect(settled?.handle.id).toBe('root:s0') + return settled?.kind === 'done' ? settled.out : undefined + }, }, - }, - 'task', - { ...context, ...common, journal }, - ) - expect(interrupted).toBe(true) - expect(creates).toBe(1) - const interruptedParent = (await context.journal.loadTree('root')) ?? [] - expect( - interruptedParent - .filter((event) => event.kind === 'metered') - .reduce( - (sum, event) => - sum + - (event.kind === 'metered' ? event.spend.tokens.input + event.spend.tokens.output : 0), - 0, - ), - ).toBe(5) - recovering = true - cutoff = setTimeout(() => abort.abort(new Error('recovery blocked parent coordination')), 500) - const restarted = createFileRunContext(runDirectory, { withDriver: true }) - const recoveryFactory = registerRetainedExecutorPreparation( - providerAsExecutor(provider(), { destroyOnSettle: false }), - ({ spawned }) => - spawned.ownedTreeRoot === undefined - ? undefined - : { - spec: ( - manager(restarted.journal) as Agent & { - executorSpec: import('../../src/runtime/supervise/types').AgentSpec - } - ).executorSpec, - factory: driverExecutorFactory, - }, - ) - const result = await createSupervisor().run( - { - name: 'root', - async act(_task, scope) { - expect(scope.view.inFlight).toBe(1) - expect(scope.budget.reservedTokens).toBe(30) - expect(scope.budget.tokensLeft).toBe(0) - const fresh = scope.spawn( - () => { - freshFactories++ - return worker() - }, - 'fresh task', - { - key: 'fresh', - budget: { maxIterations: 1, maxTokens: 1 }, - }, - ) - expect(fresh).toEqual({ ok: false, reason: 'max-live-workers' }) - expect(freshFactories).toBe(0) - releaseRoot() - const settled = await scope.next() - expect(settled?.handle.id).toBe('root:s0') - return settled?.kind === 'done' ? settled.out : undefined + 'task', + { + ...restarted, + ...common, + resume: true, + signal: abort.signal, + recoverExecutor: recoveryFactory, }, - }, - 'task', - { - ...restarted, - ...common, - resume: true, - signal: abort.signal, - recoverExecutor: recoveryFactory, - }, - ) - expect(parentActed).toBe(true) - expect(result.kind).toBe('winner') - expect(result.spentTotal.tokens.input + result.spentTotal.tokens.output).toBe(10) - if (result.kind === 'winner') expect(result.out).toMatchObject({ finalizedBy: 'manager' }) - expect(creates).toBe(1) - const events = (await context.journal.loadTree('root/root:s0')) ?? [] - expect( - events.filter((event) => event.kind === 'spawned' && event.id === 'root:s0:s0'), - ).toHaveLength(1) - expect( - events.filter((event) => event.kind === 'execution-input' && event.id === 'root:s0:s0'), - ).toHaveLength(1) - expect( - events.filter((event) => event.kind === 'settled' && event.id === 'root:s0:s0'), - ).toHaveLength(1) - } finally { - if (cutoff) clearTimeout(cutoff) - releaseChild() - releaseRoot() - abort.abort() - await rm(directory, { recursive: true, force: true }) - } -}) + ) + expect(parentActed).toBe(true) + expect(result.kind, result.kind === 'no-winner' ? result.reason : undefined).toBe('winner') + expect(result.spentTotal.tokens.input + result.spentTotal.tokens.output).toBe( + 5 + managerTokens, + ) + expect(result.spentTotal.resources).toEqual({ + compute: { unit: 'GPU-second', amount: 7, known }, + }) + const parentResourceCharges = ((await context.journal.loadTree('root')) ?? []).flatMap( + (event) => + event.kind === 'metered' && event.spend.resources?.compute + ? [event.spend.resources.compute] + : [], + ) + expect(parentResourceCharges.reduce((amount, resource) => amount + resource.amount, 0)).toBe( + 7, + ) + expect(parentResourceCharges.every((resource) => resource.known)).toBe(known) + if (result.kind === 'winner') expect(result.out).toMatchObject({ finalizedBy: 'manager' }) + expect(creates).toBe(1) + const events = (await context.journal.loadTree('root/root:s0')) ?? [] + expect( + events.filter((event) => event.kind === 'spawned' && event.id === 'root:s0:s0'), + ).toHaveLength(1) + expect( + events.filter((event) => event.kind === 'execution-input' && event.id === 'root:s0:s0'), + ).toHaveLength(1) + expect( + events.filter((event) => event.kind === 'settled' && event.id === 'root:s0:s0'), + ).toHaveLength(1) + } finally { + if (cutoff) clearTimeout(cutoff) + releaseChild() + releaseRoot() + abort.abort() + await rm(directory, { recursive: true, force: true }) + } + }, +) it('keeps nested cleanup uncertainty after the driver scope closes', async () => { const context = createInMemoryRunContext({ withDriver: true })