diff --git a/src/cli.ts b/src/cli.ts index f35cb376da..782f1b8782 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -5292,7 +5292,7 @@ async function cmdSuspend(): Promise { async function postSessionCliIpc( ipcPort: number, sessionId: string, - route: 'slash' | 'cd' | 'close' | 'preview' | 'chat-rename' | 'project', + route: 'slash' | 'cd' | 'close' | 'preview' | 'chat-rename' | 'project' | 'continuation', payload: Record, ): Promise { const requestBody: Record = { ...payload }; @@ -5324,6 +5324,68 @@ async function postSessionCliIpc( : loopbackFetch(`http://127.0.0.1:${ipcPort}${path}`, init); } +async function cmdContinuation(argv: string[]): Promise { + const action = argv[0] ?? ''; + if (!['start', 'await-user', 'cancel'].includes(action)) { + console.error('用法: botmux continuation start --readonly [--ttl-minutes N] [--max-continuations N] | await-user | cancel'); + process.exitCode = 2; + return; + } + const ctx = findAncestorSessionContext(); + if (!ctx?.sessionId || !ctx.turnId) { + console.error('✗ continuation 只能由当前 BotMux 会话的活动轮次调用'); + process.exitCode = 1; + return; + } + if (action === 'start' && !argv.includes('--readonly')) { + console.error('✗ 第一阶段只支持显式 --readonly 的只读长程任务'); + process.exitCode = 2; + return; + } + const ttlRaw = argValue(argv, '--ttl-minutes'); + const maxRaw = argValue(argv, '--max-continuations'); + const ttlMinutes = ttlRaw === undefined ? undefined : Number(ttlRaw); + const maxContinuations = maxRaw === undefined ? undefined : Number(maxRaw); + if (ttlMinutes !== undefined && (!Number.isFinite(ttlMinutes) || ttlMinutes <= 0)) { + console.error('✗ --ttl-minutes 必须是正数'); + process.exitCode = 2; + return; + } + if (maxContinuations !== undefined + && (!Number.isSafeInteger(maxContinuations) || maxContinuations <= 0)) { + console.error('✗ --max-continuations 必须是正整数'); + process.exitCode = 2; + return; + } + let discoveredPort: number | undefined; + try { discoveredPort = findDaemon(process.env.BOTMUX_LARK_APP_ID)?.ipcPort; } catch { /* isolated */ } + const ipcPort = resolveDaemonIpcPort(discoveredPort, process.env.BOTMUX_DAEMON_IPC_PORT); + if (!ipcPort) { + console.error('✗ 无法定位当前会话的 daemon'); + process.exitCode = 1; + return; + } + const response = await postSessionCliIpc(ipcPort, ctx.sessionId, 'continuation', { + action, + originTurnId: ctx.turnId, + ...(ctx.dispatchAttempt !== undefined ? { originDispatchAttempt: ctx.dispatchAttempt } : {}), + ...(action === 'start' ? { readonly: true } : {}), + ...(ttlMinutes !== undefined ? { ttlMs: Math.round(ttlMinutes * 60_000) } : {}), + ...(maxContinuations !== undefined ? { maxContinuations } : {}), + }); + const body = await response.json().catch(() => ({})) as { + ok?: boolean; + error?: string; + state?: { leaseId?: string; status?: string; expiresAt?: number; maxContinuations?: number }; + }; + if (!response.ok || !body.ok) { + console.error(`✗ continuation 被拒绝: ${body.error ?? `HTTP ${response.status}`}`); + process.exitCode = 1; + return; + } + console.log(JSON.stringify({ ok: true, ...body.state })); +} + /** `botmux preview ` registers a reachable loopback Web service for the * exact current session. There is intentionally no `--session` escape hatch: * ancestry/env resolution plus the rotating session capability prove which @@ -6306,6 +6368,9 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 同源 /preview// 访问，不暴露本机地址或任何 token。 端口必须由本会话的进程持有（在会话内直接启动，别 setsid/nohup 脱离进程树）；换代/关闭后需重新注册，远端 sandbox 后端不支持 + continuation start --readonly + （实验性）为当前 TraeX 普通会话显式开启一次只读长程任务续跑； + 可加 --ttl-minutes N / --max-continuations N，另有 await-user / cancel autostart enable 注册开机自启（macOS launchd / Linux user systemd / Windows Task Scheduler，无需 sudo） autostart disable 注销开机自启 autostart status 查看自启状态 @@ -9219,6 +9284,7 @@ async function cmdSend(rest: string[]): Promise { const marker: Record = { sentAtMs, messageId, + responseKind: effectiveResponseKind, ...(originTurnId ? { turnId: originTurnId } : {}), ...(originDispatchAttempt !== undefined ? { dispatchAttempt: originDispatchAttempt } : {}), }; @@ -9301,6 +9367,7 @@ async function cmdSend(rest: string[]): Promise { const marker: Record = { sentAtMs: Date.now(), messageId: `doc:${exactDocTarget.commentId}`, + responseKind: effectiveResponseKind, ...(originTurnId ? { turnId: originTurnId } : {}), ...(originDispatchAttempt !== undefined ? { dispatchAttempt: originDispatchAttempt } : {}), contentLength: content.length, @@ -9770,6 +9837,7 @@ async function cmdSend(rest: string[]): Promise { const marker: Record = { sentAtMs, messageId, + responseKind: effectiveResponseKind, ...(originTurnId ? { turnId: originTurnId } : {}), ...(originDispatchAttempt !== undefined ? { dispatchAttempt: originDispatchAttempt } : {}), }; @@ -13023,8 +13091,21 @@ async function cmdNativeSubagentRuntimeHook(): Promise { }); return; } - const data = JSON.parse(raw) as { ok?: unknown; invalidPolicy?: unknown; policy?: unknown }; + const data = JSON.parse(raw) as { ok?: unknown; invalidPolicy?: unknown; deny?: unknown; reason?: unknown; policy?: unknown }; if (data.ok !== true) return; + if (data.deny === true) { + nativeSubagentDiagnostic('daemon denied spawn for read-only continuation'); + await writeNativeSubagentHookDirective({ + hookSpecificOutput: { + hookEventName: 'PreToolUse', + permissionDecision: 'deny', + permissionDecisionReason: typeof data.reason === 'string' + ? data.reason + : 'Read-only continuation forbids subagents', + }, + }); + return; + } if (data.invalidPolicy === true) { nativeSubagentDiagnostic('daemon rejected invalid stored policy; allowing spawn'); return; @@ -14827,6 +14908,7 @@ switch (command) { } case 'term-link': await cmdTermLink(process.argv.slice(3)); break; case 'preview': await cmdPreview(process.argv.slice(3)); break; + case 'continuation': await cmdContinuation(process.argv.slice(3)); break; case 'schedule': await cmdSchedule(process.argv[3] ?? '', process.argv.slice(4)); break; case 'ask': { // `botmux ask buttons --options ...` → sub='buttons', rest=['--options', ...] diff --git a/src/cli/daemon-lifecycle-env.ts b/src/cli/daemon-lifecycle-env.ts index b7ce329a08..7906e852ea 100644 --- a/src/cli/daemon-lifecycle-env.ts +++ b/src/cli/daemon-lifecycle-env.ts @@ -59,6 +59,10 @@ export const DAEMON_ENV_KEYS = [ // keeps them on the deterministic resolveDaemonEnv snapshot semantics. 'BOTMUX_DASHBOARD_CONTROL_AUDIT_PATH', 'BOTMUX_DASHBOARD_TERMINAL_CONTROL_TTL_MS', + // Machine-wide emergency brake for the opt-in read-only task continuation + // lease. Missing/empty means OFF; the daemon re-checks it before every + // continuation, so a restart with false cancels restored backoff leases. + 'BOTMUX_READONLY_CONTINUATION_ENABLED', // Merlin Devbox auto-export switch (platform/devbox-dashboard-export.ts). // The dashboard resolves it (dashboard-url / control-csrf run there), so it // has to survive the allowlist copy — same reason BOTMUX_PUBLIC_URL is here. diff --git a/src/codex-rpc-engine.ts b/src/codex-rpc-engine.ts index aaa20cbe81..6aae9f9c38 100644 --- a/src/codex-rpc-engine.ts +++ b/src/codex-rpc-engine.ts @@ -28,10 +28,27 @@ import { existsSync, readFileSync, writeFileSync, rmSync, mkdirSync } from 'node import { join } from 'node:path'; import { homedir } from 'node:os'; import { WebSocket } from 'ws'; +import { + CODEX_OUTPUT_LIMIT_ERROR_CODE, + isExactCodexOutputLimitError, +} from './services/codex-transcript.js'; type Json = Record; type LogFn = (msg: string) => void; +function rpcTurnErrorCode(error: unknown, fallback = 'rpc_turn_failed'): string { + if (isExactCodexOutputLimitError(error)) return CODEX_OUTPUT_LIMIT_ERROR_CODE; + if (error && typeof error === 'object') { + const record = error as Record; + const code = String(record.code ?? '').trim(); + if (code) return code; + const message = String(record.message ?? '').trim(); + if (message) return message; + } + const text = typeof error === 'string' ? error.trim() : ''; + return text || fallback; +} + async function findFreePort(): Promise { return new Promise((resolve, reject) => { const srv = createServer(); @@ -74,6 +91,10 @@ export interface CodexRpcEngineOpts { appServerFeatures?: string[]; /** Generic process-scoped app-server config overrides. */ appServerConfig?: string[]; + /** Enable the daemon-owned read-only continuation protocol for this engine. + * Restrictions remain turn-scoped so ordinary turns in the same process keep + * their configured capabilities. */ + readonlyContinuationHardened?: boolean; /** Bridge a native request_user_input server request to the host UI. */ onRequestUserInput?: (params: unknown) => Promise; /** Override the per-request JSON-RPC timeout (default REQUEST_TIMEOUT_MS). @@ -103,6 +124,13 @@ const DEFAULT_DEPENDENCIES: CodexRpcEngineDependencies = { export interface CodexRpcTurnIdentity { turnId: string; dispatchAttempt?: number; + /** Daemon-only marker for the narrowly sandboxed continuation path. */ + readonlyContinuation?: true; +} + +export interface ReadonlyContinuationCapabilityCheck { + ok: boolean; + reason?: string; } export type CodexRpcTurnTerminalStatus = @@ -163,6 +191,10 @@ export class CodexRpcEngine { turnIdentity?: CodexRpcTurnIdentity; }>(); private readonly turnOwners = new Map(); + private readonly readonlyNativeTurns = new Set(); + private readonly pendingReadonlyTurnOwners = new Set(); + private readonly deferredPreResponseServerRequests = new Map(); + private readonly interruptingReadonlyTurns = new Set(); private readonly nativeTurnByOwner = new Map(); private readonly terminalNativeTurns = new Set(); private readonly deferredUnownedTerminals = new Map ['--enable', feature]); - const configArgs = (this.opts.appServerConfig ?? []).flatMap(value => ['-c', value]); + const configArgs = [...(this.opts.appServerConfig ?? [])] + .flatMap(value => ['-c', value]); this.child = this.dependencies.spawnProcess(this.opts.cliBin, ['app-server', ...featureArgs, ...configArgs, '--listen', `ws://127.0.0.1:${this.port}`], { cwd: this.opts.cwd, env: this.opts.env, @@ -382,24 +418,128 @@ export class CodexRpcEngine { opts?: { timeoutMs?: number; fatalOnTimeout?: boolean }, ): Promise<{ nativeTurnId: string }> { if (!this.threadId) throw new Error('sendTurn before startThread/resumeThread'); + const readonlyContinuation = identity.readonlyContinuation === true; const params: Json = { threadId: this.threadId, input: [{ type: 'text', text: content, text_elements: [] }], cwd: this.opts.cwd, approvalPolicy: 'never', - sandboxPolicy: { type: 'dangerFullAccess' }, + sandboxPolicy: readonlyContinuation + ? { type: 'readOnly', networkAccess: false } + : { type: 'dangerFullAccess' }, + ...(readonlyContinuation + ? { + environments: [], + runtimeWorkspaceRoots: [], + // TraeX capability selections are thread-sticky, not turn-local. + // Do not write `capabilities` here: disabling Skills/MCP for this + // synthetic turn would silently mutate later ordinary turns, and + // the protocol cannot read back an exact prior Skill selection. + // Eligibility proves MCP empty and Skills free of external tool + // dependencies; the turn-local sandbox and daemon hook enforce the + // remaining side-effect and native-subagent boundaries. + } + : {}), }; params.clientUserMessageId = identity.turnId; + const ownerKey = this.ownerKey(identity); + if (readonlyContinuation) this.pendingReadonlyTurnOwners.add(ownerKey); try { await this.request('turn/start', params, opts, undefined, identity); } catch (err) { + if (readonlyContinuation) this.failClosedDeferredPreResponseServerRequests(); throw err; + } finally { + this.pendingReadonlyTurnOwners.delete(ownerKey); } const nativeTurnId = this.takeNativeTurnId(identity); if (!nativeTurnId) throw new Error('turn/start ack did not bind a native turn id'); return { nativeTurnId }; } + /** Prove that this exact app-server generation has no configured MCP server + * and no enabled Skill that declares external tool dependencies. Status/list + * is deliberately used only as a conservative inventory check: it cannot + * recover the thread's effective allowlist, and a currently disconnected or + * empty server can expose tools later, so any server record makes the + * continuation ineligible. */ + async checkReadonlyContinuationCapabilities(): Promise { + if (!this.opts.readonlyContinuationHardened) { + return { ok: false, reason: 'readonly_continuation_runtime_not_hardened' }; + } + if (!this.threadId) return { ok: false, reason: 'readonly_continuation_thread_unavailable' }; + try { + let cursor: string | undefined; + do { + const result = await this.request('mcpServerStatus/list', { + threadId: this.threadId, + detail: 'full', + limit: 100, + ...(cursor ? { cursor } : {}), + }, { timeoutMs: 10_000, fatalOnTimeout: false }); + if (!result || !Array.isArray(result.data)) { + throw new Error('mcpServerStatus/list returned malformed data'); + } + if (result.nextCursor !== undefined && result.nextCursor !== null + && typeof result.nextCursor !== 'string') { + throw new Error('mcpServerStatus/list returned malformed cursor'); + } + const servers = result.data; + if (servers.length > 0) { + return { ok: false, reason: 'readonly_continuation_external_mcp_capability' }; + } + cursor = typeof result?.nextCursor === 'string' && result.nextCursor + ? result.nextCursor + : undefined; + } while (cursor); + + const skillResult = await this.request('skills/list', { + cwds: [this.opts.cwd], + forceReload: false, + }, { timeoutMs: 10_000, fatalOnTimeout: false }); + if (!skillResult || !Array.isArray(skillResult.data)) { + throw new Error('skills/list returned malformed data'); + } + const entries = skillResult.data; + for (const entry of entries) { + if (!entry || typeof entry !== 'object' + || typeof entry.cwd !== 'string' + || !Array.isArray(entry.errors) + || !Array.isArray(entry.skills)) { + throw new Error('skills/list returned malformed entry'); + } + if (entry.errors.length > 0) { + throw new Error('skills/list reported discovery errors'); + } + const skills = entry.skills; + for (const skill of skills) { + if (!skill || typeof skill !== 'object' || typeof skill.enabled !== 'boolean') { + throw new Error('skills/list returned malformed skill'); + } + if (skill.dependencies !== undefined && skill.dependencies !== null + && (!Array.isArray(skill.dependencies.tools))) { + throw new Error('skills/list returned malformed dependencies'); + } + const dependencies = skill.dependencies?.tools ?? []; + if (dependencies.some((dependency: unknown) => !dependency || typeof dependency !== 'object')) { + throw new Error('skills/list returned malformed tool dependency'); + } + if (skill.enabled === true && dependencies.length > 0) { + return { ok: false, reason: 'readonly_continuation_skill_tool_dependency' }; + } + } + } + return { ok: true }; + } catch (error) { + this.log( + `[codex-rpc] read-only continuation capability proof failed: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + return { ok: false, reason: 'readonly_continuation_capability_probe_failed' }; + } + } + /** 首条用户消息落盘后设置线程名；失败不得拖垮仍在执行的模型 turn。 */ async setThreadName(name: string): Promise { if (!this.threadId) throw new Error('setThreadName before startThread/resumeThread'); @@ -531,6 +671,7 @@ export class CodexRpcEngine { if (this.closed) return; this.closed = true; this.emitAllTurnTerminals('stopped', 'rpc_engine_stopped'); + this.clearReadonlyOwnership(); this.deferredUnownedTerminals.clear(); try { this.ws?.close(); } catch { /* already gone */ } const pid = this.child?.pid; @@ -737,6 +878,101 @@ export class CodexRpcEngine { try { this.send({ jsonrpc: '2.0', id, error: { code: -32000, message } }); } catch { /* connection gone */ } } + private serverRequestNativeTurnId(params: unknown): string | undefined { + if (!params || typeof params !== 'object') return undefined; + const p = params as Record; + const value = p.turnId ?? p.turn?.id; + return typeof value === 'string' && value ? value : undefined; + } + + private clearReadonlyOwnership(): void { + this.readonlyNativeTurns.clear(); + this.pendingReadonlyTurnOwners.clear(); + this.deferredPreResponseServerRequests.clear(); + this.interruptingReadonlyTurns.clear(); + } + + private interruptReadonlyTurn(params: unknown, reason: string): void { + const nativeTurnId = this.serverRequestNativeTurnId(params); + const p = params && typeof params === 'object' ? params as Record : {}; + const threadId = typeof p.threadId === 'string' ? p.threadId : this.threadId; + if (!threadId || !nativeTurnId) { + this.failAll(new Error(`read-only continuation request lacked exact turn coordinates: ${reason}`)); + return; + } + if (this.interruptingReadonlyTurns.has(nativeTurnId)) return; + this.interruptingReadonlyTurns.add(nativeTurnId); + this.request( + 'turn/interrupt', + { threadId, turnId: nativeTurnId }, + { timeoutMs: 10_000, fatalOnTimeout: false }, + ).then( + () => this.log(`[codex-rpc] interrupted read-only continuation ${nativeTurnId}: ${reason}`), + err => this.failAll(new Error( + `read-only continuation interrupt failed: ${err instanceof Error ? err.message : String(err)}`, + )), + ).finally(() => this.interruptingReadonlyTurns.delete(nativeTurnId)); + } + + private handleReadonlyServerRequest(msg: Json): void { + const method = String(msg.method ?? ''); + if (method === 'item/tool/requestUserInput') { + this.interruptReadonlyTurn(msg.params, method); + return; + } + if (method === 'item/commandExecution/requestApproval' + || method === 'item/fileChange/requestApproval') { + this.respond(msg.id, { decision: 'cancel' }); + } else if (method === 'execCommandApproval' || method === 'applyPatchApproval') { + this.respond(msg.id, { decision: 'abort' }); + } else if (method === 'mcpServer/elicitation/request') { + this.respond(msg.id, { action: 'cancel', content: null, _meta: null }); + } else if (method === 'item/tool/call') { + this.respond(msg.id, { contentItems: [], success: false }); + } else { + // Permission elevation and future server requests are not part of the + // reviewed read-only surface. Reject unknown protocol growth closed. + this.respondError(msg.id, `server request denied in read-only continuation: ${method}`); + } + this.interruptReadonlyTurn(msg.params, method); + } + + private releaseDeferredPreResponseServerRequests(nativeTurnId: string): void { + const deferred = this.deferredPreResponseServerRequests.get(nativeTurnId); + if (!deferred) return; + this.deferredPreResponseServerRequests.delete(nativeTurnId); + for (const msg of deferred) { + if (this.readonlyNativeTurns.has(nativeTurnId)) this.handleReadonlyServerRequest(msg); + else this.handleOrdinaryServerRequest(msg); + } + } + + private failClosedDeferredPreResponseServerRequests(): void { + for (const requests of this.deferredPreResponseServerRequests.values()) { + for (const msg of requests) { + this.respondError(msg.id, 'read-only continuation ownership could not be proven'); + this.interruptReadonlyTurn(msg.params, 'turn/start ownership could not be proven'); + } + } + this.deferredPreResponseServerRequests.clear(); + } + + private handleOrdinaryServerRequest(msg: Json): void { + if (msg.method === 'item/tool/requestUserInput' && this.opts.onRequestUserInput) { + const requestParams = msg.params; + void this.opts.onRequestUserInput(requestParams).then( + result => this.respond(msg.id, result), + err => { + const message = err instanceof Error ? err.message : String(err); + this.log(`[codex-rpc] requestUserInput bridge failed: ${message}; interrupting turn`); + this.interruptTurnFor(msg.id, requestParams, message); + }, + ); + return; + } + this.respond(msg.id, autoApproval(String(msg.method ?? ''))); + } + private send(msg: Json): void { if (!this.ws || this.ws.readyState !== WebSocket.OPEN) throw new Error('app-server ws not open'); this.ws.send(JSON.stringify(msg)); @@ -767,30 +1003,24 @@ export class CodexRpcEngine { } return; } - // Native user-input requests are the one server→client request that must - // wait for a human. In botmux this callback posts a Lark card and returns - // the protocol-shaped answers object. Keep all approval requests automatic. if (typeof msg.id === 'number' && typeof msg.method === 'string') { - if (msg.method === 'item/tool/requestUserInput' && this.opts.onRequestUserInput) { - const requestParams = msg.params; - void this.opts.onRequestUserInput(requestParams).then( - result => this.respond(msg.id, result), - err => { - // Fail VISIBLY, never silently. Verified against real traex 0.200.19: - // ANY response to this request — empty answers OR a JSON-RPC error — - // is normalized by the app-server into `{answers:{}}` and the turn - // still COMPLETES, so unsupported/broker-failed asks would be - // silently skipped. Only `turn/interrupt` (threadId+turnId, both - // carried in this request's params) actually stops the turn - // (status → 'interrupted'). So fail by interrupting the turn. - const message = err instanceof Error ? err.message : String(err); - this.log(`[codex-rpc] requestUserInput bridge failed: ${message}; interrupting turn`); - this.interruptTurnFor(msg.id, requestParams, message); - }, - ); + const nativeTurnId = this.serverRequestNativeTurnId(msg.params); + if (nativeTurnId && this.readonlyNativeTurns.has(nativeTurnId)) { + this.handleReadonlyServerRequest(msg); + return; + } + if (this.pendingReadonlyTurnOwners.size > 0) { + if (!nativeTurnId) { + this.respondError(msg.id, 'read-only continuation request lacked exact turn id'); + this.failAll(new Error('read-only continuation server request lacked exact turn id')); + return; + } + const deferred = this.deferredPreResponseServerRequests.get(nativeTurnId) ?? []; + deferred.push(msg); + this.deferredPreResponseServerRequests.set(nativeTurnId, deferred); return; } - this.respond(msg.id, autoApproval(msg.method)); + this.handleOrdinaryServerRequest(msg); return; } if (typeof msg.method === 'string') { @@ -807,7 +1037,7 @@ export class CodexRpcEngine { if (msg.method === 'turn/completed' && nativeTurnId) { const turn = params.turn ?? {}; const rawStatus = String(turn.status ?? '').toLowerCase(); - const errorCode = String(turn.error?.code ?? turn.error?.message ?? ''); + const errorCode = turn.error ? rpcTurnErrorCode(turn.error) : ''; const failed = !!turn.error || ['failed', 'error'].includes(rawStatus); const aborted = ['aborted', 'cancelled', 'canceled', 'interrupted'].includes(rawStatus); this.emitTurnTerminal( @@ -825,7 +1055,7 @@ export class CodexRpcEngine { this.emitTurnTerminal( nativeTurnId, 'failed', - String(params.error?.code ?? params.error?.message ?? 'rpc_turn_failed'), + rpcTurnErrorCode(params.error), ); return; } @@ -840,6 +1070,7 @@ export class CodexRpcEngine { if (!this.closed && !this.deadNotified) { this.deadNotified = true; this.emitAllTurnTerminals('engine-dead', 'rpc_engine_dead'); + this.clearReadonlyOwnership(); try { this.opts.onDead?.(); } catch { /* best effort */ } } } diff --git a/src/core/dashboard-ipc-server.ts b/src/core/dashboard-ipc-server.ts index 0782e54652..04783aed72 100644 --- a/src/core/dashboard-ipc-server.ts +++ b/src/core/dashboard-ipc-server.ts @@ -135,7 +135,12 @@ import { updateTaskWithOptionalPrecondition, type SchedulePreconditionMutation, } from './schedule-precondition-config.js'; -import { listActiveSessions, findActiveBySessionId, closeSession, getActiveSessionsRegistry, transferSession, deliverWriteLinkCardToOwners, forkWorker, suspendWorker, killWorker, latestPerBotEnvForRestart, latestModelForRespawn, getDaemonReplyCardUsageSnapshot, sessionSupportsWebTerminal, sendWorkerSessionInput, isSessionTransferring, mojoCloseResidualForRow, getDaemonBootId, CARD_POSTING_SENTINEL } from './worker-pool.js'; +import { listActiveSessions, findActiveBySessionId, closeSession, getActiveSessionsRegistry, transferSession, deliverWriteLinkCardToOwners, forkWorker, suspendWorker, killWorker, latestPerBotEnvForRestart, latestModelForRespawn, getDaemonReplyCardUsageSnapshot, sessionSupportsWebTerminal, sendWorkerSessionInput, isSessionTransferring, mojoCloseResidualForRow, getDaemonBootId, CARD_POSTING_SENTINEL, ensureReadonlyTaskContinuationAttached } from './worker-pool.js'; +import { + awaitReadonlyTaskContinuationUser, + cancelReadonlyTaskContinuationExplicit, + startReadonlyTaskContinuation, +} from '../services/readonly-task-continuation.js'; import { listOnlineDaemons } from '../utils/daemon-discovery.js'; import { isSessionStopped } from './session-liveness.js'; import { isRemoteBackendType, isRemoteCliId, isSuspendableBackendType } from './persistent-backend.js'; @@ -786,7 +791,7 @@ function routeHasNarrowUntrustedAuth(method: string, pathname: string): boolean // 该会话的 rotating per-turn // capability 并绑定到 URL 里的 sessionId（同 /api/asks 姿势）——capability 只 // 证明「我是这个会话当前这一轮的 CLI」，选不了别的会话。 - if (method === 'POST' && /^\/api\/sessions\/[^/]+\/(?:slash|cd|close|preview|chat-rename|project|project-dispatch-policy)$/.test(pathname)) return true; + if (method === 'POST' && /^\/api\/sessions\/[^/]+\/(?:slash|cd|close|preview|chat-rename|project|project-dispatch-policy|continuation)$/.test(pathname)) return true; // UserPromptSubmit hook 的 envelope claim：沙箱内 hook 读不到 host secret， // 走 body 里的 per-turn capability；handler 内 sessionCliIpcAuth 绑定到 URL 的 // sessionId + 按 managedTurnOrigin.turnId 权威取（同 /close 姿势）。 @@ -1524,6 +1529,17 @@ ipcRoute('POST', '/api/sessions/:sessionId/native-subagent-runtime', async (req, }; } if (!ds) return jsonRes(res, 404, { ok: false, error: 'session_not_found' }); + const readonlyOrigin = ds.readonlyContinuationTurnOrigin; + if (readonlyOrigin + && readonlyOrigin.workerGeneration === ds.workerGeneration + && ds.managedTurnOrigin?.turnId === readonlyOrigin.turnId + && ds.managedTurnOrigin.dispatchAttempt === readonlyOrigin.dispatchAttempt) { + return nativeSubagentRuntimeJsonRes({ + req, res, sessionId: params.sessionId, status: 200, + body: { ok: true, deny: true, reason: 'read-only continuation forbids subagents' }, + ...responseAuth, + }); + } let runtimeState; try { runtimeState = getBot(ds.larkAppId).nativeSubagentRuntimeState; } @@ -2243,6 +2259,89 @@ ipcRoute('POST', '/api/project-groups/:chatId/refresh-card', async (_req, res, p } }); +/** Explicit control plane for one read-only long-running task lease. The + * rotating current-turn capability binds every action to the calling session + * and turn; the daemon owns all persisted state and timers. */ +ipcRoute('POST', '/api/sessions/:sessionId/continuation', async (req, res, params) => { + type ContinuationRequestBody = { + action?: unknown; + readonly?: unknown; + ttlMs?: unknown; + maxContinuations?: unknown; + } & Record; + const body = await readJsonBody(req) + .catch(() => ({} as ContinuationRequestBody)); + const ds = findActiveBySessionId(params.sessionId); + const auth = sessionCliIpcAuth(req, ds, params.sessionId, body); + if (!auth.ok) return jsonRes(res, 403, { ok: false, error: auth.error }); + if (!ds) return jsonRes(res, 404, { ok: false, error: 'session_not_active' }); + const turnId = typeof body.originTurnId === 'string' ? body.originTurnId : undefined; + if (!turnId || ds.managedTurnOrigin?.turnId !== turnId) { + return jsonRes(res, 409, { ok: false, error: 'active_turn_required' }); + } + if (!ensureReadonlyTaskContinuationAttached(ds)) { + return jsonRes(res, 409, { ok: false, error: 'readonly_continuation_unavailable' }); + } + try { + let state; + if (body.action === 'start') { + if (body.readonly !== true) { + return jsonRes(res, 400, { ok: false, error: 'readonly_required' }); + } + if (!turnId.startsWith('om_') || ds.managedTurnOrigin?.dispatchAttempt !== undefined) { + return jsonRes(res, 409, { ok: false, error: 'ordinary_user_turn_required' }); + } + if (body.ttlMs !== undefined + && (typeof body.ttlMs !== 'number' || !Number.isSafeInteger(body.ttlMs) || body.ttlMs <= 0)) { + return jsonRes(res, 400, { ok: false, error: 'invalid_ttl_ms' }); + } + if (body.maxContinuations !== undefined + && (typeof body.maxContinuations !== 'number' + || !Number.isSafeInteger(body.maxContinuations) + || body.maxContinuations <= 0)) { + return jsonRes(res, 400, { ok: false, error: 'invalid_max_continuations' }); + } + const generation = ds.workerGeneration; + const proof = ds.readonlyContinuationRpcProof; + if (!ds.worker || ds.worker.killed || ds.worker.connected === false + || ds.workerReady !== true + || !Number.isSafeInteger(generation) || (generation ?? 0) <= 0 + || ds.session.workerGeneration !== generation + || proof?.workerGeneration !== generation) { + return jsonRes(res, 409, { ok: false, error: 'readonly_rpc_proof_required' }); + } + state = startReadonlyTaskContinuation(ds.session, { + turnId, + workerGeneration: generation!, + ...(typeof body.ttlMs === 'number' ? { ttlMs: body.ttlMs } : {}), + ...(typeof body.maxContinuations === 'number' + ? { maxContinuations: body.maxContinuations } + : {}), + }); + } else if (body.action === 'await-user') { + const before = ds.session.readonlyTaskContinuation; + state = awaitReadonlyTaskContinuationUser(ds.session, turnId); + if (state === before || state?.status !== 'awaiting_user') { + return jsonRes(res, 409, { ok: false, error: 'continuation_transition_rejected' }); + } + } else if (body.action === 'cancel') { + const before = ds.session.readonlyTaskContinuation; + state = cancelReadonlyTaskContinuationExplicit(ds.session, turnId); + if (state === before || state?.status !== 'cancelled') { + return jsonRes(res, 409, { ok: false, error: 'continuation_transition_rejected' }); + } + } else { + return jsonRes(res, 400, { ok: false, error: 'invalid_action' }); + } + return jsonRes(res, 200, { ok: true, state }); + } catch (err) { + return jsonRes(res, 409, { + ok: false, + error: err instanceof Error ? err.message : String(err), + }); + } +}); + /** Side-effect-free dispatch guard used before the CLI creates or writes a * topic. The registration route repeats this check on the host boundary. */ ipcRoute('POST', '/api/sessions/:sessionId/project-dispatch-policy', async (req, res, params) => { diff --git a/src/core/session-manager.ts b/src/core/session-manager.ts index 61cdfbd0c7..23b9742f41 100644 --- a/src/core/session-manager.ts +++ b/src/core/session-manager.ts @@ -12,7 +12,7 @@ import * as sessionStore from '../services/session-store.js'; import * as messageQueue from '../services/message-queue.js'; import { downloadMessageResource, listChatBotMembers, UserTokenMissingError } from '../im/lark/client.js'; import { logger } from '../utils/logger.js'; -import { forkWorker, sendWorkerInput, promoteQueuedActivationTail, forkAdoptWorker, adoptSandboxBlocked, killStalePids, sweepDeadPidMarkers, getCurrentCliVersion, restoreUsageLimitRuntimeState, setActiveSessionSafe, setActiveSessionIfActive, isDisposableCommandScratch, isRelayableRealSession, closeSession, getActiveSessionsRegistry, suspendWorker, withActiveSessionKeyLock, isSessionTransferring, deferUntilSessionTransferSettled, ensureOrdinaryTurnRecoveryAttached } from './worker-pool.js'; +import { forkWorker, sendWorkerInput, promoteQueuedActivationTail, forkAdoptWorker, adoptSandboxBlocked, killStalePids, sweepDeadPidMarkers, getCurrentCliVersion, restoreUsageLimitRuntimeState, setActiveSessionSafe, setActiveSessionIfActive, isDisposableCommandScratch, isRelayableRealSession, closeSession, getActiveSessionsRegistry, suspendWorker, withActiveSessionKeyLock, isSessionTransferring, deferUntilSessionTransferSettled, ensureOrdinaryTurnRecoveryAttached, ensureReadonlyTaskContinuationAttached } from './worker-pool.js'; import { createCliAdapterSync } from '../adapters/cli/registry.js'; import type { CliAdapter } from '../adapters/cli/types.js'; import { botHomePath } from '../adapters/cli/read-isolation.js'; @@ -2809,7 +2809,9 @@ export async function restoreActiveSessions( // pass has registered collision winners. A zero-delay overdue backoff must // not wake while a later row is still competing for the same route. for (const ds of restoredByThisInvocation) { - if (stillOwnsRestoreRegistration(ds)) ensureOrdinaryTurnRecoveryAttached(ds); + if (!stillOwnsRestoreRegistration(ds)) continue; + ensureOrdinaryTurnRecoveryAttached(ds); + ensureReadonlyTaskContinuationAttached(ds); } // Persistent backends: auto-fork workers for sessions whose backing session diff --git a/src/core/types.ts b/src/core/types.ts index a0c309898c..99dc72e33f 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -76,6 +76,19 @@ export interface DaemonSession { /** Monotonic within one daemon boot. Captured by durable delivery receipts * so a terminal/exit from a replaced worker cannot settle a newer attempt. */ workerGeneration?: number; + /** In-memory proof emitted by this exact worker + TraeX RPC generation. */ + readonlyContinuationRpcProof?: { + workerGeneration: number; + rpcGeneration: string; + checkedAt: number; + }; + /** Exact live synthetic turn whose hook-level native subagent requests must + * be denied. Derived only from trusted worker IPC for the current generation. */ + readonlyContinuationTurnOrigin?: { + workerGeneration: number; + turnId: string; + dispatchAttempt: number; + }; larkAppId: string; chatId: string; chatType: 'group' | 'p2p'; // p2p chats need reply_in_thread to create topics diff --git a/src/core/worker-pool.ts b/src/core/worker-pool.ts index e696e82cca..cfd66c72c5 100644 --- a/src/core/worker-pool.ts +++ b/src/core/worker-pool.ts @@ -481,6 +481,23 @@ import { requireOrdinaryTurnRecoveryAttention, type OrdinaryTurnRecoveryDispatch, } from '../services/ordinary-turn-recovery.js'; +import { + attachReadonlyTaskContinuation, + beginReadonlyTaskContinuationDelivery, + cancelReadonlyTaskContinuationForUserInput, + completeReadonlyTaskContinuationOriginalBusinessFinal, + completeReadonlyTaskContinuationWarning, + completeReadonlyTaskContinuation, + disposeReadonlyTaskContinuation, + failReadonlyTaskContinuationWarningDelivery, + failReadonlyTaskContinuationVisible, + finishReadonlyTaskContinuationAwaitUser, + handleReadonlyTaskContinuationTerminal, + parseReadonlyContinuationOutput, + readonlyTaskContinuationHandlesTerminal, + type ReadonlyTaskContinuationDispatch, + type ReadonlyTaskContinuationState, +} from '../services/readonly-task-continuation.js'; import { turnRetryOffer, shouldNotifyTurnFailure } from '../services/turn-failure-notice.js'; import { knownBotOpenIdsFromCrossRef, type BotMentionEntry } from '../utils/bot-routing.js'; import { emitSessionLifecycleHook, emitSessionStateTransitionHook } from '../services/session-lifecycle-hooks.js'; @@ -1586,6 +1603,237 @@ export function ensureOrdinaryTurnRecoveryAttached( return true; } +function readonlyTaskContinuationEnabled(): boolean { + return process.env.BOTMUX_READONLY_CONTINUATION_ENABLED?.trim().toLowerCase() === 'true'; +} + +function readonlyTaskContinuationEligible( + ds: DaemonSession, + botCfg = getBot(ds.larkAppId).config, +): boolean { + return readonlyTaskContinuationEnabled() + && sessionCliId(ds, botCfg) === 'traex' + && ds.session.status === 'active' + && !isSharedAdoptSession(ds) + && !ds.session.vcMeetingReceiver + && !ds.session.deferredScheduleRun + && !ds.session.externalTriggerTopicless + && larkTransportEnabled({ chatId: ds.chatId, apiOnly: botCfg.apiOnly }); +} + +function readonlyTaskContinuationWarning( + state: NonNullable, +): string { + const reason = state.status === 'expired' + ? '租约已过期' + : state.status === 'exhausted' + ? `达到最大续跑次数 ${state.maxContinuations}` + : state.lastErrorCode ?? '续跑交接失败'; + return `⚠️ 只读长程任务自动续跑已停止（${reason}）。请检查过程账本和 Web 终端后，再决定是否继续。`; +} + +function deliverReadonlyTaskContinuationPending( + ds: DaemonSession, + state: ReadonlyTaskContinuationState, +): void { + const pending = state.pendingDelivery; + const workerGeneration = state.currentWorkerGeneration; + if (!pending || !Number.isSafeInteger(workerGeneration) || (workerGeneration ?? 0) <= 0) { + failReadonlyTaskContinuationVisible( + ds.session, state.currentTurnId, state.currentDispatchAttempt, + workerGeneration ?? 0, 'readonly_continuation_delivery_proof_missing', + ); + return; + } + const stillOwnsDelivery = (): boolean => { + const current = ds.session.readonlyTaskContinuation; + return current?.status === 'delivering' + && current.leaseId === state.leaseId + && current.currentTurnId === state.currentTurnId + && current.currentDispatchAttempt === state.currentDispatchAttempt + && current.currentWorkerGeneration === workerGeneration; + }; + const finalMessage: Extract = { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: state.currentTurnId, + dispatchAttempt: state.currentDispatchAttempt, + content: pending.content, + lastUuid: `readonly:${state.leaseId}:${state.currentTurnId}:${state.currentDispatchAttempt ?? ''}`, + }; + deliverFinalOutput( + ds, finalMessage, tag(ds), 0, + (owned, messageId) => { + if (!owned || !messageId) { + failReadonlyTaskContinuationVisible( + ds.session, state.currentTurnId, state.currentDispatchAttempt, workerGeneration!, + owned + ? 'readonly_continuation_final_delivery_unproven' + : 'readonly_continuation_final_delivery_failed', + ); + return; + } + if (pending.kind === 'completed') { + completeReadonlyTaskContinuation( + ds.session, state.currentTurnId, state.currentDispatchAttempt, + messageId, workerGeneration!, + ); + } else { + finishReadonlyTaskContinuationAwaitUser( + ds.session, state.currentTurnId, state.currentDispatchAttempt, + messageId, workerGeneration!, + ); + } + }, + stillOwnsDelivery, + frozenReplyContextForTurn(ds, state.currentTurnId).target, + ); +} + +function deliverReadonlyTaskContinuationWarningPending( + ds: DaemonSession, + state: ReadonlyTaskContinuationState, +): void { + const stillOwnsWarning = (): boolean => { + const current = ds.session.readonlyTaskContinuation; + return current?.leaseId === state.leaseId + && !!current.pendingWarning + && current.warningDispatched !== true; + }; + deliverFinalOutput( + ds, + { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: state.logicalTurnId, + content: readonlyTaskContinuationWarning(state), + lastUuid: `readonly-warning:${state.leaseId}`, + }, + tag(ds), + 0, + (owned, messageId) => { + if (owned && messageId) { + completeReadonlyTaskContinuationWarning(ds.session, state.leaseId, messageId); + } else { + failReadonlyTaskContinuationWarningDelivery(ds.session, state.leaseId); + } + }, + stillOwnsWarning, + frozenReplyContextForTurn(ds, state.logicalTurnId).target, + ); +} + +/** Attach the opt-in read-only task continuation owner. The machine switch is + * checked both here and before every dispatch; an off switch leaves all normal + * sessions on their existing path. */ +export function ensureReadonlyTaskContinuationAttached( + ds: DaemonSession, + botCfg = getBot(ds.larkAppId).config, +): boolean { + const current = ds.session.readonlyTaskContinuation; + // A kill switch stops every new automatic dispatch, but it must not hide a + // warning that was already durably owed to the user. Attach the coordinator + // long enough to restore dashboard attention and either settle or expire the + // persisted warning outbox; deps.enabled() below still keeps execution off. + const needsWarningSettlement = !!current + && (current.pendingWarning !== undefined || current.warningDispatched === true); + if (!readonlyTaskContinuationEligible(ds, botCfg) && !needsWarningSettlement) { + disposeReadonlyTaskContinuation(ds.session); + if (current && ['active', 'backoff', 'dispatching', 'delivering'].includes(current.status)) { + ds.session.readonlyTaskContinuation = { + ...current, + status: 'cancelled', + nextAttemptAt: undefined, + lastErrorCode: readonlyTaskContinuationEnabled() + ? 'readonly_continuation_ineligible' + : 'readonly_continuation_disabled', + }; + sessionStore.updateSession(ds.session); + } + return false; + } + attachReadonlyTaskContinuation(ds.session, { + schedule: (delayMs, run) => { + const timer = setTimeout(run, delayMs); + timer.unref?.(); + return timer; + }, + cancel: timer => clearTimeout(timer as ReturnType), + persist: () => sessionStore.updateSession(ds.session), + enabled: readonlyTaskContinuationEnabled, + recoverDelivery: state => deliverReadonlyTaskContinuationPending(ds, state), + canEnqueue: () => { + const workerGeneration = ds.workerGeneration; + const proof = ds.readonlyContinuationRpcProof; + return readonlyTaskContinuationEligible(ds) + && ordinaryTurnRecoveryStillOwnsSession(ds) + && !!ds.worker + && !ds.worker.killed + && ds.worker.connected !== false + && ds.workerReady === true + && Number.isSafeInteger(workerGeneration) + && (workerGeneration ?? 0) > 0 + && ds.session.workerGeneration === workerGeneration + && proof?.workerGeneration === workerGeneration; + }, + enqueue: (dispatch: ReadonlyTaskContinuationDispatch) => { + if (!readonlyTaskContinuationEligible(ds) || !ordinaryTurnRecoveryStillOwnsSession(ds)) { + return false; + } + const workerGeneration = ds.workerGeneration; + const proof = ds.readonlyContinuationRpcProof; + if (!ds.worker || ds.worker.killed || ds.worker.connected === false + || ds.workerReady !== true + || !Number.isSafeInteger(workerGeneration) || (workerGeneration ?? 0) <= 0 + || ds.session.workerGeneration !== workerGeneration + || proof?.workerGeneration !== workerGeneration) { + return false; + } + const current = ds.session.readonlyTaskContinuation; + if (!current || current.currentTurnId !== dispatch.turnId + || current.currentDispatchAttempt !== dispatch.dispatchAttempt) return false; + try { + ds.worker.send({ + type: 'message', + content: dispatch.prompt, + turnId: dispatch.turnId, + dispatchAttempt: dispatch.dispatchAttempt, + ...(ds.crashDiagnosticParked ? { model: latestModelForRespawn(ds) } : {}), + readonlyContinuation: { + leaseId: current.leaseId, + rpcGeneration: proof!.rpcGeneration, + }, + } satisfies Extract); + recordAdmittedOrdinaryUserTurn(ds, dispatch.turnId, { + beginRecovery: false, + dispatchAttempt: dispatch.dispatchAttempt, + }); + return workerGeneration!; + } catch (err) { + logger.error( + `[${tag(ds)}] Failed to enqueue read-only task continuation ` + + `${dispatch.continuation}: ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + }, + attend: state => { + const warning = readonlyTaskContinuationWarning(state); + ds.agentAttention = { kind: 'blocked', reason: warning, at: Date.now() }; + publishAttentionPatch(ds); + emitSessionLifecycleHook(ds, 'session.requires_attention', { + reason: 'readonly_task_continuation_stopped', + errorCode: state.lastErrorCode, + logicalTurnId: state.logicalTurnId, + }); + }, + warn: state => { + deliverReadonlyTaskContinuationWarningPending(ds, state); + }, + }); + return true; +} + function sessionRuntimeDisplayName( ds: DaemonSession, botCfg?: { cliRuntime?: CliRuntimeConfig }, @@ -6899,6 +7147,7 @@ export async function closeSession( if (ds) { disposeOrdinaryTurnRecovery(ds.session); + disposeReadonlyTaskContinuation(ds.session); killWorker(ds, { ...(preparedRemoteRequestId ? { remoteCloseCommitRequestId: preparedRemoteRequestId } : {}), // The prepare above already proved this cancel; the durable row cleared the @@ -9343,31 +9592,48 @@ function rollbackWorkerForkPreInit( function recordAdmittedOrdinaryUserTurn( ds: DaemonSession, turnId: string, - opts: { beginRecovery: boolean }, + opts: { beginRecovery: boolean; dispatchAttempt?: number }, ): void { const priorRecoveryStatus = ds.session.ordinaryTurnRecovery?.status; + const priorReadonlyStatus = ds.session.readonlyTaskContinuation?.status; let recoveryBookkeepingSucceeded = false; - try { - if (opts.beginRecovery) { - // Freeze the fire's silent attribute onto the logical turn now: the - // scheduler arms the runtime registry before dispatch, so it is readable - // here, and only the persisted copy survives a restart. - const state = beginOrdinaryTurnRecovery(ds.session, turnId, { - silent: isSilentScheduledTurn(ds, turnId), - }); - recoveryBookkeepingSucceeded = state?.logicalTurnId === turnId - && state.currentTurnId === turnId; - } else { - const state = cancelOrdinaryRecoveryForUserInput(ds.session, turnId); - recoveryBookkeepingSucceeded = state?.status === 'cancelled' - && state.cancelledByTurnId === turnId; + if (isOrdinaryRecoveryTurnId(turnId)) { + try { + if (opts.beginRecovery) { + // Freeze the fire's silent attribute onto the logical turn now: the + // scheduler arms the runtime registry before dispatch, so it is readable + // here, and only the persisted copy survives a restart. + const state = beginOrdinaryTurnRecovery(ds.session, turnId, { + silent: isSilentScheduledTurn(ds, turnId), + }); + recoveryBookkeepingSucceeded = state?.logicalTurnId === turnId + && state.currentTurnId === turnId; + } else { + const state = cancelOrdinaryRecoveryForUserInput(ds.session, turnId); + recoveryBookkeepingSucceeded = state?.status === 'cancelled' + && state.cancelledByTurnId === turnId; + } + } catch (err) { + logger.error( + `[${tag(ds)}] Failed to persist ordinary-turn recovery bookkeeping ` + + `after admitting ${turnId.substring(0, 16)}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } + } + const readonlyState = ds.session.readonlyTaskContinuation; + const isCurrentReadonlyTurn = readonlyState?.currentTurnId === turnId + && readonlyState.currentDispatchAttempt === opts.dispatchAttempt; + if (!isCurrentReadonlyTurn && priorReadonlyStatus + && ['active', 'backoff', 'dispatching', 'delivering', 'awaiting_user'].includes(priorReadonlyStatus)) { + try { + cancelReadonlyTaskContinuationForUserInput(ds.session, turnId); + } catch (err) { + logger.error( + `[${tag(ds)}] Failed to cancel read-only continuation for new user input ` + + `${turnId.substring(0, 16)}: ${err instanceof Error ? err.message : String(err)}`, + ); } - } catch (err) { - logger.error( - `[${tag(ds)}] Failed to persist ordinary-turn recovery bookkeeping ` - + `after admitting ${turnId.substring(0, 16)}: ` - + `${err instanceof Error ? err.message : String(err)}`, - ); } if (recoveryBookkeepingSucceeded && (priorRecoveryStatus === 'exhausted' || priorRecoveryStatus === 'attention_required') @@ -9479,9 +9745,12 @@ export function sendWorkerInput( logger.info( `[${tag(ds)}] Staged turn ${queuedTurnId} behind queued activation ACK`, ); - if (isOrdinaryRecoveryTurnId(turnId)) { - recordAdmittedOrdinaryUserTurn(ds, turnId, { beginRecovery: false }); - } + recordAdmittedOrdinaryUserTurn(ds, queuedTurnId, { + beginRecovery: false, + ...(opts.dispatchAttempt !== undefined + ? { dispatchAttempt: opts.dispatchAttempt } + : {}), + }); return true; } catch (err) { logger.error( @@ -9576,8 +9845,12 @@ export function sendWorkerInput( ); return false; } - if (isOrdinaryRecoveryTurnId(turnId)) { - recordAdmittedOrdinaryUserTurn(ds, turnId, { beginRecovery: true }); + { + const admittedTurnId = effectiveTurnId ?? routingTurnId ?? `admitted-turn-${randomUUID()}`; + recordAdmittedOrdinaryUserTurn(ds, admittedTurnId, { + beginRecovery: true, + ...(opts.dispatchAttempt !== undefined ? { dispatchAttempt: opts.dispatchAttempt } : {}), + }); } return true; } @@ -9932,9 +10205,12 @@ export function promoteQueuedActivationTail( queuedActivationToken: token, ...(vcMeetingImTurnOrigin ? { vcMeetingImTurnOrigin } : {}), } as DaemonToWorker); - if (isOrdinaryRecoveryTurnId(head.turnId)) { - recordAdmittedOrdinaryUserTurn(ds, head.turnId, { beginRecovery: true }); - } + recordAdmittedOrdinaryUserTurn(ds, head.turnId, { + beginRecovery: true, + ...(head.dispatchAttempt !== undefined + ? { dispatchAttempt: head.dispatchAttempt } + : {}), + }); } catch (err) { // Durable ownership already moved to the journal (and Codex ledger). Never // append another owner on retry; fence this IPC generation and let recovery @@ -11088,8 +11364,13 @@ export function forkWorker( } else { worker.send(initMsg); } - if (prompt.length > 0 && isOrdinaryRecoveryTurnId(initAttributionTurnId)) { - recordAdmittedOrdinaryUserTurn(ds, initAttributionTurnId, { beginRecovery: true }); + if (prompt.length > 0) { + recordAdmittedOrdinaryUserTurn(ds, initAttributionTurnId ?? `admitted-turn-${randomUUID()}`, { + beginRecovery: true, + ...(initDispatchAttempt !== undefined + ? { dispatchAttempt: initDispatchAttempt } + : {}), + }); } ds.spawnedAt = Date.now(); // master: per-runtime-key CLI version (the init send already happened above via @@ -11329,6 +11610,8 @@ function setupWorkerHandlers( ) { throw new Error('worker generation reservation changed before IPC setup'); } + ds.readonlyContinuationRpcProof = undefined; + ds.readonlyContinuationTurnOrigin = undefined; // Tier authority belongs to this exact worker generation. Start unknown and // wait for the new worker's rollout-bound observation; this also clears a // Codex badge before a role switch starts a non-Codex worker. @@ -11476,6 +11759,7 @@ function setupWorkerHandlers( const botCfg = bot.config; const loc = botLocale(botCfg); ensureOrdinaryTurnRecoveryAttached(ds, botCfg); + ensureReadonlyTaskContinuationAttached(ds, botCfg); const notifyStartupFailure = async ( reason: string, turnId?: string, @@ -13556,12 +13840,32 @@ function setupWorkerHandlers( } case 'explicit_reply_observed': { + if (ds.worker !== worker) { + logger.warn(`[${t}] Ignored explicit_reply_observed from stale worker generation`); + break; + } if (msg.turnId.startsWith('mlrp_turn_')) { markMessageListenerRunPreviewReplied(msg.turnId, { sessionId: ds.session.sessionId, replyMessageId: msg.messageId, }); } + const continuation = ds.session.readonlyTaskContinuation; + if (msg.responseKind === 'final' + && continuation?.logicalTurnId === msg.turnId + && continuation.currentTurnId === msg.turnId + && continuation.currentDispatchAttempt === undefined) { + try { + completeReadonlyTaskContinuationOriginalBusinessFinal( + ds.session, msg.turnId, workerGeneration, msg.messageId, + ); + } catch (err) { + logger.error( + `[${t}] Failed to persist original-turn business final for read-only continuation ` + + `${msg.turnId.substring(0, 8)}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } break; } @@ -13641,7 +13945,13 @@ function setupWorkerHandlers( const isClaudeProviderFailure = msg.status !== 'completed' && sessionCliId(ds, botCfg) === 'claude-code' && (msg.errorCode?.startsWith('provider_') ?? false); - const recoveryOwnsTerminal = ordinaryTurnRecoveryHandlesTerminal(ds.session, msg); + const readonlyTerminal = { ...msg, workerGeneration }; + const readonlyContinuationOwnsTerminal = readonlyTaskContinuationHandlesTerminal( + ds.session, + readonlyTerminal, + ); + const recoveryOwnsTerminal = ordinaryTurnRecoveryHandlesTerminal(ds.session, msg) + || readonlyContinuationOwnsTerminal; let recoveryHandled = false; try { await cb.onTurnTerminal?.(ds, msg, { workerGeneration }); @@ -13664,6 +13974,16 @@ function setupWorkerHandlers( + `${err instanceof Error ? err.message : String(err)}`, ); } + try { + handleReadonlyTaskContinuationTerminal(ds.session, readonlyTerminal); + recoveryHandled ||= readonlyContinuationOwnsTerminal; + } catch (err) { + logger.error( + `[${t}] Failed to persist read-only task continuation terminal for ` + + `${msg.turnId.substring(0, 8)}: ` + + `${err instanceof Error ? err.message : String(err)}`, + ); + } let nonLarkFailureHandled = false; if (isClaudeProviderFailure && !ds.session.vcMeetingReceiver) { const failureCode = msg.errorCode ?? msg.status; @@ -13987,6 +14307,11 @@ function setupWorkerHandlers( ? { preexistingProcessIdentities } : {}), }; + ds.readonlyContinuationTurnOrigin = msg.readonlyContinuation === true + && msg.turnId?.startsWith('bmx-readonly-') + && msg.dispatchAttempt !== undefined + ? { workerGeneration, turnId: msg.turnId, dispatchAttempt: msg.dispatchAttempt } + : undefined; break; } @@ -14029,6 +14354,11 @@ function setupWorkerHandlers( logger.warn(`[${t}] Ignored stale policy capability in managed turn origin revoke`); } if (!revokeLive && !revokePolicy) break; + if (ds.readonlyContinuationTurnOrigin?.workerGeneration === workerGeneration + && ds.readonlyContinuationTurnOrigin.turnId === msg.turnId + && ds.readonlyContinuationTurnOrigin.dispatchAttempt === msg.dispatchAttempt) { + ds.readonlyContinuationTurnOrigin = undefined; + } if (revokeLive) { ds.managedTurnOrigin = origin.policyCapability && !revokePolicy ? { @@ -14130,7 +14460,73 @@ function setupWorkerHandlers( break; } + case 'readonly_continuation_rpc_status': { + if (ds.worker !== worker + || msg.sessionId !== ds.session.sessionId + || ds.workerGeneration !== workerGeneration + || ds.session.workerGeneration !== workerGeneration) { + logger.warn(`[${t}] Ignored read-only RPC proof from stale worker generation`); + break; + } + ds.readonlyContinuationRpcProof = msg.eligible + ? { workerGeneration, rpcGeneration: msg.rpcGeneration, checkedAt: Date.now() } + : undefined; + break; + } + case 'final_output': { + const continuation = ds.session.readonlyTaskContinuation; + const exactReadonlyFinal = continuation + && ['active', 'backoff'].includes(continuation.status) + && msg.turnId.startsWith('bmx-readonly-') + && msg.dispatchAttempt !== undefined + && continuation.currentTurnId === msg.turnId + && continuation.currentDispatchAttempt === msg.dispatchAttempt + && continuation.currentWorkerGeneration === workerGeneration; + if (exactReadonlyFinal) { + // MR1 deliberately surfaces failed-turn fallback text before the + // structured terminal. For a synthetic continuation that text is a + // transport diagnostic, not the model's strict JSON business result; + // let the following terminal decide whether the allowlisted output + // limit should continue or another failure should stop visibly. + if (msg.turnFailed) break; + const output = parseReadonlyContinuationOutput(msg.content); + if (!output) { + failReadonlyTaskContinuationVisible( + ds.session, msg.turnId, msg.dispatchAttempt, workerGeneration, + 'readonly_continuation_invalid_output', + ); + break; + } + if (output.status === 'continue') break; + const deliveryState = beginReadonlyTaskContinuationDelivery( + ds.session, msg.turnId, msg.dispatchAttempt, workerGeneration, + { kind: output.status, content: output.content }, + ); + if (deliveryState?.status !== 'delivering') break; + deliverReadonlyTaskContinuationPending(ds, deliveryState); + break; + } + if (msg.turnId.startsWith('bmx-readonly-')) { + logger.warn(`[${t}] Dropped stale/unbound read-only continuation final_output`); + break; + } + const originalContinuation = ds.session.readonlyTaskContinuation; + if (originalContinuation?.logicalTurnId === msg.turnId + && originalContinuation.currentTurnId === msg.turnId + && originalContinuation.currentDispatchAttempt === undefined + && msg.turnFailed !== true) { + try { + completeReadonlyTaskContinuationOriginalBusinessFinal( + ds.session, msg.turnId, workerGeneration, msg.lastUuid, + ); + } catch (err) { + logger.error( + `[${t}] Failed to persist original-turn final output for read-only continuation ` + + `${msg.turnId.substring(0, 8)}: ${err instanceof Error ? err.message : String(err)}`, + ); + } + } if (msg.codexAppSettlement) { const settlement = msg.codexAppSettlement; const acknowledge = (ok: boolean, error?: string): void => { @@ -14735,7 +15131,7 @@ function deliverFinalOutput( msg: Extract, t: string, attempt: number, - onComplete?: (owned: boolean) => void, + onComplete?: (owned: boolean, messageId?: string) => void, isStillOwned: () => boolean = () => true, frozenReplyTarget?: FrozenSessionReplyTarget, frozenUsage?: CardUsageSnapshot, @@ -15167,7 +15563,7 @@ function deliverFinalOutput( `[${t}] VC listener fallback replayed existing provider result ` + `(turn ${msg.turnId.substring(0, 8)})`, ); - onComplete?.(true); + onComplete?.(true, preparedListenerReply.messageId); return; } @@ -15220,7 +15616,7 @@ function deliverFinalOutput( if (feedbackPolicy && baseFeedbackCard && messageId) { await persistFinalOutputFeedback(ds, msg, safeAssistantText, effectiveCliId, messageId, feedbackPolicy!, baseFeedbackCard, feedbackRequesterSubjectId, getBot(ds.larkAppId).config.feedbackWebhooks?.destinations, t); } - onComplete?.(true); + onComplete?.(true, messageId); } catch (err: any) { if (!isStillOwned()) { onComplete?.(false); return; } if (err instanceof MessageWithdrawnError) { @@ -15346,6 +15742,7 @@ export function forkAdoptWorker(ds: DaemonSession, opts?: { restoredFromMetadata } if (!canForkRegisteredSession(ds)) return; ds.workerReady = false; + ds.readonlyContinuationRpcProof = undefined; const cb = requireCallbacks(); const t = tag(ds); const adopted = ds.adoptedFrom; diff --git a/src/services/bridge-fallback-gate.ts b/src/services/bridge-fallback-gate.ts index 310b311925..7854806707 100644 --- a/src/services/bridge-fallback-gate.ts +++ b/src/services/bridge-fallback-gate.ts @@ -194,6 +194,7 @@ export function bridgePostText(finalText: string, adoptMode: boolean): string { export interface BridgeSendMarker { sentAtMs: number; messageId?: string; + responseKind?: 'progress' | 'final' | 'auxiliary'; turnId?: string; dispatchAttempt?: number; contentLength?: number; diff --git a/src/services/codex-transcript.ts b/src/services/codex-transcript.ts index 46bb325fd5..aedd17bd98 100644 --- a/src/services/codex-transcript.ts +++ b/src/services/codex-transcript.ts @@ -354,7 +354,9 @@ export const CODEX_CONNECTION_ERROR_CODE = 'codex_connection_failed'; * so the user-facing card can say "server-side transient, just retry later" * instead of pointing at the local network. */ export const CODEX_UPSTREAM_ERROR_CODE = 'codex_upstream_error'; +export const CODEX_OUTPUT_LIMIT_ERROR_CODE = 'codex_output_limit_exceeded'; export const CODEX_TASK_FAILED_ERROR_CODE = 'codex_task_failed'; +export const CODEX_OUTPUT_LIMIT_ERROR_MESSAGE = 'model output limit exceeded: max_output_tokens'; const CODEX_FAILURE_SUMMARY_MAX_CHARS = 320; /** Pre-scan bound applied BEFORE the redaction regexes run. Well above the @@ -547,6 +549,19 @@ export function codexTaskFailureCode(error: unknown): string { return CODEX_TASK_FAILED_ERROR_CODE; } +/** Exact opt-in classifier used only by the TraeX read-only continuation + * path. Keeping it out of codexTaskFailureCode preserves every other CLI's + * existing public error taxonomy. */ +export function isExactCodexOutputLimitError(error: unknown): boolean { + const leaf = codexFailureLeaf(error); + const leafMessage = typeof leaf === 'string' + ? leaf + : leaf && typeof leaf === 'object' && typeof (leaf as Record).message === 'string' + ? String((leaf as Record).message) + : ''; + return leafMessage.trim().toLowerCase() === CODEX_OUTPUT_LIMIT_ERROR_MESSAGE; +} + export function isCodexRateLimitEvent(event: CodexBridgeEvent): boolean { return event.kind === 'assistant_final' && event.terminalStatus === 'failed' diff --git a/src/services/readonly-task-continuation.ts b/src/services/readonly-task-continuation.ts new file mode 100644 index 0000000000..3497bd431a --- /dev/null +++ b/src/services/readonly-task-continuation.ts @@ -0,0 +1,982 @@ +export const READONLY_TASK_CONTINUATION_OUTPUT_LIMIT_CODE = 'codex_output_limit_exceeded'; + +export const READONLY_TASK_CONTINUATION_PROMPT = [ + '[BOTMUX_READONLY_CONTINUATION]', + '这是同一个只读长程任务的受限自动续跑。请读取当前会话与工作区中的过程账本，从最后一个可验证检查点继续；', + '保持只读，不执行任何写入、发布、重启、配置修改或其他外部副作用，也不要重复已经完成的查询。', + '本轮最终输出必须是单个 JSON 对象且不要使用代码块：任务完成时输出 {"status":"completed","content":"给用户的最终结论"}；仍可继续时输出 {"status":"continue"}；需要用户输入时输出 {"status":"await_user","content":"要问用户的问题"}。', + '不要调用 botmux send，不要用自然语言猜测或声明内部完成状态；daemon 只接受上述严格结构并负责最终投递。', +].join('\n'); + +export const READONLY_TASK_CONTINUATION_DEFAULT_TTL_MS = 60 * 60_000; +export const READONLY_TASK_CONTINUATION_MAX_TTL_MS = 4 * 60 * 60_000; +export const READONLY_TASK_CONTINUATION_DEFAULT_MAX = 6; +export const READONLY_TASK_CONTINUATION_HARD_MAX = 12; +export const READONLY_TASK_CONTINUATION_WARNING_RETRY_MS = 60_000; +/** Feishu UUID idempotency is guaranteed for one hour. Leave a five-minute + * margin so a restored daemon never replays an uncertain delivery outside + * that provider-side dedupe window. */ +export const READONLY_TASK_CONTINUATION_DELIVERY_RECOVERY_MS = 55 * 60_000; + +export type ReadonlyTaskContinuationStatus = + | 'active' + | 'backoff' + | 'dispatching' + | 'delivering' + | 'completed' + | 'awaiting_user' + | 'cancelled' + | 'expired' + | 'exhausted' + | 'failed'; + +export interface ReadonlyTaskContinuationState { + leaseId: string; + logicalTurnId: string; + currentTurnId: string; + currentDispatchAttempt?: number; + currentWorkerGeneration?: number; + createdAt: number; + expiresAt: number; + maxContinuations: number; + continuationsStarted: number; + status: ReadonlyTaskContinuationStatus; + nextAttemptAt?: number; + lastErrorCode?: string; + completedMessageId?: string; + pendingDelivery?: { + kind: 'completed' | 'await_user'; + content: string; + startedAt: number; + }; + cancelledByTurnId?: string; + pendingWarning?: { + startedAt: number; + deliveryAttempts: number; + nextAttemptAt?: number; + }; + warningDispatched?: boolean; + warningMessageId?: string; +} + +export interface ReadonlyTaskContinuationTerminal { + turnId: string; + dispatchAttempt?: number; + status: 'completed' | 'failed' | 'cancelled' | 'ambiguous'; + errorCode?: string; + workerGeneration?: number; +} + +export interface ReadonlyTaskContinuationDispatch { + logicalTurnId: string; + turnId: string; + dispatchAttempt: number; + prompt: string; + continuation: number; +} + +export interface ReadonlyTaskContinuationDeps { + schedule: (delayMs: number, run: () => void) => TTimer; + cancel: (timer: TTimer) => void; + persist: (state: ReadonlyTaskContinuationState) => void; + /** A restored lease may outlive its worker. Keep the durable backoff pending + * until the exact worker/RPC proof is ready instead of consuming an attempt. */ + canEnqueue?: () => boolean; + enqueue: (dispatch: ReadonlyTaskContinuationDispatch) => number | false; + /** Deliver one already-persisted warning with a stable provider key. */ + warn: (state: ReadonlyTaskContinuationState) => void; + /** Restore daemon-local attention independently from provider delivery. */ + attend?: (state: ReadonlyTaskContinuationState) => void; + enabled: () => boolean; + /** Re-send one already-persisted daemon-owned final with its stable UUID. */ + recoverDelivery?: (state: ReadonlyTaskContinuationState) => void; + /** Keep a local fail-closed fence when the durable store is unavailable. */ + retain?: (state: ReadonlyTaskContinuationState) => void; + now?: () => number; + randomId?: () => string; + delayMs?: number; +} + +export interface ReadonlyTaskContinuationSession { + sessionId: string; + readonlyTaskContinuation?: ReadonlyTaskContinuationState; + turnReplyContexts?: Record; + replyTargets?: Record; +} + +export interface StartReadonlyTaskContinuationInput { + turnId: string; + workerGeneration: number; + ttlMs?: number; + maxContinuations?: number; +} + +type AttachedContinuation = { + session: ReadonlyTaskContinuationSession; + coordinator: ReadonlyTaskContinuationCoordinator; + dispose: () => void; +}; + +const attachedContinuations = new Map(); + +function isLiveStatus(status: ReadonlyTaskContinuationStatus): boolean { + return status === 'active' || status === 'backoff' || status === 'dispatching' + || status === 'delivering'; +} + +function isOpenStatus(status: ReadonlyTaskContinuationStatus): boolean { + return isLiveStatus(status) || status === 'awaiting_user'; +} + +export type ReadonlyContinuationOutput = + | { status: 'continue' } + | { status: 'completed' | 'await_user'; content: string }; + +export function parseReadonlyContinuationOutput(text: string): ReadonlyContinuationOutput | undefined { + let value: unknown; + try { value = JSON.parse(text.trim()); } catch { return undefined; } + if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined; + const record = value as Record; + const keys = Object.keys(record).sort(); + if (record.status === 'continue') { + return keys.length === 1 && keys[0] === 'status' ? { status: 'continue' } : undefined; + } + if (record.status !== 'completed' && record.status !== 'await_user') return undefined; + if (keys.length !== 2 || keys[0] !== 'content' || keys[1] !== 'status') return undefined; + if (typeof record.content !== 'string' || !record.content.trim()) return undefined; + return { status: record.status, content: record.content.trim() }; +} + +export function readonlyTaskContinuationRecoversTerminal( + state: ReadonlyTaskContinuationState | undefined, + terminal: ReadonlyTaskContinuationTerminal, +): boolean { + if (!state || state.status !== 'active' + || terminal.turnId !== state.currentTurnId + || terminal.dispatchAttempt !== state.currentDispatchAttempt + || terminal.workerGeneration !== state.currentWorkerGeneration) return false; + return terminal.status === 'completed' + || (terminal.status === 'failed' + && terminal.errorCode === READONLY_TASK_CONTINUATION_OUTPUT_LIMIT_CODE); +} + +/** A deliberately narrow task lease. It never infers completion from prose: + * only the daemon-validated strict JSON terminal can complete it. A normal CLI + * terminal therefore means "continue" while the lease remains live. */ +export class ReadonlyTaskContinuationCoordinator { + private state: ReadonlyTaskContinuationState | undefined; + private timer: TTimer | undefined; + private readonly now: () => number; + private readonly randomId: () => string; + private readonly delayMs: number; + + constructor(private readonly deps: ReadonlyTaskContinuationDeps) { + this.now = deps.now ?? Date.now; + this.randomId = deps.randomId ?? (() => Math.random().toString(36).slice(2)); + this.delayMs = deps.delayMs ?? 1_000; + } + + restore(state: ReadonlyTaskContinuationState): void { + this.cancelTimer(); + this.state = { ...state }; + if (this.state.pendingWarning && this.state.warningDispatched !== true) { + this.publishAttention(this.state); + if (!Number.isFinite(this.state.pendingWarning.startedAt) + || this.now() - this.state.pendingWarning.startedAt >= READONLY_TASK_CONTINUATION_DELIVERY_RECOVERY_MS) { + this.expireWarningDelivery(this.state); + return; + } + if ((this.state.pendingWarning.nextAttemptAt ?? 0) > this.now()) this.armWarningRetry(); + else this.requestWarningDelivery({ ...this.state, pendingWarning: { ...this.state.pendingWarning } }); + return; + } + if (this.state.warningDispatched === true) this.publishAttention(this.state); + if (!this.deps.enabled()) { + if (isLiveStatus(this.state.status)) { + this.commit({ + ...this.state, + status: 'cancelled', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_disabled', + }); + } + return; + } + if (this.state.status === 'dispatching') { + this.warnOnce({ + ...this.state, + status: 'failed', + lastErrorCode: 'readonly_continuation_dispatch_interrupted', + }); + return; + } + if (this.state.status === 'delivering') { + const pending = this.state.pendingDelivery; + if (!pending + || !Number.isFinite(pending.startedAt) + || this.now() - pending.startedAt >= READONLY_TASK_CONTINUATION_DELIVERY_RECOVERY_MS + || !this.deps.recoverDelivery) { + this.warnOnce({ + ...this.state, + status: 'failed', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_delivery_recovery_unavailable', + }); + return; + } + try { + this.deps.recoverDelivery({ ...this.state, pendingDelivery: { ...pending } }); + } catch { + this.warnOnce({ + ...this.state, + status: 'failed', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_delivery_recovery_failed', + }); + } + return; + } + if (isLiveStatus(this.state.status) && this.now() >= this.state.expiresAt) { + this.warnOnce({ + ...this.state, + status: 'expired', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_expired', + }); + return; + } + if (this.state.status === 'backoff') this.armBackoff(); + else if (this.state.status === 'active') this.armExpiry(); + } + + start(input: StartReadonlyTaskContinuationInput): ReadonlyTaskContinuationState { + if (!this.deps.enabled()) throw new Error('readonly_continuation_disabled'); + const current = this.state; + if (current && (isLiveStatus(current.status) + || (!!current.pendingWarning && current.warningDispatched !== true))) { + if (current.currentTurnId === input.turnId && current.logicalTurnId === input.turnId) { + return current; + } + throw new Error('readonly_continuation_already_active'); + } + const ttlMs = Math.min( + Math.max(1, input.ttlMs ?? READONLY_TASK_CONTINUATION_DEFAULT_TTL_MS), + READONLY_TASK_CONTINUATION_MAX_TTL_MS, + ); + const maxContinuations = Math.min( + Math.max(1, input.maxContinuations ?? READONLY_TASK_CONTINUATION_DEFAULT_MAX), + READONLY_TASK_CONTINUATION_HARD_MAX, + ); + const createdAt = this.now(); + this.cancelTimer(); + const started = this.commit({ + leaseId: `readonly-${this.randomId()}`, + logicalTurnId: input.turnId, + currentTurnId: input.turnId, + currentWorkerGeneration: input.workerGeneration, + createdAt, + expiresAt: createdAt + ttlMs, + maxContinuations, + continuationsStarted: 0, + status: 'active', + }); + this.armExpiry(); + return started; + } + + onTerminal( + current: ReadonlyTaskContinuationState, + terminal: ReadonlyTaskContinuationTerminal, + ): ReadonlyTaskContinuationState { + if (terminal.turnId !== current.currentTurnId + || terminal.dispatchAttempt !== current.currentDispatchAttempt + || terminal.workerGeneration !== current.currentWorkerGeneration + || current.status !== 'active') return current; + if (!readonlyTaskContinuationRecoversTerminal(current, terminal)) { + this.cancelTimer(); + const stopped = { + ...current, + status: (terminal.status === 'cancelled' ? 'cancelled' : 'failed') as 'cancelled' | 'failed', + nextAttemptAt: undefined, + ...(terminal.errorCode ? { lastErrorCode: terminal.errorCode } : {}), + }; + try { return this.commit(stopped); } catch { this.retain(stopped); return stopped; } + } + return this.scheduleContinuation(current); + } + + complete( + turnId: string, + dispatchAttempt: number | undefined, + messageId: string, + workerGeneration: number, + ): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || !isLiveStatus(current.status) + || current.currentTurnId !== turnId + || current.currentDispatchAttempt !== dispatchAttempt + || current.currentWorkerGeneration !== workerGeneration) return current; + this.cancelTimer(); + const settled = { + ...current, + status: 'completed' as const, + nextAttemptAt: undefined, + completedMessageId: messageId, + pendingDelivery: undefined, + }; + try { return this.commit(settled); } catch { this.retain(settled); return settled; } + } + + completeOriginalBusinessFinal( + turnId: string, + workerGeneration: number, + messageId?: string, + ): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || current.status !== 'active' + || current.logicalTurnId !== turnId + || current.currentTurnId !== turnId + || current.currentDispatchAttempt !== undefined + || current.currentWorkerGeneration !== workerGeneration) return current; + this.cancelTimer(); + const settled = { + ...current, + status: 'completed' as const, + nextAttemptAt: undefined, + ...(messageId ? { completedMessageId: messageId } : {}), + }; + try { + return this.commit(settled); + } catch (error) { + this.retain({ + ...current, + status: 'failed', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_business_final_persist_failed', + }); + throw error; + } + } + + beginDelivery( + turnId: string, + dispatchAttempt: number | undefined, + workerGeneration: number, + delivery: { kind: 'completed' | 'await_user'; content: string }, + ): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || (current.status !== 'active' && current.status !== 'backoff') + || current.currentTurnId !== turnId + || current.currentDispatchAttempt !== dispatchAttempt + || current.currentWorkerGeneration !== workerGeneration) return current; + this.cancelTimer(); + try { + return this.commit({ + ...current, + status: 'delivering', + nextAttemptAt: undefined, + pendingDelivery: { ...delivery, startedAt: this.now() }, + }); + } catch { + return this.retainFailed(current, 'readonly_continuation_delivery_persist_failed'); + } + } + + finishAwaitUser( + turnId: string, + dispatchAttempt: number | undefined, + messageId: string, + workerGeneration: number, + ): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || current.status !== 'delivering' + || current.currentTurnId !== turnId + || current.currentDispatchAttempt !== dispatchAttempt + || current.currentWorkerGeneration !== workerGeneration + || current.pendingDelivery?.kind !== 'await_user') return current; + const settled = { + ...current, + status: 'awaiting_user' as const, + completedMessageId: messageId, + pendingDelivery: undefined, + }; + try { return this.commit(settled); } catch { this.retain(settled); return settled; } + } + + completeWarning(leaseId: string, messageId: string): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || current.leaseId !== leaseId || !current.pendingWarning + || current.warningDispatched === true) return current; + const settled = { + ...current, + pendingWarning: undefined, + warningDispatched: true, + warningMessageId: messageId, + }; + this.cancelTimer(); + try { return this.commit(settled); } catch { this.retain(settled); return settled; } + } + + warningDeliveryFailed(leaseId: string): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || current.leaseId !== leaseId || !current.pendingWarning + || current.warningDispatched === true) return current; + const deadline = current.pendingWarning.startedAt + + READONLY_TASK_CONTINUATION_DELIVERY_RECOVERY_MS; + if (this.now() >= deadline) return this.expireWarningDelivery(current); + const pending = { + ...current, + pendingWarning: { + ...current.pendingWarning, + deliveryAttempts: current.pendingWarning.deliveryAttempts + 1, + nextAttemptAt: Math.min(this.now() + READONLY_TASK_CONTINUATION_WARNING_RETRY_MS, deadline), + }, + }; + try { this.commit(pending); } catch { this.retain(pending); } + this.armWarningRetry(); + return this.state; + } + + failVisible( + turnId: string, + dispatchAttempt: number | undefined, + workerGeneration: number, + errorCode: string, + ): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || !isLiveStatus(current.status) + || current.currentTurnId !== turnId + || current.currentDispatchAttempt !== dispatchAttempt + || current.currentWorkerGeneration !== workerGeneration) return current; + this.cancelTimer(); + this.warnOnce({ ...current, status: 'failed', lastErrorCode: errorCode }); + return this.state; + } + + awaitUser(turnId: string): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || !isLiveStatus(current.status) || current.currentTurnId !== turnId) return current; + this.cancelTimer(); + const settled = { ...current, status: 'awaiting_user' as const, nextAttemptAt: undefined }; + try { return this.commit(settled); } catch { this.retain(settled); return settled; } + } + + cancelForUserInput(turnId: string): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || !isOpenStatus(current.status)) return current; + this.cancelTimer(); + const cancelled = { + ...current, + status: 'cancelled' as const, + nextAttemptAt: undefined, + cancelledByTurnId: turnId, + }; + try { + return this.commit(cancelled); + } catch (error) { + this.retain({ + ...current, + status: 'failed', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_user_cancel_persist_failed', + }); + throw error; + } + } + + cancelExplicit(turnId: string): ReadonlyTaskContinuationState | undefined { + const current = this.state; + if (!current || !isOpenStatus(current.status) || current.currentTurnId !== turnId) return current; + this.cancelTimer(); + const cancelled = { + ...current, + status: 'cancelled' as const, + nextAttemptAt: undefined, + cancelledByTurnId: turnId, + }; + try { + return this.commit(cancelled); + } catch (error) { + this.retain({ + ...current, + status: 'failed', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_explicit_cancel_persist_failed', + }); + throw error; + } + } + + private scheduleContinuation( + current: ReadonlyTaskContinuationState, + ): ReadonlyTaskContinuationState { + if (!this.deps.enabled()) { + const cancelled = { + ...current, + status: 'cancelled' as const, + lastErrorCode: 'readonly_continuation_disabled', + }; + try { return this.commit(cancelled); } catch { this.retain(cancelled); return cancelled; } + } + if (this.now() >= current.expiresAt) { + const expired = { + ...current, + status: 'expired' as const, + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_expired', + }; + this.warnOnce(expired); + return this.state ?? expired; + } + if (current.continuationsStarted >= current.maxContinuations) { + const exhausted = { + ...current, + status: 'exhausted' as const, + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_exhausted', + }; + this.warnOnce(exhausted); + return this.state ?? exhausted; + } + let next: ReadonlyTaskContinuationState; + try { + next = this.commit({ + ...current, + status: 'backoff', + nextAttemptAt: this.now() + this.delayMs, + }); + } catch { + return this.retainFailed(current, 'readonly_continuation_backoff_persist_failed'); + } + this.armBackoff(); + return next; + } + + private armBackoff(): void { + const current = this.state; + if (!current || current.status !== 'backoff') return; + this.cancelTimer(); + const delayMs = Math.max(0, (current.nextAttemptAt ?? this.now()) - this.now()); + this.timer = this.deps.schedule(delayMs, () => { + this.timer = undefined; + const live = this.state; + if (!live || live.status !== 'backoff') return; + if (!this.deps.enabled()) { + const cancelled = { + ...live, + status: 'cancelled' as const, + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_disabled', + }; + try { this.commit(cancelled); } catch { this.retain(cancelled); } + return; + } + if (this.now() >= live.expiresAt) { + this.warnOnce({ + ...live, + status: 'expired', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_expired', + }); + return; + } + if (this.deps.canEnqueue && !this.deps.canEnqueue()) { + const waiting = { + ...live, + nextAttemptAt: Math.min(this.now() + this.delayMs, live.expiresAt), + }; + try { + this.commit(waiting); + this.armBackoff(); + } catch { + this.retainFailed(live, 'readonly_continuation_readiness_persist_failed'); + } + return; + } + const continuation = live.continuationsStarted + 1; + const turnId = `bmx-readonly-${this.randomId()}`; + let dispatching: ReadonlyTaskContinuationState; + try { + dispatching = this.commit({ + ...live, + currentTurnId: turnId, + currentDispatchAttempt: continuation, + continuationsStarted: continuation, + status: 'dispatching', + nextAttemptAt: undefined, + }); + } catch { + this.warnOnce({ + ...live, + status: 'failed' as const, + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_dispatch_persist_failed', + }); + return; + } + let enqueued: number | false = false; + try { + enqueued = this.deps.enqueue({ + logicalTurnId: dispatching.logicalTurnId, + turnId, + dispatchAttempt: continuation, + prompt: READONLY_TASK_CONTINUATION_PROMPT, + continuation, + }); + } catch { + enqueued = false; + } + if (!enqueued) { + this.warnOnce({ + ...dispatching, + status: 'failed', + lastErrorCode: 'readonly_continuation_enqueue_failed', + }); + return; + } + try { + this.commit({ + ...dispatching, + status: 'active', + currentWorkerGeneration: enqueued, + }); + this.armExpiry(); + } catch { + // The child already owns the prompt, so replay is unsafe. Keep the + // local state failed closed even if the durable store is unavailable. + // A later daemon restore sees the durable dispatching fence and also + // refuses replay, so this can never become an automatic duplicate. + const failed = { + ...dispatching, + status: 'failed' as const, + lastErrorCode: 'readonly_continuation_activation_persist_failed', + }; + this.retain(failed); + this.warnOnce(failed); + } + }); + } + + private armExpiry(): void { + const current = this.state; + if (!current || current.status !== 'active') return; + this.cancelTimer(); + const delayMs = Math.max(0, current.expiresAt - this.now()); + this.timer = this.deps.schedule(delayMs, () => { + this.timer = undefined; + const live = this.state; + if (!live || live.status !== 'active') return; + if (this.now() < live.expiresAt) { + this.armExpiry(); + return; + } + this.warnOnce({ + ...live, + status: 'expired', + nextAttemptAt: undefined, + lastErrorCode: 'readonly_continuation_expired', + }); + }); + } + + private warnOnce(state: ReadonlyTaskContinuationState): void { + const prior = this.state; + if (prior?.warningDispatched || prior?.pendingWarning + || state.warningDispatched || state.pendingWarning) return; + const pending = { + ...state, + pendingDelivery: undefined, + pendingWarning: { startedAt: this.now(), deliveryAttempts: 0 }, + }; + try { + this.commit(pending); + } catch { + // A warning is an outbox item: never perform the external send unless the + // exact payload is durable first. Keep only a local fail-closed fence. + this.retain(pending); + return; + } + this.publishAttention(pending); + this.requestWarningDelivery(pending); + } + + private requestWarningDelivery(state: ReadonlyTaskContinuationState): void { + try { this.deps.warn(state); } catch { /* pending outbox remains recoverable */ } + } + + private publishAttention(state: ReadonlyTaskContinuationState): void { + try { this.deps.attend?.(state); } catch { /* local projection is best effort */ } + } + + private armWarningRetry(): void { + const current = this.state; + if (!current?.pendingWarning || current.warningDispatched === true) return; + this.cancelTimer(); + const deadline = current.pendingWarning.startedAt + + READONLY_TASK_CONTINUATION_DELIVERY_RECOVERY_MS; + const retryAt = Math.min(current.pendingWarning.nextAttemptAt ?? this.now(), deadline); + this.timer = this.deps.schedule(Math.max(0, retryAt - this.now()), () => { + this.timer = undefined; + const live = this.state; + if (!live?.pendingWarning || live.warningDispatched === true) return; + if (this.now() >= live.pendingWarning.startedAt + + READONLY_TASK_CONTINUATION_DELIVERY_RECOVERY_MS) { + this.expireWarningDelivery(live); + return; + } + this.requestWarningDelivery({ ...live, pendingWarning: { ...live.pendingWarning } }); + }); + } + + private expireWarningDelivery( + state: ReadonlyTaskContinuationState, + ): ReadonlyTaskContinuationState { + this.cancelTimer(); + const expired = { + ...state, + pendingWarning: undefined, + lastErrorCode: 'readonly_continuation_warning_delivery_expired', + }; + try { return this.commit(expired); } catch { this.retain(expired); return expired; } + } + + private retainFailed( + state: ReadonlyTaskContinuationState, + errorCode: string, + ): ReadonlyTaskContinuationState { + const failed = { + ...state, + status: 'failed' as const, + nextAttemptAt: undefined, + lastErrorCode: errorCode, + }; + this.retain(failed); + this.warnOnce(failed); + return failed; + } + + private commit(state: ReadonlyTaskContinuationState): ReadonlyTaskContinuationState { + const prior = this.state; + const next = { ...state }; + this.state = next; + try { + this.deps.persist(next); + } catch (err) { + this.state = prior; + throw err; + } + return next; + } + + private retain(state: ReadonlyTaskContinuationState): void { + this.state = { ...state }; + this.deps.retain?.(this.state); + } + + private cancelTimer(): void { + if (this.timer !== undefined) this.deps.cancel(this.timer); + this.timer = undefined; + } + + dispose(): void { + this.cancelTimer(); + } +} + +export function attachReadonlyTaskContinuation( + session: ReadonlyTaskContinuationSession, + deps: ReadonlyTaskContinuationDeps, +): void { + if (attachedContinuations.get(session.sessionId)?.session === session) return; + disposeReadonlyTaskContinuation(session); + let coordinator!: ReadonlyTaskContinuationCoordinator; + const wrapped: ReadonlyTaskContinuationDeps = { + ...deps, + persist: state => { + const prior = session.readonlyTaskContinuation; + session.readonlyTaskContinuation = structuredClone(state); + try { + deps.persist(state); + } catch (err) { + session.readonlyTaskContinuation = prior; + throw err; + } + }, + retain: state => { + session.readonlyTaskContinuation = structuredClone(state); + deps.retain?.(state); + }, + enqueue: dispatch => { + let contextCopied = false; + const sourceContext = session.turnReplyContexts?.[dispatch.logicalTurnId]; + if (sourceContext !== undefined) { + session.turnReplyContexts = { + ...(session.turnReplyContexts ?? {}), + [dispatch.turnId]: structuredClone(sourceContext), + }; + contextCopied = true; + } + const sourceTarget = session.replyTargets?.[dispatch.logicalTurnId]; + if (sourceTarget !== undefined) { + session.replyTargets = { + ...(session.replyTargets ?? {}), + [dispatch.turnId]: structuredClone(sourceTarget), + }; + contextCopied = true; + } + if (contextCopied && session.readonlyTaskContinuation) { + deps.persist(session.readonlyTaskContinuation); + } + return deps.enqueue(dispatch); + }, + }; + coordinator = new ReadonlyTaskContinuationCoordinator(wrapped); + attachedContinuations.set(session.sessionId, { + session, + coordinator, + dispose: () => coordinator.dispose(), + }); + const restored = session.readonlyTaskContinuation; + if (restored) coordinator.restore(restored); +} + +export function startReadonlyTaskContinuation( + session: ReadonlyTaskContinuationSession, + input: StartReadonlyTaskContinuationInput, +): ReadonlyTaskContinuationState | undefined { + const attached = attachedContinuations.get(session.sessionId); + if (!attached) return session.readonlyTaskContinuation; + if (session.readonlyTaskContinuation) attached.coordinator.restore(session.readonlyTaskContinuation); + return attached.coordinator.start(input); +} + +export function handleReadonlyTaskContinuationTerminal( + session: ReadonlyTaskContinuationSession, + terminal: ReadonlyTaskContinuationTerminal, +): ReadonlyTaskContinuationState | undefined { + const attached = attachedContinuations.get(session.sessionId); + const current = session.readonlyTaskContinuation; + if (!attached || !current) return current; + return attached.coordinator.onTerminal(current, terminal); +} + +export function readonlyTaskContinuationHandlesTerminal( + session: ReadonlyTaskContinuationSession, + terminal: ReadonlyTaskContinuationTerminal, +): boolean { + return attachedContinuations.get(session.sessionId)?.session === session + && readonlyTaskContinuationRecoversTerminal(session.readonlyTaskContinuation, terminal); +} + +export function completeReadonlyTaskContinuation( + session: ReadonlyTaskContinuationSession, + turnId: string, + dispatchAttempt: number | undefined, + messageId: string, + workerGeneration: number, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.complete( + turnId, + dispatchAttempt, + messageId, + workerGeneration, + ) + ?? session.readonlyTaskContinuation; +} + +export function completeReadonlyTaskContinuationOriginalBusinessFinal( + session: ReadonlyTaskContinuationSession, + turnId: string, + workerGeneration: number, + messageId?: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.completeOriginalBusinessFinal( + turnId, workerGeneration, messageId, + ) ?? session.readonlyTaskContinuation; +} + +export function beginReadonlyTaskContinuationDelivery( + session: ReadonlyTaskContinuationSession, + turnId: string, + dispatchAttempt: number | undefined, + workerGeneration: number, + delivery: { kind: 'completed' | 'await_user'; content: string }, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.beginDelivery( + turnId, dispatchAttempt, workerGeneration, delivery, + ) ?? session.readonlyTaskContinuation; +} + +export function failReadonlyTaskContinuationVisible( + session: ReadonlyTaskContinuationSession, + turnId: string, + dispatchAttempt: number | undefined, + workerGeneration: number, + errorCode: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.failVisible( + turnId, dispatchAttempt, workerGeneration, errorCode, + ) ?? session.readonlyTaskContinuation; +} + +export function completeReadonlyTaskContinuationWarning( + session: ReadonlyTaskContinuationSession, + leaseId: string, + messageId: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.completeWarning(leaseId, messageId) + ?? session.readonlyTaskContinuation; +} + +export function failReadonlyTaskContinuationWarningDelivery( + session: ReadonlyTaskContinuationSession, + leaseId: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.warningDeliveryFailed(leaseId) + ?? session.readonlyTaskContinuation; +} + +export function finishReadonlyTaskContinuationAwaitUser( + session: ReadonlyTaskContinuationSession, + turnId: string, + dispatchAttempt: number | undefined, + messageId: string, + workerGeneration: number, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.finishAwaitUser( + turnId, dispatchAttempt, messageId, workerGeneration, + ) ?? session.readonlyTaskContinuation; +} + +export function awaitReadonlyTaskContinuationUser( + session: ReadonlyTaskContinuationSession, + turnId: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.awaitUser(turnId) + ?? session.readonlyTaskContinuation; +} + +export function cancelReadonlyTaskContinuationExplicit( + session: ReadonlyTaskContinuationSession, + turnId: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.cancelExplicit(turnId) + ?? session.readonlyTaskContinuation; +} + +export function cancelReadonlyTaskContinuationForUserInput( + session: ReadonlyTaskContinuationSession, + turnId: string, +): ReadonlyTaskContinuationState | undefined { + return attachedContinuations.get(session.sessionId)?.coordinator.cancelForUserInput(turnId) + ?? session.readonlyTaskContinuation; +} + +export function disposeReadonlyTaskContinuation( + session: Pick, +): void { + const attached = attachedContinuations.get(session.sessionId); + if (!attached) return; + attached.dispose(); + attachedContinuations.delete(session.sessionId); +} diff --git a/src/services/traex-transcript.ts b/src/services/traex-transcript.ts index 742a75f086..2c4d6b8e93 100644 --- a/src/services/traex-transcript.ts +++ b/src/services/traex-transcript.ts @@ -51,6 +51,8 @@ import { type CodexDrainResult, codexSessionIdFromRolloutPath, codexTaskFailureCode, + CODEX_OUTPUT_LIMIT_ERROR_CODE, + isExactCodexOutputLimitError, safeFailureSummary, } from './codex-transcript.js'; import { @@ -552,7 +554,9 @@ export function drainTraexRollout( // the real reason instead of an empty-final alert. ...(failed ? { terminalStatus: 'failed' as const, - terminalErrorCode: codexTaskFailureCode(payload.error), + terminalErrorCode: isExactCodexOutputLimitError(payload.error) + ? CODEX_OUTPUT_LIMIT_ERROR_CODE + : codexTaskFailureCode(payload.error), terminalErrorSummary: safeFailureSummary(payload.error), } : {}), }); diff --git a/src/types.ts b/src/types.ts index a468d18967..dc0396fae3 100644 --- a/src/types.ts +++ b/src/types.ts @@ -317,6 +317,9 @@ export interface Session { /** Crash-safe bounded recovery state for an ordinary Claude/Lark logical * turn. Timer ownership is runtime-only; this record re-arms it on restore. */ ordinaryTurnRecovery?: import('./services/ordinary-turn-recovery.js').OrdinaryTurnRecoveryState; + /** Explicit opt-in lease for one read-only long-running task. Disabled + * globally unless BOTMUX_READONLY_CONTINUATION_ENABLED=true. */ + readonlyTaskContinuation?: import('./services/readonly-task-continuation.js').ReadonlyTaskContinuationState; /** Dashboard 看板视图的手动放置：列 id（backlog/todo/in_progress/in_review/done）。 * 未设置时前端按运行状态推导默认列；一旦用户拖拽过就以此为准。 */ kanbanColumn?: string; @@ -834,6 +837,13 @@ export interface Session { }; } +/** Private daemon→worker capability stamp for one continuation dispatch. The + * worker accepts it only on the exact RPC generation that produced the proof. */ +export interface ReadonlyContinuationDispatchMarker { + leaseId: string; + rpcGeneration: string; +} + export interface SessionCliLaunchSnapshotV1 { version: 1; state: 'pending' | 'resolved'; @@ -1246,7 +1256,7 @@ type DaemonToWorkerBase = * the next message, with no restart IPC to refresh the snapshot. Same * three-state contract (undefined = not carried → keep snapshot; null = launch * with no model). It never affects the CLI already running. */ - | { type: 'message'; content: string; codexAppInput?: CodexAppTurnInput; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; turnId?: string; replyTurnId?: string; dispatchAttempt?: number; codexAppDispatchId?: string; codexAppSteerable?: true; queuedActivationToken?: string; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; trustedCaller?: TrustedCaller; atMostOnce?: true; mojoLivePatch?: MojoLivePatch; model?: string | null } + | { type: 'message'; content: string; codexAppInput?: CodexAppTurnInput; nativeSessionTitle?: string; nativeSessionTitlePrompt?: string; turnId?: string; replyTurnId?: string; dispatchAttempt?: number; codexAppDispatchId?: string; codexAppSteerable?: true; queuedActivationToken?: string; vcMeetingImTurnOrigin?: VcMeetingImTurnOrigin; trustedCaller?: TrustedCaller; atMostOnce?: true; mojoLivePatch?: MojoLivePatch; model?: string | null; readonlyContinuation?: ReadonlyContinuationDispatchMarker } | { type: 'codex_app_dispatch_persisted'; requestId: string; ok: boolean; error?: string } /** Literal slash-command passthrough. `followUpContent` rides along so the * worker enqueues it strictly AFTER the slash command's Enter — two separate @@ -1475,6 +1485,13 @@ export type WorkerToDaemon = /** Worker-side close handler has crossed the point where it will no longer * read bridge send markers or emit transcript fallback for this session. */ | { type: 'session_close_ready'; sessionId: string } + | { + type: 'readonly_continuation_rpc_status'; + sessionId: string; + rpcGeneration: string; + eligible: boolean; + reason?: string; + } | { type: 'prompt_ready' } | { type: 'runner_build_ready'; runnerBuildId: string } | { @@ -1504,7 +1521,7 @@ export type WorkerToDaemon = /** Worker observed a successful explicit `botmux send` for this turn, so * the daemon should treat listener-preview runs as visibly replied even * though transcript fallback output is suppressed to avoid duplicates. */ - | { type: 'explicit_reply_observed'; turnId: string; messageId?: string } + | { type: 'explicit_reply_observed'; turnId: string; messageId?: string; responseKind?: 'progress' | 'final' | 'auxiliary' } | { type: 'tui_prompt'; description: string; options: Array<{ label?: string; text: string; selected: boolean; type?: string; keys?: string[] }>; multiSelect?: boolean; turnId?: string; dispatchAttempt?: number } | { type: 'tui_prompt_resolved'; selectedText?: string; cardMessageId?: string; turnId?: string; dispatchAttempt?: number } | { type: 'tui_prompt_submit_failed'; cardMessageId?: string; stuckNonce?: number; turnId?: string; dispatchAttempt?: number } @@ -1527,7 +1544,7 @@ export type WorkerToDaemon = dispatchAttempt: number; disposition: 'queued_removed' | 'cli_fenced'; } - | { type: 'managed_turn_origin'; sessionId: string; capability: string; policyCapability?: string; originChannelId?: string; turnId?: string; dispatchAttempt?: number } + | { type: 'managed_turn_origin'; sessionId: string; capability: string; policyCapability?: string; originChannelId?: string; turnId?: string; dispatchAttempt?: number; readonlyContinuation?: true } /** An in-worker CLI restart rotates the managed-send authority without * replacing the Node worker. Carry the old token so the daemon can revoke * exactly that generation and ignore a delayed revoke after the next turn diff --git a/src/utils/pending-input-queue.ts b/src/utils/pending-input-queue.ts index e6100a2a00..e04160bc9d 100644 --- a/src/utils/pending-input-queue.ts +++ b/src/utils/pending-input-queue.ts @@ -1,4 +1,9 @@ -import type { CodexAppTurnInput, TrustedCaller, VcMeetingImTurnOrigin } from '../types.js'; +import type { + CodexAppTurnInput, + ReadonlyContinuationDispatchMarker, + TrustedCaller, + VcMeetingImTurnOrigin, +} from '../types.js'; export interface PendingCliInput { content: string; @@ -42,6 +47,7 @@ export interface PendingCliInput { * session is NOT dropped (codex #776 round-8). The worker's CLI-exit carry * predicate and pending-drop both honor it. */ noReplay?: boolean; + readonlyContinuation?: ReadonlyContinuationDispatchMarker; } /** @@ -104,7 +110,8 @@ export function mergeQueuedCliInput( || tail.codexAppInput || next.codexAppInput || tail.nativeSessionTitle || next.nativeSessionTitle || tail.nativeSessionTitlePrompt || next.nativeSessionTitlePrompt - || tail.logicalContent || next.logicalContent) return false; + || tail.logicalContent || next.logicalContent + || tail.readonlyContinuation || next.readonlyContinuation) return false; tail.content = `${tail.content}\n\n${next.content}`; tail.turnId = next.turnId ?? tail.turnId; return true; diff --git a/src/worker.ts b/src/worker.ts index 447392f57f..b860b25e7e 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -557,6 +557,7 @@ let remoteWsUrl: string | undefined; let remoteThreadId: string | undefined; let rpcDialogDismissTimer: ReturnType | null = null; let rpcEnginePidMarker: string | null = null; +let readonlyContinuationRpcGeneration: string | undefined; const piInitialPromptCleanupPaths: string[] = []; const piInitialPromptCleanupDirs: string[] = []; let piInitialPromptReadonlyRoots: string[] = []; @@ -713,6 +714,7 @@ function stopCodexRpcEngine(): void { // a restart. That stale continuation must never republish the stopped engine. rpcEngagementFence.invalidate(); const engine = codexRpcEngine; + readonlyContinuationRpcGeneration = undefined; const ownedRpcTurns = new Set([ ...rpcTurnsAwaitingActivation.keys(), ...rpcLifecycleFailClosedOwners.keys(), @@ -1222,6 +1224,8 @@ async function engageCodexRpc(cfg: Extract): P let engine: CodexRpcEngine | undefined; let enginePidMarker: string | null = null; let freshDeliveryOwned = false; + const readonlyContinuationEnabled = cfg.cliId === 'traex' + && process.env.BOTMUX_READONLY_CONTINUATION_ENABLED?.trim().toLowerCase() === 'true'; const assertRpcEngagementCurrent = (): void => { if (!rpcEngagementFence.isCurrent(engagementLease)) { throw new CliSpawnSupersededError(); @@ -1273,6 +1277,7 @@ async function engageCodexRpc(cfg: Extract): P appServerConfig: cfg.cliId === 'traex' ? [traexNativeSubagentHookConfig(nativeSubagentRuntimeHookCommand())] : undefined, + readonlyContinuationHardened: readonlyContinuationEnabled, onRequestUserInput: cfg.cliId === 'traex' ? (params: unknown) => bridgeTraexUserInput(cfg, params) : undefined, @@ -1439,6 +1444,19 @@ async function engageCodexRpc(cfg: Extract): P outcome = first.outcome; // accepted | ambiguous — both stay engaged, prompt never re-queued } codexRpcEngine = engine; + const capability = readonlyContinuationEnabled + ? await engine.checkReadonlyContinuationCapabilities() + : { ok: false, reason: 'readonly_continuation_disabled' }; + readonlyContinuationRpcGeneration = capability.ok + ? randomBytes(16).toString('hex') + : undefined; + send({ + type: 'readonly_continuation_rpc_status', + sessionId: cfg.sessionId, + rpcGeneration: readonlyContinuationRpcGeneration ?? 'unavailable', + eligible: capability.ok, + ...(capability.reason ? { reason: capability.reason } : {}), + }); remoteWsUrl = engine.wsUrl; remoteThreadId = threadId; persistCliSessionId(threadId); @@ -3023,6 +3041,10 @@ function publishSandboxRelayCapability(opts: { failClosed?: boolean } = {}): boo ...(capability.dispatchAttempt !== undefined ? { dispatchAttempt: capability.dispatchAttempt } : {}), + ...(currentBotmuxTurnId?.startsWith('bmx-readonly-') + && currentBotmuxDispatchAttempt !== undefined + ? { readonlyContinuation: true as const } + : {}), }); } return true; @@ -4710,12 +4732,16 @@ function explicitReplyMarkerForTurnWindow( return inWindow.at(-1); } -function notifyExplicitReplyObserved(turnId: string, marker: BridgeSendMarker | undefined): void { +function notifyExplicitReplyObserved( + turnId: string, + marker: BridgeSendMarker | undefined, +): void { if (!marker) return; send({ type: 'explicit_reply_observed', turnId, ...(marker.messageId ? { messageId: marker.messageId } : {}), + ...(marker.responseKind ? { responseKind: marker.responseKind } : {}), }); } @@ -11738,12 +11764,47 @@ async function flushPending(): Promise { let rpcTurnIdentity: CodexRpcTurnIdentity | undefined; let rpcTurnGeneration: RpcTurnGeneration | undefined; try { + if (item.readonlyContinuation && !writeRpcEngine) { + emitTurnTerminal( + item.turnId ?? 'readonly-continuation-unknown', + 'failed', + 'readonly_continuation_rpc_unavailable', + item.dispatchAttempt, + ); + break; + } if (writeRpcEngine) { + if (item.readonlyContinuation) { + const exactRestrictedInput = item.turnId?.startsWith('bmx-readonly-') + && item.dispatchAttempt !== undefined + && item.readonlyContinuation.rpcGeneration === readonlyContinuationRpcGeneration; + if (!exactRestrictedInput) { + emitTurnTerminal( + item.turnId ?? 'readonly-continuation-unknown', + 'failed', + 'readonly_continuation_rpc_proof_mismatch', + item.dispatchAttempt, + ); + break; + } + const capability = await writeRpcEngine.checkReadonlyContinuationCapabilities(); + if (!capability.ok) { + readonlyContinuationRpcGeneration = undefined; + emitTurnTerminal( + item.turnId!, + 'failed', + capability.reason ?? 'readonly_continuation_capability_probe_failed', + item.dispatchAttempt, + ); + break; + } + } rpcTurnIdentity = { turnId: item.turnId ?? `codex-rpc-${randomBytes(8).toString('hex')}`, ...(item.dispatchAttempt !== undefined ? { dispatchAttempt: item.dispatchAttempt } : {}), + ...(item.readonlyContinuation ? { readonlyContinuation: true } : {}), }; rpcTurnGeneration = { engine: writeRpcEngine, @@ -12149,6 +12210,7 @@ async function flushPending(): Promise { // adjacent IM turns wait for separate idle edges so neither can be // HOL-dropped or steered into the other. if (rpcLifecycleFailClosedOwners.size > 0) break; + if (item.readonlyContinuation) break; if (item.trustedCaller && lastInitConfig?.cliId === 'codex') break; if (shouldStopPendingBatch(item, pendingMessages[0])) break; } @@ -12212,6 +12274,7 @@ function sendToPty( * path's `atMostOnce → noReplay` for a keyed follow-up delivered to a LIVE * worker via `type: 'message'` (codex #776 round-8; turn-level PR #71). */ atMostOnce?: true; + readonlyContinuation?: import('./types.js').ReadonlyContinuationDispatchMarker; } = {}, ): boolean { const next: PendingCliInput = { @@ -12228,6 +12291,7 @@ function sendToPty( ...(opts.trustedCaller ? { trustedCaller: opts.trustedCaller } : {}), ...(opts.dispatchAttempt !== undefined ? { dispatchAttempt: opts.dispatchAttempt } : {}), ...(opts.atMostOnce ? { noReplay: true } : {}), + ...(opts.readonlyContinuation ? { readonlyContinuation: opts.readonlyContinuation } : {}), ...(opts.vcMeetingImTurnOrigin ? { vcMeetingImTurnOrigin: opts.vcMeetingImTurnOrigin } : {}), @@ -19902,6 +19966,7 @@ process.on('message', async (raw: unknown) => { trustedCaller: msg.trustedCaller, // Applied when THIS item is written, not on receipt. ...(msg.mojoLivePatch ? { mojoLivePatch: msg.mojoLivePatch } : {}), + ...(msg.readonlyContinuation ? { readonlyContinuation: msg.readonlyContinuation } : {}), ...(postSubmitNativeSessionTitle ? { nativeSessionTitle: postSubmitNativeSessionTitle } : {}), ...(msg.nativeSessionTitlePrompt ? { nativeSessionTitlePrompt: msg.nativeSessionTitlePrompt } : {}), }); diff --git a/test/codex-rpc-engine.test.ts b/test/codex-rpc-engine.test.ts index 63fe3c9814..a73fd88aee 100644 --- a/test/codex-rpc-engine.test.ts +++ b/test/codex-rpc-engine.test.ts @@ -28,6 +28,11 @@ const owner = (turnId: string, dispatchAttempt?: number) => ({ turnId, ...(dispatchAttempt !== undefined ? { dispatchAttempt } : {}), }); +const readonlyOwner = (turnId: string, dispatchAttempt: number) => ({ + turnId, + dispatchAttempt, + readonlyContinuation: true as const, +}); describe('CodexRpcEngine — happy-path lifecycle against a fake app-server', () => { it('passes exact argv and env to the model-owning app-server through a portable spawn seam', async () => { @@ -70,6 +75,161 @@ describe('CodexRpcEngine — happy-path lifecycle against a fake app-server', () } }, 20_000); + it('does not weaken ordinary turns with process-wide multi-agent overrides', async () => { + const launches: Array<{ args: string[] }> = []; + const engine = makeEngine({ readonlyContinuationHardened: true }, { + spawnProcess(command: string, args: string[], options: SpawnOptions): ChildProcess { + launches.push({ args: [...args] }); + return spawn(command, args, options); + }, + }); + try { + await engine.start(); + expect(launches[0]?.args).not.toContain('features.multi_agent=false'); + expect(launches[0]?.args).not.toContain('features.multi_agent_v2=false'); + } finally { + engine.stop(); + } + }, 20_000); + + it('uses turn-scoped read-only restrictions without changing thread capability selections', async () => { + const turnFile = join(tmpdir(), `fake-turn-cfg-${Math.round(performance.now())}.jsonl`); + const engine = makeEngine({ + readonlyContinuationHardened: true, + env: { ...process.env, FAKE_TURN_CONFIG_FILE: turnFile }, + }); + await engine.start(); + await engine.startThread(); + await engine.sendTurn('continue safely', readonlyOwner('readonly-1', 1)); + await engine.sendTurn('ordinary user turn', owner('ordinary-2', 2)); + engine.stop(); + const turns = readFileSync(turnFile, 'utf8').trim().split('\n').map(line => JSON.parse(line)); + rmSync(turnFile, { force: true }); + expect(turns[0]).toMatchObject({ + approvalPolicy: 'never', + sandboxPolicy: { type: 'readOnly', networkAccess: false }, + environments: [], + runtimeWorkspaceRoots: [], + }); + expect(turns[0].capabilities).toBeUndefined(); + expect(turns[0].multiAgentMode).toBeUndefined(); + expect(turns[1]).toMatchObject({ + approvalPolicy: 'never', + sandboxPolicy: { type: 'dangerFullAccess' }, + }); + expect(turns[1].environments).toBeUndefined(); + expect(turns[1].runtimeWorkspaceRoots).toBeUndefined(); + expect(turns[1].capabilities).toBeUndefined(); + expect(turns[1].multiAgentMode).toBeUndefined(); + }, 20_000); + + it('fails capability proof closed for external MCP, enabled tool skills, or unhardened runtime', async () => { + const unhardened = makeEngine(); + expect(await unhardened.checkReadonlyContinuationCapabilities()).toEqual({ + ok: false, reason: 'readonly_continuation_runtime_not_hardened', + }); + unhardened.stop(); + + for (const [env, reason] of [ + [{ FAKE_MCP_CAPABILITY: 'tools' }, 'readonly_continuation_external_mcp_capability'], + [{ FAKE_MCP_CAPABILITY: 'empty' }, 'readonly_continuation_external_mcp_capability'], + [{ FAKE_SKILL_TOOL_DEPENDENCY: '1' }, 'readonly_continuation_skill_tool_dependency'], + ] as const) { + const engine = makeEngine({ + readonlyContinuationHardened: true, + env: { ...process.env, ...env }, + }); + await engine.start(); + await engine.startThread(); + expect(await engine.checkReadonlyContinuationCapabilities()).toEqual({ ok: false, reason }); + engine.stop(); + } + + const eligible = makeEngine({ readonlyContinuationHardened: true }); + await eligible.start(); + await eligible.startThread(); + expect(await eligible.checkReadonlyContinuationCapabilities()).toEqual({ ok: true }); + eligible.stop(); + }, 20_000); + + it.each([ + { FAKE_MCP_RESPONSE: 'missing-data' }, + { FAKE_MCP_RESPONSE: 'malformed-data' }, + { FAKE_MCP_RESPONSE: 'malformed-cursor' }, + { FAKE_SKILLS_RESPONSE: 'missing-data' }, + { FAKE_SKILLS_RESPONSE: 'malformed-data' }, + { FAKE_SKILLS_RESPONSE: 'malformed-entry' }, + { FAKE_SKILLS_RESPONSE: 'errors' }, + { FAKE_SKILLS_RESPONSE: 'malformed-skill' }, + { FAKE_SKILLS_RESPONSE: 'malformed-dependencies' }, + { FAKE_SKILLS_RESPONSE: 'malformed-tool-dependency' }, + ])('fails malformed capability inventory closed: %o', async env => { + const engine = makeEngine({ + readonlyContinuationHardened: true, + env: { ...process.env, ...env }, + }); + try { + await engine.start(); + await engine.startThread(); + expect(await engine.checkReadonlyContinuationCapabilities()).toEqual({ + ok: false, reason: 'readonly_continuation_capability_probe_failed', + }); + } finally { + engine.stop(); + } + }, 20_000); + + it('fails every restricted server request closed even when it precedes turn/start ack', async () => { + const methods = [ + 'item/commandExecution/requestApproval', + 'item/fileChange/requestApproval', + 'execCommandApproval', + 'applyPatchApproval', + 'mcpServer/elicitation/request', + 'item/tool/call', + 'item/tool/requestUserInput', + 'item/permissions/requestApproval', + 'future/unknown/request', + ]; + const responseFile = join(tmpdir(), `fake-server-response-${Math.round(performance.now())}.jsonl`); + const terminals: any[] = []; + const engine = makeEngine({ + readonlyContinuationHardened: true, + env: { + ...process.env, + FAKE_SERVER_REQUEST_METHODS: methods.join(','), + FAKE_SERVER_RESPONSE_FILE: responseFile, + }, + onTurnTerminal: terminal => terminals.push(terminal), + }); + await engine.start(); + await engine.startThread(); + for (let index = 0; index < methods.length; index++) { + await engine.sendTurn(`restricted ${index}`, readonlyOwner(`restricted-${index}`, index + 1)); + await new Promise(resolve => setTimeout(resolve, 30)); + } + engine.stop(); + const responses = readFileSync(responseFile, 'utf8').trim().split('\n') + .filter(Boolean).map(line => JSON.parse(line)); + rmSync(responseFile, { force: true }); + expect(responses.map(response => response.result ?? response.error)).toEqual([ + { decision: 'cancel' }, + { decision: 'cancel' }, + { decision: 'abort' }, + { decision: 'abort' }, + { action: 'cancel', content: null, _meta: null }, + { contentItems: [], success: false }, + // requestUserInput is cancelled by turn/interrupt without a direct reply. + { code: -32000, message: 'server request denied in read-only continuation: item/permissions/requestApproval' }, + { code: -32000, message: 'server request denied in read-only continuation: future/unknown/request' }, + ]); + expect(terminals).toHaveLength(methods.length); + expect(terminals.every(terminal => terminal.status === 'aborted')).toBe(true); + expect((engine as any).readonlyNativeTurns.size).toBe(0); + expect((engine as any).pendingReadonlyTurnOwners.size).toBe(0); + expect((engine as any).deferredPreResponseServerRequests.size).toBe(0); + }, 20_000); + it('start (spawn → /readyz → connect → initialize) then startThread → sendTurn → stop', async () => { const engine = makeEngine(); await engine.start(); @@ -608,6 +768,30 @@ describe('CodexRpcEngine — failure/recovery paths', () => { engine.stop(); }, 20_000); + it('recognises the exact output-limit message even when a generic code is also present', async () => { + const terminals: any[] = []; + const engine = makeEngine({ + sessionId: 'terminal-output-limit', + env: { + ...process.env, + FAKE_TURN_STATUS: 'failed', + FAKE_TURN_ERROR_CODE: 'generic_turn_failed', + FAKE_TURN_ERROR_MESSAGE: 'model output limit exceeded: max_output_tokens', + }, + onTurnTerminal: terminal => terminals.push(terminal), + }); + await engine.start(); + await engine.startThread(); + await engine.sendTurn('output limit', owner('turn-output-limit', 7)); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(terminals).toEqual([expect.objectContaining({ + identity: { turnId: 'turn-output-limit', dispatchAttempt: 7 }, + status: 'failed', + errorCode: 'codex_output_limit_exceeded', + })]); + engine.stop(); + }, 20_000); + it('P1-1 sendFirstTurn: dispatched, no ack, NO rollout evidence → ambiguous (never downgraded to safe)', async () => { const engine = makeEngine({ sessionId: 'first-amb', env: { ...process.env, FAKE_HANG_TURN: '1' }, requestTimeoutMs: 400 }); await engine.start(); diff --git a/test/codex-transcript.test.ts b/test/codex-transcript.test.ts index 57583a7b8a..635c39b83f 100644 --- a/test/codex-transcript.test.ts +++ b/test/codex-transcript.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { mkdtempSync, writeFileSync, appendFileSync, rmSync, statSync, mkdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { CODEX_AUTH_ERROR_CODE, CODEX_CONNECTION_ERROR_CODE, CODEX_INVALID_REQUEST_ERROR_CODE, CODEX_RATE_LIMIT_ERROR_CODE, CODEX_TASK_FAILED_ERROR_CODE, CODEX_UPSTREAM_ERROR_CODE, codexTaskFailureCode, drainCodexRollout, codexSessionIdFromRolloutPath, findCodexRolloutBySessionId, findCodexSessionIdByBotmuxSessionId, codexHistorySidIsOwned, isCodexRateLimitEvent, splitCodexEventsByCutoff, extractLastCodexTurn, scanCodexThreadSettings, readLatestCodexRuntime, codexCotEntriesFromResponseItem, type CodexBridgeEvent } from '../src/services/codex-transcript.js'; +import { CODEX_AUTH_ERROR_CODE, CODEX_CONNECTION_ERROR_CODE, CODEX_INVALID_REQUEST_ERROR_CODE, CODEX_RATE_LIMIT_ERROR_CODE, CODEX_TASK_FAILED_ERROR_CODE, CODEX_UPSTREAM_ERROR_CODE, codexTaskFailureCode, drainCodexRollout, codexSessionIdFromRolloutPath, findCodexRolloutBySessionId, findCodexSessionIdByBotmuxSessionId, codexHistorySidIsOwned, isCodexRateLimitEvent, isExactCodexOutputLimitError, splitCodexEventsByCutoff, extractLastCodexTurn, scanCodexThreadSettings, readLatestCodexRuntime, codexCotEntriesFromResponseItem, type CodexBridgeEvent } from '../src/services/codex-transcript.js'; let dir: string; let path: string; @@ -273,6 +273,14 @@ describe('extractLastCodexTurn', () => { }); describe('codexTaskFailureCode (shared Codex-family failure classifier)', () => { + it('keeps the exact output-limit discriminator separate from the shared classifier', () => { + const exact = 'model output limit exceeded: max_output_tokens'; + expect(isExactCodexOutputLimitError(exact)).toBe(true); + expect(isExactCodexOutputLimitError({ message: ` ${exact.toUpperCase()} ` })).toBe(true); + expect(isExactCodexOutputLimitError(`${exact}: extra`)).toBe(false); + expect(codexTaskFailureCode(exact)).toBe(CODEX_TASK_FAILED_ERROR_CODE); + }); + it('classifies model gateway / upstream failures as codex_upstream_error', () => { // Live incident shape: the model gateway cancelled the stream mid-turn. expect(codexTaskFailureCode( diff --git a/test/dashboard-create-session.test.ts b/test/dashboard-create-session.test.ts index f22fd0a95f..a7b79bed61 100644 --- a/test/dashboard-create-session.test.ts +++ b/test/dashboard-create-session.test.ts @@ -71,6 +71,7 @@ vi.mock('../src/core/worker-pool.js', () => ({ getCurrentCliVersion: vi.fn(() => 'test-cli-v1'), restoreUsageLimitRuntimeState: vi.fn(), ensureOrdinaryTurnRecoveryAttached: vi.fn(), + ensureReadonlyTaskContinuationAttached: vi.fn(), setActiveSessionIfActive: vi.fn((map: Map, k: string, ds: any) => { if (map.has(k) && map.get(k) !== ds) return false; map.set(k, ds); diff --git a/test/dashboard-ipc.test.ts b/test/dashboard-ipc.test.ts index 46a60ebfb4..01a4c82145 100644 --- a/test/dashboard-ipc.test.ts +++ b/test/dashboard-ipc.test.ts @@ -547,6 +547,27 @@ describe('POST /api/sessions/:sessionId/native-subagent-runtime', () => { }); }); + it('denies native subagents for the exact live read-only continuation turn', async () => { + const active = installRuntimeSession({ model: { mode: 'custom', value: 'session-model' } }); + active.workerGeneration = 3; + active.managedTurnOrigin = { + ...active.managedTurnOrigin, turnId: 'bmx-readonly-exact', dispatchAttempt: 2, + }; + active.readonlyContinuationTurnOrigin = { + workerGeneration: 3, turnId: 'bmx-readonly-exact', dispatchAttempt: 2, + }; + setIpcAuthSecret(TEST_IPC_SECRET); + handle = await startIpcServer({ port: 0, host: '127.0.0.1', authRequired: true }); + const path = `/api/sessions/${SESSION_ID}/native-subagent-runtime`; + + const res = await post({}, trustedHostHeaders('POST', path, handle.port)); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + ok: true, deny: true, reason: 'read-only continuation forbids subagents', + }); + }); + it('signs the exact trusted-host response with the request challenge', async () => { installRuntimeSession({ model: { mode: 'custom', value: 'session-model' } }); setIpcAuthSecret(TEST_IPC_SECRET); diff --git a/test/fixtures/fake-codex-rpc-server.mjs b/test/fixtures/fake-codex-rpc-server.mjs index ffc932e4f1..7480360860 100755 --- a/test/fixtures/fake-codex-rpc-server.mjs +++ b/test/fixtures/fake-codex-rpc-server.mjs @@ -16,12 +16,14 @@ // on resume) import { createServer } from 'node:http'; import { WebSocketServer } from 'ws'; -import { writeFileSync } from 'node:fs'; +import { appendFileSync, writeFileSync } from 'node:fs'; const listenArg = process.argv[process.argv.indexOf('--listen') + 1] || ''; const m = listenArg.match(/ws:\/\/127\.0\.0\.1:(\d+)/); const port = m ? Number(m[1]) : 0; const HANG_TURN = process.env.FAKE_HANG_TURN === '1'; +const TURN_ERROR_CODE = process.env.FAKE_TURN_ERROR_CODE || 'fake_failed'; +const TURN_ERROR_MESSAGE = process.env.FAKE_TURN_ERROR_MESSAGE || 'fake failure'; const HANG_TURN_NOTIFY = process.env.FAKE_HANG_TURN_NOTIFY === '1'; const TERMINAL_BEFORE_RESPONSE = process.env.FAKE_TERMINAL_BEFORE_RESPONSE === '1'; const ERROR_AFTER_STARTED = process.env.FAKE_ERROR_AFTER_STARTED === '1'; @@ -36,6 +38,9 @@ const UPDATED_AFTER = Number(process.env.FAKE_UPDATED_AFTER ?? '101'); let threadReadAttempt = 0; let currentThreadName; const REQUEST_USER_INPUT = process.env.FAKE_REQUEST_USER_INPUT === '1'; +const SERVER_REQUEST_METHODS = (process.env.FAKE_SERVER_REQUEST_METHODS ?? '') + .split(',').map(value => value.trim()).filter(Boolean); +const serverRequestIds = new Set(); let turnCount = 0; const httpServer = createServer((req, res) => { @@ -60,7 +65,7 @@ wss.on('connection', (ws) => { id: nativeTurnId, ...(status ? { status } : {}), ...(status === 'failed' - ? { error: { code: 'fake_failed', message: 'fake failure' } } + ? { error: { code: TURN_ERROR_CODE, message: TURN_ERROR_MESSAGE } } : {}), }; const completed = JSON.stringify({ @@ -77,6 +82,13 @@ wss.on('connection', (ws) => { }; ws.on('message', (data) => { let msg; try { msg = JSON.parse(data.toString()); } catch { return; } + if (serverRequestIds.has(msg.id) && (msg.result !== undefined || msg.error !== undefined)) { + if (process.env.FAKE_SERVER_RESPONSE_FILE) { + try { appendFileSync(process.env.FAKE_SERVER_RESPONSE_FILE, `${JSON.stringify(msg)}\n`); } catch { /* test-only */ } + } + serverRequestIds.delete(msg.id); + return; + } if (REQUEST_USER_INPUT && msg.id === 900 && (msg.result !== undefined || msg.error !== undefined)) { if (!pendingTurnReply) return; // Real traex 0.200.19 normalizes ANY reply to requestUserInput (empty @@ -122,6 +134,38 @@ wss.on('connection', (ws) => { updatedAt: threadReadAttempt > UPDATED_DELAY_READS ? UPDATED_AFTER : UPDATED_BEFORE, } }); case 'thread/name/set': currentThreadName = msg.params?.name; return reply({}); + case 'mcpServerStatus/list': { + const responseMode = process.env.FAKE_MCP_RESPONSE ?? ''; + if (responseMode === 'missing-data') return reply({}); + if (responseMode === 'malformed-data') return reply({ data: {} }); + if (responseMode === 'malformed-cursor') return reply({ data: [], nextCursor: 7 }); + const mcpMode = process.env.FAKE_MCP_CAPABILITY ?? ''; + return reply({ data: mcpMode === 'tools' + ? [{ name: 'fake', tools: { mutate: {} } }] + : mcpMode === 'empty' ? [{ name: 'fake', tools: {} }] : [] }); + } + case 'skills/list': { + const responseMode = process.env.FAKE_SKILLS_RESPONSE ?? ''; + if (responseMode === 'missing-data') return reply({}); + if (responseMode === 'malformed-data') return reply({ data: {} }); + if (responseMode === 'malformed-entry') return reply({ data: [{}] }); + if (responseMode === 'errors') return reply({ + data: [{ cwd: process.cwd(), errors: [{ path: '/fake', message: 'broken' }], skills: [] }], + }); + if (responseMode === 'malformed-skill') return reply({ + data: [{ cwd: process.cwd(), errors: [], skills: [{}] }], + }); + if (responseMode === 'malformed-dependencies') return reply({ + data: [{ cwd: process.cwd(), errors: [], skills: [{ name: 'fake', enabled: true, dependencies: {} }] }], + }); + if (responseMode === 'malformed-tool-dependency') return reply({ + data: [{ cwd: process.cwd(), errors: [], skills: [{ name: 'fake', enabled: true, dependencies: { tools: ['shell'] } }] }], + }); + const hasToolDependency = process.env.FAKE_SKILL_TOOL_DEPENDENCY === '1'; + return reply({ data: [{ cwd: process.cwd(), errors: [], skills: hasToolDependency + ? [{ name: 'fake', enabled: true, dependencies: { tools: [{ type: 'command', value: 'shell' }] } }] + : [] }] }); + } case 'turn/interrupt': { // FAKE_INTERRUPT_ERROR=1 models an interrupt that itself fails: the // app-server rejects turn/interrupt with a JSON-RPC error. The engine @@ -154,6 +198,9 @@ wss.on('connection', (ws) => { turnCount++; const nativeTurnId = `turn-fake-${turnCount}`; const threadId = msg.params?.threadId; + if (process.env.FAKE_TURN_CONFIG_FILE) { + try { appendFileSync(process.env.FAKE_TURN_CONFIG_FILE, `${JSON.stringify(msg.params ?? {})}\n`); } catch { /* test-only */ } + } if (HANG_TURN) { if (HANG_TURN_NOTIFY) emitTurnLifecycle(threadId, nativeTurnId); return; @@ -178,6 +225,24 @@ wss.on('connection', (ws) => { })); return; } + if (SERVER_REQUEST_METHODS.length > 0) { + pendingTurnReply = msg.id; + pendingNativeTurnId = nativeTurnId; + pendingThreadId = threadId; + const requestId = 1900 + turnCount; + const method = SERVER_REQUEST_METHODS[(turnCount - 1) % SERVER_REQUEST_METHODS.length]; + serverRequestIds.add(requestId); + ws.send(JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method, + params: { threadId, turnId: nativeTurnId, itemId: `item-fake-${turnCount}` }, + })); + // Deliberately acknowledge after the server request. WebSocket frame + // ordering locks the pre-response ownership race under test. + reply({ turn: { id: nativeTurnId } }); + return; + } if (ERROR_AFTER_STARTED) { emitTurnStarted(threadId, nativeTurnId); ws.send(JSON.stringify({ diff --git a/test/ipc-readonly-continuation-route.test.ts b/test/ipc-readonly-continuation-route.test.ts new file mode 100644 index 0000000000..c6843fc2d5 --- /dev/null +++ b/test/ipc-readonly-continuation-route.test.ts @@ -0,0 +1,173 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + setIpcAuthSecret, + startIpcServer, + type IpcServerHandle, +} from '../src/core/dashboard-ipc-server.js'; +import * as botRegistry from '../src/bot-registry.js'; +import * as sessionStore from '../src/services/session-store.js'; +import * as workerPool from '../src/core/worker-pool.js'; +import { disposeReadonlyTaskContinuation } from '../src/services/readonly-task-continuation.js'; + +const CAP = 'ab12cd34'.repeat(8); +const SESSION_ID = 's-readonly-continuation'; +let handle: IpcServerHandle | null = null; + +function session(overrides: Record = {}) { + const worker = { killed: false, connected: true }; + return { + session: { + sessionId: SESSION_ID, + status: 'active', + cliId: 'traex', + workerGeneration: 3, + ...((overrides.session as Record | undefined) ?? {}), + }, + managedTurnOrigin: { capability: CAP, turnId: 'om_original' }, + worker, + workerReady: true, + workerGeneration: 3, + readonlyContinuationRpcProof: { workerGeneration: 3, rpcGeneration: 'rpc-proof', checkedAt: 1 }, + larkAppId: 'app-1', + chatId: 'oc-chat', + chatType: 'group', + scope: 'thread', + ...overrides, + } as any; +} + +async function post(body: Record): Promise { + if (!handle) { + handle = await startIpcServer({ port: 0, host: '127.0.0.1', authRequired: true }); + } + return fetch(`http://127.0.0.1:${handle.port}/api/sessions/${SESSION_ID}/continuation`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + originCapability: CAP, + originTurnId: 'om_original', + ...body, + }), + }); +} + +afterEach(async () => { + if (handle) await handle.close(); + handle = null; + setIpcAuthSecret(null); + delete process.env.BOTMUX_READONLY_CONTINUATION_ENABLED; + disposeReadonlyTaskContinuation({ sessionId: SESSION_ID }); + vi.restoreAllMocks(); +}); + +describe('POST /api/sessions/:sessionId/continuation', () => { + it('starts, hands off to the user, and cancels only the current ordinary TraeX turn', async () => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + const ds = session(); + vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue(ds); + vi.spyOn(botRegistry, 'getBot').mockReturnValue({ config: { cliId: 'traex' } } as any); + vi.spyOn(sessionStore, 'updateSession').mockImplementation(() => undefined); + + const started = await post({ + action: 'start', + readonly: true, + ttlMs: 120_000, + maxContinuations: 2, + }); + expect(started.status).toBe(200); + expect(await started.json()).toMatchObject({ + ok: true, + state: { + logicalTurnId: 'om_original', + currentTurnId: 'om_original', + status: 'active', + maxContinuations: 2, + }, + }); + + const awaiting = await post({ action: 'await-user' }); + expect(awaiting.status).toBe(200); + expect(await awaiting.json()).toMatchObject({ ok: true, state: { status: 'awaiting_user' } }); + + const restarted = await post({ action: 'start', readonly: true }); + expect(restarted.status).toBe(200); + const cancelled = await post({ action: 'cancel' }); + expect(cancelled.status).toBe(200); + expect(await cancelled.json()).toMatchObject({ ok: true, state: { status: 'cancelled' } }); + }); + + it('rejects missing capability, stale turn, synthetic turn, and dispatch attempts', async () => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + const ds = session(); + vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue(ds); + vi.spyOn(botRegistry, 'getBot').mockReturnValue({ config: { cliId: 'traex' } } as any); + vi.spyOn(sessionStore, 'updateSession').mockImplementation(() => undefined); + + const missingCapability = await post({ originCapability: undefined, action: 'start', readonly: true }); + expect(missingCapability.status).toBe(403); + + const staleTurn = await post({ originTurnId: 'om_stale', action: 'start', readonly: true }); + expect(staleTurn.status).toBe(409); + expect(await staleTurn.json()).toMatchObject({ ok: false, error: 'active_turn_required' }); + + ds.managedTurnOrigin = { capability: CAP, turnId: 'bmx-synthetic' }; + const synthetic = await post({ + originTurnId: 'bmx-synthetic', + action: 'start', + readonly: true, + }); + expect(synthetic.status).toBe(409); + expect(await synthetic.json()).toMatchObject({ ok: false, error: 'ordinary_user_turn_required' }); + + ds.managedTurnOrigin = { capability: CAP, turnId: 'om_attempt', dispatchAttempt: 2 }; + const retryAttempt = await post({ + originTurnId: 'om_attempt', + originDispatchAttempt: 2, + action: 'start', + readonly: true, + }); + expect(retryAttempt.status).toBe(409); + expect(await retryAttempt.json()).toMatchObject({ ok: false, error: 'ordinary_user_turn_required' }); + }); + + it('is unavailable by default and for non-TraeX sessions', async () => { + const ds = session(); + vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue(ds); + const botSpy = vi.spyOn(botRegistry, 'getBot') + .mockReturnValue({ config: { cliId: 'traex' } } as any); + + const disabled = await post({ action: 'start', readonly: true }); + expect(disabled.status).toBe(409); + expect(await disabled.json()).toMatchObject({ + ok: false, error: 'readonly_continuation_unavailable', + }); + + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + botSpy.mockReturnValue({ config: { cliId: 'codex' } } as any); + ds.session.cliId = 'codex'; + const nonTraex = await post({ action: 'start', readonly: true }); + expect(nonTraex.status).toBe(409); + expect(await nonTraex.json()).toMatchObject({ + ok: false, error: 'readonly_continuation_unavailable', + }); + }); + + it('returns non-200 and retains a failed fence when cancel persistence fails', async () => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + const ds = session(); + vi.spyOn(workerPool, 'findActiveBySessionId').mockReturnValue(ds); + vi.spyOn(botRegistry, 'getBot').mockReturnValue({ config: { cliId: 'traex' } } as any); + const persist = vi.spyOn(sessionStore, 'updateSession').mockImplementation(() => undefined); + + expect((await post({ action: 'start', readonly: true })).status).toBe(200); + persist.mockImplementation(() => { throw new Error('session store unavailable'); }); + const cancelled = await post({ action: 'cancel' }); + + expect(cancelled.status).toBe(409); + expect(await cancelled.json()).toEqual({ ok: false, error: 'session store unavailable' }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'failed', + lastErrorCode: 'readonly_continuation_explicit_cancel_persist_failed', + }); + }); +}); diff --git a/test/native-subagent-runtime-hook.test.ts b/test/native-subagent-runtime-hook.test.ts index 0e68d12435..83c7cafb0f 100644 --- a/test/native-subagent-runtime-hook.test.ts +++ b/test/native-subagent-runtime-hook.test.ts @@ -471,6 +471,22 @@ describe('native-subagent-runtime-hook CLI', () => { expect(overloaded.stderr).toContain('policy service overloaded; denying spawn'); }); + it('denies spawn when the authenticated daemon binds the turn to read-only continuation', async () => { + const denied = await runHook(JSON.stringify(spawnPayload), { + response: { ok: true, deny: true, reason: 'read-only continuation forbids subagents' }, + }); + + expect(denied.status).toBe(0); + expect(JSON.parse(denied.stdout)).toEqual({ + hookSpecificOutput: { + hookEventName: 'PreToolUse', + permissionDecision: 'deny', + permissionDecisionReason: 'read-only continuation forbids subagents', + }, + }); + expect(denied.stderr).toContain('daemon denied spawn for read-only continuation'); + }); + it('cancels oversized and timed-out never-ending response streams', async () => { const oversized = await runHook(JSON.stringify(spawnPayload), { responseMode: 'oversized-never-ending', diff --git a/test/readonly-task-continuation.test.ts b/test/readonly-task-continuation.test.ts new file mode 100644 index 0000000000..9e4cedd5e3 --- /dev/null +++ b/test/readonly-task-continuation.test.ts @@ -0,0 +1,550 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + attachReadonlyTaskContinuation, + awaitReadonlyTaskContinuationUser, + cancelReadonlyTaskContinuationForUserInput, + completeReadonlyTaskContinuation, + disposeReadonlyTaskContinuation, + handleReadonlyTaskContinuationTerminal, + READONLY_TASK_CONTINUATION_OUTPUT_LIMIT_CODE, + READONLY_TASK_CONTINUATION_PROMPT, + ReadonlyTaskContinuationCoordinator, + startReadonlyTaskContinuation, + type ReadonlyTaskContinuationSession, + type ReadonlyTaskContinuationState, +} from '../src/services/readonly-task-continuation.js'; + +function state(overrides: Partial = {}): ReadonlyTaskContinuationState { + return { + leaseId: 'readonly-lease', + logicalTurnId: 'om_original', + currentTurnId: 'om_original', + createdAt: 1_000, + expiresAt: 61_000, + maxContinuations: 2, + continuationsStarted: 0, + currentWorkerGeneration: 1, + status: 'active', + ...overrides, + }; +} + +afterEach(() => { + disposeReadonlyTaskContinuation({ sessionId: 'session' }); +}); + +describe('ReadonlyTaskContinuationCoordinator', () => { + it('continues a completed turn without replaying the original prompt', () => { + const timers: Array<() => void> = []; + const enqueue = vi.fn(() => 7); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist: vi.fn(), + enqueue, + warn: vi.fn(), + enabled: () => true, + now: () => 2_000, + randomId: () => 'next', + delayMs: 1_000, + }); + coordinator.restore(state()); + + expect(coordinator.onTerminal(state(), { + turnId: 'om_original', + status: 'completed', + workerGeneration: 1, + }).status).toBe('backoff'); + timers.at(-1)!(); + expect((coordinator as any).state).toMatchObject({ currentWorkerGeneration: 7 }); + + expect(enqueue).toHaveBeenCalledWith({ + logicalTurnId: 'om_original', + turnId: 'bmx-readonly-next', + dispatchAttempt: 1, + prompt: READONLY_TASK_CONTINUATION_PROMPT, + continuation: 1, + }); + expect(enqueue.mock.calls[0][0].prompt).not.toContain('original user prompt'); + }); + + it('continues only the exact allowlisted failed terminal', () => { + const timers: Array<() => void> = []; + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist: vi.fn(), + enqueue: vi.fn(() => 1), + warn: vi.fn(), + enabled: () => true, + now: () => 2_000, + now: () => 2_000, + }); + coordinator.restore(state()); + + expect(coordinator.onTerminal(state(), { + turnId: 'om_original', + status: 'failed', + errorCode: READONLY_TASK_CONTINUATION_OUTPUT_LIMIT_CODE, + workerGeneration: 1, + }).status).toBe('backoff'); + expect(timers).toHaveLength(2); + + const ordinaryFailure = state({ leaseId: 'other' }); + coordinator.restore(ordinaryFailure); + expect(coordinator.onTerminal(ordinaryFailure, { + turnId: 'om_original', + status: 'failed', + errorCode: 'codex_connection_failed', + workerGeneration: 1, + }).status).toBe('failed'); + expect(timers).toHaveLength(3); + }); + + it('waits for worker/RPC readiness without consuming a continuation attempt', () => { + const timers: Array<() => void> = []; + let ready = false; + const enqueue = vi.fn(() => 9); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist: vi.fn(), + canEnqueue: () => ready, + enqueue, + warn: vi.fn(), + enabled: () => true, + now: () => 2_000, + randomId: () => 'next', + delayMs: 1_000, + }); + coordinator.restore(state()); + coordinator.onTerminal(state(), { + turnId: 'om_original', status: 'completed', workerGeneration: 1, + }); + + timers.at(-1)!(); + expect(enqueue).not.toHaveBeenCalled(); + expect((coordinator as any).state).toMatchObject({ + status: 'backoff', continuationsStarted: 0, nextAttemptAt: 3_000, + }); + + ready = true; + timers.at(-1)!(); + expect(enqueue).toHaveBeenCalledOnce(); + expect((coordinator as any).state).toMatchObject({ + status: 'active', continuationsStarted: 1, currentWorkerGeneration: 9, + }); + }); + + it.each([ + ['new user input', (coordinator: ReadonlyTaskContinuationCoordinator) => + coordinator.cancelForUserInput('om_new'), 'readonly_continuation_user_cancel_persist_failed'], + ['explicit cancel', (coordinator: ReadonlyTaskContinuationCoordinator) => + coordinator.cancelExplicit('om_original'), 'readonly_continuation_explicit_cancel_persist_failed'], + ] as const)('retains a failed fence when %s cannot persist', (_label, cancel, errorCode) => { + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist: vi.fn(() => { throw new Error('store unavailable'); }), + enqueue: vi.fn(() => 1), + warn: vi.fn(), + enabled: () => true, + now: () => 2_000, + }); + coordinator.restore(state()); + + expect(() => cancel(coordinator)).toThrow('store unavailable'); + expect((coordinator as any).state).toMatchObject({ + status: 'failed', lastErrorCode: errorCode, + }); + }); + + it('settles only on an explicit final proof for the current turn', () => { + const persist = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist, + enqueue: vi.fn(() => 1), + warn: vi.fn(), + enabled: () => true, + now: () => 2_000, + }); + coordinator.restore(state()); + + expect(coordinator.complete('stale-turn', undefined, 'om_reply', 1)).toEqual(state()); + expect(coordinator.complete('om_original', 1, 'om_wrong_attempt', 1)).toEqual(state()); + expect(coordinator.complete('om_original', undefined, 'om_reply', 1)).toMatchObject({ + status: 'completed', + completedMessageId: 'om_reply', + }); + expect(persist).toHaveBeenCalledTimes(1); + }); + + it('stops for user input or an explicit await-user handoff', () => { + const cancel = vi.fn(); + const persist = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel, + persist, + enqueue: vi.fn(() => 1), + warn: vi.fn(), + enabled: () => true, + now: () => 2_000, + }); + coordinator.restore(state({ status: 'backoff', nextAttemptAt: 12_000 })); + expect(coordinator.cancelForUserInput('om_new')).toMatchObject({ + status: 'cancelled', + cancelledByTurnId: 'om_new', + }); + + coordinator.restore(state()); + expect(coordinator.awaitUser('om_original')).toMatchObject({ status: 'awaiting_user' }); + expect(coordinator.cancelForUserInput('om_answer')).toMatchObject({ + status: 'cancelled', + cancelledByTurnId: 'om_answer', + }); + expect(cancel).toHaveBeenCalledTimes(2); + expect(persist).toHaveBeenCalledTimes(3); + }); + + it('allows explicit cancellation while awaiting user input', () => { + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist: vi.fn(), + enqueue: vi.fn(() => 1), + warn: vi.fn(), + enabled: () => true, + }); + coordinator.restore(state({ status: 'awaiting_user' })); + + expect(coordinator.cancelExplicit('om_original')).toMatchObject({ + status: 'cancelled', + cancelledByTurnId: 'om_original', + }); + }); + + it('expires and exhausts with one warning instead of dispatching forever', () => { + const warn = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist: vi.fn(), + enqueue: vi.fn(() => 1), + warn, + enabled: () => true, + now: () => 61_000, + }); + coordinator.restore(state()); + expect(coordinator.onTerminal(state(), { + turnId: 'om_original', status: 'completed', workerGeneration: 1, + }).status).toBe('expired'); + expect(warn).toHaveBeenCalledTimes(1); + + const exhausted = state({ expiresAt: 120_000, continuationsStarted: 2 }); + coordinator.restore(exhausted); + expect(coordinator.onTerminal(exhausted, { + turnId: 'om_original', status: 'completed', workerGeneration: 1, + }).status).toBe('exhausted'); + expect(warn).toHaveBeenCalledTimes(2); + }); + + it('fails closed when the kill switch is off', () => { + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist: vi.fn(), + enqueue: vi.fn(() => 1), + warn: vi.fn(), + enabled: () => false, + }); + + expect(() => coordinator.start({ turnId: 'om_original', workerGeneration: 1 })) + .toThrow('readonly_continuation_disabled'); + coordinator.restore(state({ status: 'backoff', nextAttemptAt: Date.now() + 10_000 })); + expect(coordinator.onTerminal(state(), { + turnId: 'om_original', status: 'completed', workerGeneration: 1, + }).status).toBe('cancelled'); + }); + + it('expires an active lease even when the CLI never emits another terminal', () => { + const timers: Array<() => void> = []; + let now = 1_000; + const warn = vi.fn(); + const persist = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist, + enqueue: vi.fn(() => 1), + warn, + enabled: () => true, + now: () => now, + randomId: () => 'lease', + }); + + expect(coordinator.start({ turnId: 'om_original', workerGeneration: 1, ttlMs: 5_000 }).status).toBe('active'); + expect(timers).toHaveLength(1); + now = 6_000; + timers.at(-1)!(); + expect(persist).toHaveBeenLastCalledWith(expect.objectContaining({ + status: 'expired', + lastErrorCode: 'readonly_continuation_expired', + })); + expect(warn).toHaveBeenCalledOnce(); + }); + + it('fails closed without crashing when activation persistence fails after enqueue', () => { + const timers: Array<() => void> = []; + let persistCalls = 0; + const warn = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist: vi.fn(() => { + persistCalls++; + if (persistCalls === 3) throw new Error('store unavailable'); + }), + enqueue: vi.fn(() => 1), + warn, + enabled: () => true, + now: () => 2_000, + randomId: () => 'next', + }); + coordinator.restore(state()); + coordinator.onTerminal(state(), { turnId: 'om_original', status: 'completed', workerGeneration: 1 }); + + expect(() => timers.at(-1)!()).not.toThrow(); + expect(warn).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + lastErrorCode: 'readonly_continuation_activation_persist_failed', + })); + expect((coordinator as any).state).toMatchObject({ + status: 'failed', + lastErrorCode: 'readonly_continuation_activation_persist_failed', + pendingWarning: { startedAt: 2_000, deliveryAttempts: 0 }, + }); + }); + + it('never enqueues when the dispatching fence cannot be persisted', () => { + const timers: Array<() => void> = []; + let persistCalls = 0; + const enqueue = vi.fn(() => 1); + const warn = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist: vi.fn(() => { + persistCalls++; + if (persistCalls === 2) throw new Error('store unavailable'); + }), + enqueue, + warn, + enabled: () => true, + now: () => 2_000, + randomId: () => 'next', + }); + coordinator.restore(state()); + coordinator.onTerminal(state(), { turnId: 'om_original', status: 'completed', workerGeneration: 1 }); + + expect(() => timers.at(-1)!()).not.toThrow(); + expect(enqueue).not.toHaveBeenCalled(); + expect((coordinator as any).state).toMatchObject({ + status: 'failed', + lastErrorCode: 'readonly_continuation_dispatch_persist_failed', + }); + expect(warn).toHaveBeenCalledOnce(); + }); + + it('recovers only a recent persisted daemon-owned delivery', () => { + const recoverDelivery = vi.fn(); + const warn = vi.fn(); + const persist = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist, + enqueue: vi.fn(() => 1), + warn, + recoverDelivery, + enabled: () => true, + now: () => 2_000, + }); + const delivering = state({ + status: 'delivering', + pendingDelivery: { kind: 'completed', content: 'done', startedAt: 1_500 }, + }); + + coordinator.restore(delivering); + + expect(recoverDelivery).toHaveBeenCalledWith(delivering); + expect(warn).not.toHaveBeenCalled(); + expect(persist).not.toHaveBeenCalled(); + }); + + it('fails visibly instead of replaying a stale or legacy delivery', () => { + const warn = vi.fn(); + const persist = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist, + enqueue: vi.fn(() => 1), + warn, + recoverDelivery: vi.fn(), + enabled: () => true, + now: () => 60 * 60_000, + }); + + coordinator.restore(state({ + status: 'delivering', + pendingDelivery: { kind: 'completed', content: 'done', startedAt: 1_000 }, + })); + + expect(warn).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + lastErrorCode: 'readonly_continuation_delivery_recovery_unavailable', + })); + expect(persist).toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); + }); + + it('replays a recent persisted warning until delivery is acknowledged', () => { + const warn = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist: vi.fn(), + enqueue: vi.fn(() => 1), + warn, + enabled: () => true, + now: () => 2_000, + }); + const pending = state({ + status: 'failed', + lastErrorCode: 'readonly_continuation_enqueue_failed', + pendingWarning: { startedAt: 1_500, deliveryAttempts: 0 }, + }); + + coordinator.restore(pending); + + expect(warn).toHaveBeenCalledWith(pending); + expect((coordinator as any).state.warningDispatched).toBeUndefined(); + expect(coordinator.completeWarning(pending.leaseId, 'om_warning')).toMatchObject({ + warningDispatched: true, + warningMessageId: 'om_warning', + pendingWarning: undefined, + }); + }); + + it('does not publish a warning whose outbox state failed to persist', () => { + const warn = vi.fn(); + const attend = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => run, + cancel: vi.fn(), + persist: vi.fn(() => { throw new Error('store unavailable'); }), + enqueue: vi.fn(() => 1), + warn, + attend, + enabled: () => true, + now: () => 2_000, + }); + coordinator.restore(state()); + + coordinator.failVisible('om_original', undefined, 1, 'terminal_failure'); + + expect((coordinator as any).state).toMatchObject({ + status: 'failed', + lastErrorCode: 'terminal_failure', + pendingWarning: expect.any(Object), + }); + expect(attend).not.toHaveBeenCalled(); + expect(warn).not.toHaveBeenCalled(); + }); + + it('keeps retrying a pending warning until its delivery window expires', () => { + const timers: Array<() => void> = []; + let now = 2_000; + const warn = vi.fn(); + const persist = vi.fn(); + const attend = vi.fn(); + const coordinator = new ReadonlyTaskContinuationCoordinator({ + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist, + enqueue: vi.fn(() => 1), + warn, + attend, + enabled: () => true, + now: () => now, + }); + const pending = state({ + status: 'failed', + pendingWarning: { startedAt: 1_500, deliveryAttempts: 0 }, + }); + coordinator.restore(pending); + expect(attend).toHaveBeenCalledOnce(); + expect(warn).toHaveBeenCalledOnce(); + + coordinator.warningDeliveryFailed(pending.leaseId); + expect((coordinator as any).state.pendingWarning).toMatchObject({ + deliveryAttempts: 1, + nextAttemptAt: 62_000, + }); + now = 62_000; + timers.at(-1)!(); + expect(warn).toHaveBeenCalledTimes(2); + + now = 1_500 + 55 * 60_000; + coordinator.warningDeliveryFailed(pending.leaseId); + expect((coordinator as any).state).toMatchObject({ + lastErrorCode: 'readonly_continuation_warning_delivery_expired', + pendingWarning: undefined, + }); + }); +}); + +describe('attached read-only continuation', () => { + it('persists the lease, copies routing context, and exposes explicit terminal controls', () => { + const session: ReadonlyTaskContinuationSession = { + sessionId: 'session', + turnReplyContexts: { om_original: { inThread: true } }, + replyTargets: { om_original: { rootMessageId: 'om_root' } }, + }; + const timers: Array<() => void> = []; + const enqueue = vi.fn(() => 1); + attachReadonlyTaskContinuation(session, { + schedule: (_delayMs, run) => { timers.push(run); return run; }, + cancel: vi.fn(), + persist: vi.fn(), + enqueue, + warn: vi.fn(), + enabled: () => true, + now: () => 1_000, + randomId: vi.fn().mockReturnValueOnce('lease').mockReturnValueOnce('next'), + }); + + expect(startReadonlyTaskContinuation(session, { turnId: 'om_original', workerGeneration: 1 })) + .toMatchObject({ leaseId: 'readonly-lease', status: 'active' }); + expect(handleReadonlyTaskContinuationTerminal(session, { + turnId: 'om_original', status: 'completed', workerGeneration: 1, + })?.status).toBe('backoff'); + timers.at(-1)!(); + expect(session.turnReplyContexts?.['bmx-readonly-next']).toEqual({ inThread: true }); + expect(session.replyTargets?.['bmx-readonly-next']).toEqual({ rootMessageId: 'om_root' }); + expect(enqueue).toHaveBeenCalledOnce(); + + expect(awaitReadonlyTaskContinuationUser(session, 'bmx-readonly-next')) + .toMatchObject({ status: 'awaiting_user' }); + + startReadonlyTaskContinuation(session, { turnId: 'om_new', workerGeneration: 1 }); + expect(cancelReadonlyTaskContinuationForUserInput(session, 'om_interrupt')) + .toMatchObject({ status: 'cancelled', cancelledByTurnId: 'om_interrupt' }); + + startReadonlyTaskContinuation(session, { turnId: 'om_final', workerGeneration: 1 }); + expect(completeReadonlyTaskContinuation(session, 'om_final', undefined, 'om_reply', 1)) + .toMatchObject({ status: 'completed', completedMessageId: 'om_reply' }); + }); +}); diff --git a/test/restore-zombie-close.test.ts b/test/restore-zombie-close.test.ts index b92234fe97..8db277e6b5 100644 --- a/test/restore-zombie-close.test.ts +++ b/test/restore-zombie-close.test.ts @@ -101,6 +101,7 @@ vi.mock('../src/core/worker-pool.js', () => ({ getCurrentCliVersion: vi.fn(() => '1.0.0-test'), restoreUsageLimitRuntimeState: vi.fn(), ensureOrdinaryTurnRecoveryAttached: vi.fn(), + ensureReadonlyTaskContinuationAttached: vi.fn(), withActiveSessionKeyLock: vi.fn(async (_map: Map, _key: string, action: () => any) => action()), setActiveSessionSafe: vi.fn(async (map: Map, key: string, ds: any) => { const prev = map.get(key); diff --git a/test/session-lifecycle-start.test.ts b/test/session-lifecycle-start.test.ts index 4e432396f8..5cf2f7ca21 100644 --- a/test/session-lifecycle-start.test.ts +++ b/test/session-lifecycle-start.test.ts @@ -43,6 +43,8 @@ vi.mock('../src/im/lark/card-builder.js', () => ({ buildSessionCard: vi.fn(() => '{"type":"session"}'), buildTuiPromptCard: vi.fn(() => '{"type":"tui"}'), buildTuiPromptResolvedCard: vi.fn(() => '{"type":"tui-resolved"}'), + buildCanonicalFinalReplyCard: vi.fn(() => '{"type":"final"}'), + buildContextualReplyCard: vi.fn(() => '{"type":"contextual"}'), getCliDisplayName: vi.fn(() => 'Codex'), // Echo the inputs the failure-notice assertions care about (error code + // retry action) rather than a fixed blob, so those assertions test the @@ -70,6 +72,7 @@ vi.mock('../src/bot-registry.js', () => ({ botOpenId: 'ou_bot', botName: 'TestBot', })), + resolveBrandLabel: vi.fn(() => 'TestBot'), getAllBots: vi.fn(() => []), getLoadedConfigPath: vi.fn(() => '/home/u/.botmux/bots.json'), // Provenance travels with the path (see core/config-dir.ts): 'loaded' = the @@ -184,11 +187,17 @@ import { forkWorker, getDaemonBootId, initWorkerPool, + ensureReadonlyTaskContinuationAttached, promoteQueuedActivationTail, restartCounts, sendWorkerInput, + setActiveSessionsRegistry, suspendWorker, } from '../src/core/worker-pool.js'; +import { + disposeReadonlyTaskContinuation, + startReadonlyTaskContinuation, +} from '../src/services/readonly-task-continuation.js'; import { managedOriginCapabilityDirectory, readManagedOriginCapability, @@ -262,6 +271,7 @@ function defaultBot(overrides: Record = {}) { beforeEach(() => { vi.useRealTimers(); vi.clearAllMocks(); + delete process.env.BOTMUX_READONLY_CONTINUATION_ENABLED; vi.mocked(sessionStore.updateSession).mockImplementation(() => undefined); __testOnly_resetOrdinaryImDeliveries(); vi.mocked(getBot).mockImplementation(() => defaultBot()); @@ -285,6 +295,13 @@ beforeEach(() => { getActiveCount: () => 1, closeSession: vi.fn(), }); + setActiveSessionsRegistry(undefined); +}); + +afterEach(() => { + delete process.env.BOTMUX_READONLY_CONTINUATION_ENABLED; + disposeReadonlyTaskContinuation({ sessionId: 'sid-start-test' }); + setActiveSessionsRegistry(undefined); }); describe('host memory pressure worker admission', () => { @@ -1207,6 +1224,671 @@ describe('ordinary IM worker receipt acknowledgement', () => { }); }); +describe('TraeX opt-in read-only task continuation', () => { + function startTraexLease() { + vi.useFakeTimers(); + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const ds = makeDs(); + forkWorker(ds, 'original task', 'om_original'); + const worker = forkMock.mock.results.at(-1)!.value; + ds.workerReady = true; + worker.emit('message', { + type: 'readonly_continuation_rpc_status', + sessionId: ds.session.sessionId, + rpcGeneration: 'rpc-proof', + eligible: true, + }); + expect(startReadonlyTaskContinuation(ds.session, { + turnId: 'om_original', + workerGeneration: ds.workerGeneration!, + ttlMs: 60_000, + maxContinuations: 2, + })).toMatchObject({ status: 'active', continuationsStarted: 0 }); + return { ds, worker }; + } + + it('continues a completed turn and the exact output-limit failure without replaying the task', async () => { + const { ds, worker } = startTraexLease(); + + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: 'om_original', + status: 'completed', + }); + await Promise.resolve(); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ status: 'backoff' }); + + await vi.advanceTimersByTimeAsync(1_000); + const first = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .find(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(first).toEqual(expect.objectContaining({ + content: expect.stringContaining('[BOTMUX_READONLY_CONTINUATION]'), + })); + expect(first.content).not.toContain('original task'); + + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: first.turnId, + dispatchAttempt: first.dispatchAttempt, + status: 'failed', + errorCode: 'codex_output_limit_exceeded', + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + + const continuations = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(continuations).toHaveLength(2); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + continuationsStarted: 2, + currentTurnId: continuations[1].turnId, + }); + }); + + it('ignores the MR1 failed-turn fallback and lets the exact output-limit terminal continue', async () => { + const { ds, worker } = startTraexLease(); + + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: 'om_original', + status: 'completed', + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + + const first = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .find(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(first).toEqual(expect.objectContaining({ dispatchAttempt: 1 })); + + worker.emit('message', { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: first.turnId, + dispatchAttempt: first.dispatchAttempt, + lastUuid: 'mr1-failed-turn-fallback', + content: 'model output limit exceeded: max_output_tokens', + turnFailed: true, + }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + currentTurnId: first.turnId, + continuationsStarted: 1, + }); + + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: first.turnId, + dispatchAttempt: first.dispatchAttempt, + status: 'failed', + errorCode: 'codex_output_limit_exceeded', + }); + await Promise.resolve(); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'backoff', + continuationsStarted: 1, + }); + + await vi.advanceTimersByTimeAsync(1_000); + const continuations = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(continuations).toHaveLength(2); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + continuationsStarted: 2, + currentTurnId: continuations[1].turnId, + }); + }); + + it('does not mistake the original turn MR1 failed fallback for business-final proof', async () => { + const { ds, worker } = startTraexLease(); + + worker.emit('message', { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: 'om_original', + lastUuid: 'mr1-original-failed-turn-fallback', + content: 'model output limit exceeded: max_output_tokens', + turnFailed: true, + }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + currentTurnId: 'om_original', + continuationsStarted: 0, + }); + + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: 'om_original', + status: 'failed', + errorCode: 'codex_output_limit_exceeded', + }); + await Promise.resolve(); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'backoff', + continuationsStarted: 0, + }); + + await vi.advanceTimersByTimeAsync(1_000); + const continuations = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(continuations).toHaveLength(1); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + continuationsStarted: 1, + currentTurnId: continuations[0].turnId, + }); + }); + + it('keeps an overdue restored backoff waiting for the live worker RPC proof', async () => { + vi.useFakeTimers(); + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const now = Date.now(); + const ds = makeDs(); + ds.session.cliId = 'traex'; + ds.session.workerGeneration = 7; + ds.session.readonlyTaskContinuation = { + leaseId: 'readonly-cold-backoff', + logicalTurnId: 'om_original', + currentTurnId: 'om_original', + currentWorkerGeneration: 7, + createdAt: now - 10_000, + expiresAt: now + 60_000, + maxContinuations: 2, + continuationsStarted: 0, + status: 'backoff', + nextAttemptAt: now - 1, + }; + + forkWorker(ds, '', { resume: true }); + const worker = forkMock.mock.results.at(-1)!.value; + await vi.advanceTimersByTimeAsync(2_000); + + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'backoff', + continuationsStarted: 0, + }); + expect(vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.turnId?.startsWith('bmx-readonly-'))).toHaveLength(0); + + worker.emit('message', { type: 'ready', port: 3456, token: 'token' }); + worker.emit('message', { + type: 'readonly_continuation_rpc_status', + sessionId: ds.session.sessionId, + rpcGeneration: 'rpc-proof-after-restore', + eligible: true, + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + + const continuations = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(continuations).toHaveLength(1); + expect(continuations[0]).toEqual(expect.objectContaining({ + dispatchAttempt: 1, + readonlyContinuation: { + leaseId: 'readonly-cold-backoff', + rpcGeneration: 'rpc-proof-after-restore', + }, + })); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + continuationsStarted: 1, + currentTurnId: continuations[0].turnId, + currentWorkerGeneration: 8, + }); + + await vi.advanceTimersByTimeAsync(5_000); + expect(vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.turnId?.startsWith('bmx-readonly-'))).toHaveLength(1); + }); + + it('does not continue another failure code', async () => { + const { ds, worker } = startTraexLease(); + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: 'om_original', + status: 'failed', + errorCode: 'codex_connection_failed', + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(5_000); + + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'failed', + lastErrorCode: 'codex_connection_failed', + }); + expect(vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.turnId?.startsWith('bmx-readonly-'))).toHaveLength(0); + }); + + it('settles an original turn only on an explicit final send marker', async () => { + const { ds, worker } = startTraexLease(); + worker.emit('message', { + type: 'explicit_reply_observed', + turnId: 'om_original', + messageId: 'om_final_reply', + responseKind: 'final', + }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'completed', + completedMessageId: 'om_final_reply', + }); + }); + + it('does not settle on progress or a final marker from another turn', () => { + const { ds, worker } = startTraexLease(); + worker.emit('message', { + type: 'explicit_reply_observed', + turnId: 'om_original', + messageId: 'om_progress', + responseKind: 'progress', + }); + worker.emit('message', { + type: 'explicit_reply_observed', + turnId: 'om_other', + messageId: 'om_other_final', + responseKind: 'final', + }); + + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + currentTurnId: 'om_original', + }); + }); + + it('keeps a continuation live for its exact synthetic turn and cancels it for any other admitted turn', async () => { + const { ds, worker } = startTraexLease(); + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: 'om_original', + status: 'completed', + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + + const continuation = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .find(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + expect(continuation).toEqual(expect.objectContaining({ dispatchAttempt: 1 })); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'active', + currentTurnId: continuation.turnId, + currentDispatchAttempt: 1, + }); + + expect(sendWorkerInput(ds, 'same continuation', continuation.turnId, { + dispatchAttempt: 1, + })).toBe(true); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ status: 'active' }); + + expect(sendWorkerInput(ds, 'scheduled side turn', 'schedule-other-turn')).toBe(true); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'cancelled', + cancelledByTurnId: 'schedule-other-turn', + }); + }); + + it('treats the original leased turn final output as business-final proof', () => { + const { ds, worker } = startTraexLease(); + worker.emit('message', { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: 'om_original', + lastUuid: 'original-final', + content: '{"status":"completed","content":"ordinary output"}', + }); + + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'completed', + currentTurnId: 'om_original', + completedMessageId: 'original-final', + }); + expect(ds.session.readonlyTaskContinuation?.currentDispatchAttempt).toBeUndefined(); + expect(ds.session.readonlyTaskContinuation?.pendingDelivery).toBeUndefined(); + }); + + it('keeps a local failed fence when cancelling for new input cannot persist', () => { + const { ds, worker } = startTraexLease(); + vi.mocked(sessionStore.updateSession).mockImplementation(() => { + throw new Error('session store unavailable'); + }); + + expect(sendWorkerInput(ds, 'new instruction', 'om_interrupt')).toBe(true); + + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'failed', + currentTurnId: 'om_original', + lastErrorCode: 'readonly_continuation_user_cancel_persist_failed', + }); + expect(vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .filter(message => message?.turnId?.startsWith('bmx-readonly-'))).toHaveLength(0); + }); + + it('settles a synthetic continuation only for the exact dispatch attempt', async () => { + const sessionReply = vi.fn(async () => 'om_readonly_final'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const { ds, worker } = startTraexLease(); + setActiveSessionsRegistry(new Map([['om_root::app_test', ds]])); + worker.emit('message', { + type: 'turn_terminal', + sessionId: ds.session.sessionId, + turnId: 'om_original', + status: 'completed', + }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1_000); + + const continuation = vi.mocked(worker.send).mock.calls + .map(call => call[0]) + .find(message => message?.type === 'message' + && message?.turnId?.startsWith('bmx-readonly-')); + worker.emit('message', { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: continuation.turnId, + dispatchAttempt: 2, + lastUuid: 'wrong-attempt', + content: '{"status":"completed","content":"wrong"}', + }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ status: 'active' }); + + vi.useRealTimers(); + worker.emit('message', { + type: 'final_output', + sessionId: ds.session.sessionId, + turnId: continuation.turnId, + dispatchAttempt: 1, + lastUuid: 'exact-attempt', + content: '{"status":"completed","content":"done"}', + }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ status: 'delivering' }); + await vi.waitFor(() => expect(sessionReply).toHaveBeenCalledTimes(1)); + expect(ds.session.readonlyTaskContinuation?.lastErrorCode).toBeUndefined(); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'completed', + completedMessageId: 'om_readonly_final', + pendingDelivery: undefined, + }); + expect(sessionReply).toHaveBeenCalledWith( + 'om_root', + expect.any(String), + 'interactive', + 'app_test', + continuation.turnId, + expect.objectContaining({ uuid: expect.stringMatching(/^bf_/) }), + ); + setActiveSessionsRegistry(undefined); + }); + + it('replays one recent persisted daemon-owned delivery after daemon restore', async () => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const sessionReply = vi.fn(async () => 'om_recovered_final'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.session.cliId = 'traex'; + ds.session.workerGeneration = 7; + ds.session.readonlyTaskContinuation = { + leaseId: 'readonly-restored', + logicalTurnId: 'om_original', + currentTurnId: 'bmx-readonly-restored', + currentDispatchAttempt: 2, + currentWorkerGeneration: 7, + createdAt: Date.now() - 10_000, + expiresAt: Date.now() + 60_000, + maxContinuations: 3, + continuationsStarted: 2, + status: 'delivering', + pendingDelivery: { + kind: 'completed', + content: 'restored result', + startedAt: Date.now() - 1_000, + }, + }; + setActiveSessionsRegistry(new Map([['om_root::app_test', ds]])); + + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(true); + await vi.waitFor(() => expect(sessionReply).toHaveBeenCalledTimes(1)); + expect(ds.session.readonlyTaskContinuation?.lastErrorCode).toBeUndefined(); + + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'completed', + completedMessageId: 'om_recovered_final', + }); + expect(sessionReply).toHaveBeenCalledTimes(1); + setActiveSessionsRegistry(undefined); + }); + + it('keeps a rejected warning pending and replays it with one stable UUID after restore', async () => { + vi.useFakeTimers(); + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + let reject = true; + const sessionReply = vi.fn(async () => { + if (reject) throw new Error('lark unavailable'); + return 'om_recovered_warning'; + }); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.session.cliId = 'traex'; + ds.session.readonlyTaskContinuation = { + leaseId: 'readonly-warning-restored', + logicalTurnId: 'om_original', + currentTurnId: 'bmx-readonly-failed', + currentDispatchAttempt: 2, + currentWorkerGeneration: 7, + createdAt: Date.now() - 10_000, + expiresAt: Date.now() + 60_000, + maxContinuations: 3, + continuationsStarted: 2, + status: 'failed', + lastErrorCode: 'readonly_continuation_enqueue_failed', + pendingWarning: { startedAt: Date.now() - 1_000, deliveryAttempts: 0 }, + }; + setActiveSessionsRegistry(new Map([['om_root::app_test', ds]])); + + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(true); + await vi.advanceTimersByTimeAsync(20_000); + expect(sessionReply).toHaveBeenCalledTimes(3); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'failed', + pendingWarning: expect.any(Object), + }); + expect(ds.session.readonlyTaskContinuation?.warningDispatched).toBeUndefined(); + const firstUuid = sessionReply.mock.calls[0]?.[5]?.uuid; + expect(firstUuid).toMatch(/^bf_/); + expect(sessionReply.mock.calls.every(call => call[5]?.uuid === firstUuid)).toBe(true); + + reject = false; + await vi.advanceTimersByTimeAsync(60_000); + await vi.advanceTimersByTimeAsync(1); + + expect(sessionReply).toHaveBeenCalledTimes(4); + expect(sessionReply.mock.calls[3]?.[5]?.uuid).toBe(firstUuid); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + warningDispatched: true, + warningMessageId: 'om_recovered_warning', + pendingWarning: undefined, + }); + setActiveSessionsRegistry(undefined); + }); + + it('expires a stale warning outbox without replaying it and restores dashboard attention', () => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const sessionReply = vi.fn(async () => 'om_unexpected'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.session.cliId = 'traex'; + ds.session.readonlyTaskContinuation = { + leaseId: 'readonly-warning-expired', + logicalTurnId: 'om_original', + currentTurnId: 'bmx-readonly-failed', + currentDispatchAttempt: 2, + currentWorkerGeneration: 7, + createdAt: Date.now() - 60 * 60_000, + expiresAt: Date.now() - 1, + maxContinuations: 3, + continuationsStarted: 2, + status: 'failed', + lastErrorCode: 'readonly_continuation_enqueue_failed', + pendingWarning: { + startedAt: Date.now() - 55 * 60_000, + deliveryAttempts: 4, + nextAttemptAt: Date.now() - 1, + }, + }; + + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(true); + + expect(sessionReply).not.toHaveBeenCalled(); + expect(ds.agentAttention).toMatchObject({ kind: 'blocked' }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'failed', + lastErrorCode: 'readonly_continuation_warning_delivery_expired', + pendingWarning: undefined, + }); + }); + + it('settles a persisted warning after cold restore even when the kill switch is off', async () => { + vi.useFakeTimers(); + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const sessionReply = vi.fn(async () => 'om_disabled_recovered_warning'); + initWorkerPool({ + sessionReply, + getSessionWorkingDir: () => '/repo', + getActiveCount: () => 1, + closeSession: vi.fn(), + }); + const ds = makeDs(); + ds.session.cliId = 'traex'; + ds.session.readonlyTaskContinuation = { + leaseId: 'readonly-warning-disabled', + logicalTurnId: 'om_original', + currentTurnId: 'bmx-readonly-failed', + currentDispatchAttempt: 1, + currentWorkerGeneration: 7, + createdAt: Date.now() - 10_000, + expiresAt: Date.now() + 60_000, + maxContinuations: 3, + continuationsStarted: 1, + status: 'failed', + lastErrorCode: 'readonly_continuation_enqueue_failed', + pendingWarning: { startedAt: Date.now() - 1_000, deliveryAttempts: 0 }, + }; + + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(true); + await vi.advanceTimersByTimeAsync(1); + + expect(sessionReply).toHaveBeenCalledOnce(); + expect(ds.agentAttention).toMatchObject({ kind: 'blocked' }); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + warningDispatched: true, + warningMessageId: 'om_disabled_recovered_warning', + pendingWarning: undefined, + }); + expect(vi.mocked(forkMock)).not.toHaveBeenCalled(); + }); + + it('cancels a restored live lease when the feature becomes ineligible', () => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const ds = makeDs(); + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(true); + startReadonlyTaskContinuation(ds.session, { + turnId: 'om_original', workerGeneration: ds.workerGeneration ?? 1, + }); + + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'codex' })); + ds.session.cliId = 'codex'; + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(false); + expect(ds.session.readonlyTaskContinuation).toMatchObject({ + status: 'cancelled', + lastErrorCode: 'readonly_continuation_ineligible', + }); + }); + + it.each([ + ['non-TraeX', defaultBot({ cliId: 'codex' }), {}], + ['adopt', defaultBot({ cliId: 'traex' }), { adoptedFrom: { sessionId: 'external' } }], + ['VC receiver', defaultBot({ cliId: 'traex' }), { + session: { vcMeetingReceiver: { listenerAppId: 'app', meetingId: 'm', memberId: 'u', memberEpoch: 1 } }, + }], + ['deferred schedule', defaultBot({ cliId: 'traex' }), { + session: { deferredScheduleRun: { turnId: 'schedule-turn' } }, + }], + ['external topicless trigger', defaultBot({ cliId: 'traex' }), { + session: { externalTriggerTopicless: true }, + }], + ['no Lark transport', defaultBot({ cliId: 'traex' }), { chatId: 'http_async_test' }], + ])('does not attach for %s sessions', (_label, bot, shape) => { + process.env.BOTMUX_READONLY_CONTINUATION_ENABLED = 'true'; + vi.mocked(getBot).mockImplementation(() => bot as any); + const { session: sessionShape, ...daemonShape } = shape as any; + const ds = makeDs(daemonShape); + if (sessionShape) Object.assign(ds.session, sessionShape); + + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(false); + expect(startReadonlyTaskContinuation(ds.session, { + turnId: 'om_original', workerGeneration: ds.workerGeneration ?? 1, + })).toBeUndefined(); + }); + + it('stays disabled unless the machine-wide switch is explicitly true', () => { + vi.mocked(getBot).mockImplementation(() => defaultBot({ cliId: 'traex' })); + const ds = makeDs(); + expect(ensureReadonlyTaskContinuationAttached(ds)).toBe(false); + expect(ds.session.readonlyTaskContinuation).toBeUndefined(); + }); +}); + describe('ordinary Claude semantic recovery', () => { it('continues twice without another user message, then warns exactly once', async () => { vi.useFakeTimers(); diff --git a/test/session-resume.test.ts b/test/session-resume.test.ts index 62ae2c6e43..792ef2ae8c 100644 --- a/test/session-resume.test.ts +++ b/test/session-resume.test.ts @@ -56,6 +56,7 @@ vi.mock('../src/core/worker-pool.js', () => ({ getCurrentCliVersion: vi.fn(() => '1.0.0-test'), restoreUsageLimitRuntimeState: vi.fn(), ensureOrdinaryTurnRecoveryAttached: vi.fn(), + ensureReadonlyTaskContinuationAttached: vi.fn(), // Default: promotion succeeds. A specific test overrides this to false to // exercise the restore-time transient-failure quarantine path. promoteQueuedActivationTail: vi.fn(() => true), diff --git a/test/traex-transcript.test.ts b/test/traex-transcript.test.ts index 9cb2b28934..bef4515b5e 100644 --- a/test/traex-transcript.test.ts +++ b/test/traex-transcript.test.ts @@ -703,6 +703,18 @@ describe('drainTraexRollout', () => { }); }); + it('maps only the exact output-limit failure to the continuation allowlist code', () => { + writeFileSync(path, [ + line(user('long task')), + line(taskCompleteWithError({ message: 'model output limit exceeded: max_output_tokens' })), + ].join('')); + + expect(drainTraexRollout(path, 0).events.at(-1)).toMatchObject({ + terminalStatus: 'failed', + terminalErrorCode: 'codex_output_limit_exceeded', + }); + }); + it('synthesises the bare sentinel when the last commentary ends with BOTMUX_NO_REPLY (path B: deliberate silence)', () => { writeFileSync(path, [ line(user('do the work')),