From eabe7ce602af27e098f9bb40b5cb2130b4d7f371 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 7 Sep 2026 07:36:21 +0900 Subject: [PATCH 1/6] docs: plan axis1 bounded bug fixes [skip ci] --- .../_plan/260907_axis1_bugfixes/000_plan.md | 22 +++++++++++++++++++ .../260907_axis1_bugfixes/010_roadmap.md | 3 +++ .../_plan/260907_axis1_bugfixes/011_audit.md | 5 +++++ .../260907_axis1_bugfixes/012_roadmap_lock.md | 5 +++++ .../020_bounded_fixes.md | 20 +++++++++++++++++ .../260907_axis1_bugfixes/030_delivery.md | 7 ++++++ 6 files changed, 62 insertions(+) create mode 100644 devlog/_plan/260907_axis1_bugfixes/000_plan.md create mode 100644 devlog/_plan/260907_axis1_bugfixes/010_roadmap.md create mode 100644 devlog/_plan/260907_axis1_bugfixes/011_audit.md create mode 100644 devlog/_plan/260907_axis1_bugfixes/012_roadmap_lock.md create mode 100644 devlog/_plan/260907_axis1_bugfixes/020_bounded_fixes.md create mode 100644 devlog/_plan/260907_axis1_bugfixes/030_delivery.md diff --git a/devlog/_plan/260907_axis1_bugfixes/000_plan.md b/devlog/_plan/260907_axis1_bugfixes/000_plan.md new file mode 100644 index 0000000000..1fa7537164 --- /dev/null +++ b/devlog/_plan/260907_axis1_bugfixes/000_plan.md @@ -0,0 +1,22 @@ +# Axis 1: measured bug fixes and failure diagnostics + +Archetype: satisfy existing contracts. Trigger: owner assigned axis 1 (#3809, #3464, #3661). Goal: deliver reviewable fixes through a manual PR chain and merge the verified scope. Non-goals: new account/retry policy, auth defaults, multipart recovery, releases, native stacks, sibling edits. Stop: merged feasible scope plus explicit unresolved dispositions. Escalation: defer a policy-dependent or unreproducible slice; reclaim a worker slice after two failed packets. Evidence: this unit plus ignored `.tmp/axis1/` and `.codexclaw` receipts. Resources: task-owned worktree/branches and GitHub repository access; Astra high leaves within host capacity; no caller-specified token or wall-clock budget. + +Baseline: origin/dev 137d6a727; source PR #3809 at 4a1012359a522ddd6d7ff77203c9e5f3632d605c. Assigned 5cc8 checkout has pre-existing changes and remains untouched. Code lives in /tmp/ocx-axis1-20260907. + +## Cycle map +1. wp0: docs-only scope, source audit and dependency roadmap; no runtime changes. +2. wp1: bounded quota, version-guidance and recovery-diagnostic changes; independent source/security review and structural checks. Runtime verification deferred explicitly to wp2. +3. wp2: publish ordinary PR chain, run final cumulative hosted CI, resolve findings, admin merge bottom-up and verify dev ancestry. Lower CI only if final CI fails. + +## Delivery contract +The owner explicitly requests a manual delivery chain even where units are independent: quota -> CLI guidance -> recovery reasons, with each layer carrying its own tests and credit. This order is an integration order, not a fabricated runtime dependency. No native registration. Lower commits carry [skip ci] to defer duplicate workflow runs; final head does not. Skipped lower runs are never called passing. No local tests/typecheck/build suites and no hook-triggered suites; task pushes use --no-verify. Hosted ci.yml on the final head must cover all changed runtime/tests; lower-level runs are diagnostic only after final failure. Merge with --admin under the explicit owner exception; preserve original commits/trailers with merge commits, retarget each child to dev, and check integration trees against final evidence. Concurrent dev changes require fresh combined verification. + +## Work boundaries +- Quota: src/providers/quota.ts, src/oauth/anthropic-routing.ts, src/oauth/health.ts, src/server/responses/core.ts, src/images/loop.ts, src/web-search/loop.ts, focused quota tests/layout, provider documentation. +- CLI: src/cli/version-skew.ts and relevant status/doctor consumers, tests/cli/cli-version-skew.test.ts, troubleshooting documentation. No service restart or repair behavior changes. +- Recovery: src/server/responses/agent-task-recovery.ts, agent-task-recovery-cache.ts, src/lib/bounded-body.ts and existing focused tests, Responses error projection if needed, recovery documentation. No expanded admission/retry. +- Main owns shared core.ts integration and test-layout files. Workers must not touch each other's paths or git index. + +## Verification and acceptance +No local suite commands are executed. Source mapping, git diff --check and documentation structural checks are local evidence only. Hosted Cross-platform CI at final head provides runtime/typecheck/privacy and affected platform proof; inspect jobs for skipped coverage. Build completion is provisional until that run and independent audit succeed. Original PR author(s) must be named in commit Co-authored-by trailers, sourced from original commits/API; report authors may also be acknowledged accurately. Source-of-truth sync uses relevant existing structure and docs-site pages. diff --git a/devlog/_plan/260907_axis1_bugfixes/010_roadmap.md b/devlog/_plan/260907_axis1_bugfixes/010_roadmap.md new file mode 100644 index 0000000000..75e07478c0 --- /dev/null +++ b/devlog/_plan/260907_axis1_bugfixes/010_roadmap.md @@ -0,0 +1,3 @@ +# wp0: scope roadmap + +Read current source, prior issue disposition and PR #3809 before choosing changes. Independent Astra high reviewers map each bounded issue. Confirm existing launcher behavior and bounded recovery reasons are already in dev; plan only residual fixes. Record exact file boundaries and acceptance scenarios in 020. Success: all three slices have verifiable requirements, main-owned shared files, original author anchors and explicit policy exclusions. Local evidence is documentation and source inspection; no runtime claim. diff --git a/devlog/_plan/260907_axis1_bugfixes/011_audit.md b/devlog/_plan/260907_axis1_bugfixes/011_audit.md new file mode 100644 index 0000000000..cc1cb1d51e --- /dev/null +++ b/devlog/_plan/260907_axis1_bugfixes/011_audit.md @@ -0,0 +1,5 @@ +# wp0 audit disposition + +Independent Astra high reviewer Hooke: VERDICT: GO-WITH-FIXES (blockers=1). Shared-flight failure propagation was the blocker. Accepted: 000/020 now assign cache and bounded-body ownership and define shared typed outcomes, success-only cache, caller-local cancellation and capacity semantics. Source scouts independently identified and confirmed these requirements. Fixed stale CLI test path. Windows runtime proof requires final workflow_dispatch, now explicit in 030. + +No runtime code changed. Documentation source/ownership inspection and git diff --check are the wp0 evidence. Runtime verification remains wp2. diff --git a/devlog/_plan/260907_axis1_bugfixes/012_roadmap_lock.md b/devlog/_plan/260907_axis1_bugfixes/012_roadmap_lock.md new file mode 100644 index 0000000000..cf2bf0afce --- /dev/null +++ b/devlog/_plan/260907_axis1_bugfixes/012_roadmap_lock.md @@ -0,0 +1,5 @@ +# Roadmap lock + +The second independent audit returned VERDICT: PASS with no remaining blockers. The three accepted slices are ready for scoped implementation. Original quota author: Éverton Toffanetto (everton-dgn), commit identity from 4f3779c04753 and 3ef0ade296c3. Issue reporters: garysassano (10464497) and Hu9956 (282876394). Reporter acknowledgement is separate from code authorship. + +Preserve raw unequal version diagnostics. Detailed recovery outcomes must travel in the shared flight, not caller-local closures. Quota observations use immutable dispatch identity. Final verification is hosted workflow_dispatch for full Windows coverage; local suites remain prohibited. diff --git a/devlog/_plan/260907_axis1_bugfixes/020_bounded_fixes.md b/devlog/_plan/260907_axis1_bugfixes/020_bounded_fixes.md new file mode 100644 index 0000000000..836f743a73 --- /dev/null +++ b/devlog/_plan/260907_axis1_bugfixes/020_bounded_fixes.md @@ -0,0 +1,20 @@ +# wp1: implement bounded bug fixes + +## Quota +Carry only the source PR diff onto current dev, with original-author trailer. Header utilization fraction -> percentage; reset epoch -> timestamp. Creation: parser; serialization: account quota cache; deserialization: existing hydration; consumers: account ranking/health and management reading. Account-bound writer generation is captured with serving credentials, including retry/sidecar/continuation rebinds. Header observations merge model-specific windows and cannot indefinitely postpone probes. Existing 429 eligibility and retry count stay unchanged. Explicit reset evidence must not be truncated by an invented six-hour policy; any unresolved policy piece is deferred. +Scenarios: 200 and 429 on main/sidecar/continuation attribute only the serving account; generation invalidation discards writes; partial/malformed headers preserve known fields; no prior probe means model-window probe is still due; weekly rejected reset outlasts five-hour reset; absent evidence retains existing fallback. Verify with focused tests included in final hosted CI. + +## Version guidance +Compare CLI and running proxy using existing semantic-version utilities if present. CLI newer points to service restart; proxy newer points to upgrading/PATH resolution of CLI; equal/unknown retain suppression; incomparable differing builds use neutral wording. status and doctor share advice. Preserve whether requests are allowed and do not perform repair. Test both directions, prereleases, placeholders, malformed versions and consumer projection. + +## Recovery reasons +Keep existing public wrapper returning boolean and typed detailed result. Classify actual upstream HTTP refusal, transport error, timeout/caller cancellation, response-body/decode failures with a bounded vocabulary. Creation: request/collector; propagation: detailed recovery result; consumers: existing response reason projection/tests/docs. No raw upstream body/errors/tokens/ciphertext in output. Strict admission, one attempt, same credential and unchanged request mutation guarantees. Exercise each failure branch, cancellation races, malformed terminal output and successful recovery in final hosted CI. + +Main owns src/server/responses/core.ts and layout metadata. Source/security review must check public boundaries and negative cases, not only implementation-mirroring tests. Source-only C evidence does not claim runtime correctness; wp2 is mandatory. + +## Source-map clarification from independent #3464 research +Use src/lib/strict-semver.ts unchanged. Raw unequal versions remain skewed; equal precedence with different build metadata and invalid/whitespace/v-prefixed values get neutral wording, not normalization or a guessed direction. Placeholder suppression is unchanged. src/cli/doctor.ts must not call suppressed placeholders a confirmed match. Focused files: tests/cli/cli-version-skew.test.ts, tests/cli/cli-status-json.test.ts, tests/codex-integration/doctor.test.ts. Documentation: reference/cli/lifecycle.md and directly affected Korean/Russian pages. Existing launcher landed via #3616 (4e2246c32); no service runtime changes. + +## Audit refinements +Quota: observe physical responses at the existing oauthDispatch boundary before any main/continuation replacement or return. Use immutable request binding to pair response with selected account; skip when final authorization headers do not prove that bearer or credentialGeneration has changed. An active-account switch alone does not invalidate another account's in-flight observation. Native Claude passthrough and single-account expansion remain outside #3809 carry. Preserve Retry-After precedence; only reject nonfinite/unrepresentable deadlines rather than invent an anomaly ceiling. Header-only rows are probe-due; hydrated Anthropic observations must be probe-due unless probe time is proven. Failed probes settle with the most recent committed observation for all joiners. +Recovery: worker owns agent-task-recovery-cache.ts and bounded-body.ts narrow decode discriminator alongside focused tests. Shared flight carries typed outcome, cache retains only success plaintext, cancelled waiters remain local. Recognized caller cancellation precedes owned timeout, which precedes decode/transport classification. Fatal UTF-8 discriminator must identify actual decoder exceptions without reclassifying fetch/body-reader TypeErrors. Rejected-response cancellation is nonblocking best effort. Keep current public wrappers and combo error projection. Update documented reason lists in structure/04_transports-and-sidecars.md and docs-site/reference/architecture.md. diff --git a/devlog/_plan/260907_axis1_bugfixes/030_delivery.md b/devlog/_plan/260907_axis1_bugfixes/030_delivery.md new file mode 100644 index 0000000000..be851f530d --- /dev/null +++ b/devlog/_plan/260907_axis1_bugfixes/030_delivery.md @@ -0,0 +1,7 @@ +# wp2: hosted proof and manual-stack landing + +Publish task-owned branches with --no-verify. Standard PR template, source links, truthful skipped-local/lower-CI disclosure and contributor trailers. Lower layers use [skip ci], final cumulative head runs existing Cross-platform CI; never modify shared workflow filters or fabricate checks. On final failure inspect failing jobs, fix owned defects, and only then use lower CI to localize ambiguity. Leave unrelated/unresolvable slices unmerged with evidence. + +Before admin merge: source/security review findings resolved, final CI SHA/run pinned, current PR head and manual membership inspected. Record owner-authorized admin review/lower-CI exception. Merge bottom-up with original commits preserved; do not delete parent branches while children depend on them. Retarget child to dev after parent landing. Reconcile concurrent dev before claiming final integrated proof. Verify every merge SHA is ancestor of refreshed origin/dev. Close #3809 only after its accepted replacement scope lands; keep #3661 open for multipart/retry and #3464 open if broader original acceptance remains unresolved. No release/deploy. + +Final full platform evidence uses workflow_dispatch ci.yml on the final cumulative branch, because ordinary PR CI excludes the Windows runtime job. Cancel only duplicate task-owned PR CI runs; skipped/cancelled runs are not passing evidence. From f215f79b4562735029ad5672a68bc6104e534b98 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 7 Sep 2026 07:41:57 +0900 Subject: [PATCH 2/6] fix(anthropic): attribute quota headers and honor measured reset deadlines [skip ci] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carry and refine #3809: observe each request-bound physical response, preserve probe clocks and model-specific windows, and retain valid multi-day upstream reset deadlines. Preserve credential ownership and skip unprovable observations. Runtime checks are deferred to the final cumulative hosted CI at owner request; no local suite was run. Co-authored-by: Éverton Toffanetto --- .../src/content/docs/guides/claude-code.md | 13 +- .../docs/reference/configuration/providers.md | 20 +- scripts/test-layout/layout.json | 2 + src/images/loop.ts | 12 +- src/oauth/anthropic-routing.ts | 73 ++- src/oauth/health.ts | 3 + src/providers/quota.ts | 77 ++- src/server/responses/core.ts | 35 +- src/web-search/loop.ts | 12 +- .../anthropic-quota-dispatch.test.ts | 283 ++++++++++ .../anthropic-ratelimit-headers.test.ts | 528 ++++++++++++++++++ ...anthropic-sidecar-account-failover.test.ts | 55 +- tests/fixtures/test-layout-expected.json | 2 + 13 files changed, 1074 insertions(+), 41 deletions(-) create mode 100644 tests/adapters/anthropic/anthropic-quota-dispatch.test.ts create mode 100644 tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 5ed946c72a..2e841e205c 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -27,9 +27,16 @@ rotation does not protect against provider enforcement. Operational contract when enabled: -- Upstream **429** cools that account using `Retry-After` when present (else a default backoff), - clears its affinities, and may rotate to another eligible account within the same request - (bounded). +- Upstream **429** cools that account, clears its affinities, and may rotate to another eligible + account within the same request (bounded). The cooldown uses a usable `Retry-After` when present, + otherwise the latest valid reset time among windows Anthropic marks `rejected`, including + weekly windows. Valid upstream deadlines are not shortened to a fixed cooldown ceiling. + A refusal with no usable deadline falls back to a default backoff. +- Responses report the serving account's 5-hour and weekly utilization, and whichever of those + two the response carries is recorded for that account — each window independently, and a + refusal counts as well as a success. Usage-aware selection works from ordinary traffic, + without waiting for a dashboard poll. Headers preserve model-specific quota windows and do + not postpone usage probes or clear a failed usage probe's unavailable status. - Affinity is **process-local** (lost on proxy restart). - **401/403** credential failures quarantine the account (`needsReauth`) so it is excluded from selection until re-authenticated. diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 02d5ba4323..c80176868e 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -430,14 +430,26 @@ rotation may trigger provider restrictions. | `anthropicAccountPool.enabled?` | `boolean` | `false` | Enable sticky session affinity and quota-ranked new-session selection. **429 failover is not gated here**: it activates whenever two or more usable accounts are stored, exactly like every other multi-credential provider, and cannot be switched off. | | `anthropicAccountPool.autoSwitchThreshold?` | `number` | `80` | For new sessions, when the active account reaches this threshold, choose the lowest known cached usage in the configured window; the account chosen does not itself have to be at or above the threshold. `0` disables **proactive** usage-based switching only — new-session selection and routing recovery after an eligible 429 still consult `quotaWindow`. | | `anthropicAccountPool.strategy?` | `"quota" \| "round-robin" \| "fill-first"` | `"quota"` | New-session strategy; `quota` ranks accounts by the window set by `quotaWindow`, and `fill-first` evaluates its drain threshold in that same window. | -| `anthropicAccountPool.quotaWindow?` | `"five-hour" \| "weekly" \| "max-utilization"` | `"five-hour"` | The cached provider-reported utilization bar used for usage-aware account selection. `five-hour` keeps the original behavior. `weekly` scores the weekly bar and skips accounts whose 5-hour bar is exhausted while another eligible account remains, but falls back to exhausted candidates when none do. `max-utilization` scores the highest known bar, so it can use 5-hour usage before weekly usage is available; if neither is known, the account follows unknown-usage ordering. Known usage ranks before unknown usage under the opt-in `weekly` and `max-utilization` windows only; an omitted or explicit `five-hour` preserves the legacy ordering. If every eligible account is unknown, selection still returns one in eligible order. After the documented lower-5-hour tie-break, exact ties preserve eligible order. A healthy affinity-bound session is not proactively rebalanced. For new-session assignment and routing recovery after an eligible 429 replacement, `quota` ranks eligible candidates directly with this window; `fill-first` advances in stable order using this window's threshold and exhaustion rules; `round-robin` ignores it. Cooldown, failover limits, and reauthentication eligibility remain separate local state. Per-account weekly bars are only known once the dashboard Providers page has polled them. | +| `anthropicAccountPool.quotaWindow?` | `"five-hour" \| "weekly" \| "max-utilization"` | `"five-hour"` | The cached provider-reported utilization bar used for usage-aware account selection. `five-hour` keeps the original behavior. `weekly` scores the weekly bar and skips accounts whose 5-hour bar is exhausted while another eligible account remains, but falls back to exhausted candidates when none do. `max-utilization` scores the highest known bar, so it can use 5-hour usage before weekly usage is available; if neither is known, the account follows unknown-usage ordering. Known usage ranks before unknown usage under the opt-in `weekly` and `max-utilization` windows only; an omitted or explicit `five-hour` preserves the legacy ordering. If every eligible account is unknown, selection still returns one in eligible order. After the documented lower-5-hour tie-break, exact ties preserve eligible order. A healthy affinity-bound session is not proactively rebalanced. For new-session assignment and routing recovery after an eligible 429 replacement, `quota` ranks eligible candidates directly with this window; `fill-first` advances in stable order using this window's threshold and exhaustion rules; `round-robin` ignores it. Cooldown, failover limits, and reauthentication eligibility remain separate local state. Per-account weekly bars come from usage probes or observed response headers. | | `anthropicAccountPool.stickyLimit?` | `number` | `1` | Successful new-session binds retained on one round-robin selection. Range 1–100. | -When enabled, 429 records bounded cooldown from `Retry-After` or a default backoff and may rotate -within the request. Affinity is process-local and size-bounded. Credential 401/403 marks the account -as needing reauthentication. If all eligible accounts are cooling, clients receive 429 with +When enabled, 429 records a cooldown and may rotate within the request. The cooldown length comes +from a usable `Retry-After`, otherwise from the latest valid reset time among rate-limit windows +Anthropic reports as `rejected`, including weekly windows. Valid upstream deadlines are not +shortened to a fixed cooldown ceiling; non-finite or unrepresentable deadlines are ignored. +A refusal with no usable deadline falls back to a short default backoff. Affinity is process-local +and size-bounded. Credential 401/403 marks the account as needing reauthentication. If all eligible accounts are cooling, clients receive 429 with `Retry-After` when known, not an authentication error. +Anthropic responses also report the serving account's 5-hour and weekly utilization, and whichever +of those two a given response carries is recorded against that account — each window independently, +on refusals as well as successes. Usage-aware selection therefore works from the accounts you +actually use, without waiting for the dashboard Providers page to poll them. These readings refresh +the existing row rather than replacing it, so the model-scoped weekly bars that only the usage +endpoint reports are preserved. Header observations do not postpone usage probes or clear a failed +probe's unavailable status. After restart, cached Anthropic observations remain available while +the next quota read probes again, because the saved observations do not include the probe clock. + :::caution[Experimental] Leave this disabled unless you understand Anthropic account policy risk. Prefer manual `ocx account use anthropic ` switching when unsure. diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index d309073a3f..f560dffeb5 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -204,6 +204,8 @@ "anthropic-image-retry.test.ts": "adapters/anthropic", "anthropic-pool-toggle-copy.test.ts": "adapters/anthropic", "anthropic-quorum-cache.test.ts": "routing", + "anthropic-quota-dispatch.test.ts": "adapters/anthropic", + "anthropic-ratelimit-headers.test.ts": "adapters/anthropic", "anthropic-reasoning.test.ts": "adapters/anthropic", "anthropic-sidecar-account-failover.test.ts": "adapters/anthropic", "anthropic-stream-hardening.test.ts": "adapters/anthropic", diff --git a/src/images/loop.ts b/src/images/loop.ts index e3a7f8252f..7d4855f91b 100644 --- a/src/images/loop.ts +++ b/src/images/loop.ts @@ -263,8 +263,16 @@ export interface ImageBridgeDeps { * Optional 429 failover for the routed (non-xAI) model. Return a rebuilt adapter for the * rotated credential, or null when the pool is exhausted. Async hooks support OAuth refresh; * existing synchronous key-pool hooks remain valid. + * + * `responseHeaders` carries the whole refusal, not just Retry-After, because an Anthropic + * 429 states the window's reset epoch even when it omits Retry-After -- and a rotation that + * cannot see it cools the drained account for the short default instead of until the window + * actually reopens. Optional so existing callers keep compiling. */ - on429?: (retryAfterHeader: string | null) => ProviderAdapter | null | Promise; + on429?: ( + retryAfterHeader: string | null, + responseHeaders?: Headers, + ) => ProviderAdapter | null | Promise; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; /** Called when the bridged Responses stream completes (parity with runTurn / routed paths). */ @@ -579,7 +587,7 @@ export async function runWithImageBridge(deps: ImageBridgeDeps): Promise {}); } catch { /* already closed */ } adapter = rotated; diff --git a/src/oauth/anthropic-routing.ts b/src/oauth/anthropic-routing.ts index a029207be5..6b2eea5a3b 100644 --- a/src/oauth/anthropic-routing.ts +++ b/src/oauth/anthropic-routing.ts @@ -10,9 +10,10 @@ * Intentionally narrower than the Codex pool: no mid-session quota rotation, * soft-avoid ladders, or probe leases. Anthropic OAuth is ToS-sensitive. * - * Affinity is process-local (lost on restart). Cooldown uses Retry-After when present, - * otherwise a default backoff. 401/403 credential failures should set needsReauth on the - * store (existing OAuth path) so the account is excluded from eligibility. + * Affinity is process-local (lost on restart). Cooldown uses Retry-After when present, else + * the reset time of whichever rate-limit window upstream reports as rejected, else a default + * backoff. 401/403 credential failures should set needsReauth on the store (existing OAuth + * path) so the account is excluded from eligibility. */ import { createHash } from "node:crypto"; import { captureOAuthAccountSelection, commitOAuthAccountSelection, credentialGeneration, getAccountSet, getAccountCredential, getAccountCredentialWithStatus } from "./store"; @@ -33,9 +34,16 @@ import type { OcxAccountPoolQuotaWindow, OcxAccountPoolRotationStrategy, OcxConf import { sweepExpiredOnWrite } from "../lib/state-store-sweeper"; import { retainedUtf8Bytes } from "../lib/admission"; +/** + * The read side of a `Headers` object, so a caller can pass the live upstream response's + * headers without this module importing anything from the server layer -- and so a test can + * hand it a plain `new Headers({...})`. + */ +export type AnthropicRateLimitHeaders = Pick; + const PROVIDER = "anthropic"; +/** Backoff only when upstream supplies no usable deadline. */ const DEFAULT_COOLDOWN_MS = 60_000; -const MAX_COOLDOWN_MS = 15 * 60_000; const AFFINITY_IDLE_TTL_MS = 24 * 60 * 60_000; const MAX_AFFINITY_ENTRIES = 2_000; const MAX_AFFINITY_COMPONENT_BYTES = 512; @@ -58,9 +66,19 @@ export interface AnthropicAccountPoolConfig { quotaWindow?: OcxAccountPoolQuotaWindow; } +/** + * Where a cooldown's length came from. Same vocabulary as `CodexCooldownSource`, because it + * answers the same question for the same reason: `retry-after` is upstream answering THIS + * refusal, `reset-derived` is upstream stating when the spent window reopens, and `default` + * is our own guess. The dashboard renders the first as a rate limit and the rest as quota, + * which is exactly the distinction a reset-derived cooldown carries -- collapsing it into + * `retry-after` would report a drained five-hour window as request-rate throttling. + */ +type AnthropicCooldownSource = "retry-after" | "reset-derived" | "default"; + interface AccountHealth { cooldownUntil: number; - cooldownSource: "retry-after" | "default"; + cooldownSource: AnthropicCooldownSource; } interface AffinityEntry { @@ -112,19 +130,38 @@ export function anthropicQuotaWindow(config: AnthropicAccountPoolConfig): OcxAcc return normalizeAccountPoolQuotaWindow(config.quotaWindow); } +/** Accept upstream deadlines within the runtime's date range, without a policy ceiling. */ +function delayUntil(timestamp: number, now: number): number | undefined { + const delay = timestamp - now; + return Number.isFinite(new Date(timestamp).getTime()) && Number.isFinite(delay) && delay > 0 + ? delay : undefined; +} + function parseRetryAfterMs(value: string | null | undefined, now: number): number | undefined { const text = value?.trim(); if (!text) return undefined; if (/^\d+(?:\.\d+)?$/.test(text)) { const seconds = Number(text); - if (Number.isFinite(seconds) && seconds > 0) { - return Math.min(Math.max(Math.ceil(seconds * 1000), 1), MAX_COOLDOWN_MS); - } + if (!Number.isFinite(seconds) || seconds <= 0) return undefined; + return delayUntil(now + Math.max(Math.ceil(seconds * 1000), 1), now); } - const timestamp = Date.parse(text); - if (!Number.isFinite(timestamp)) return undefined; - const delay = timestamp - now; - return delay > 0 ? Math.min(delay, MAX_COOLDOWN_MS) : undefined; + return delayUntil(Date.parse(text), now); +} + +/** Only rejected windows constrain recovery; all must reopen, so take the latest reset. */ +function parseRateLimitResetMs(headers: AnthropicRateLimitHeaders | null | undefined, now: number): number | undefined { + if (!headers) return undefined; + let latest: number | undefined; + for (const window of ["5h", "7d"] as const) { + if (headers.get(`anthropic-ratelimit-unified-${window}-status`)?.trim() !== "rejected") continue; + const resetSeconds = Number(headers.get(`anthropic-ratelimit-unified-${window}-reset`)?.trim()); + if (!Number.isFinite(resetSeconds) || resetSeconds <= 0) continue; + const resetAt = resetSeconds * 1000; + if (delayUntil(resetAt, now) === undefined) continue; + if (latest === undefined || resetAt > latest) latest = resetAt; + } + if (latest === undefined) return undefined; + return latest - now; } export function getAnthropicAccountHealthSnapshot( @@ -669,6 +706,7 @@ export function rotateAnthropicAccountOn429( retryAfterHeader: string | null | undefined, sessionKey?: string | null, now = Date.now(), + rateLimitHeaders?: AnthropicRateLimitHeaders | null, ): string | null { // Reactive 429 failover is NOT gated on the pool flag. That flag buys PROACTIVE routing -- // session affinity, quota-ranked new-session selection, autoSwitchThreshold, strategy -- all @@ -678,11 +716,18 @@ export function rotateAnthropicAccountOn429( // Presence is the activation rule, the same one an apiKeyPool of two keys already uses. if (!isAnthropicAccountPoolEnabled(config) && !hasAnthropicFailoverQuorum(now)) return null; + // Retry-After first: it is the header written FOR this decision. The rejected window's + // reset is the fallback, because a 429 that omits Retry-After still carries it -- and + // without that fallback such a refusal cools for the 60s default and the exhausted + // account is back in the rotation a minute later. const parsedRetry = parseRetryAfterMs(retryAfterHeader, now); - const cooldownMs = parsedRetry ?? DEFAULT_COOLDOWN_MS; + const resetDerived = parsedRetry === undefined ? parseRateLimitResetMs(rateLimitHeaders, now) : undefined; + const cooldownMs = parsedRetry ?? resetDerived ?? DEFAULT_COOLDOWN_MS; upstreamHealth.set(failedAccountId, { cooldownUntil: now + cooldownMs, - cooldownSource: parsedRetry ? "retry-after" : "default", + cooldownSource: parsedRetry !== undefined + ? "retry-after" + : resetDerived !== undefined ? "reset-derived" : "default", }); sweepExpiredOnWrite(now); clearAnthropicSessionAffinityForAccount(failedAccountId); diff --git a/src/oauth/health.ts b/src/oauth/health.ts index 4c997c47cc..011ebd8f41 100644 --- a/src/oauth/health.ts +++ b/src/oauth/health.ts @@ -184,6 +184,9 @@ export function projectStoredOAuthAccountHealth( needsReauth: account.needsReauth === true, reauthReason: account.needsReauth === true ? "refresh_failed" : undefined, cooldownUntilMs: anthropicSnap?.cooldownUntil, + // Same mapping as the Codex pool's `cooldownReasonFromSource`: only a Retry-After is + // request-rate throttling. A reset-derived cooldown means a usage window is spent, which + // is quota, and reporting it as a rate limit would tell the operator to retry shortly. cooldownReason: anthropicSnap?.cooldownSource === "retry-after" ? "rate_limit" : anthropicSnap ? "quota" : undefined, warningReason: detectOAuthWarning(provider, account, opts.observeOnly === true, now), now, diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 7136cd3c70..dbdb6699f7 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -1568,7 +1568,10 @@ function hydrateAccountQuotaCache(): void { if (diskHydrated) return; diskHydrated = true; for (const [key, quota] of readPersistedAccountQuotas()) { - if (!accountQuotaCache.has(key)) accountQuotaCache.set(key, { ts: quota.updatedAt, quota }); + // Disk stores observation time, not the Anthropic usage probe's clock. + if (!accountQuotaCache.has(key)) { + accountQuotaCache.set(key, { ts: key.startsWith("anthropic\u0000") ? 0 : quota.updatedAt, quota }); + } } } @@ -1642,6 +1645,66 @@ export function setCachedProviderAccountQuotaForTests( accountQuotaCache.set(key, { ts: Date.now(), quota }); } +/** Unified headers report utilization fractions and epoch-second reset times. */ +function anthropicHeaderResetAt(value: string | null): number | undefined { + const seconds = toFiniteNumber(value); + if (seconds === undefined || seconds <= 0) return undefined; + const timestamp = seconds * 1000; + return Number.isFinite(new Date(timestamp).getTime()) ? timestamp : undefined; +} + +export function parseAnthropicRateLimitHeaders(headers: Headers): ProviderQuota | null { + const fiveHourPercent = normalizeUtilizationFraction(headers.get("anthropic-ratelimit-unified-5h-utilization")); + const weeklyPercent = normalizeUtilizationFraction(headers.get("anthropic-ratelimit-unified-7d-utilization")); + if (fiveHourPercent === undefined && weeklyPercent === undefined) return null; + const fiveHourResetAt = anthropicHeaderResetAt(headers.get("anthropic-ratelimit-unified-5h-reset")); + const weeklyResetAt = anthropicHeaderResetAt(headers.get("anthropic-ratelimit-unified-7d-reset")); + return { + ...(fiveHourPercent !== undefined ? { fiveHourPercent } : {}), + ...(fiveHourResetAt !== undefined ? { fiveHourResetAt } : {}), + ...(weeklyPercent !== undefined ? { weeklyPercent } : {}), + ...(weeklyResetAt !== undefined ? { weeklyResetAt } : {}), + updatedAt: Date.now(), + }; +} + +/** Reject unknown scales; round fraction conversion for persisted/displayed percentages. */ +function normalizeUtilizationFraction(value: string | null): number | undefined { + const numeric = toFiniteNumber(value); + if (numeric === undefined || numeric < 0 || numeric > 1) return undefined; + return Math.round(numeric * 10_000) / 100; +} + +/** + * Merge serving-account observations without advancing the usage probe's clock or + * erasing model-specific windows. The caller owns credential attribution; this guard + * prevents a retired account key from being revived by an older config generation. + */ +export function recordAnthropicAccountQuotaFromHeaders( + accountId: string, + headers: Headers, + writerGeneration: number, +): void { + if (!accountId) return; + const observed = parseAnthropicRateLimitHeaders(headers); + if (!observed) return; + const key = accountCacheKey("anthropic", accountId); + if (!mayCommitAccountQuotaKey(key, writerGeneration)) return; + // Hydrate before writing, for the same reason `recordPassiveAccountQuota` does: this write + // arrives unprompted from the request path, and `persistAccountQuotaCache` serializes the + // whole map. Landing before any reader has hydrated would persist this single row and erase + // every other provider's saved row. + hydrateAccountQuotaCache(); + const previous = accountQuotaCache.get(key); + accountQuotaCache.set(key, { + ...previous, + // Headers do not prove that the last usage probe succeeded. + ts: previous?.ts ?? 0, + quota: { ...(previous?.quota ?? {}), ...observed }, + }); + persistAccountQuotaCache(); +} + /** * Providers whose per-account quota is OBSERVED in-band, never probed. * @@ -1714,7 +1777,11 @@ export function readPassiveProviderAccountQuotas(provider: string): ProviderAcco export function sweepExpiredProviderAccountQuotaRows(now = Date.now()): number { let removed = 0; for (const [key, entry] of accountQuotaCache) { - if (entry.ts + ACCOUNT_QUOTA_TTL_MS > now) continue; + // Anthropic observations extend retention, never the usage probe's eligibility clock. + const retainedAt = key.startsWith("anthropic\u0000") + ? Math.max(entry.ts, entry.quota?.updatedAt ?? 0) + : entry.ts; + if (retainedAt + ACCOUNT_QUOTA_TTL_MS > now) continue; accountQuotaCache.delete(key); removed += 1; } @@ -1907,6 +1974,7 @@ async function fetchAccountQuota( ): Promise { if (!supportsPerAccountQuota(provider)) return { ts: Date.now(), quota: null, unavailable: true }; if (explicitAccountReader(provider)) return fetchExplicitAccountQuota(provider, accountId, forceRefresh, providerConfig); + if (provider === "anthropic") hydrateAccountQuotaCache(); const key = accountCacheKey(provider, accountId); const writerGeneration = captureConfigGeneration(); const cached = accountQuotaCache.get(key); @@ -1947,7 +2015,8 @@ async function fetchAccountQuota( // negative-cache instead of re-probing on every GUI poll. const entry: AccountQuotaCacheEntry = { ts: Date.now(), - quota: cached?.quota ?? null, + // Settle once for all joiners against observations committed during the probe. + quota: (provider === "anthropic" ? accountQuotaCache.get(key)?.quota : cached?.quota) ?? null, unavailable: true, }; if (mayCommitAccountQuotaKey(key, writerGeneration)) { @@ -1969,7 +2038,7 @@ async function fetchAccountQuota( } catch { const entry: AccountQuotaCacheEntry = { ts: Date.now(), - quota: cached?.quota ?? null, + quota: (provider === "anthropic" ? accountQuotaCache.get(key)?.quota : cached?.quota) ?? null, unavailable: true, }; if (mayCommitAccountQuotaKey(key, writerGeneration)) { diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 3c539c6d8e..fc2195f427 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -230,7 +230,7 @@ import { } from "../../providers/request-pacing"; import { slugsEquivalent } from "../../providers/slug-codec"; import { isMuseSubscriptionUsagePayload, parseMuseSubscriptionUsage } from "../../providers/muse-subscription-usage"; -import { hasPassiveAccountQuota, recordPassiveAccountQuota } from "../../providers/quota"; +import { hasPassiveAccountQuota, recordAnthropicAccountQuotaFromHeaders, recordPassiveAccountQuota } from "../../providers/quota"; import { captureConfigGeneration } from "../../lib/state-store-sweeper"; import { applyOpenAiVirtualModel, resolveOpenAiCompactModel } from "../../providers/openai-virtual-models"; import { isUsageDebugEnabled } from "../../usage/debug"; @@ -3946,7 +3946,27 @@ async function handleResponsesInner( for (let attempt = 0; attempt < 3; attempt++) { if (selectionIsCurrent(requestBindings.get(wireRequest))) { const fetchImpl = (route.provider as OcxProviderConfig & { fetch?: typeof globalThis.fetch }).fetch ?? execute; - return fetchImpl(destination, dispatchInit); + const binding = requestBindings.get(wireRequest); + const snapshot = route.providerName === "anthropic" && anthropicPoolAccountId && binding?.kind === "oauth" + ? binding.snapshot : undefined; + const writerGeneration = snapshot ? captureConfigGeneration() : 0; + const sentHeaders = snapshot ? new Headers(dispatchInit.headers) : undefined; + const ownsBearer = snapshot !== undefined + && sentHeaders?.get("authorization") === `Bearer ${snapshot.accessToken}` + && !sentHeaders?.has("x-api-key"); + const response = await fetchImpl(destination, dispatchInit); + // Observe each physical response before retries replace it. The binding belongs to + // this dispatch, so a manual switch cannot file A's headers against B. Header + // overrides and credential replacement make ownership unprovable: skip those writes. + if (ownsBearer && snapshot) { + try { + const current = getAccountCredentialWithStatus("anthropic", snapshot.accountId); + if (current && !current.needsReauth && credentialGeneration(current.credential) === snapshot.generation) { + recordAnthropicAccountQuotaFromHeaders(snapshot.accountId, response.headers, writerGeneration); + } + } catch { /* best-effort observation cannot fail the response */ } + } + return response; } const nextAdapter = await refreshDispatchAdapter(requestParsed); const rebuilt = await nextAdapter.buildRequest(requestParsed, { @@ -5956,7 +5976,10 @@ async function handleResponsesInner( const imgPlan = !routedCompaction ? await planImageBridge(config, parsed, route.provider) : undefined; const vidPlan = !routedCompaction ? await planVideoBridge(config, parsed, route.provider) : undefined; const canRunWebSearch = !!wsPlan && !adapter.runTurn; - const rotateSidecarProviderOn429 = async (retryAfter: string | null): Promise => { + const rotateSidecarProviderOn429 = async ( + retryAfter: string | null, + responseHeaders?: Headers, + ): Promise => { const rotated = rotateProviderTransportOn429(config, route.providerName, route.provider, { retryAfter, now: Date.now(), @@ -6000,6 +6023,8 @@ async function handleResponsesInner( anthropicPoolAccountId, retryAfter, anthropicSessionKey, + Date.now(), + responseHeaders, ); if (!nextAccountId) return null; try { @@ -7032,6 +7057,8 @@ async function handleResponsesInner( anthropicPoolAccountId, upstreamResponse.headers.get("retry-after"), anthropicSessionKey, + Date.now(), + upstreamResponse.headers, ); if (!nextAccountId) break; try { void upstreamResponse.body?.cancel().catch(() => {}); } catch { /* already consumed/closed */ } @@ -7445,6 +7472,8 @@ async function handleResponsesInner( anthropicPoolAccountId, response.headers.get("retry-after"), anthropicSessionKey, + Date.now(), + response.headers, ); if (nextAccountId) { try { void response.body?.cancel().catch(() => {}); } catch { /* already closed */ } diff --git a/src/web-search/loop.ts b/src/web-search/loop.ts index 3a2c5e99b4..0c957e1c17 100644 --- a/src/web-search/loop.ts +++ b/src/web-search/loop.ts @@ -309,8 +309,16 @@ export interface WebSearchLoopDeps { * 429 failover hook: rotate the provider's active credential and return a rebuilt adapter, * or null when the pool is exhausted. Async hooks support OAuth refresh; existing synchronous * key-pool hooks remain valid. + * + * `responseHeaders` carries the whole refusal, not just Retry-After, because an Anthropic + * 429 states the window's reset epoch even when it omits Retry-After -- and a rotation that + * cannot see it cools the drained account for the short default instead of until the window + * actually reopens. Optional so existing callers keep compiling. */ - on429?: (retryAfterHeader: string | null) => ProviderAdapter | null | Promise; + on429?: ( + retryAfterHeader: string | null, + responseHeaders?: Headers, + ) => ProviderAdapter | null | Promise; /** Opt-in same-target 429 policy (key-auth providers). When present, 429 replays on the SAME key before on429 rotation. */ retryOn429Policy?: Required | null; /** Called only when the final bridged Responses stream reaches completed or incomplete. */ @@ -521,7 +529,7 @@ export async function runWithWebSearch(deps: WebSearchLoopDeps): Promise }[]; + +beforeEach(() => { + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-quota-dispatch-")); + process.env.OPENCODEX_HOME = home; + sent = []; + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearGenericFailoverHealth(); + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); +}); + +afterEach(() => { + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearGenericFailoverHealth(); + // Cancel the debounced persistence before restoring the real home. + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + removeTreeWithRetry(home); +}); + +function credential(index: number) { + return { + access: `synthetic-anthropic-access-${index}`, + refresh: `synthetic-anthropic-refresh-${index}`, + expires: Date.now() + 3_600_000, + accountId: `synthetic-account-${index}`, + }; +} + +async function seed(count = 2): Promise { + for (let index = 0; index < count; index++) { + await saveCredential("anthropic", credential(index)); + } + const ids = getAccountSet("anthropic")!.accounts.map(account => account.id); + await setActiveAccount("anthropic", ids[0]!); + return ids; +} + +function quotaHeaders(fiveHour: string, weekly: string): Record { + return { + "anthropic-ratelimit-unified-5h-utilization": fiveHour, + "anthropic-ratelimit-unified-7d-utilization": weekly, + }; +} + +function limited(fiveHour = "1", weekly = "0.61"): Response { + return Response.json({ type: "error", error: { type: "rate_limit_error", message: "synthetic quota exhausted" } }, { + status: 429, + headers: { ...quotaHeaders(fiveHour, weekly), "retry-after": "30" }, + }); +} + +function answer(stream: boolean, fiveHour = "0.23", weekly = "0.47", text = "The answer is complete."): Response { + const usage = { input_tokens: 8, output_tokens: 6 }; + const message = { id: "msg_synthetic", type: "message", role: "assistant", model: "claude-sonnet-4-5", content: [{ type: "text", text }], stop_reason: "end_turn", usage }; + if (!stream) return Response.json(message, { headers: quotaHeaders(fiveHour, weekly) }); + const frames = [ + { type: "message_start", message: { ...message, content: [], stop_reason: null } }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text } }, + { type: "content_block_stop", index: 0 }, + { type: "message_delta", delta: { stop_reason: "end_turn" }, usage }, + { type: "message_stop" }, + ]; + return new Response(frames.map(frame => `event: ${frame.type}\ndata: ${JSON.stringify(frame)}\n\n`).join(""), { + headers: { ...quotaHeaders(fiveHour, weekly), "content-type": "text/event-stream" }, + }); +} + +function configFor(reply: (body: Record) => Response | Promise, headers?: Record): OcxConfig { + const transport = (async (_input, init) => { + const wireHeaders = new Headers(init?.headers); + const body = JSON.parse(String(init?.body)) as Record; + sent.push({ authorization: wireHeaders.get("authorization"), apiKey: wireHeaders.get("x-api-key"), body }); + return reply(body); + }) as typeof fetch; + const provider: OcxProviderConfig & { fetch: typeof fetch } = { + adapter: "anthropic", baseUrl: "https://anthropic-quota.test", authMode: "oauth", + models: ["claude-sonnet-4-5"], fetch: transport, ...(headers ? { headers } : {}), + }; + return { + port: 0, defaultProvider: "anthropic", + anthropicAccountPool: { enabled: false, strategy: "round-robin" }, + providers: { anthropic: provider }, + }; +} + +function post(config: OcxConfig, body: Record = {}) { + return handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "anthropic/claude-sonnet-4-5", input: "Answer briefly", stream: false, ...body }), + }), config, { model: "", provider: "" }); +} + +function expectQuota(id: string, fiveHourPercent: number, weeklyPercent: number) { + expect(getCachedProviderAccountQuota("anthropic", id)).toMatchObject({ fiveHourPercent, weeklyPercent }); +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(done => { resolve = done; }); + return { promise, resolve }; +} + +test("main A429 -> B200 records both physical responses against their sending accounts", async () => { + const [a, b] = await seed(); + const config = configFor(body => { + if (sent.length === 1) return limited(); + expect(sent.length).toBe(2); + // A must already be measured before the replacement response exists. + expectQuota(a!, 100, 61); + expect(getCachedProviderAccountQuota("anthropic", b!)).toBeNull(); + return answer(body.stream === true); + }); + const response = await post(config); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expectQuota(a!, 100, 61); + expectQuota(b!, 23, 47); +}); + +test("terminal 429 after both accounts are exhausted records both refused physical responses", async () => { + const [a, b] = await seed(); + const response = await post(configFor(() => { + if (sent.length === 1) return limited(); + expect(sent.length).toBe(2); + expectQuota(a!, 100, 61); + return limited("0.89", "1"); + })); + expect(response.status).toBe(429); + await response.text(); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expectQuota(a!, 100, 61); + expectQuota(b!, 89, 100); +}); + +test("manual active switch while A is pending keeps A's measurement off B", async () => { + const [a, b] = await seed(); + const entered = deferred(); + const returned = deferred(); + const config = configFor(() => { entered.resolve(); return returned.promise; }); + const pending = post(config); + await entered.promise; + try { + expect(sent[0]!.authorization).toBe(`Bearer ${credential(0).access}`); + expect(await setActiveAccount("anthropic", b!)).toBe(true); + } finally { + returned.resolve(answer(false, "0.37", "0.53")); + } + const response = await pending; + expect(response.status).toBe(200); + await response.text(); + expect(sent).toHaveLength(1); + expect(getAccountSet("anthropic")!.activeAccountId).toBe(b!); + expectQuota(a!, 37, 53); + expect(getCachedProviderAccountQuota("anthropic", b!)).toBeNull(); +}); + +test("credential replacement while A is pending skips its old-generation response", async () => { + const [a, b] = await seed(); + const entered = deferred(); + const returned = deferred(); + const pending = post(configFor(() => { entered.resolve(); return returned.promise; })); + await entered.promise; + try { + expect(sent[0]!.authorization).toBe(`Bearer ${credential(0).access}`); + await saveAccountCredential("anthropic", a!, { ...credential(0), access: "synthetic-replacement-access", refresh: "synthetic-replacement-refresh" }); + } finally { + returned.resolve(answer(false)); + } + const response = await pending; + expect(response.status).toBe(200); + await response.text(); + expect(sent).toHaveLength(1); + expect(getAccountSet("anthropic")!.accounts.find(row => row.id === a)!.credential.access).toBe("synthetic-replacement-access"); + expect(getCachedProviderAccountQuota("anthropic", a!)).toBeNull(); + expect(getCachedProviderAccountQuota("anthropic", b!)).toBeNull(); +}); + +const overriddenHeaders: { label: string; headers: Record; authorization: string; apiKey: string | null }[] = [ + { label: "overridden bearer", headers: { Authorization: "Bearer synthetic-override" }, authorization: "Bearer synthetic-override", apiKey: null }, + { label: "additional x-api-key", headers: { "x-api-key": "synthetic-api-key" }, authorization: `Bearer ${credential(0).access}`, apiKey: "synthetic-api-key" }, +]; +test.each(overriddenHeaders)("$label skips quota attribution even when a selected OAuth account exists", async ({ headers, authorization, apiKey }) => { + const ids = await seed(); + const response = await post(configFor(body => answer(body.stream === true), headers)); + expect(response.status).toBe(200); + await response.text(); + expect(sent).toHaveLength(1); + expect(sent[0]).toMatchObject({ authorization, apiKey }); + for (const id of ids) expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); +}); + +test("real web-search routed loop records A429 and B200 through fetchForRequest", async () => { + const [a, b] = await seed(); + const config = configFor(body => { + // The search loop forces upstream streaming although the client asks for JSON. + expect(body.stream).toBe(true); + if (sent.length === 1) return limited(); + expect(sent.length).toBe(2); + expectQuota(a!, 100, 61); + return answer(true); + }); + config.webSearchSidecar = { backend: "anthropic", enabled: true }; + const response = await post(config, { tools: [{ type: "web_search" }] }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expectQuota(a!, 100, 61); + expectQuota(b!, 23, 47); +}); + +test("real terminal continuation records A429 before retrying the continuation on B", async () => { + const [a, b] = await seed(); + const config = configFor(body => { + // The real guard recognizes an actionable request plus a short execution announcement, + // with available tools and no tool call. A normal completed answer does not trigger it. + if (sent.length === 1) return answer(body.stream === true, "0.11", "0.31", "I will modify the file now."); + if (sent.length === 2) { + expectQuota(a!, 11, 31); + return limited(); + } + expect(sent.length).toBe(3); + expectQuota(a!, 100, 61); + return answer(body.stream === true); + }); + const response = await post(config, { + input: "Please modify the file now", + tools: [{ type: "function", name: "read_file", description: "read a file", parameters: { type: "object" } }], + }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expectQuota(a!, 100, 61); + expectQuota(b!, 23, 47); +}); + +test("real image bridge routed loop records A429 and B200 through fetchForRequest", async () => { + const [a, b] = await seed(); + const config = configFor(body => { + expect(body.stream).toBe(true); + // Only the bridge installs this synthetic tool for the hosted image_generation input. + expect(body.tools).toEqual(expect.arrayContaining([expect.objectContaining({ name: "custom_image_gen" })])); + if (sent.length === 1) return limited(); + expect(sent.length).toBe(2); + expectQuota(a!, 100, 61); + return answer(true); + }); + config.images = { bridgeEnabled: true }; + config.providers.xai = { + adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "key", apiKey: "synthetic-image-key", + }; + const response = await post(config, { stream: true, tools: [{ type: "image_generation" }] }); + expect(response.status).toBe(200); + expect(await response.text()).toContain("The answer is complete."); + expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); + expectQuota(a!, 100, 61); + expectQuota(b!, 23, 47); +}); diff --git a/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts b/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts new file mode 100644 index 0000000000..21ae2d8489 --- /dev/null +++ b/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts @@ -0,0 +1,528 @@ +/** Anthropic response observations must preserve account usage and probe semantics. */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + clearAnthropicAccountCooldown, + clearAnthropicAccountPoolState, + forgetAnthropicFailoverQuorum, + getAnthropicAccountHealthSnapshot, + rotateAnthropicAccountOn429, +} from "../../../src/oauth/anthropic-routing"; +import { projectStoredOAuthAccountHealth } from "../../../src/oauth/health"; +import { + clearAccountQuotaCache, + fetchProviderAccountQuotas, + getCachedProviderAccountQuota, + parseAnthropicRateLimitHeaders, + recordAnthropicAccountQuotaFromHeaders, + reconcileProviderAccountQuotaRows, + resetProviderQuotaReconcileStateForTests, + setCachedProviderAccountQuotaForTests, + sweepExpiredProviderAccountQuotaRows, +} from "../../../src/providers/quota"; +import { getAccountSet, saveCredential } from "../../../src/oauth/store"; +import { clearPoolRotationState } from "../../../src/codex/pool-rotation"; +import { removeTreeWithRetry } from "../../helpers/remove-tree"; +import type { OcxConfig } from "../../../src/types"; + +const originalHome = process.env.OPENCODEX_HOME; +const originalFetch = globalThis.fetch; +const originalNow = Date.now; +let home: string; + +beforeEach(() => { + globalThis.fetch = (async () => { throw new Error("Unexpected network request in quota test"); }) as typeof fetch; + home = mkdtempSync(join(tmpdir(), "ocx-anthropic-ratelimit-")); + process.env.OPENCODEX_HOME = home; + clearAnthropicAccountPoolState(); + clearPoolRotationState(); + clearAccountQuotaCache(); + // `lastReconciledGeneration` is module-global and survives a cache clear, so the fence case + // below would otherwise raise the floor for every test that runs after it in this file. + resetProviderQuotaReconcileStateForTests(); + forgetAnthropicFailoverQuorum(); +}); + +afterEach(() => { + globalThis.fetch = originalFetch; + Date.now = originalNow; + clearAnthropicAccountPoolState(); + clearPoolRotationState(); + // The argument-less form, deliberately: only it calls cancelPendingAccountQuotaPersist. + // The observer ends in a 250ms-debounced write that resolves OPENCODEX_HOME at fire time, + // so a provider-scoped clear would leave that write to land in whatever home is current a + // quarter second later — the next test's sandbox, or the developer's real one. + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); + forgetAnthropicFailoverQuorum(); + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + removeTreeWithRetry(home); +}); + +/** The store assigns its own slot ids, so the seeded `accountId` is never the cache key. */ +async function seed(count: number): Promise { + for (let i = 0; i < count; i++) { + await saveCredential("anthropic", { + access: `access-${i}`, + refresh: `refresh-${i}`, + expires: Date.now() + 3_600_000, + accountId: `uuid-${i}`, + email: `user${i}@example.test`, + } as never); + } + return getAccountSet("anthropic")?.accounts.map(a => a.id) ?? []; +} + +function poolEnabled(): OcxConfig { + return { + port: 0, + defaultProvider: "anthropic", + providers: { + anthropic: { adapter: "anthropic", baseUrl: "https://api.anthropic.com", authMode: "oauth" }, + }, + anthropicAccountPool: { enabled: true }, + } as OcxConfig; +} + +/** A real 429 from a drained five-hour window, captured from api.anthropic.com. */ +function drainedFiveHour(resetEpochSeconds: number): Headers { + return new Headers({ + "anthropic-ratelimit-unified-status": "rejected", + "anthropic-ratelimit-unified-5h-status": "rejected", + "anthropic-ratelimit-unified-5h-reset": String(resetEpochSeconds), + "anthropic-ratelimit-unified-5h-utilization": "1.0", + "anthropic-ratelimit-unified-7d-status": "allowed", + "anthropic-ratelimit-unified-7d-reset": String(resetEpochSeconds + 86_400), + "anthropic-ratelimit-unified-7d-utilization": "0.36", + }); +} + +describe("Anthropic cooldown honours the stated window", () => { + test("a multi-hour Retry-After is not truncated to the guessed-backoff ceiling", async () => { + const start = Date.now(); + const ids = await seed(2); + // 7999s is what a drained five-hour window actually answers; the old 15-minute clamp + // turned a single refusal into sixteen wasted retries before the window reopened. + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, "7999", null, start); + const health = getAnthropicAccountHealthSnapshot(ids[0]!, start); + expect(health?.cooldownUntil).toBe(start + 7_999_000); + expect(health?.cooldownSource).toBe("retry-after"); + }); + + test("a week-long Retry-After retains its stated deadline", async () => { + const start = Date.now(); + const ids = await seed(2); + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, "604800", null, start); + expect(getAnthropicAccountHealthSnapshot(ids[0]!, start)?.cooldownUntil) + .toBe(start + 604_800_000); + }); + + test("an HTTP-date Retry-After is honoured beyond six hours", async () => { + const start = Date.now(); + const ids = await seed(2); + // RFC 9110 allows either form, and both are upstream STATING when it will serve again -- + // the date branch had its own clamp and would have kept the 15-minute truncation. + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, new Date(start + 2 * 60 * 60_000).toUTCString(), null, start); + const cooldown = getAnthropicAccountHealthSnapshot(ids[0]!, start)?.cooldownUntil; + // toUTCString drops sub-second precision, so the deadline lands within a second of target. + expect(cooldown).toBeGreaterThan(start + 2 * 60 * 60_000 - 1_000); + expect(cooldown).toBeLessThanOrEqual(start + 2 * 60 * 60_000); + + const reset = Math.floor(start / 1000) * 1000 + 48 * 60 * 60_000; + rotateAnthropicAccountOn429(poolEnabled(), ids[1]!, new Date(reset).toUTCString(), null, start); + expect(getAnthropicAccountHealthSnapshot(ids[1]!, start)?.cooldownUntil).toBe(reset); + }); + + test("a 429 without Retry-After cools until the rejected window reopens", async () => { + const start = Date.now(); + const ids = await seed(2); + // The wire carries whole seconds, so the reset is built from an epoch second and the + // expectation is derived from the same value rather than from `start + 90min` — an + // assertion on the un-truncated millisecond would be testing the fixture, not the code. + const resetEpochSeconds = Math.floor((start + 90 * 60_000) / 1000); + // Retry-After is not guaranteed on an Anthropic 429; the rejected window's reset is. + // Without reading it this refusal cooled for the 60s default and the drained account + // was back in the rotation a minute later. + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, null, null, start, drainedFiveHour(resetEpochSeconds)); + const health = getAnthropicAccountHealthSnapshot(ids[0]!, start); + expect(health?.cooldownUntil).toBe(resetEpochSeconds * 1000); + // Its own source, not "retry-after": the dashboard renders that one as request-rate + // throttling, and a spent five-hour window is quota. Same vocabulary the Codex pool uses. + expect(health?.cooldownSource).toBe("reset-derived"); + }); + + test("an ALLOWED window's reset never cools the account", async () => { + const start = Date.now(); + const ids = await seed(2); + // Every response names when the current period ends, including a healthy one. Treating + // that as a cooldown would bench an account with 4% used for the rest of its window. + const healthy = new Headers({ + "anthropic-ratelimit-unified-status": "allowed", + "anthropic-ratelimit-unified-5h-status": "allowed", + "anthropic-ratelimit-unified-5h-reset": String(Math.floor((start + 3 * 60 * 60_000) / 1000)), + "anthropic-ratelimit-unified-5h-utilization": "0.04", + }); + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, null, null, start, healthy); + const health = getAnthropicAccountHealthSnapshot(ids[0]!, start); + expect(health?.cooldownUntil).toBe(start + 60_000); + expect(health?.cooldownSource).toBe("default"); + }); + + test("both windows rejected cools until the LAST one reopens", async () => { + const start = Date.now(); + const ids = await seed(2); + // The limiter is AND-composed: upstream refuses while ANY window rejects. An account whose + // 5-hour bucket rolls in three minutes is still refused for the days its weekly window + // needs, so cooling to the earliest reset would re-offer it every three minutes until the + // weekly window finally reopens -- the exact loop this path exists to end. + const fiveHourReset = Math.floor((start + 3 * 60_000) / 1000); + const weeklyReset = Math.floor((start + 5 * 24 * 60 * 60_000) / 1000); + const bothDrained = new Headers({ + "anthropic-ratelimit-unified-status": "rejected", + "anthropic-ratelimit-unified-5h-status": "rejected", + "anthropic-ratelimit-unified-5h-reset": String(fiveHourReset), + "anthropic-ratelimit-unified-7d-status": "rejected", + "anthropic-ratelimit-unified-7d-reset": String(weeklyReset), + }); + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, null, null, start, bothDrained); + expect(getAnthropicAccountHealthSnapshot(ids[0]!, start)?.cooldownUntil).toBe(weeklyReset * 1000); + }); + + test("a reset-derived cooldown surfaces as quota, a Retry-After as a rate limit", async () => { + const start = Date.now(); + const ids = await seed(2); + const account = getAccountSet("anthropic")!.accounts.find(a => a.id === ids[0]!)!; + // The distinction is not cosmetic: the dashboard tells an operator to wait out a rate + // limit and to switch accounts on spent quota. A drained five-hour window is the second. + rotateAnthropicAccountOn429( + poolEnabled(), + ids[0]!, + null, + null, + start, + drainedFiveHour(Math.floor((start + 90 * 60_000) / 1000)), + ); + expect(projectStoredOAuthAccountHealth("anthropic", account, start)).toMatchObject({ + status: "cooldown", + reason: "quota", + }); + + clearAnthropicAccountCooldown(ids[0]!); + rotateAnthropicAccountOn429(poolEnabled(), ids[0]!, "300", null, start); + expect(projectStoredOAuthAccountHealth("anthropic", account, start)).toMatchObject({ + status: "cooldown", + reason: "rate_limit", + }); + }); + + test("Retry-After wins over the header reset", async () => { + const start = Date.now(); + const ids = await seed(2); + // Retry-After is written for this decision; the reset epoch is a fallback for the + // refusals that omit it. A disagreement must not silently prefer the fallback. + rotateAnthropicAccountOn429( + poolEnabled(), + ids[0]!, + "120", + null, + start, + drainedFiveHour(Math.floor((start + 4 * 60 * 60_000) / 1000)), + ); + expect(getAnthropicAccountHealthSnapshot(ids[0]!, start)?.cooldownUntil).toBe(start + 120_000); + }); +}); + +describe("Anthropic rate-limit headers feed the routing cache", () => { + test("utilization is read as a fraction, not as a percent", () => { + // The header sends 0.74 for a 74%-spent window while the probe endpoint sends 74.0 for + // the same account. Passing the header value through unscaled would file the emptiest + // account as the freshest and route every new session straight at it. + const quota = parseAnthropicRateLimitHeaders(new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0.42", + "anthropic-ratelimit-unified-7d-utilization": "0.74", + })); + expect(quota?.fiveHourPercent).toBe(42); + expect(quota?.weeklyPercent).toBe(74); + }); + + test("reset epochs are promoted from seconds to milliseconds", () => { + const quota = parseAnthropicRateLimitHeaders(new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0.5", + "anthropic-ratelimit-unified-5h-reset": "1788717000", + })); + expect(quota?.fiveHourResetAt).toBe(1_788_717_000_000); + }); + + test("a header set with no utilization yields no measurement", () => { + // A renamed or dropped header must degrade to "unmeasured", which the router already + // has a defined behaviour for -- never to a fabricated zero, which reads as a fresh + // account and would pull traffic toward whichever account stopped reporting. + expect(parseAnthropicRateLimitHeaders(new Headers({ + "anthropic-ratelimit-unified-5h-reset": "1788717000", + }))).toBeNull(); + }); + + test("a utilization above 1 is rejected rather than clamped", () => { + // Above one is a wire change, not a full window. Inventing 100 from it would cool a + // healthy account on a misread. + expect(parseAnthropicRateLimitHeaders(new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "42", + }))).toBeNull(); + }); + + test("an observed turn makes the serving account's usage known to the router", async () => { + const ids = await seed(2); + // Before the observation the account has no reading at all, which is what left a + // two-account pool scoring both at UNKNOWN_USAGE_SCORE and picking between them blind. + expect(getCachedProviderAccountQuota("anthropic", ids[0]!)).toBeNull(); + recordAnthropicAccountQuotaFromHeaders(ids[0]!, drainedFiveHour(Math.floor(Date.now() / 1000) + 3600), 0); + expect(getCachedProviderAccountQuota("anthropic", ids[0]!)?.fiveHourPercent).toBe(100); + // The other account stays unmeasured: an observation is attributed to the account that + // served the turn, never spread across the roster. + expect(getCachedProviderAccountQuota("anthropic", ids[1]!)).toBeNull(); + }); + + test("headers with nothing parseable leave the previous reading intact", async () => { + const ids = await seed(1); + recordAnthropicAccountQuotaFromHeaders(ids[0]!, new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0.25", + }), 0); + recordAnthropicAccountQuotaFromHeaders(ids[0]!, new Headers({ "content-type": "application/json" }), 0); + // A response that says nothing about quota is not evidence that the quota is gone. + expect(getCachedProviderAccountQuota("anthropic", ids[0]!)?.fiveHourPercent).toBe(25); + }); + + test("an empty account id writes nothing", () => { + // API-key providers and single-account installs below failover quorum reach the observer + // with no account to attribute; that is an ordinary state, not an error. Asserting only + // that it does not throw would pass with the guard deleted -- an empty-string cache key + // is perfectly writable -- so this asserts the absence of the row instead. + recordAnthropicAccountQuotaFromHeaders("", drainedFiveHour(Math.floor(Date.now() / 1000) + 3600), 0); + expect(getCachedProviderAccountQuota("anthropic", "")).toBeNull(); + }); + + test("a stale writer generation is refused", async () => { + const ids = await seed(1); + // The fence exists because a turn is a long await: an account or config change that lands + // mid-turn must not be overwritten by a measurement taken before it. Every other test here + // passes 0, which a fresh worker always accepts, so without this case the parameter is + // carried but never actually exercised as a fence. + reconcileProviderAccountQuotaRows({ + generation: 5, + providerNames: new Set(), + comboIds: new Set(), + comboTargets: new Set(), + codexAccountIds: new Set(), + oauthAccountKeys: new Set(), + configRoots: new Set(), + }); + recordAnthropicAccountQuotaFromHeaders(ids[0]!, new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0.5", + }), 1); + expect(getCachedProviderAccountQuota("anthropic", ids[0]!)).toBeNull(); + }); + + test("an observation keeps the model-scoped bars the probe filled", async () => { + const ids = await seed(1); + // The probe reports per-model weekly limits (Opus, Sonnet, Fable) that no header carries. + // They are read by the manual-preference exhaustion check and by `headroomOf`, so a + // wholesale replace would not merely blank the dashboard: it would route an Opus request + // to an account whose Opus allowance is spent. + setCachedProviderAccountQuotaForTests("anthropic", ids[0]!, { + fiveHourPercent: 10, + weeklyPercent: 20, + customWindows: [{ label: "Opus", percent: 96 }], + updatedAt: Date.now(), + }); + recordAnthropicAccountQuotaFromHeaders(ids[0]!, new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0.41", + }), 0); + const quota = getCachedProviderAccountQuota("anthropic", ids[0]!); + expect(quota?.fiveHourPercent).toBe(41); + // Untouched by this observation, not erased by it. + expect(quota?.weeklyPercent).toBe(20); + expect(quota?.customWindows).toEqual([{ label: "Opus", percent: 96 }]); + }); + + test("a percent that is not exactly representable is rounded, not left as an artifact", () => { + // `0.29 * 100` is 28.999999999999996 in binary floating point, and the CLI interpolates the + // percent raw. A user reading `5h 28.999999999999996%` would reasonably file a bug. + expect(parseAnthropicRateLimitHeaders(new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0.29", + }))?.fiveHourPercent).toBe(29); + }); +}); + +describe("Anthropic observation and probe clocks", () => { + function observe(accountId: string, percent = "0.41"): void { + recordAnthropicAccountQuotaFromHeaders(accountId, new Headers({ + "anthropic-ratelimit-unified-5h-utilization": percent, + }), 0); + } + + function usageResponse(): Response { + return Response.json({ five_hour: { utilization: 12 }, seven_day_opus: { utilization: 63 } }); + } + + test("a cold header-only row does not defer the first usage probe", async () => { + const [id] = await seed(1); + let calls = 0; + globalThis.fetch = (async () => { calls++; return usageResponse(); }) as typeof fetch; + observe(id!); + expect(getCachedProviderAccountQuota("anthropic", id!)?.fiveHourPercent).toBe(41); + const [row] = await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(1); + expect(row?.quota).toMatchObject({ fiveHourPercent: 12, customWindows: [{ label: "Opus", percent: 63 }] }); + expect(row?.unavailable).toBeUndefined(); + }); + + test("fresh header observations survive sweeping until their own TTL expires", async () => { + const [id] = await seed(1); + const observedAt = originalNow(); + Date.now = () => observedAt; + observe(id!); + expect(sweepExpiredProviderAccountQuotaRows(observedAt + 1)).toBe(0); + expect(getCachedProviderAccountQuota("anthropic", id!)?.fiveHourPercent).toBe(41); + expect(sweepExpiredProviderAccountQuotaRows(observedAt + 10 * 60_000 - 1)).toBe(0); + expect(sweepExpiredProviderAccountQuotaRows(observedAt + 10 * 60_000)).toBe(1); + expect(getCachedProviderAccountQuota("anthropic", id!)).toBeNull(); + }); + + test("headers preserve the probe TTL instead of renewing it", async () => { + const [id] = await seed(1); + let now = originalNow(); + Date.now = () => now; + let calls = 0; + globalThis.fetch = (async () => { calls++; return usageResponse(); }) as typeof fetch; + await fetchProviderAccountQuotas("anthropic"); + now += 9 * 60_000; + observe(id!); + expect((await fetchProviderAccountQuotas("anthropic"))[0]?.quota?.fiveHourPercent).toBe(41); + expect(calls).toBe(1); + now += 60_001; + await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(2); + }); + + for (const observeAfterRestart of [false, true]) { + test(`restart keeps Anthropic probes due with new headers: ${observeAfterRestart}`, async () => { + const [id] = await seed(1); + const updatedAt = Date.now(); + const saved = { fiveHourPercent: 41, customWindows: [{ label: "Opus", percent: 63 }], updatedAt }; + writeFileSync(join(home, "provider-account-quota-cache.json"), JSON.stringify({ + version: 1, + rows: { [`anthropic\u0000${id}`]: saved, "kiro\u0000other": { monthlyPercent: 17, updatedAt } }, + })); + clearAccountQuotaCache(); + // Cover both dashboard-first and response-first hydration after restart. + if (observeAfterRestart) observe(id!, "0.52"); + let calls = 0; + globalThis.fetch = (async () => { calls++; return new Response("busy", { status: 429 }); }) as typeof fetch; + const [row] = await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(1); + expect(row?.quota).toMatchObject({ fiveHourPercent: observeAfterRestart ? 52 : 41, customWindows: saved.customWindows }); + expect(getCachedProviderAccountQuota("kiro", "other")?.monthlyPercent).toBe(17); + expect(row?.unavailable).toBe(true); + }); + } + + for (const [failure, warm] of [["http", true], ["network", true], ["http", false]] as const) { + test(`joined ${failure} probe failures preserve in-flight headers (warm cache: ${warm})`, async () => { + const [id] = await seed(1); + if (warm) setCachedProviderAccountQuotaForTests("anthropic", id!, { + fiveHourPercent: 10, weeklyPercent: 20, customWindows: [{ label: "Opus", percent: 63 }], updatedAt: Date.now(), + }); + let started!: () => void; + const dispatched = new Promise(resolve => { started = resolve; }); + let finish!: (response: Response) => void; + let fail!: (error: Error) => void; + const response = new Promise((resolve, reject) => { finish = resolve; fail = reject; }); + let calls = 0; + globalThis.fetch = (async () => { calls++; started(); return response; }) as typeof fetch; + const first = fetchProviderAccountQuotas("anthropic", true); + await dispatched; + const second = fetchProviderAccountQuotas("anthropic", true); + observe(id!); + const latest = getCachedProviderAccountQuota("anthropic", id!); + if (failure === "http") finish(new Response("busy", { status: 429 })); + else fail(new Error("offline")); + const [a, b] = await Promise.all([first, second]); + expect(calls).toBe(1); + expect(a).toEqual(b); + expect(a[0]?.quota).toEqual(latest); + expect(a[0]?.quota?.fiveHourPercent).toBe(41); + if (warm) expect(a[0]?.quota).toMatchObject({ weeklyPercent: 20, customWindows: [{ label: "Opus", percent: 63 }] }); + expect(a[0]?.unavailable).toBe(true); + expect(getCachedProviderAccountQuota("anthropic", id!)).toEqual(latest); + // A later partial observation cannot claim that the failed usage probe succeeded. + observe(id!, "0.53"); + const [cached] = await fetchProviderAccountQuotas("anthropic"); + expect(cached?.unavailable).toBe(true); + expect(cached?.quota?.fiveHourPercent).toBe(53); + expect(calls).toBe(1); + globalThis.fetch = (async () => usageResponse()) as typeof fetch; + expect((await fetchProviderAccountQuotas("anthropic", true))[0]?.unavailable).toBeUndefined(); + }); + } +}); + +describe("Anthropic malformed deadlines and partial windows", () => { + for (const invalid of ["NaN", "Infinity", "1e309", "1e308", "8640000000001", "not-a-date", "-1", "0"]) { + test(`invalid reset ${invalid} cannot establish a cooldown deadline`, async () => { + const start = Date.now(); + const [id] = await seed(1); + const headers = new Headers({ + "anthropic-ratelimit-unified-7d-status": "rejected", + "anthropic-ratelimit-unified-7d-reset": invalid, + "anthropic-ratelimit-unified-7d-utilization": "0.74", + }); + rotateAnthropicAccountOn429(poolEnabled(), id!, null, null, start, headers); + expect(getAnthropicAccountHealthSnapshot(id!, start)).toMatchObject({ + cooldownUntil: start + 60_000, cooldownSource: "default", + }); + expect(parseAnthropicRateLimitHeaders(headers)?.weeklyResetAt).toBeUndefined(); + }); + } + + test("overflowing Retry-After falls back to a valid rejected reset", async () => { + const start = Date.now(); + const [id] = await seed(1); + const reset = Math.floor(start / 1000) + 432_000; + for (const invalid of ["9".repeat(400), "8640000000001", "invalid-date"]) { + rotateAnthropicAccountOn429(poolEnabled(), id!, invalid, null, start, drainedFiveHour(reset)); + expect(getAnthropicAccountHealthSnapshot(id!, start)).toMatchObject({ + cooldownUntil: reset * 1000, cooldownSource: "reset-derived", + }); + } + }); + + test("a malformed weekly deadline cannot hide a valid five-hour reset", async () => { + const start = Date.now(); + const [id] = await seed(1); + const reset = Math.floor(start / 1000) + 180; + const headers = drainedFiveHour(reset); + headers.set("anthropic-ratelimit-unified-7d-status", "rejected"); + headers.set("anthropic-ratelimit-unified-7d-reset", "1e308"); + rotateAnthropicAccountOn429(poolEnabled(), id!, null, null, start, headers); + expect(getAnthropicAccountHealthSnapshot(id!, start)?.cooldownUntil).toBe(reset * 1000); + }); + + test("partial zero utilization preserves other and model-specific windows", async () => { + const [id] = await seed(1); + const customWindows = [{ label: "Opus", percent: 63 }]; + setCachedProviderAccountQuotaForTests("anthropic", id!, { + fiveHourPercent: 10, weeklyPercent: 20, weeklyResetAt: 1_800_000_000_000, customWindows, updatedAt: Date.now(), + }); + recordAnthropicAccountQuotaFromHeaders(id!, new Headers({ + "anthropic-ratelimit-unified-5h-utilization": "0", + "anthropic-ratelimit-unified-7d-utilization": "NaN", + "anthropic-ratelimit-unified-7d-reset": "1e308", + }), 0); + expect(getCachedProviderAccountQuota("anthropic", id!)).toMatchObject({ + fiveHourPercent: 0, weeklyPercent: 20, weeklyResetAt: 1_800_000_000_000, customWindows, + }); + }); +}); diff --git a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts index 31b2389fb8..8d094db631 100644 --- a/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts +++ b/tests/adapters/anthropic/anthropic-sidecar-account-failover.test.ts @@ -8,10 +8,11 @@ import { afterAll, afterEach, beforeAll, beforeEach, expect, mock, test } from " import { mkdtempSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import type { ProviderAdapter } from "../../../src/adapters/base"; +import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "../../../src/adapters/base"; import { clearAnthropicAccountPoolState } from "../../../src/oauth/anthropic-routing"; import { clearGenericFailoverHealth } from "../../../src/oauth/generic-account-failover"; import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; +import { clearAccountQuotaCache, getCachedProviderAccountQuota, resetProviderQuotaReconcileStateForTests } from "../../../src/providers/quota"; import type { OcxConfig, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; @@ -66,15 +67,24 @@ beforeAll(async () => { runWithWebSearch: async (args: { parsed: OcxParsedRequest; adapter: ProviderAdapter; + incomingMeta: IncomingMeta; + fetchForRequest: (request: AdapterRequest, parsed: OcxParsedRequest) => typeof fetch; on429?: (retryAfter: string | null) => Promise; }) => { - const first = await args.adapter.buildRequest(args.parsed); - observedKeys.push(new Headers(first.headers).get("authorization") ?? ""); - const rotated = await args.on429?.("30"); + // This is a dispatch seam test. The real loop is covered in anthropic-quota-dispatch. + const first = await args.adapter.buildRequest(args.parsed, args.incomingMeta); + const refused = await args.fetchForRequest(first, args.parsed)(first.url, { + method: first.method, headers: first.headers, body: first.body, + }); + expect(refused.status).toBe(429); + const retryAfter = refused.headers.get("retry-after"); + await refused.body?.cancel(); + const rotated = await args.on429?.(retryAfter); if (!rotated) throw new Error("Anthropic sidecar did not rotate after 429"); - const second = await rotated.buildRequest(args.parsed); - observedKeys.push(new Headers(second.headers).get("authorization") ?? ""); - return new Response("sidecar-ok", { status: 200 }); + const second = await rotated.buildRequest(args.parsed, args.incomingMeta); + return args.fetchForRequest(second, args.parsed)(second.url, { + method: second.method, headers: second.headers, body: second.body, + }); }, })); @@ -88,11 +98,15 @@ beforeEach(() => { sidecarMode = false; clearAnthropicAccountPoolState(); clearGenericFailoverHealth(); + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); }); afterEach(() => { clearAnthropicAccountPoolState(); clearGenericFailoverHealth(); + clearAccountQuotaCache(); + resetProviderQuotaReconcileStateForTests(); removeTreeWithRetry(testHome); }); @@ -102,7 +116,7 @@ afterAll(() => { mock.restore(); }); -test("Anthropic web-search sidecar rotates on 429 when proactive pooling is disabled", async () => { +test("Anthropic sidecar dispatch seam records A429 and B200 when proactive pooling is disabled", async () => { sidecarMode = true; for (let index = 0; index < 2; index += 1) { await saveCredential("anthropic", { @@ -110,7 +124,7 @@ test("Anthropic web-search sidecar rotates on 429 when proactive pooling is disa refresh: `anthropic-refresh-${index}`, expires: Date.now() + 3_600_000, accountId: `anthropic-account-${index}`, - } as never, { addAccount: true }); + }); } const ids = getAccountSet("anthropic")!.accounts.map(account => account.id); await setActiveAccount("anthropic", ids[0]!); @@ -125,6 +139,27 @@ test("Anthropic web-search sidecar rotates on 429 when proactive pooling is disa baseUrl: "https://anthropic-sidecar.test/v1", authMode: "oauth", models: ["model"], + fetch: (async (_input, init) => { + observedKeys.push(new Headers(init?.headers).get("authorization") ?? ""); + if (observedKeys.length === 1) { + return new Response("rate limited", { + status: 429, + headers: { + "retry-after": "30", + "anthropic-ratelimit-unified-5h-utilization": "1", + "anthropic-ratelimit-unified-7d-utilization": "0.61", + }, + }); + } + expect(observedKeys).toHaveLength(2); + expect(getCachedProviderAccountQuota("anthropic", ids[0]!)).toMatchObject({ fiveHourPercent: 100, weeklyPercent: 61 }); + return new Response("sidecar-ok", { + headers: { + "anthropic-ratelimit-unified-5h-utilization": "0.23", + "anthropic-ratelimit-unified-7d-utilization": "0.47", + }, + }); + }) as typeof fetch, }, }, } as unknown as OcxConfig; @@ -146,4 +181,6 @@ test("Anthropic web-search sidecar rotates on 429 when proactive pooling is disa "Bearer anthropic-access-0", "Bearer anthropic-access-1", ]); + expect(getCachedProviderAccountQuota("anthropic", ids[0]!)).toMatchObject({ fiveHourPercent: 100, weeklyPercent: 61 }); + expect(getCachedProviderAccountQuota("anthropic", ids[1]!)).toMatchObject({ fiveHourPercent: 23, weeklyPercent: 47 }); }); diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index 6565f12821..2a1cac7ec3 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -39,6 +39,8 @@ "anthropic-image-retry.test.ts": "adapters/anthropic", "anthropic-pool-toggle-copy.test.ts": "adapters/anthropic", "anthropic-quorum-cache.test.ts": "routing", + "anthropic-quota-dispatch.test.ts": "adapters/anthropic", + "anthropic-ratelimit-headers.test.ts": "adapters/anthropic", "anthropic-reasoning.test.ts": "adapters/anthropic", "anthropic-sidecar-account-failover.test.ts": "adapters/anthropic", "anthropic-stream-hardening.test.ts": "adapters/anthropic", From a3c2eb56216ac4f3331e7bc4b43b47e1d5d2ac6c Mon Sep 17 00:00:00 2001 From: t Date: Mon, 7 Sep 2026 09:15:21 +0900 Subject: [PATCH 3/6] fix(anthropic): expire retained quota measurements at known resets [skip ci] Address #3825 review discussion_r3945728864. Retained standard and model-specific measurements become unknown after their known reset, including idle reads, hydration, persistence and joined failed probes. Reset-only headers cannot renew old usage. Keep unknown-reset behavior, probe clocks, unavailability and credential policy unchanged. Add real quota-evidence/manual-selection and persistence regressions; no local suites run per maintainer instruction. Original #3809 credit remains in ancestor f215f79b4. --- .../src/content/docs/guides/claude-code.md | 4 +- .../docs/reference/configuration/providers.md | 7 +- src/providers/quota.ts | 67 ++++- .../anthropic-ratelimit-headers.test.ts | 237 +++++++++++++++++- 4 files changed, 299 insertions(+), 16 deletions(-) diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 2e841e205c..720a303b00 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -36,7 +36,9 @@ Operational contract when enabled: two the response carries is recorded for that account — each window independently, and a refusal counts as well as a success. Usage-aware selection works from ordinary traffic, without waiting for a dashboard poll. Headers preserve model-specific quota windows and do - not postpone usage probes or clear a failed usage probe's unavailable status. + not postpone usage probes or clear a failed usage probe's unavailable status. Measurements + whose known reset time has passed are discarded as unknown, including retained model-specific + windows. Values without a known reset are preserved; missing data is never reported as zero usage. - Affinity is **process-local** (lost on proxy restart). - **401/403** credential failures quarantine the account (`needsReauth`) so it is excluded from selection until re-authenticated. diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index c80176868e..710c069b24 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -446,7 +446,12 @@ of those two a given response carries is recorded against that account — each on refusals as well as successes. Usage-aware selection therefore works from the accounts you actually use, without waiting for the dashboard Providers page to poll them. These readings refresh the existing row rather than replacing it, so the model-scoped weekly bars that only the usage -endpoint reports are preserved. Header observations do not postpone usage probes or clear a failed +endpoint reports are preserved until their known reset time passes. Expired measurements become +unknown, including retained standard windows omitted by later headers. A reset-only header cannot +extend an older utilization measurement. Values with no known reset retain their existing behavior; +missing measurements are never replaced with zero usage. + +Header observations do not postpone usage probes or clear a failed probe's unavailable status. After restart, cached Anthropic observations remain available while the next quota read probes again, because the saved observations do not include the probe clock. diff --git a/src/providers/quota.ts b/src/providers/quota.ts index dbdb6699f7..5dcb69beb3 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -1552,6 +1552,37 @@ type AccountQuotaCacheEntry = { identity?: string; isCurrent?: () => boolean; }; +/** Expired measurements become unknown; missing reset evidence never implies a fresh allowance. */ +function normalizeAnthropicQuota(quota: ProviderQuota | null | undefined, now: number): ProviderQuota | null { + if (!quota) return null; + const expired = (resetAt: number | undefined): boolean => typeof resetAt === "number" + && Number.isFinite(resetAt) && resetAt > 0 && resetAt <= now; + let result = quota; + for (const [percent, reset] of [ + ["fiveHourPercent", "fiveHourResetAt"], + ["weeklyPercent", "weeklyResetAt"], + ["monthlyPercent", "monthlyResetAt"], + ] as const) { + if (!expired(quota[reset])) continue; + if (result === quota) result = { ...quota }; + delete result[percent]; + delete result[reset]; + } + // Persisted rows validate only the outer quota object, so custom data may be malformed. + if (quota.customWindows !== undefined) { + const windows = Array.isArray(quota.customWindows) ? quota.customWindows : []; + const retained = windows.filter(window => window !== null && typeof window === "object" + && typeof window.label === "string" && typeof window.percent === "number" + && Number.isFinite(window.percent) && !expired(window.resetAt)); + if (!Array.isArray(quota.customWindows) || retained.length !== windows.length) { + if (result === quota) result = { ...quota }; + if (retained.length) result.customWindows = retained; + else delete result.customWindows; + } + } + return hasQuotaRows(result) ? result : null; +} + const accountQuotaCache = new Map(); let explicitAccountEpoch = 0; @@ -1570,15 +1601,21 @@ function hydrateAccountQuotaCache(): void { for (const [key, quota] of readPersistedAccountQuotas()) { // Disk stores observation time, not the Anthropic usage probe's clock. if (!accountQuotaCache.has(key)) { - accountQuotaCache.set(key, { ts: key.startsWith("anthropic\u0000") ? 0 : quota.updatedAt, quota }); + const anthropic = key.startsWith("anthropic\u0000"); + accountQuotaCache.set(key, { + ts: anthropic ? 0 : quota.updatedAt, + quota: anthropic ? normalizeAnthropicQuota(quota, Date.now()) : quota, + }); } } } function persistAccountQuotaCache(): void { schedulePersistAccountQuotas(function* () { + const now = Date.now(); for (const [key, entry] of accountQuotaCache) { - if (entry.quota) yield [key, entry.quota] as [string, ProviderQuota]; + const quota = key.startsWith("anthropic\u0000") ? normalizeAnthropicQuota(entry.quota, now) : entry.quota; + if (quota) yield [key, quota] as [string, ProviderQuota]; } }); } @@ -1628,7 +1665,7 @@ function accountCacheKey(provider: string, accountId: string): string { export function getCachedProviderAccountQuota(provider: string, accountId: string): ProviderQuota | null { const entry = accountQuotaCache.get(accountCacheKey(provider, accountId)); if (entry?.isCurrent && !entry.isCurrent()) return null; - return entry?.quota ?? null; + return provider === "anthropic" ? normalizeAnthropicQuota(entry?.quota, Date.now()) : entry?.quota ?? null; } /** Test-only: seed or clear the per-account quota cache without probing upstream. */ @@ -1661,9 +1698,9 @@ export function parseAnthropicRateLimitHeaders(headers: Headers): ProviderQuota const weeklyResetAt = anthropicHeaderResetAt(headers.get("anthropic-ratelimit-unified-7d-reset")); return { ...(fiveHourPercent !== undefined ? { fiveHourPercent } : {}), - ...(fiveHourResetAt !== undefined ? { fiveHourResetAt } : {}), + ...(fiveHourPercent !== undefined && fiveHourResetAt !== undefined ? { fiveHourResetAt } : {}), ...(weeklyPercent !== undefined ? { weeklyPercent } : {}), - ...(weeklyResetAt !== undefined ? { weeklyResetAt } : {}), + ...(weeklyPercent !== undefined && weeklyResetAt !== undefined ? { weeklyResetAt } : {}), updatedAt: Date.now(), }; } @@ -1700,7 +1737,9 @@ export function recordAnthropicAccountQuotaFromHeaders( ...previous, // Headers do not prove that the last usage probe succeeded. ts: previous?.ts ?? 0, - quota: { ...(previous?.quota ?? {}), ...observed }, + quota: normalizeAnthropicQuota({ + ...normalizeAnthropicQuota(previous?.quota, observed.updatedAt), ...observed, + }, observed.updatedAt), }); persistAccountQuotaCache(); } @@ -1978,7 +2017,9 @@ async function fetchAccountQuota( const key = accountCacheKey(provider, accountId); const writerGeneration = captureConfigGeneration(); const cached = accountQuotaCache.get(key); - if (!forceRefresh && cached && Date.now() - cached.ts < ACCOUNT_QUOTA_TTL_MS) return cached; + if (!forceRefresh && cached && Date.now() - cached.ts < ACCOUNT_QUOTA_TTL_MS) { + return provider === "anthropic" ? { ...cached, quota: normalizeAnthropicQuota(cached.quota, Date.now()) } : cached; + } const joinable = accountQuotaInflight.get(key); if (joinable) return joinable; @@ -2016,7 +2057,8 @@ async function fetchAccountQuota( const entry: AccountQuotaCacheEntry = { ts: Date.now(), // Settle once for all joiners against observations committed during the probe. - quota: (provider === "anthropic" ? accountQuotaCache.get(key)?.quota : cached?.quota) ?? null, + quota: provider === "anthropic" + ? normalizeAnthropicQuota(accountQuotaCache.get(key)?.quota, Date.now()) : cached?.quota ?? null, unavailable: true, }; if (mayCommitAccountQuotaKey(key, writerGeneration)) { @@ -2026,7 +2068,9 @@ async function fetchAccountQuota( } return entry; } - const entry: AccountQuotaCacheEntry = { ts: Date.now(), quota }; + const entry: AccountQuotaCacheEntry = { + ts: Date.now(), quota: provider === "anthropic" ? normalizeAnthropicQuota(quota, Date.now()) : quota, + }; if (mayCommitAccountQuotaKey(key, writerGeneration)) { accountQuotaCache.set(key, entry); // Exhaustion state rides the SAME commit guard as the quota row: a probe from a @@ -2038,7 +2082,8 @@ async function fetchAccountQuota( } catch { const entry: AccountQuotaCacheEntry = { ts: Date.now(), - quota: (provider === "anthropic" ? accountQuotaCache.get(key)?.quota : cached?.quota) ?? null, + quota: provider === "anthropic" + ? normalizeAnthropicQuota(accountQuotaCache.get(key)?.quota, Date.now()) : cached?.quota ?? null, unavailable: true, }; if (mayCommitAccountQuotaKey(key, writerGeneration)) { @@ -2070,7 +2115,7 @@ export async function fetchProviderAccountQuotas( const entry = await fetchAccountQuota(provider, account.id, forceRefresh, providerConfig); const result: ProviderAccountQuota = { accountId: account.id, - quota: entry.quota, + quota: provider === "anthropic" ? normalizeAnthropicQuota(entry.quota, Date.now()) : entry.quota, ...(entry.unavailable ? { unavailable: true as const } : {}), }; if (!explicitAccountReader(provider)) return result; diff --git a/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts b/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts index 21ae2d8489..77989064ff 100644 --- a/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts +++ b/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts @@ -1,6 +1,6 @@ /** Anthropic response observations must preserve account usage and probe semantics. */ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, writeFileSync } from "node:fs"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -9,8 +9,11 @@ import { forgetAnthropicFailoverQuorum, getAnthropicAccountHealthSnapshot, rotateAnthropicAccountOn429, + resetAnthropicRoutingForManualSelection, + resolveAnthropicAccountForSession, } from "../../../src/oauth/anthropic-routing"; import { projectStoredOAuthAccountHealth } from "../../../src/oauth/health"; +import { quotaEvidenceForCandidate } from "../../../src/routing/quota"; import { clearAccountQuotaCache, fetchProviderAccountQuotas, @@ -22,7 +25,7 @@ import { setCachedProviderAccountQuotaForTests, sweepExpiredProviderAccountQuotaRows, } from "../../../src/providers/quota"; -import { getAccountSet, saveCredential } from "../../../src/oauth/store"; +import { getAccountSet, saveCredential, setActiveAccount } from "../../../src/oauth/store"; import { clearPoolRotationState } from "../../../src/codex/pool-rotation"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; import type { OcxConfig } from "../../../src/types"; @@ -526,3 +529,231 @@ describe("Anthropic malformed deadlines and partial windows", () => { }); }); }); + +describe("Anthropic known-reset expiry", () => { + const start = 1_800_000_000_000; + let now: number; + + beforeEach(() => { + now = start; + Date.now = () => now; + }); + + function observe(id: string, headers: Record = { + "anthropic-ratelimit-unified-5h-utilization": "0.41", + }): void { + recordAnthropicAccountQuotaFromHeaders(id, new Headers(headers), 0); + } + + test("headers expire only known elapsed custom windows without mutating their source", async () => { + const [id] = await seed(1); + const saved = { + fiveHourPercent: 10, + customWindows: [ + { label: "Opus", percent: 100, resetAt: start + 60_000 }, + { label: "Sonnet", percent: 90, resetAt: start + 600_000 }, + { label: "Fable", percent: 70 }, + { label: "Unknown reset", percent: 60, resetAt: 0 }, + ], + updatedAt: start, + }; + setCachedProviderAccountQuotaForTests("anthropic", id!, saved); + now += 120_000; + observe(id!); + const quota = getCachedProviderAccountQuota("anthropic", id!); + expect(quota?.customWindows).toEqual(saved.customWindows.slice(1)); + expect(quota?.fiveHourPercent).toBe(41); + expect(quota?.updatedAt).toBe(now); + expect(saved.customWindows).toHaveLength(4); + expect(saved.updatedAt).toBe(start); + now += 30_000; + observe(id!); + expect(getCachedProviderAccountQuota("anthropic", id!)?.customWindows).toEqual(saved.customWindows.slice(1)); + }); + + for (const [percent, reset, observedWindow] of [ + ["fiveHourPercent", "fiveHourResetAt", "7d"], + ["weeklyPercent", "weeklyResetAt", "5h"], + ["monthlyPercent", "monthlyResetAt", "5h"], + ] as const) { + test(`partial headers remove the expired ${percent} pair without inventing zero`, async () => { + const [id] = await seed(1); + setCachedProviderAccountQuotaForTests("anthropic", id!, { + [percent]: 100, [reset]: start + 60_000, updatedAt: start, + }); + now += 60_000; + observe(id!, { [`anthropic-ratelimit-unified-${observedWindow}-utilization`]: "0.2" }); + const quota = getCachedProviderAccountQuota("anthropic", id!); + expect(quota).not.toBeNull(); + expect(quota?.[percent]).toBeUndefined(); + expect(quota?.[reset]).toBeUndefined(); + }); + } + + test("standard windows without reset evidence remain known", async () => { + const [id] = await seed(1); + setCachedProviderAccountQuotaForTests("anthropic", id!, { weeklyPercent: 100, updatedAt: start }); + now += 120_000; + observe(id!); + expect(getCachedProviderAccountQuota("anthropic", id!)?.weeklyPercent).toBe(100); + }); + + test("a reset-only header cannot extend retained usage even before the original reset", async () => { + const [id] = await seed(1); + setCachedProviderAccountQuotaForTests("anthropic", id!, { + fiveHourPercent: 10, weeklyPercent: 100, weeklyResetAt: start + 60_000, updatedAt: start, + }); + now += 30_000; + observe(id!, { + "anthropic-ratelimit-unified-5h-utilization": "0.2", + "anthropic-ratelimit-unified-7d-utilization": "invalid", + "anthropic-ratelimit-unified-7d-reset": String((start + 600_000) / 1000), + }); + expect(getCachedProviderAccountQuota("anthropic", id!)?.weeklyResetAt).toBe(start + 60_000); + now += 30_000; + expect(getCachedProviderAccountQuota("anthropic", id!)?.weeklyPercent).toBeUndefined(); + expect(getCachedProviderAccountQuota("anthropic", id!)?.weeklyResetAt).toBeUndefined(); + observe(id!, { + "anthropic-ratelimit-unified-7d-utilization": "0.3", + "anthropic-ratelimit-unified-7d-reset": String((start + 600_000) / 1000), + }); + expect(getCachedProviderAccountQuota("anthropic", id!)).toMatchObject({ + weeklyPercent: 30, weeklyResetAt: start + 600_000, + }); + }); + + test("idle cache reads cross a reset without another observation or probe", async () => { + const [id] = await seed(1); + const quota = { customWindows: [{ label: "Opus", percent: 100, resetAt: start + 60_000 }], updatedAt: start }; + setCachedProviderAccountQuotaForTests("anthropic", id!, quota); + setCachedProviderAccountQuotaForTests("kiro", "untouched", quota); + const candidate = { provider: "anthropic", model: "claude-opus-4-6", accountRef: id! }; + now += 59_999; + expect(getCachedProviderAccountQuota("anthropic", id!)).toEqual(quota); + expect(quotaEvidenceForCandidate(candidate)).toMatchObject({ known: true, exhausted: true, headroom: 0 }); + now++; + expect(getCachedProviderAccountQuota("anthropic", id!)).toBeNull(); + expect(quotaEvidenceForCandidate(candidate)).toEqual({ known: false }); + const [row] = await fetchProviderAccountQuotas("anthropic"); + expect(row?.quota).toBeNull(); + expect(row?.unavailable).toBeUndefined(); + expect(getCachedProviderAccountQuota("kiro", "untouched")).toBe(quota); + }); + + test("expired Opus evidence stops suppressing an otherwise healthy manual selection", async () => { + const [a, b] = await seed(2); + setCachedProviderAccountQuotaForTests("anthropic", a!, { + fiveHourPercent: 30, customWindows: [{ label: "Opus", percent: 100, resetAt: start + 60_000 }], updatedAt: start, + }); + setCachedProviderAccountQuotaForTests("anthropic", b!, { fiveHourPercent: 11, updatedAt: start }); + await setActiveAccount("anthropic", a!); + resetAnthropicRoutingForManualSelection(a!); + const config = poolEnabled(); + config.anthropicAccountPool = { enabled: true, strategy: "quota", autoSwitchThreshold: 20 }; + const candidate = { provider: "anthropic", model: "claude-opus-4-6", accountRef: a! }; + expect(resolveAnthropicAccountForSession(null, config, now).accountId).toBe(b); + expect(quotaEvidenceForCandidate(candidate)).toMatchObject({ known: true, exhausted: true, headroom: 0 }); + now += 60_000; + expect(resolveAnthropicAccountForSession(null, config, now)).toMatchObject({ accountId: a, reason: "manual" }); + expect(quotaEvidenceForCandidate(candidate)).toMatchObject({ known: true, exhausted: false, headroom: 0.7 }); + }); + + for (const failure of ["http", "network"] as const) { + test(`joined ${failure} failures remove windows expiring during the shared probe`, async () => { + const [id] = await seed(1); + setCachedProviderAccountQuotaForTests("anthropic", id!, { + fiveHourPercent: 10, weeklyPercent: 100, weeklyResetAt: start + 60_000, + customWindows: [{ label: "Opus", percent: 100, resetAt: start + 60_000 }, { label: "Fable", percent: 63 }], + updatedAt: start, + }); + let started!: () => void; + const dispatched = new Promise(resolve => { started = resolve; }); + let finish!: (response: Response) => void; + let fail!: (error: Error) => void; + const response = new Promise((resolve, reject) => { finish = resolve; fail = reject; }); + let calls = 0; + globalThis.fetch = (async () => { calls++; started(); return response; }) as typeof fetch; + const first = fetchProviderAccountQuotas("anthropic", true); + await dispatched; + const second = fetchProviderAccountQuotas("anthropic", true); + now += 30_000; + observe(id!); + now += 30_000; + if (failure === "http") finish(new Response("busy", { status: 429 })); + else fail(new Error("offline")); + const [a, b] = await Promise.all([first, second]); + expect(calls).toBe(1); + expect(a).toEqual(b); + expect(a[0]?.unavailable).toBe(true); + expect(a[0]?.quota).toEqual({ fiveHourPercent: 41, customWindows: [{ label: "Fable", percent: 63 }], updatedAt: start + 30_000 }); + expect(getCachedProviderAccountQuota("anthropic", id!)).toEqual(a[0]?.quota); + expect((await fetchProviderAccountQuotas("anthropic"))[0]).toEqual(a[0]); + expect(calls).toBe(1); + }); + } + + test("restart cannot revive expired bars from a recently updated disk row", async () => { + const [id] = await seed(1); + now += 120_000; + writeFileSync(join(home, "provider-account-quota-cache.json"), JSON.stringify({ version: 1, rows: { + [`anthropic\u0000${id}`]: { + fiveHourPercent: 41, weeklyPercent: 100, weeklyResetAt: start + 60_000, + customWindows: [{ label: "Opus", percent: 100, resetAt: start + 60_000 }, { label: "Fable", percent: 63 }], + updatedAt: now, + }, + } })); + clearAccountQuotaCache(); + let calls = 0; + globalThis.fetch = (async () => { calls++; return new Response("busy", { status: 429 }); }) as typeof fetch; + const [row] = await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(1); + expect(row?.unavailable).toBe(true); + expect(row?.quota).toEqual({ fiveHourPercent: 41, customWindows: [{ label: "Fable", percent: 63 }], updatedAt: now }); + }); + + for (const malformed of [null, {}, [null, "bad", { label: "invalid", percent: "100" }]]) { + test(`malformed persisted custom windows stay unknown without breaking other rows: ${JSON.stringify(malformed)}`, async () => { + const [id] = await seed(1); + writeFileSync(join(home, "provider-account-quota-cache.json"), JSON.stringify({ version: 1, rows: { + [`anthropic\u0000${id}`]: { customWindows: malformed, updatedAt: now }, + "kiro\u0000untouched": { monthlyPercent: 17, updatedAt: now }, + } })); + clearAccountQuotaCache(); + let calls = 0; + globalThis.fetch = (async () => { calls++; return new Response("busy", { status: 429 }); }) as typeof fetch; + const [row] = await fetchProviderAccountQuotas("anthropic"); + expect(calls).toBe(1); + expect(row?.quota).toBeNull(); + expect(row?.unavailable).toBe(true); + expect(getCachedProviderAccountQuota("kiro", "untouched")).toEqual({ monthlyPercent: 17, updatedAt: now }); + }); + } + + test("fresh utilization without a reset does not inherit an expired reset", async () => { + const [id] = await seed(1); + setCachedProviderAccountQuotaForTests("anthropic", id!, { + fiveHourPercent: 100, fiveHourResetAt: start + 60_000, updatedAt: start, + }); + now += 60_000; + observe(id!); + expect(getCachedProviderAccountQuota("anthropic", id!)).toEqual({ fiveHourPercent: 41, updatedAt: now }); + }); + + test("deferred persistence evaluates expiry at write time and leaves other providers intact", async () => { + const [id] = await seed(1); + const saved = { weeklyPercent: 100, weeklyResetAt: start + 60_000, updatedAt: start }; + setCachedProviderAccountQuotaForTests("anthropic", id!, saved); + setCachedProviderAccountQuotaForTests("kiro", "untouched", saved); + let flush!: () => void; + const timer = spyOn(globalThis, "setTimeout").mockImplementation(((callback: () => void) => { + flush = callback; + return 0 as unknown as ReturnType; + }) as typeof setTimeout); + try { observe(id!); } finally { timer.mockRestore(); } + now += 60_000; + flush(); + const disk = JSON.parse(readFileSync(join(home, "provider-account-quota-cache.json"), "utf8")); + expect(disk.rows[`anthropic\u0000${id}`]).toEqual({ fiveHourPercent: 41, updatedAt: start }); + expect(disk.rows["kiro\u0000untouched"]).toEqual(saved); + }); +}); From 54fcc688d7b23677892a01803fc07663fbdd2074 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 7 Sep 2026 09:22:08 +0900 Subject: [PATCH 4/6] test(anthropic): keep probe fixtures inside live reset windows [skip ci] The attribution/cache tests used July 2026 quota reset dates, which correctly expire under the known-reset fix. Generate future reset dates from one clock snapshot; preserve all existing assertions. Explicit expiry tests retain fixed simulated boundaries. No local suites run. --- tests/providers/provider-account-quota.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/providers/provider-account-quota.test.ts b/tests/providers/provider-account-quota.test.ts index 989f55a410..1b940d71d0 100644 --- a/tests/providers/provider-account-quota.test.ts +++ b/tests/providers/provider-account-quota.test.ts @@ -33,9 +33,11 @@ async function seedTwoAccounts(): Promise { } function usageBody(fiveHour: number, sevenDay: number): string { + // These tests exercise current account measurements, not expired historical windows. + const now = Date.now(); return JSON.stringify({ - five_hour: { utilization: fiveHour, resets_at: "2026-07-05T12:00:00Z" }, - seven_day: { utilization: sevenDay, resets_at: "2026-07-08T12:00:00Z" }, + five_hour: { utilization: fiveHour, resets_at: new Date(now + 5 * 60 * 60_000).toISOString() }, + seven_day: { utilization: sevenDay, resets_at: new Date(now + 7 * 24 * 60 * 60_000).toISOString() }, }); } From d3c70f9d8c8cc6fced7a93577b93e8b141473ea3 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 7 Sep 2026 09:44:37 +0900 Subject: [PATCH 5/6] fix(anthropic): normalize retained quota metadata and guard test transport [skip ci] Reject unusable custom labels/percentages, omit invalid reset metadata while retaining valid usage, and state the existing 60-second fallback. Guard unexpected test network calls with teardown-safe restoration. No new TTL or synthetic quota values. Addresses #3825 review findings; no local suite was run. --- .../src/content/docs/guides/claude-code.md | 2 +- .../docs/reference/configuration/providers.md | 2 +- src/providers/quota.ts | 38 ++++++++--- .../anthropic-quota-dispatch.test.ts | 64 +++++++++++++------ .../anthropic-ratelimit-headers.test.ts | 63 +++++++++++++++++- 5 files changed, 137 insertions(+), 32 deletions(-) diff --git a/docs-site/src/content/docs/guides/claude-code.md b/docs-site/src/content/docs/guides/claude-code.md index 720a303b00..4c5cc44b34 100644 --- a/docs-site/src/content/docs/guides/claude-code.md +++ b/docs-site/src/content/docs/guides/claude-code.md @@ -31,7 +31,7 @@ Operational contract when enabled: account within the same request (bounded). The cooldown uses a usable `Retry-After` when present, otherwise the latest valid reset time among windows Anthropic marks `rejected`, including weekly windows. Valid upstream deadlines are not shortened to a fixed cooldown ceiling. - A refusal with no usable deadline falls back to a default backoff. + A refusal with no usable deadline falls back to a 60-second default backoff. - Responses report the serving account's 5-hour and weekly utilization, and whichever of those two the response carries is recorded for that account — each window independently, and a refusal counts as well as a success. Usage-aware selection works from ordinary traffic, diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 710c069b24..b4d7cb2094 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -437,7 +437,7 @@ When enabled, 429 records a cooldown and may rotate within the request. The cool from a usable `Retry-After`, otherwise from the latest valid reset time among rate-limit windows Anthropic reports as `rejected`, including weekly windows. Valid upstream deadlines are not shortened to a fixed cooldown ceiling; non-finite or unrepresentable deadlines are ignored. -A refusal with no usable deadline falls back to a short default backoff. Affinity is process-local +A refusal with no usable deadline falls back to a 60-second default backoff. Affinity is process-local and size-bounded. Credential 401/403 marks the account as needing reauthentication. If all eligible accounts are cooling, clients receive 429 with `Retry-After` when known, not an authentication error. diff --git a/src/providers/quota.ts b/src/providers/quota.ts index 5dcb69beb3..71644a9eae 100644 --- a/src/providers/quota.ts +++ b/src/providers/quota.ts @@ -1555,26 +1555,48 @@ type AccountQuotaCacheEntry = { /** Expired measurements become unknown; missing reset evidence never implies a fresh allowance. */ function normalizeAnthropicQuota(quota: ProviderQuota | null | undefined, now: number): ProviderQuota | null { if (!quota) return null; - const expired = (resetAt: number | undefined): boolean => typeof resetAt === "number" - && Number.isFinite(resetAt) && resetAt > 0 && resetAt <= now; + const validReset = (resetAt: unknown): resetAt is number => typeof resetAt === "number" + && Number.isFinite(resetAt) && resetAt > 0 && Number.isFinite(new Date(resetAt).getTime()); let result = quota; for (const [percent, reset] of [ ["fiveHourPercent", "fiveHourResetAt"], ["weeklyPercent", "weeklyResetAt"], ["monthlyPercent", "monthlyResetAt"], ] as const) { - if (!expired(quota[reset])) continue; + const resetAt = quota[reset]; + if (resetAt === undefined) continue; + const valid = validReset(resetAt); + if (valid && resetAt > now) continue; if (result === quota) result = { ...quota }; - delete result[percent]; + if (valid) delete result[percent]; delete result[reset]; } // Persisted rows validate only the outer quota object, so custom data may be malformed. if (quota.customWindows !== undefined) { const windows = Array.isArray(quota.customWindows) ? quota.customWindows : []; - const retained = windows.filter(window => window !== null && typeof window === "object" - && typeof window.label === "string" && typeof window.percent === "number" - && Number.isFinite(window.percent) && !expired(window.resetAt)); - if (!Array.isArray(quota.customWindows) || retained.length !== windows.length) { + const retained: ProviderQuotaWindow[] = []; + let changed = !Array.isArray(quota.customWindows); + for (const window of windows) { + if (!window || typeof window !== "object" || typeof window.label !== "string" || !window.label.trim() + || typeof window.percent !== "number" || !Number.isFinite(window.percent) + || window.percent < 0 || window.percent > 100) { + changed = true; + continue; + } + if (validReset(window.resetAt) && window.resetAt <= now) { + changed = true; + continue; + } + if (window.resetAt !== undefined && !validReset(window.resetAt)) { + const normalized = { ...window }; + delete normalized.resetAt; + retained.push(normalized); + changed = true; + } else { + retained.push(window); + } + } + if (changed) { if (result === quota) result = { ...quota }; if (retained.length) result.customWindows = retained; else delete result.customWindows; diff --git a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts index 6442436fbb..09952ea5ab 100644 --- a/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts +++ b/tests/adapters/anthropic/anthropic-quota-dispatch.test.ts @@ -13,10 +13,19 @@ import type { OcxConfig, OcxProviderConfig } from "../../../src/types"; import { removeTreeWithRetry } from "../../helpers/remove-tree"; const originalHome = process.env.OPENCODEX_HOME; +let originalFetch: typeof globalThis.fetch; +let unexpectedGlobalFetches = 0; let home: string; let sent: { authorization: string | null; apiKey: string | null; body: Record }[]; beforeEach(() => { + home = ""; + originalFetch = globalThis.fetch; + unexpectedGlobalFetches = 0; + globalThis.fetch = (async () => { + unexpectedGlobalFetches += 1; + throw new Error("Unexpected global fetch in Anthropic quota dispatch test"); + }) as typeof fetch; home = mkdtempSync(join(tmpdir(), "ocx-anthropic-quota-dispatch-")); process.env.OPENCODEX_HOME = home; sent = []; @@ -29,16 +38,25 @@ beforeEach(() => { }); afterEach(() => { - clearAnthropicAccountPoolState(); - forgetAnthropicFailoverQuorum(); - clearGenericFailoverHealth(); - // Cancel the debounced persistence before restoring the real home. - clearAccountQuotaCache(); - resetProviderQuotaReconcileStateForTests(); - clearResponseStateForTests(); - if (originalHome === undefined) delete process.env.OPENCODEX_HOME; - else process.env.OPENCODEX_HOME = originalHome; - removeTreeWithRetry(home); + try { + // Provider code may catch the guard's rejection; the attempted network call still fails the test. + expect(unexpectedGlobalFetches).toBe(0); + } finally { + try { + // Cancel the debounced persistence before restoring the real home. + clearAccountQuotaCache(); + clearAnthropicAccountPoolState(); + forgetAnthropicFailoverQuorum(); + clearGenericFailoverHealth(); + resetProviderQuotaReconcileStateForTests(); + clearResponseStateForTests(); + } finally { + globalThis.fetch = originalFetch; + if (originalHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = originalHome; + if (home) removeTreeWithRetry(home); + } + } }); function credential(index: number) { @@ -136,8 +154,9 @@ test("main A429 -> B200 records both physical responses against their sending ac return answer(body.stream === true); }); const response = await post(config); + const responseText = await response.text(); expect(response.status).toBe(200); - expect(await response.text()).toContain("The answer is complete."); + expect(responseText).toContain("The answer is complete."); expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); expectQuota(a!, 100, 61); expectQuota(b!, 23, 47); @@ -151,8 +170,8 @@ test("terminal 429 after both accounts are exhausted records both refused physic expectQuota(a!, 100, 61); return limited("0.89", "1"); })); - expect(response.status).toBe(429); await response.text(); + expect(response.status).toBe(429); expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); expectQuota(a!, 100, 61); expectQuota(b!, 89, 100); @@ -165,15 +184,16 @@ test("manual active switch while A is pending keeps A's measurement off B", asyn const config = configFor(() => { entered.resolve(); return returned.promise; }); const pending = post(config); await entered.promise; + let response!: Response; try { expect(sent[0]!.authorization).toBe(`Bearer ${credential(0).access}`); expect(await setActiveAccount("anthropic", b!)).toBe(true); } finally { returned.resolve(answer(false, "0.37", "0.53")); + response = await pending; + await response.text(); } - const response = await pending; expect(response.status).toBe(200); - await response.text(); expect(sent).toHaveLength(1); expect(getAccountSet("anthropic")!.activeAccountId).toBe(b!); expectQuota(a!, 37, 53); @@ -186,15 +206,16 @@ test("credential replacement while A is pending skips its old-generation respons const returned = deferred(); const pending = post(configFor(() => { entered.resolve(); return returned.promise; })); await entered.promise; + let response!: Response; try { expect(sent[0]!.authorization).toBe(`Bearer ${credential(0).access}`); await saveAccountCredential("anthropic", a!, { ...credential(0), access: "synthetic-replacement-access", refresh: "synthetic-replacement-refresh" }); } finally { returned.resolve(answer(false)); + response = await pending; + await response.text(); } - const response = await pending; expect(response.status).toBe(200); - await response.text(); expect(sent).toHaveLength(1); expect(getAccountSet("anthropic")!.accounts.find(row => row.id === a)!.credential.access).toBe("synthetic-replacement-access"); expect(getCachedProviderAccountQuota("anthropic", a!)).toBeNull(); @@ -208,8 +229,8 @@ const overriddenHeaders: { label: string; headers: Record; autho test.each(overriddenHeaders)("$label skips quota attribution even when a selected OAuth account exists", async ({ headers, authorization, apiKey }) => { const ids = await seed(); const response = await post(configFor(body => answer(body.stream === true), headers)); - expect(response.status).toBe(200); await response.text(); + expect(response.status).toBe(200); expect(sent).toHaveLength(1); expect(sent[0]).toMatchObject({ authorization, apiKey }); for (const id of ids) expect(getCachedProviderAccountQuota("anthropic", id)).toBeNull(); @@ -227,8 +248,9 @@ test("real web-search routed loop records A429 and B200 through fetchForRequest" }); config.webSearchSidecar = { backend: "anthropic", enabled: true }; const response = await post(config, { tools: [{ type: "web_search" }] }); + const responseText = await response.text(); expect(response.status).toBe(200); - expect(await response.text()).toContain("The answer is complete."); + expect(responseText).toContain("The answer is complete."); expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); expectQuota(a!, 100, 61); expectQuota(b!, 23, 47); @@ -252,8 +274,9 @@ test("real terminal continuation records A429 before retrying the continuation o input: "Please modify the file now", tools: [{ type: "function", name: "read_file", description: "read a file", parameters: { type: "object" } }], }); + const responseText = await response.text(); expect(response.status).toBe(200); - expect(await response.text()).toContain("The answer is complete."); + expect(responseText).toContain("The answer is complete."); expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); expectQuota(a!, 100, 61); expectQuota(b!, 23, 47); @@ -275,8 +298,9 @@ test("real image bridge routed loop records A429 and B200 through fetchForReques adapter: "openai-chat", baseUrl: "https://api.x.ai/v1", authMode: "key", apiKey: "synthetic-image-key", }; const response = await post(config, { stream: true, tools: [{ type: "image_generation" }] }); + const responseText = await response.text(); expect(response.status).toBe(200); - expect(await response.text()).toContain("The answer is complete."); + expect(responseText).toContain("The answer is complete."); expect(sent.map(row => row.authorization)).toEqual([`Bearer ${credential(0).access}`, `Bearer ${credential(1).access}`]); expectQuota(a!, 100, 61); expectQuota(b!, 23, 47); diff --git a/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts b/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts index 77989064ff..f1ad70542e 100644 --- a/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts +++ b/tests/adapters/anthropic/anthropic-ratelimit-headers.test.ts @@ -561,14 +561,56 @@ describe("Anthropic known-reset expiry", () => { now += 120_000; observe(id!); const quota = getCachedProviderAccountQuota("anthropic", id!); - expect(quota?.customWindows).toEqual(saved.customWindows.slice(1)); + const retained = [saved.customWindows[1], saved.customWindows[2], { label: "Unknown reset", percent: 60 }]; + expect(quota?.customWindows).toEqual(retained); expect(quota?.fiveHourPercent).toBe(41); expect(quota?.updatedAt).toBe(now); expect(saved.customWindows).toHaveLength(4); expect(saved.updatedAt).toBe(start); now += 30_000; observe(id!); - expect(getCachedProviderAccountQuota("anthropic", id!)?.customWindows).toEqual(saved.customWindows.slice(1)); + expect(getCachedProviderAccountQuota("anthropic", id!)?.customWindows).toEqual(retained); + }); + + test("custom windows reject empty labels and invalid percentages while preserving valid objects", async () => { + const [id] = await seed(1); + const valid = [{ label: "Opus", percent: 0 }, { label: "Sonnet", percent: 100, resetAt: start + 60_000 }]; + const saved = { customWindows: [ + ...valid, + { label: "", percent: 50 }, { label: " ", percent: 50 }, + { label: "negative", percent: -1 }, { label: "too high", percent: 101 }, + { label: "not finite", percent: Number.NaN }, { label: "infinite", percent: Infinity }, + ], updatedAt: start }; + setCachedProviderAccountQuotaForTests("anthropic", id!, saved); + const normalized = getCachedProviderAccountQuota("anthropic", id!); + expect(normalized?.customWindows).toEqual(valid); + expect(normalized?.customWindows?.[0]).toBe(valid[0]); + expect(saved.customWindows).toHaveLength(8); + setCachedProviderAccountQuotaForTests("anthropic", id!, normalized!); + expect(getCachedProviderAccountQuota("anthropic", id!)).toBe(normalized); + }); + + test("invalid reset metadata is removed without discarding valid usage", async () => { + const [id] = await seed(1); + const invalidResets = [0, -1, Number.NaN, Infinity, 8_640_000_000_000_001]; + const saved = { + fiveHourPercent: 40, fiveHourResetAt: 0, + weeklyPercent: 50, weeklyResetAt: Infinity, + monthlyPercent: 60, monthlyResetAt: 8_640_000_000_000_001, + customWindows: invalidResets.map((resetAt, index) => ({ label: `window-${index}`, percent: 70, resetAt })), + updatedAt: start, + }; + setCachedProviderAccountQuotaForTests("anthropic", id!, saved); + const normalized = getCachedProviderAccountQuota("anthropic", id!); + expect(normalized).toEqual({ + fiveHourPercent: 40, weeklyPercent: 50, monthlyPercent: 60, + customWindows: invalidResets.map((_, index) => ({ label: `window-${index}`, percent: 70 })), + updatedAt: start, + }); + expect(saved.customWindows[0]?.resetAt).toBe(0); + expect(saved.fiveHourResetAt).toBe(0); + setCachedProviderAccountQuotaForTests("anthropic", id!, normalized!); + expect(getCachedProviderAccountQuota("anthropic", id!)).toBe(normalized); }); for (const [percent, reset, observedWindow] of [ @@ -729,6 +771,23 @@ describe("Anthropic known-reset expiry", () => { }); } + test("persisted nonnumeric reset metadata does not erase otherwise valid windows", async () => { + const [id] = await seed(1); + writeFileSync(join(home, "provider-account-quota-cache.json"), JSON.stringify({ version: 1, rows: { + [`anthropic\u0000${id}`]: { + weeklyPercent: 80, weeklyResetAt: "unknown", + customWindows: [{ label: "Opus", percent: 70, resetAt: null }, { label: "Sonnet", percent: 60, resetAt: "later" }], + updatedAt: now, + }, + } })); + clearAccountQuotaCache(); + globalThis.fetch = (async () => new Response("busy", { status: 429 })) as typeof fetch; + const [row] = await fetchProviderAccountQuotas("anthropic"); + expect(row?.quota).toEqual({ weeklyPercent: 80, + customWindows: [{ label: "Opus", percent: 70 }, { label: "Sonnet", percent: 60 }], updatedAt: now }); + expect(row?.unavailable).toBe(true); + }); + test("fresh utilization without a reset does not inherit an expired reset", async () => { const [id] = await seed(1); setCachedProviderAccountQuotaForTests("anthropic", id!, { From 872f0e5aa714f6a2e757510195d1c038ac70e26d Mon Sep 17 00:00:00 2001 From: t Date: Mon, 7 Sep 2026 07:41:57 +0900 Subject: [PATCH 6/6] fix(cli): explain which side of a version mismatch is older [skip ci] Address the diagnostic residual of #3464. Keep raw mismatch and placeholder behavior, compare valid SemVer precedence, and avoid false doctor match claims. No automatic service repair or request-policy change. Local suites omitted by owner instruction; final cumulative hosted CI pending. Reported-by: garysassano <10464497+garysassano@users.noreply.github.com> --- .../docs/ko/reference/cli/lifecycle.md | 13 +++ .../content/docs/reference/cli/lifecycle.md | 20 ++++- .../docs/ru/reference/cli/lifecycle.md | 13 +++ src/cli/doctor.ts | 4 +- src/cli/version-skew.ts | 39 ++++++++- tests/cli/cli-status-json.test.ts | 82 ++++++++++++++++++- tests/cli/cli-version-skew.test.ts | 70 +++++++++++++++- tests/codex-integration/doctor.test.ts | 63 +++++++++++++- 8 files changed, 291 insertions(+), 13 deletions(-) diff --git a/docs-site/src/content/docs/ko/reference/cli/lifecycle.md b/docs-site/src/content/docs/ko/reference/cli/lifecycle.md index 4847614674..068807025b 100644 --- a/docs-site/src/content/docs/ko/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ko/reference/cli/lifecycle.md @@ -82,6 +82,19 @@ dedicated-provider history도 포함됩니다. 상태를 백업하고 이 전체 ### `ocx status [--json]` +status와 `ocx doctor`는 현재 CLI와 실행 중인 프록시의 버전을 비교합니다. CLI가 더 새로우면 +원하는 최신 설치로 프록시를 재시작하십시오. 백그라운드 서비스라면 `ocx service repair`를 +실행합니다(`ocx service restart`는 별칭). 프록시가 더 새로우면 CLI를 업그레이드하거나 +`PATH`가 원하는 설치를 가리키도록 수정하십시오. 이 진단은 서비스를 복구하거나 요청 허용 +여부를 바꾸지 않습니다. + +버전 문자열이 같거나 어느 쪽이 `unknown` / `0.0.0`이면 경고하지 않으며, 프록시 버전이 없어도 +경고하지 않습니다. doctor는 placeholder를 버전 일치로 확정하지 않습니다. 엄격한 SemVer로 +해석할 수 없는 서로 다른 문자열이나 build metadata만 다른 버전은 어느 쪽이 오래됐다고 +단정하지 않는 중립 경고를 표시합니다. 공백을 제거하거나 앞의 `v`를 정규화하지 않습니다. +JSON의 `versionSkew`에도 같은 안내가 들어가며 필드는 `cliVersion`, `proxyVersion`, `skewed`, +`warning` 그대로입니다. + 읽기 전용 진단 요약을 출력합니다. 프록시 PID, `/healthz` 도달 가능 여부, 대시보드 URL, 설정 경로, 기본 공급자, Codex 자동 시작 설정, 서비스 상태, shim 상태, 그리고 마스킹된 실제로 적용되는 Codex 홈이 포함됩니다. 명시적이고 높은 신뢰도의 Windows Orca 런타임 홈 시그니처만 diff --git a/docs-site/src/content/docs/reference/cli/lifecycle.md b/docs-site/src/content/docs/reference/cli/lifecycle.md index e75a2b6241..0dda487b3a 100644 --- a/docs-site/src/content/docs/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/reference/cli/lifecycle.md @@ -88,6 +88,19 @@ are left in place. ### `ocx status [--json]` +Status and `ocx doctor` compare this CLI's version with the running proxy. If the CLI is newer, +restart the proxy using the intended current installation; for a background service, run +`ocx service repair` (`ocx service restart` is an alias). If the proxy is newer, upgrade the CLI +or resolve `PATH` to the intended installation. These diagnostics do not repair the service or +change whether requests are allowed. + +Identical version strings and the `unknown` / `0.0.0` placeholders suppress the warning, as does +an absent proxy version. Doctor does not report placeholders as a confirmed match. Different +strings still produce a neutral warning when they cannot be strictly parsed as SemVer or differ +only in build metadata; neither side is called older. Versions are not trimmed and a leading `v` +is not normalized. JSON exposes the same advice in `versionSkew`, whose fields remain +`cliVersion`, `proxyVersion`, `skewed`, and `warning`. + Print a read-only diagnostic summary: proxy PID, `/healthz` reachability, dashboard URL, config path, default provider, Codex autostart setting, service state, shim state, and the redacted effective Codex home. Only the explicit, high-confidence Windows Orca runtime-home signature adds an actionable App-home @@ -261,9 +274,10 @@ bundled Bun paths are deliberately rediscovered after upgrades instead of being Definitions installed before this change still carry the old versioned paths and cannot migrate themselves — once the old executable is deleted, no opencodex code runs to fix it. Run `ocx service repair` once after upgrading; after that, each service start follows the launcher. -An already-running proxy is not replaced by an external upgrade: restart the service (or run -`ocx service repair`) so the new build serves, and treat a CLI/proxy version mismatch warning as -exactly that signal. +An already-running proxy is not replaced by an external upgrade: when the installed CLI is newer +than the running proxy, restart the service (or run `ocx service repair`) so the new build serves. +If the proxy is newer instead, check the CLI installation and `PATH` as described under +[`ocx status`](#ocx-status---json). | Subcommand | Action | | --- | --- | diff --git a/docs-site/src/content/docs/ru/reference/cli/lifecycle.md b/docs-site/src/content/docs/ru/reference/cli/lifecycle.md index 1ace7cc10f..7be5d5ad77 100644 --- a/docs-site/src/content/docs/ru/reference/cli/lifecycle.md +++ b/docs-site/src/content/docs/ru/reference/cli/lifecycle.md @@ -89,6 +89,19 @@ ocx eject back ### `ocx status [--json]` +Status и `ocx doctor` сравнивают версии текущего CLI и работающего прокси. Если CLI новее, +перезапустите прокси из нужной актуальной установки. Для фоновой службы используйте +`ocx service repair` (`ocx service restart` — её псевдоним). Если новее прокси, обновите CLI +или исправьте `PATH`, чтобы он указывал на нужную установку. Диагностика не ремонтирует службу +и не меняет разрешение запросов. + +При одинаковых строках версий, значениях `unknown` / `0.0.0` или отсутствии версии прокси +предупреждение подавляется. Doctor не считает placeholder подтверждённым совпадением. +Разные строки, которые нельзя строго разобрать как SemVer, и версии, отличающиеся только +build metadata, вызывают нейтральное предупреждение без указания устаревшей стороны. +Пробелы не удаляются, префикс `v` не нормализуется. JSON содержит ту же рекомендацию в +`versionSkew` с прежними полями `cliVersion`, `proxyVersion`, `skewed` и `warning`. + Печатает read-only диагностическую сводку: PID прокси, достижимость `/healthz`, URL дашборда, путь к конфигу, провайдера по умолчанию, настройку автозапуска Codex, состояние службы, состояние shim'а и redacted effective Codex home. Только явная и высокоуверенная сигнатура mismatch diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index 1ab4fe9f1b..d7148530a0 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -1157,11 +1157,11 @@ export async function runDoctor(args: string[] = []): Promise { // No extra probe -- findLiveProxy already carried the version back. { const { packageVersion } = await import("./help"); - const { computeVersionSkew } = await import("./version-skew"); + const { computeVersionSkew, isConfirmedVersionMatch } = await import("./version-skew"); const skew = computeVersionSkew(packageVersion(), live?.version); if (skew.skewed && skew.warning) { console.log(`!! ${skew.warning}`); - } else if (skew.proxyVersion !== null) { + } else if (isConfirmedVersionMatch(skew)) { console.log(`ok ocx ${skew.cliVersion} matches the running proxy`); } } diff --git a/src/cli/version-skew.ts b/src/cli/version-skew.ts index 588d29a307..48b71a51ee 100644 --- a/src/cli/version-skew.ts +++ b/src/cli/version-skew.ts @@ -1,5 +1,5 @@ /** - * CLI-versus-proxy version skew (#2701). + * CLI-versus-proxy version skew (#2701, #3464). * * The reported failure: `ocx` on PATH is an older install than the running proxy, so its * help describes commands the proxy does not have and its output describes a different @@ -9,6 +9,7 @@ * comparison instead of reimplementing it -- two diagnostics disagreeing about whether an * install is stale would be worse than neither reporting it. */ +import { parseStrictSemver, type StrictSemver } from "../lib/strict-semver"; /** Placeholder versions that mean "unknown", not "different". */ const PLACEHOLDERS = new Set(["unknown", "0.0.0"]); @@ -22,6 +23,30 @@ export interface VersionSkew { readonly warning: string | null; } +/** Suppressed comparisons are not confirmed matches, even when both placeholders agree. */ +export function isConfirmedVersionMatch(skew: VersionSkew): boolean { + return skew.proxyVersion === skew.cliVersion && !PLACEHOLDERS.has(skew.cliVersion); +} + +/** SemVer precedence ignores build metadata; raw equality is handled separately. */ +function compareVersions(cli: StrictSemver, proxy: StrictSemver): number { + for (let i = 0; i < cli.core.length; i++) { + if (cli.core[i]! !== proxy.core[i]!) return cli.core[i]! > proxy.core[i]! ? 1 : -1; + } + if (cli.prerelease.length === 0) return proxy.prerelease.length === 0 ? 0 : 1; + if (proxy.prerelease.length === 0) return -1; + for (let i = 0; i < Math.max(cli.prerelease.length, proxy.prerelease.length); i++) { + const left = cli.prerelease[i]; + const right = proxy.prerelease[i]; + if (left === right) continue; + if (left === undefined) return -1; + if (right === undefined) return 1; + if (typeof left !== typeof right) return typeof left === "bigint" ? -1 : 1; + return left > right ? 1 : -1; + } + return 0; +} + /** * Compare the running CLI against the live proxy. * @@ -36,11 +61,19 @@ export function computeVersionSkew(cliVersion: string, proxyVersion: string | un if (proxy === null || PLACEHOLDERS.has(proxy) || PLACEHOLDERS.has(cliVersion) || proxy === cliVersion) { return { cliVersion, proxyVersion: proxy, skewed: false, warning: null }; } + const cliSemver = parseStrictSemver(cliVersion); + const proxySemver = parseStrictSemver(proxy); + const order = cliSemver && proxySemver ? compareVersions(cliSemver, proxySemver) : 0; + const advice = order > 0 + ? "the running proxy is older than this CLI. Restart the proxy using the intended current installation. " + + "For a background service, run ocx service repair (ocx service restart is an alias)." + : order < 0 + ? "this ocx on PATH is older than the running proxy. Upgrade the CLI or resolve PATH to the intended installation." + : "the versions differ, but neither can be identified as older. Check which installations the CLI and proxy use."; return { cliVersion, proxyVersion: proxy, skewed: true, - warning: `CLI ${cliVersion} does not match the running proxy ${proxy} — this ocx on PATH is stale. ` - + "Its help and features describe a different build. Reinstall, or run the proxy's own binary.", + warning: `CLI ${cliVersion} does not match the running proxy ${proxy} — ${advice}`, }; } diff --git a/tests/cli/cli-status-json.test.ts b/tests/cli/cli-status-json.test.ts index 31371baa33..10ab4f110e 100644 --- a/tests/cli/cli-status-json.test.ts +++ b/tests/cli/cli-status-json.test.ts @@ -10,9 +10,11 @@ import { fileURLToPath } from "node:url"; import { isConnectionRefused, isUncleanExitEvidence, proxyHealthFailureReason, resolveStatusPid, selectListenTarget } from "../../src/cli/status"; import * as statusFacade from "../../src/cli/status"; import * as statusProbes from "../../src/cli/status-probes"; +import { packageVersion } from "../../src/cli/help"; +import { getDefaultConfig } from "../../src/config"; import { findDeadPid } from "../helpers/dead-pid"; import { removeTreeWithRetry } from "../helpers/remove-tree"; -import { STORE_BUDGET_MS } from "../helpers/test-budget"; +import { INTERNAL_DEADLINE_MS, SPAWN_BUDGET_MS, STORE_BUDGET_MS } from "../helpers/test-budget"; import { inspectClientRotationRecoveryGate, readClientConnectionState } from "../../src/client/state"; import * as lifecycleLock from "../../src/client/lifecycle-lock"; import { writeDesktopDisconnectReceipt } from "../../src/claude/desktop-remote-store"; @@ -28,6 +30,84 @@ function runStatusJson(opencodexHome: string) { }); } +describe("status version skew projection", () => { + test.each([ + ["0.0.1", "the running proxy is older"], + ["999999.0.0", "this ocx on PATH is older"], + [packageVersion(), null], + [`${packageVersion()}+skew-fixture`, "neither can be identified as older"], + ["not-a-version", "neither can be identified as older"], + ["unknown", null], + ["0.0.0", null], + [undefined, null], + ] as const)("projects proxy %s in JSON and human output", async (proxyVersion, expected) => { + const home = mkdtempSync(join(tmpdir(), "ocx-status-skew-")); + const codexHome = join(home, "codex"); + let server: ReturnType | undefined; + try { + // Explicit CODEX_HOME must exist before the CLI imports codex/paths.ts. + mkdirSync(codexHome, { recursive: true }); + server = Bun.serve({ + hostname: "127.0.0.1", port: 0, + fetch(request) { + return new URL(request.url).pathname === "/healthz" + ? Response.json({ service: "opencodex", status: "ok", version: proxyVersion, uptime: 1 }) + : new Response("not found", { status: 404 }); + }, + }); + writeFileSync(join(home, "config.json"), JSON.stringify({ + ...getDefaultConfig(), port: server.port, hostname: "127.0.0.1", codexAutoStart: false, + })); + for (const json of [true, false]) { + // Async child execution lets the fixture answer the real identity/health probes. + const child = Bun.spawn([process.execPath, cliPath, "status", ...(json ? ["--json"] : [])], { + cwd: repoRoot, + env: { ...process.env, OPENCODEX_HOME: home, CODEX_HOME: codexHome }, + stdout: "pipe", stderr: "pipe", + }); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGKILL"); + }, INTERNAL_DEADLINE_MS); + try { + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), new Response(child.stderr).text(), child.exited, + ]); + expect(timedOut).toBe(false); + // Preserve both gates while surfacing the child error when startup fails. + expect({ exitCode, stderr }).toEqual({ exitCode: 0, stderr: "" }); + if (json) { + const parsed = JSON.parse(stdout); + expect(parsed.schemaVersion).toBe(1); + expect(Object.keys(parsed.versionSkew).sort()).toEqual(["cliVersion", "proxyVersion", "skewed", "warning"]); + expect(parsed.versionSkew.cliVersion).toBe(packageVersion()); + expect(parsed.versionSkew.proxyVersion).toBe(proxyVersion ?? null); + expect(parsed.versionSkew.skewed).toBe(expected !== null); + if (expected === null) expect(parsed.versionSkew.warning).toBeNull(); + else expect(parsed.versionSkew.warning).toContain(expected); + } else if (expected === null) { + expect(stdout).not.toContain("does not match the running proxy"); + } else { + expect(stdout).toContain(expected); + } + } finally { + clearTimeout(timer); + if (child.exitCode === null) child.kill("SIGKILL"); + await child.exited; + } + } + expect(existsSync(join(home, "ocx.pid"))).toBe(false); + } finally { + try { + await server?.stop(true); + } finally { + removeTreeWithRetry(home); + } + } + }, SPAWN_BUDGET_MS); +}); + function withRecoveryStatusFixture(work: (fixture: { home: string; lockDeps: { lockPath: string }; diff --git a/tests/cli/cli-version-skew.test.ts b/tests/cli/cli-version-skew.test.ts index 6e45f83c28..36fb6845f9 100644 --- a/tests/cli/cli-version-skew.test.ts +++ b/tests/cli/cli-version-skew.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { computeVersionSkew } from "../../src/cli/version-skew"; +import { computeVersionSkew, isConfirmedVersionMatch } from "../../src/cli/version-skew"; import { packageVersion } from "../../src/cli/help"; /** @@ -7,20 +7,83 @@ import { packageVersion } from "../../src/cli/help"; * build, and nothing surfaced it because the CLI never compared the two versions. */ describe("version skew detection", () => { - test("reports skew when the proxy reports a different version", () => { + test("directs an older CLI to upgrade or resolve PATH", () => { const skew = computeVersionSkew("2.35.0", "2.36.1"); expect(skew.skewed).toBe(true); expect(skew.cliVersion).toBe("2.35.0"); expect(skew.proxyVersion).toBe("2.36.1"); expect(skew.warning).toContain("2.35.0"); expect(skew.warning).toContain("2.36.1"); - expect(skew.warning).toContain("stale"); + expect(skew.warning).toContain("this ocx on PATH is older"); + expect(skew.warning).toContain("Upgrade the CLI or resolve PATH"); + expect(skew.warning).not.toContain("ocx service repair"); + }); + + test("#3464 directs a newer CLI to restart the older proxy", () => { + const skew = computeVersionSkew("2.42.0", "2.10.1-preview.20260805"); + expect(skew).toEqual({ + cliVersion: "2.42.0", + proxyVersion: "2.10.1-preview.20260805", + skewed: true, + warning: "CLI 2.42.0 does not match the running proxy 2.10.1-preview.20260805 — " + + "the running proxy is older than this CLI. Restart the proxy using the intended current installation. " + + "For a background service, run ocx service repair (ocx service restart is an alias).", + }); + expect(skew.warning).not.toContain("this ocx on PATH is older"); + }); + + test.each([ + ["2.43.0", "2.43.0-preview.1"], + ["2.43.0-preview.10", "2.43.0-preview.2"], + ["2.43.0-preview.beta", "2.43.0-preview.10"], + ["2.43.0-preview.1", "2.43.0-preview"], + ["2.43.0-beta", "2.43.0-alpha"], + ["2.44.0-preview.1", "2.43.0"], + ["10.0.0", "9.99.99"], + ["2.43.1", "2.43.0"], + ["2.43.0-preview.9007199254740993", "2.43.0-preview.9007199254740992"], + ])("orders %s above %s in both directions", (newer, older) => { + expect(computeVersionSkew(newer, older).warning).toContain("the running proxy is older"); + expect(computeVersionSkew(older, newer).warning).toContain("this ocx on PATH is older"); + }); + + test.each([ + ["2.43.0+build.1", "2.43.0+build.2"], + ["2.43.0", "2.43.0+build.1"], + ["2.43.0-preview.1+a", "2.43.0-preview.1+b"], + ["invalid", "2.43.0"], + ["2.43", "2.43.0"], + ["v2.43.0", "2.43.0"], + [" 2.43.0", "2.43.0"], + ["2.43.0 ", "2.43.0"], + ["2.43.0-preview.01", "2.43.0-preview.1"], + ["", "2.43.0"], + ])("keeps raw unequal %s / %s neutral in both directions", (left, right) => { + for (const [cli, proxy] of [[left, right], [right, left]]) { + const skew = computeVersionSkew(cli!, proxy!); + expect(skew.cliVersion).toBe(cli); + expect(skew.proxyVersion).toBe(proxy); + expect(skew.skewed).toBe(true); + expect(skew.warning).toContain("neither can be identified as older"); + expect(skew.warning).not.toContain("ocx service repair"); + expect(isConfirmedVersionMatch(skew)).toBe(false); + } + }); + + test.each(["unknown", "0.0.0"])("suppresses %s on either side without confirming a match", placeholder => { + for (const [cli, proxy] of [[placeholder, "2.43.0"], ["2.43.0", placeholder], [placeholder, placeholder]]) { + const skew = computeVersionSkew(cli!, proxy!); + expect(skew.skewed).toBe(false); + expect(skew.warning).toBeNull(); + expect(isConfirmedVersionMatch(skew)).toBe(false); + } }); test("stays quiet when the versions match", () => { const skew = computeVersionSkew("2.35.0", "2.35.0"); expect(skew.skewed).toBe(false); expect(skew.warning).toBeNull(); + expect(isConfirmedVersionMatch(skew)).toBe(true); }); test("stays quiet when nothing is live", () => { @@ -28,6 +91,7 @@ describe("version skew detection", () => { expect(skew.skewed).toBe(false); expect(skew.proxyVersion).toBeNull(); expect(skew.warning).toBeNull(); + expect(isConfirmedVersionMatch(skew)).toBe(false); }); test("suppresses the warning when the proxy reports the 0.0.0 placeholder", () => { diff --git a/tests/codex-integration/doctor.test.ts b/tests/codex-integration/doctor.test.ts index 9fdb7ee30d..acb0f1b87e 100644 --- a/tests/codex-integration/doctor.test.ts +++ b/tests/codex-integration/doctor.test.ts @@ -1,4 +1,7 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; +import * as proxyLiveness from "../../src/server/proxy-liveness"; +import * as cliHelp from "../../src/cli/help"; +import { getDefaultConfig } from "../../src/config"; import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, mkdtempSync, utimesSync, writeFileSync } from "node:fs"; import { join } from "node:path"; @@ -32,6 +35,7 @@ import { } from "../../src/lib/local-management-capability"; import { findDeadPid } from "../helpers/dead-pid"; import { removeTreeWithRetry } from "../helpers/remove-tree"; +import { STORE_BUDGET_MS } from "../helpers/test-budget"; const TEST_DIR = join(import.meta.dir, ".tmp-doctor-test"); const TEST_CODEX_HOME = join(TEST_DIR, "codex"); @@ -780,6 +784,63 @@ describe("doctor abandoned response-state temps", () => { }); }); +describe("doctor version skew projection", () => { + test.each([ + ["2.42.0", "2.10.1-preview.20260805", "the running proxy is older"], + ["2.35.0", "2.36.1", "this ocx on PATH is older"], + ["2.43.0", "2.43.0", "ok ocx 2.43.0 matches the running proxy"], + ["2.43.0+a", "2.43.0+b", "neither can be identified as older"], + ["v2.43.0", "2.43.0", "neither can be identified as older"], + ["2.43.0", "unknown", null], + ["unknown", "2.43.0", null], + ["2.43.0", "0.0.0", null], + ["0.0.0", "0.0.0", null], + ["unknown", "unknown", null], + ["2.43.0", undefined, null], + ] as const)("projects CLI %s / proxy %s without false matches", async (cli, proxy, expected) => { + const home = mkdtempSync(join(tmpdir(), "ocx-doctor-skew-")); + const codexHome = join(home, "codex"); + const previousHome = process.env.OPENCODEX_HOME; + const previousCodexHome = process.env.CODEX_HOME; + const previousExitCode = process.exitCode; + const restore: Array<() => void> = []; + try { + // Runtime history diagnostics resolve and stat an explicit CODEX_HOME. + mkdirSync(codexHome, { recursive: true }); + process.env.OPENCODEX_HOME = home; + process.env.CODEX_HOME = codexHome; + writeFileSync(join(home, "config.json"), JSON.stringify({ ...getDefaultConfig(), port: 9, codexAutoStart: false })); + const logged: string[] = []; + const log = spyOn(console, "log").mockImplementation((...args: unknown[]) => { logged.push(args.map(String).join(" ")); }); + restore.push(() => log.mockRestore()); + const version = spyOn(cliHelp, "packageVersion").mockReturnValue(cli); + restore.push(() => version.mockRestore()); + // Other doctor sections probe upstream health; this diagnostic fixture must stay offline. + const fetch = spyOn(globalThis, "fetch").mockImplementation(async () => new Response(null, { status: 503 })); + restore.push(() => fetch.mockRestore()); + const proxyInfo: proxyLiveness.LiveProxy = { + pid: null, port: 9, hostname: "127.0.0.1", source: "config", ...(proxy === undefined ? {} : { version: proxy }), + }; + const live = spyOn(proxyLiveness, "findLiveProxy").mockResolvedValue(proxyInfo); + restore.push(() => live.mockRestore()); + await runDoctor([]); + const output = logged.join("\n"); + if (expected !== null) expect(output).toContain(expected); + else expect(output).not.toContain("does not match the running proxy"); + if (cli !== "2.43.0" || proxy !== "2.43.0") expect(output).not.toContain("matches the running proxy"); + if (expected === "the running proxy is older") expect(output).toContain("ocx service repair"); + } finally { + for (const cleanup of restore.reverse()) cleanup(); + process.exitCode = previousExitCode; + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + removeTreeWithRetry(home); + } + }, STORE_BUDGET_MS); +}); + describe("doctor reclaim wiring (end to end)", () => { // The formatter tests above cannot observe deletion. This covers the call site itself: // inverting the report/reclaim ternary in runDoctor must fail a test.