From a5805626579201cd75c36f9ec4d22d36b2aae426 Mon Sep 17 00:00:00 2001 From: "Derek J. Russell" Date: Sun, 14 Jun 2026 21:09:32 -0700 Subject: [PATCH] feat(rail): Sovereign Evidence Rail + L3 Memory Governor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Evidence-driven graduation of the REVIEW/PLAN shadow subagents to authoritative status, plus a worktree-RAM-budget governor for L3. Unit D — L3 Memory Governor (l3_memory_governor.py + subagent_scheduler.py): reuses MemoryPressureGate; caps concurrent worktrees by live RAM, strictest-wins with the existing fan-out gate. Off-is-inert. Unit B — Evaluator (shadow_evaluator.py): pure REVIEW binary agreement + PLAN refinement (coverage ∧ acyclic ∧ disjoint); malformed→block. Unit A — Telemetry store (shadow_telemetry_store.py): async SQLite, to_thread non-blocking writer, two-phase upsert, rolling FIFO cap. Unit C — Gate + breaker (shadow_graduation_gate.py): event-driven 50-soak → flips _AUTHORITATIVE via existing persist_flag_to_env; PlanBreaker (CRITICAL-pressure pre-empt → legacy); AGENT_DEGRADATION SSE. Live wiring: producers in orchestrator _run_{plan,review}_shadow, authoritative REVIEW tier-raise, GLS construct/teardown. OFF-inert: store disabled → orchestrator _shadow_store None → FSM byte-identical. Rebased onto main (Slice 255); keep-both resolution of the EVENT_TYPE_* collision with the loop's orthogonal Slice 252 Shadow-Telemetry (spec §14 documents the two unrelated "shadows"). Endorsed-DAG-drives-execution is deferred (observer-only today). 45 rail/governor tests + 49 SSE green. Co-Authored-By: Claude Opus 4.8 (1M context) [integrity-verified: daaafc3768bc] --- .../governance/autonomy/l3_memory_governor.py | 80 + .../governance/autonomy/subagent_scheduler.py | 79 + .../governance/governed_loop_service.py | 45 + .../governance/ide_observability_stream.py | 28 + .../core/ouroboros/governance/orchestrator.py | 273 ++- .../ouroboros/governance/shadow_evaluator.py | 107 ++ .../governance/shadow_graduation_gate.py | 173 ++ .../governance/shadow_telemetry_store.py | 263 +++ .../plans/2026-06-14-l3-memory-governor.md | 485 +++++ .../2026-06-14-sovereign-evidence-rail.md | 1568 +++++++++++++++++ .../2026-06-14-sovereign-evidence-rail.md | 507 ++++++ .../autonomy/test_l3_memory_governor.py | 93 + .../test_scheduler_memory_governor.py | 100 ++ .../autonomy/test_subagent_scheduler.py | 10 + tests/governance/test_shadow_evaluator.py | 122 ++ .../governance/test_shadow_graduation_gate.py | 198 +++ .../test_shadow_rail_integration.py | 79 + .../governance/test_shadow_telemetry_store.py | 93 + 18 files changed, 4292 insertions(+), 11 deletions(-) create mode 100644 backend/core/ouroboros/governance/autonomy/l3_memory_governor.py create mode 100644 backend/core/ouroboros/governance/shadow_evaluator.py create mode 100644 backend/core/ouroboros/governance/shadow_graduation_gate.py create mode 100644 backend/core/ouroboros/governance/shadow_telemetry_store.py create mode 100644 docs/superpowers/plans/2026-06-14-l3-memory-governor.md create mode 100644 docs/superpowers/plans/2026-06-14-sovereign-evidence-rail.md create mode 100644 docs/superpowers/specs/2026-06-14-sovereign-evidence-rail.md create mode 100644 tests/governance/autonomy/test_l3_memory_governor.py create mode 100644 tests/governance/autonomy/test_scheduler_memory_governor.py create mode 100644 tests/governance/test_shadow_evaluator.py create mode 100644 tests/governance/test_shadow_graduation_gate.py create mode 100644 tests/governance/test_shadow_rail_integration.py create mode 100644 tests/governance/test_shadow_telemetry_store.py diff --git a/backend/core/ouroboros/governance/autonomy/l3_memory_governor.py b/backend/core/ouroboros/governance/autonomy/l3_memory_governor.py new file mode 100644 index 0000000000..62810dc003 --- /dev/null +++ b/backend/core/ouroboros/governance/autonomy/l3_memory_governor.py @@ -0,0 +1,80 @@ +"""L3 worktree-RAM-budget governor (pure math). + +Composes ON TOP of MemoryPressureGate's free-%-based fan-out caps: +the gate answers "is the box under pressure?"; this module answers +"given the absolute RAM cost of a worktree, how many fit right now?". +Strictest-wins between the two. No IO, no scheduler import — every +decision is a deterministic function of its arguments so it can be +proven at all pressure levels in isolation. +""" +from __future__ import annotations + +import math +import os +from dataclasses import dataclass + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in ("true", "1", "yes") + + +def _env_int(name: str, default: int, *, minimum: int) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + return max(minimum, int(raw)) + except (TypeError, ValueError): + return default + + +def governor_enabled() -> bool: + """Master flag. Default TRUE; inert until an L3 graph actually runs.""" + return _env_bool("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", True) + + +def worktree_ram_budget_mb() -> int: + """Assumed peak RAM per concurrent worktree. Default 1500MB.""" + return _env_int("JARVIS_L3_WORKTREE_RAM_BUDGET_MB", 1500, minimum=64) + + +@dataclass(frozen=True) +class GovernorDecision: + requested: int + ram_cap: int + level_cap: int + n_allowed: int + avail_mb: float + budget_mb: int + disposition: str # compute_worktree_cap emits "allow"|"clamp"; the + # scheduler layer may instead report "disabled"/"probe_fail" (this pure + # function never produces those — it is only reached with a live probe). + + +def compute_worktree_cap( + *, + requested: int, + avail_mb: float, + budget_mb: int, + level_cap: int, +) -> GovernorDecision: + """Pure clamp. ``ram_cap = floor(avail_mb / budget_mb)`` (>=1); + final allowance is the strictest of requested / ram_cap / level_cap.""" + # Fail-safe: a bad/non-positive avail_mb (e.g. a garbage probe reading) + # floors to ram_cap=1 — the most conservative non-zero fan-out — rather + # than 0 or negative. Clamping down on bad input is the safe direction. + ram_cap = max(1, int(math.floor(avail_mb / float(budget_mb)))) + n_allowed = max(0, min(requested, ram_cap, level_cap)) + disposition = "clamp" if n_allowed < requested else "allow" + return GovernorDecision( + requested=requested, + ram_cap=ram_cap, + level_cap=level_cap, + n_allowed=n_allowed, + avail_mb=avail_mb, + budget_mb=budget_mb, + disposition=disposition, + ) diff --git a/backend/core/ouroboros/governance/autonomy/subagent_scheduler.py b/backend/core/ouroboros/governance/autonomy/subagent_scheduler.py index 542c01afb5..78308462c5 100644 --- a/backend/core/ouroboros/governance/autonomy/subagent_scheduler.py +++ b/backend/core/ouroboros/governance/autonomy/subagent_scheduler.py @@ -35,6 +35,11 @@ from backend.core.ouroboros.governance.saga.merge_coordinator import MergeCoordinator from backend.core.ouroboros.governance.saga.saga_types import FileOp, PatchedFile, RepoPatch from backend.core.ouroboros.governance.test_runner import BlockedPathError +# Module-level (not lazy like _consult_memory_gate's in-method import) so +# tests can monkeypatch this symbol as a seam; the governor path below uses it. +from backend.core.ouroboros.governance.memory_pressure_gate import ( + get_default_gate, +) logger = logging.getLogger("Ouroboros.SubagentScheduler") @@ -497,6 +502,9 @@ async def _run_graph(self, graph_id: str) -> None: # Zero work loss. Gate-disabled → pass-through (no clamp). Every # decision is logged + SSE-published (allow / clamp / disabled / # probe_fail) so operators have a §8 audit trail. + # Bound before the guard so the Unit D block below can safely + # read it even if a future edit repopulates `selected`. + decision = None if selected: decision = self._consult_memory_gate( len(selected), graph_id=graph_id, @@ -506,6 +514,23 @@ async def _run_graph(self, graph_id: str) -> None: selected = list(selected[:decision.n_allowed]) deferred = sorted(list(deferred) + overflow) + # Unit D — worktree-RAM-budget governor composes on top of + # the fan-out gate (strictest-wins). Same zero-work-loss + # defer-overflow mechanism; disabled/probe-fail -> no clamp. + if selected: + level_cap = ( + decision.n_allowed + if decision is not None + else len(selected) + ) + gov = self._consult_memory_governor( + len(selected), graph_id=graph_id, level_cap=level_cap, + ) + if gov is not None and gov.n_allowed < len(selected): + overflow = list(selected[gov.n_allowed:]) + selected = list(selected[:gov.n_allowed]) + deferred = sorted(list(deferred) + overflow) + if not selected: state = self._update_state( state, @@ -755,6 +780,60 @@ def _consult_memory_gate( return decision + def _consult_memory_governor( + self, + n_requested: int, + *, + graph_id: str, + level_cap: int, + ) -> Optional[Any]: + """Unit D — worktree-RAM-budget clamp composed on top of the + Slice 5 Arc B fan-out gate. + + Returns a ``GovernorDecision`` (or ``None`` when disabled / on + any probe failure — the scheduler must never break on the + governor). ``level_cap`` is the allowance already granted by + the free-%-based fan-out gate; the governor takes the strictest + of that and the absolute RAM budget. + """ + from backend.core.ouroboros.governance.autonomy.l3_memory_governor import ( + compute_worktree_cap, + governor_enabled, + worktree_ram_budget_mb, + ) + + if not governor_enabled(): + return None + try: + gate = get_default_gate() + probe = gate.probe() + avail_mb = float(probe.available_bytes) / (1024.0 * 1024.0) + except Exception: # noqa: BLE001 — governor must not break scheduler + logger.debug( + "[SubagentScheduler] memory governor probe failed " + "(non-fatal)", exc_info=True, + ) + return None + + decision = compute_worktree_cap( + requested=n_requested, + avail_mb=avail_mb, + budget_mb=worktree_ram_budget_mb(), + level_cap=level_cap, + ) + log_fn = ( + logger.warning if decision.disposition == "clamp" else logger.info + ) + log_fn( + "[SubagentScheduler] ram_governor: graph=%s disposition=%s " + "requested=%d allowed=%d ram_cap=%d level_cap=%d avail_mb=%.0f " + "budget_mb=%d", + graph_id, decision.disposition, decision.requested, + decision.n_allowed, decision.ram_cap, decision.level_cap, + decision.avail_mb, decision.budget_mb, + ) + return decision + def _select_ready_batch( self, graph: ExecutionGraph, diff --git a/backend/core/ouroboros/governance/governed_loop_service.py b/backend/core/ouroboros/governance/governed_loop_service.py index 053f4206d1..66802e9a28 100644 --- a/backend/core/ouroboros/governance/governed_loop_service.py +++ b/backend/core/ouroboros/governance/governed_loop_service.py @@ -1139,6 +1139,9 @@ def __init__( self._graph_coalescer: Optional[Any] = None self._advanced_autonomy: Optional[Any] = None self._mcp_client: Optional[Any] = None # Phase A: GovernanceMCPClient, wired in start() + # Evidence Rail (Unit A/C) — durable shadow-vs-legacy ledger. + # Initialized in _build_components when store_enabled(); None = inert. + self._shadow_store_ref: Optional[Any] = None # Compute-class admission gate (set externally after fetching /v1/capability; # None = gate disabled — backward-compatible default) @@ -2089,6 +2092,15 @@ async def stop(self) -> None: except Exception: pass + # Evidence Rail — close the durable telemetry store + _ssr = getattr(self, "_shadow_store_ref", None) + if _ssr is not None: + try: + await _ssr.aclose() + except Exception: # noqa: BLE001 + pass + self._shadow_store_ref = None + # Detach from stack self._detach_from_stack() self._state = ServiceState.INACTIVE @@ -4870,6 +4882,39 @@ def _on_streaming_token(token: str) -> None: except Exception as exc: logger.debug("[GLS] SubagentOrchestrator skipped: %s", exc) + # ---- Evidence Rail (Unit A/C) — shadow-vs-legacy telemetry store + graduation gate ---- + # Inert + byte-identical when JARVIS_SHADOW_TELEMETRY_STORE_ENABLED=false: + # store_enabled() is False → no store constructed → set_shadow_rail never + # called → orchestrator's _shadow_store stays None → FSM byte-identical. + try: + from backend.core.ouroboros.governance.shadow_telemetry_store import ( + ShadowTelemetryStore, store_enabled, + ) + from backend.core.ouroboros.governance.shadow_graduation_gate import ( + ShadowGraduationGate, build_rail_evaluator, gate_enabled, + ) + if store_enabled(): + _shadow_store = ShadowTelemetryStore( + evaluator=build_rail_evaluator(), + ) + await _shadow_store.start() + self._shadow_store_ref = _shadow_store + _shadow_gate = ( + ShadowGraduationGate(store=_shadow_store) + if gate_enabled() else None + ) + if self._orchestrator is not None and hasattr( + self._orchestrator, "set_shadow_rail" + ): + self._orchestrator.set_shadow_rail( + _shadow_store, _shadow_gate, + ) + except Exception: # noqa: BLE001 — rail attach must never break boot + logger.warning( + "[GovernedLoopService] shadow rail attach failed (non-fatal)", + exc_info=True, + ) + # ---- Wire Self-Critique Engine (Phase 3a — post-VERIFY quality signal) ---- # Cheap DW critique over the applied diff against the original goal. # Poor ratings become FEEDBACK memories; excellent ratings reinforce diff --git a/backend/core/ouroboros/governance/ide_observability_stream.py b/backend/core/ouroboros/governance/ide_observability_stream.py index d9ad60f925..25306ab7da 100644 --- a/backend/core/ouroboros/governance/ide_observability_stream.py +++ b/backend/core/ouroboros/governance/ide_observability_stream.py @@ -197,6 +197,11 @@ EVENT_TYPE_GOVERNOR_EMERGENCY_BRAKE = "governor_emergency_brake" EVENT_TYPE_MEMORY_PRESSURE_CHANGED = "memory_pressure_changed" +# Evidence Rail Unit C — emitted by the circuit breaker when an authoritative +# subagent trips back to legacy (degradation event). Payload carries agent + +# trip_reason + pressure_level so IDE consumers can render a "degraded" banner. +EVENT_TYPE_AGENT_DEGRADATION = "agent_degradation" + # Dynamic Risk-State Convergence Engine (Slice 98 Phase 2) — emitted # ONLY on a convergence-band TRANSITION (NORMAL/ELEVATED/PARANOIA). # Payload carries the ConvergenceVerdict.to_dict() snapshot. Keyed by @@ -1535,6 +1540,9 @@ # in-process on_trip callback drives harness # graceful shutdown, this SSE is the IDE # observability surface) + EVENT_TYPE_AGENT_DEGRADATION, # Evidence Rail Unit C — authoritative + # subagent tripped back to legacy; payload: + # agent + trip_reason + pressure_level. }) @@ -4441,6 +4449,26 @@ def publish_review_branch_event( return None +# --------------------------------------------------------------------------- +# Evidence Rail Unit C — AGENT_DEGRADATION SSE publisher +# --------------------------------------------------------------------------- + + +def publish_agent_degradation_event( + *, broker: Any, agent: str, op_id: str, trip_reason: str, + pressure_level: str, +) -> None: + """Best-effort AGENT_DEGRADATION frame. Never raises/blocks.""" + try: + broker.publish( + EVENT_TYPE_AGENT_DEGRADATION, op_id, + {"agent": agent, "trip_reason": trip_reason, + "pressure_level": pressure_level}, + ) + except Exception: # noqa: BLE001 + pass + + # --------------------------------------------------------------------------- # FlagRegistry self-registration (auto-discovered by §33.3 walker) # --------------------------------------------------------------------------- diff --git a/backend/core/ouroboros/governance/orchestrator.py b/backend/core/ouroboros/governance/orchestrator.py index c9b2b70f8d..7798321ab0 100644 --- a/backend/core/ouroboros/governance/orchestrator.py +++ b/backend/core/ouroboros/governance/orchestrator.py @@ -1213,6 +1213,13 @@ def __init__( # stable. None until governed_loop_service wires it. self._subagent_orchestrator: Any = None + # Evidence Rail (Unit A + C) — harness-attached via set_shadow_rail(). + # Both default to None; when None the rail is fully inert and the FSM + # is byte-identical to its pre-rail state. + self._shadow_store: Any = None + self._shadow_gate: Any = None + self._shadow_gate_tasks: set = set() + # ── Phase 1 Step 3C: reload-hostile state hoisted to _governance_state ── # Every field that would otherwise get re-allocated on # ``importlib.reload(orchestrator)`` now lives on an @@ -1569,17 +1576,52 @@ def set_subagent_orchestrator(self, orch: Any) -> None: """ self._subagent_orchestrator = orch - async def _run_review_shadow(self, ctx: Any, best_candidate: Any) -> None: - """Phase B — post-VALIDATE REVIEW subagent in OBSERVER MODE. + def set_shadow_rail(self, store: Any, gate: Any) -> None: + """Attach the Evidence Rail store + graduation gate (Unit A/C). + + Both default to None → rail is inert and the FSM is byte-identical + to the pre-rail state. Mirrors the ``set_subagent_orchestrator`` + pattern so harness wiring stays uniform. + """ + self._shadow_store = store + self._shadow_gate = gate + + def _fire_shadow_gate(self, agent: str) -> None: + """Fire-and-forget graduation check after a comparison row lands. + + Strong task ref prevents GC; fully guarded + fail-soft. No-ops + when ``_shadow_gate`` is None (the default) so the FSM is + byte-identical when the rail is not attached. + """ + gate = getattr(self, "_shadow_gate", None) + if gate is None: + return + try: + import asyncio as _asyncio + _task = _asyncio.ensure_future(gate.maybe_promote(agent)) + self._shadow_gate_tasks.add(_task) + _task.add_done_callback(self._shadow_gate_tasks.discard) + except Exception: # noqa: BLE001 — observer contract + pass + + async def _run_review_shadow( + self, ctx: Any, best_candidate: Any, + ) -> "Optional[RiskTier]": + """Phase B — post-VALIDATE REVIEW subagent (observer → authoritative). Gated by ``JARVIS_REVIEW_SUBAGENT_SHADOW`` (default **``true``**, graduated 2026-04-20). When on, dispatches a REVIEW subagent per candidate file and emits the verdict to telemetry. **The FSM - proceeds to GATE regardless of verdict** — no risk-tier change, - no retry routing, no state mutation. The contract stays - observer-only even post-graduation; promoting REVIEW into - authority-carrying gate logic is a separate slice with its own - graduation arc. + proceeds to GATE regardless of verdict** — no retry routing, no + state mutation beyond risk-tier escalation. + + **Authoritative mode** (Unit C): when + ``JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE=true`` (default ``false``), + a REJECT aggregate verdict escalates the risk tier to + APPROVAL_REQUIRED via the return value (strictest-wins: only + increases, never decreases). The call site captures the return + and applies it to the GATE-local ``risk_tier`` variable. + Defaults false → return None → byte-identical FSM behaviour. Graduation evidence (2026-04-20): * 28-test regression spine green (test_review_subagent.py + @@ -1707,6 +1749,44 @@ async def _run_review_shadow(self, ctx: Any, best_candidate: Any) -> None: # can build rollup counters (aggregate-verdict distribution, # approve/reject rates, per-session verdict sanity) across the # graduation arc. Matches the [SemanticGuard] log convention. + + # Evidence Rail (Unit A) — record shadow + legacy outcomes. + # Fully guarded: inert + byte-identical when no rail is attached. + # NOTE: ctx.risk_tier is the pre-SemanticGuardian tier (guardian + # upgrade is stored in the outer local only, not back-propagated to + # ctx). This is the correct legacy signal: it reflects the vanilla + # VALIDATE-phase risk classification before any pattern-detector + # intervention, giving a stable like-for-like comparison baseline. + # semantic_guard_hard is NOT accessible here (held in outer scope); + # we leave it False (the conservative default) — the alignment + # evaluator treats this as a softer legacy block signal, which is + # correct for the pre-guardian tier. + if getattr(self, "_shadow_store", None) is not None: + try: + self._shadow_store.record_shadow_nowait( + op_id=getattr(ctx, "op_id", "?"), + agent="review", + ts=time.time(), + shadow_outcome={"aggregate": _aggregate}, + ) + _rt = getattr(ctx, "risk_tier", None) + self._shadow_store.record_legacy_nowait( + op_id=getattr(ctx, "op_id", "?"), + agent="review", + ts=time.time(), + legacy_outcome={ + "risk_tier": str(_rt.name if _rt is not None else ""), + "semantic_guard_hard": False, + }, + ) + except Exception: # noqa: BLE001 — observer contract + pass + # Gate fires before the async writer commits THIS op's row, so + # the streak it reads reflects N-1 ops (eventual-consistency). + # This is intentionally conservative — graduation lands one op + # later than the bare threshold, never earlier. + self._fire_shadow_gate("review") + logger.info( "[REVIEW-SHADOW] op=%s aggregate=%s files_reviewed=%d " "approved=%d reservations=%d rejected=%d failed=%d " @@ -1720,12 +1800,32 @@ async def _run_review_shadow(self, ctx: Any, best_candidate: Any) -> None: _counts["failed"], _duration_ms, ) + + # Authoritative REVIEW (Unit C, post-graduation): a REJECT + # verdict raises the risk tier to APPROVAL_REQUIRED. + # Composes strictest-wins with SemanticGuardian/Iron Gate — + # only ADDS friction, never removes it. + # Flag defaults false -> return None -> byte-identical until + # graduation. + if os.environ.get( + "JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE", "false", + ).strip().lower() in ("true", "1", "yes") and _aggregate == "REJECT": + try: + logger.info( + "[AUTHORITATIVE] agent=review op=%s REJECT -> " + "APPROVAL_REQUIRED", + getattr(ctx, "op_id", "?"), + ) + return RiskTier.APPROVAL_REQUIRED + except Exception: # noqa: BLE001 + pass except Exception: # Observer contract: shadow must never break the FSM. logger.debug( "[Orchestrator] REVIEW shadow dispatch skipped", exc_info=True, ) + return None async def _run_plan_shadow(self, ctx: Any) -> Any: """Phase B PLAN-shadow — AgenticPlanSubagent dispatch running @@ -1824,6 +1924,149 @@ async def _run_plan_shadow(self, ctx: Any) -> Any: else str(_result.status) ) + # Evidence Rail (Unit A) — record legacy + shadow for graduation soak. + # Fully guarded: inert + byte-identical when no rail is attached. + # + # Legacy flat: the target_files tuple is the canonical set of paths + # the operation was scoped to. It's stable, always present, and + # represents what the legacy flat-plan generator was asked to cover. + # Using ctx.implementation_plan's ordered_changes would be slightly + # more precise but risks a JSON parse failure on a skipped plan; the + # target_files path is simpler and correct for alignment purposes. + # + # Shadow DAG: coerce the tuple-of-tuples _execution_graph into the + # dict-of-dicts shape the evaluator expects. The inner unit tuples + # carry ("unit_id", ...), ("dependency_ids", ...), ("owned_paths", ...) + # which we remap to the evaluator's canonical keys ("id", "deps", + # "owned_paths") so shadow_evaluator._has_cycle + evaluate_plan work. + if getattr(self, "_shadow_store", None) is not None: + try: + _legacy_flat = [ + p for p in (getattr(ctx, "target_files", ()) or ()) if p + ] + self._shadow_store.record_legacy_nowait( + op_id=getattr(ctx, "op_id", "?"), + agent="plan", + ts=time.time(), + legacy_outcome={"flat": _legacy_flat}, + ) + if _execution_graph is not None: + try: + _eg_dict = dict(_execution_graph) + _units_payload = _eg_dict.get("units") or () + _shadow_units = [ + { + "id": str(dict(_ut).get("unit_id", "")), + "deps": list( + dict(_ut).get("dependency_ids", ()) or () + ), + "owned_paths": list( + dict(_ut).get("owned_paths", ()) or () + ), + } + for _ut in _units_payload + ] + except Exception: # noqa: BLE001 + _shadow_units = [] + self._shadow_store.record_shadow_nowait( + op_id=getattr(ctx, "op_id", "?"), + agent="plan", + ts=time.time(), + shadow_outcome={"units": _shadow_units}, + ) + except Exception: # noqa: BLE001 — observer contract + pass + # Gate fires before the async writer commits THIS op's row, so + # the streak it reads reflects N-1 ops (eventual-consistency). + # This is intentionally conservative — graduation lands one op + # later than the bare threshold, never earlier. + self._fire_shadow_gate("plan") + + # Authoritative PLAN (Unit C, post-graduation): run the + # graceful-degradation breaker. On trip (cyclical/empty DAG + # or CRITICAL memory pressure), emit AGENT_DEGRADATION and + # KEEP legacy as the authoritative input. + # Flag defaults false -> byte-identical until graduation. + # + # NOTE on _shadow_units scoping: the producer block computes + # _shadow_units inside a nested if/try (guarded by + # _shadow_store + _execution_graph). Rather than referencing + # a possibly-unbound local, we independently rebuild the + # units list from _execution_graph here — same transform, + # no dependency on the producer's inner scope. + # + # NOTE on downstream DAG consumption: ctx.execution_graph + # (stashed above) is currently OBSERVER-ONLY — it is stashed + # on ctx, not on best_candidate, so _materialize_execution_ + # graph_candidate (line ~10523) which gates on + # "execution_graph" in best_candidate never sees it. + # Making the shadow DAG drive actual execution is a + # follow-up slice. When AUTHORITATIVE=true, the breaker-trip + # path correctly leaves legacy authoritative (it already is, + # since the DAG is observer-only). The endorsed path logs an + # INFO and proceeds — no execution change yet. + if os.environ.get( + "JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "false", + ).strip().lower() in ("true", "1", "yes"): + try: + from backend.core.ouroboros.governance.shadow_graduation_gate import ( + PlanBreaker, + ) + # Rebuild units from _execution_graph independent of + # producer scope (safe even if _shadow_store is None). + _breaker_units: list = [] + if _execution_graph is not None: + try: + _beg_dict = dict(_execution_graph) + _beg_payload = _beg_dict.get("units") or () + _breaker_units = [ + { + "id": str(dict(_bu).get("unit_id", "")), + "deps": list( + dict(_bu).get("dependency_ids", ()) or () + ), + "owned_paths": list( + dict(_bu).get("owned_paths", ()) or () + ), + } + for _bu in _beg_payload + ] + except Exception: # noqa: BLE001 + _breaker_units = [] + _decision = PlanBreaker().should_use_legacy( + dag={"units": _breaker_units}, + ) + if _decision.trip: + logger.warning( + "[BREAKER] agent=plan op=%s trip=%s pressure=%s" + " -> legacy (authoritative path retained)", + getattr(ctx, "op_id", "?"), + _decision.reason, + _decision.pressure_level, + ) + try: + from backend.core.ouroboros.governance.ide_observability_stream import ( # noqa: E501 + get_default_broker as _get_broker, + publish_agent_degradation_event, + ) + publish_agent_degradation_event( + broker=_get_broker(), + agent="plan", + op_id=getattr(ctx, "op_id", "?"), + trip_reason=_decision.reason, + pressure_level=_decision.pressure_level, + ) + except Exception: # noqa: BLE001 + pass + else: + logger.info( + "[AUTHORITATIVE] agent=plan op=%s DAG endorsed" + " (execution follow-up slice pending)", + getattr(ctx, "op_id", "?"), + ) + except Exception: # noqa: BLE001 + pass + logger.info( "[PLAN-SHADOW] op=%s status=%s dag_units=%d edges=%d " "roots=%d parallel_pairs=%d validation_valid=%s " @@ -7736,10 +7979,18 @@ async def _validate_one(cand: Dict[str, Any]) -> Tuple[Dict[str, Any], "Validati exc_info=True, ) - # ---- REVIEW subagent (Slice 1a — SHADOW MODE observer only) ---- - # Gated by JARVIS_REVIEW_SUBAGENT_SHADOW. Emits verdict telemetry - # only; FSM proceeds to GATE unchanged. See _run_review_shadow. - await self._run_review_shadow(ctx, best_candidate) + # ---- REVIEW subagent (Slice 1a — observer; Unit C — authoritative) ---- + # Gated by JARVIS_REVIEW_SUBAGENT_SHADOW. Emits verdict telemetry. + # When JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE=true (default false), + # a REJECT aggregate returns RiskTier.APPROVAL_REQUIRED here and + # is applied strictest-wins to the GATE-local risk_tier. Inert + # (returns None) until graduation. See _run_review_shadow. + _review_escalated_tier = await self._run_review_shadow(ctx, best_candidate) + if _review_escalated_tier is not None and ( + risk_tier is None + or risk_tier.value < _review_escalated_tier.value + ): + risk_tier = _review_escalated_tier # ---- MutationGate: APPLY-phase execution boundary (cached) ---- # diff --git a/backend/core/ouroboros/governance/shadow_evaluator.py b/backend/core/ouroboros/governance/shadow_evaluator.py new file mode 100644 index 0000000000..ce177371ad --- /dev/null +++ b/backend/core/ouroboros/governance/shadow_evaluator.py @@ -0,0 +1,107 @@ +"""Deterministic shadow-vs-legacy alignment evaluator (Unit B). + +Pure functions: no IO, no LLM, no imports of the store or orchestrator. +Every function returns a structured ``Alignment`` even on malformed +input — malformed maps to ``aligned=False`` (the conservative default +that BLOCKS graduation rather than risking a false promotion). +""" +from __future__ import annotations + +from dataclasses import dataclass + +_BLOCK = "BLOCK" +_ALLOW = "ALLOW" +_BLOCKING_TIERS = frozenset({"APPROVAL_REQUIRED", "BLOCKED"}) + + +@dataclass(frozen=True) +class Alignment: + aligned: bool + reason: str # "" when aligned; divergence/malformed detail otherwise + + +def _legacy_review_binary(legacy: dict) -> str: + tier = str(legacy.get("risk_tier", "")).upper() + hard = bool(legacy.get("semantic_guard_hard", False)) + if hard or tier in _BLOCKING_TIERS: + return _BLOCK + return _ALLOW + + +def _shadow_review_binary(shadow: dict) -> str: + agg = str(shadow.get("aggregate", "")).lower() + # reservations are advisory -> ALLOW; only outright reject BLOCKs. + return _BLOCK if agg == "reject" else _ALLOW + + +def evaluate_review(legacy: object, shadow: object) -> Alignment: + if not isinstance(legacy, dict) or not isinstance(shadow, dict): + return Alignment(False, "malformed:non_dict_input") + if "risk_tier" not in legacy or "aggregate" not in shadow: + return Alignment(False, "malformed:missing_keys") + lb = _legacy_review_binary(legacy) + sb = _shadow_review_binary(shadow) + if lb == sb: + return Alignment(True, "") + return Alignment(False, f"shadow={sb} legacy={lb}") + + +def _has_cycle(units: list) -> bool: + """Kahn's algorithm — True if any cycle remains.""" + ids = {u["id"] for u in units} + indeg = {u["id"]: 0 for u in units} + adj: dict = {u["id"]: [] for u in units} + for u in units: + for dep in u.get("deps", []): + if dep in ids: # deps on external/unknown units are treated as pre-satisfied + adj[dep].append(u["id"]) + indeg[u["id"]] += 1 + queue = [i for i, d in indeg.items() if d == 0] + visited = 0 + while queue: + n = queue.pop() + visited += 1 + for m in adj[n]: + indeg[m] -= 1 + if indeg[m] == 0: + queue.append(m) + return visited != len(units) + + +def _owned_path_overlap(units: list) -> str: + seen: dict = {} + for u in units: + for p in u.get("owned_paths", []): + if p in seen and seen[p] != u["id"]: + return p + seen[p] = u["id"] + return "" + + +def evaluate_plan(legacy_flat: object, shadow_dag: object) -> Alignment: + if not isinstance(legacy_flat, list) or not isinstance(shadow_dag, dict): + return Alignment(False, "malformed:non_collection_input") + units = shadow_dag.get("units") + if not isinstance(units, list) or not all( + isinstance(u, dict) and "id" in u for u in units + ): + return Alignment(False, "malformed:bad_units") + + # 1. Coverage — DAG must touch every legacy task (extra is OK). + dag_paths = set() + for u in units: + dag_paths.update(u.get("owned_paths", [])) + dropped = [t for t in legacy_flat if t not in dag_paths] + if dropped: + return Alignment(False, "dropped_tasks:" + ",".join(sorted(dropped))) + + # 2. Acyclicity. + if _has_cycle(units): + return Alignment(False, "cyclical_dag") + + # 3. Disjoint ownership. + overlap = _owned_path_overlap(units) + if overlap: + return Alignment(False, "owned_path_overlap:" + overlap) + + return Alignment(True, "") diff --git a/backend/core/ouroboros/governance/shadow_graduation_gate.py b/backend/core/ouroboros/governance/shadow_graduation_gate.py new file mode 100644 index 0000000000..1294792aa3 --- /dev/null +++ b/backend/core/ouroboros/governance/shadow_graduation_gate.py @@ -0,0 +1,173 @@ +"""Event-driven graduation gate + graceful-degradation circuit breaker +(Unit C). + +Reads the telemetry store at each op boundary; once an agent has N +consecutive aligned ops it flips that agent's ``_AUTHORITATIVE`` flag +and persists it via the existing credential-safe ``persist_flag_to_env`` +writer. Promotion is idempotent and honors explicit operator settings. +""" +from __future__ import annotations + +import logging +import os +from dataclasses import dataclass +from typing import Any + +from backend.core.ouroboros.governance.graduation_orchestrator import ( + persist_flag_to_env, +) +# Intentional reuse of shadow_evaluator's cycle detector (private by +# convention but co-owned); keeps a single Kahn's implementation. +from backend.core.ouroboros.governance.shadow_evaluator import _has_cycle + +logger = logging.getLogger("Ouroboros.ShadowGraduationGate") + +_AUTH_FLAG = { + "plan": "JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", + "review": "JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE", +} +_SHADOW_FLAG = { + "plan": "JARVIS_PLAN_SUBAGENT_SHADOW", + "review": "JARVIS_REVIEW_SUBAGENT_SHADOW", +} + + +def gate_enabled() -> bool: + raw = os.environ.get("JARVIS_SHADOW_GRADUATION_GATE_ENABLED") + return raw is None or raw.strip().lower() in ("true", "1", "yes") + + +def _threshold() -> int: + try: + return max(1, int(os.environ.get( + "JARVIS_SHADOW_GRADUATION_THRESHOLD", "50"))) + except (TypeError, ValueError): + return 50 + + +def _is_authoritative(agent: str) -> bool: + return os.environ.get(_AUTH_FLAG[agent], "false").strip().lower() in ( + "true", "1", "yes") + + +class ShadowGraduationGate: + def __init__(self, *, store: Any) -> None: + self._store = store + + async def maybe_promote(self, agent: str) -> bool: + if not gate_enabled() or agent not in _AUTH_FLAG: + return False + if _is_authoritative(agent): + return False # idempotent — already graduated + try: + streak = await self._store.recent_aligned_streak(agent) + except Exception: # noqa: BLE001 — gate must not break the FSM + logger.warning( + "[ShadowGraduationGate] streak read failed (non-fatal)", + exc_info=True) + return False + if streak < _threshold(): + return False + return self._promote(agent, streak) + + def _promote(self, agent: str, streak: int) -> bool: + auth = _AUTH_FLAG[agent] + shadow = _SHADOW_FLAG[agent] + ok1 = persist_flag_to_env(auth, "true") + ok2 = persist_flag_to_env(shadow, "false") + if ok1: + os.environ[auth] = "true" + if ok2: + os.environ[shadow] = "false" + logger.info( + "[GRADUATION] agent=%s streak=%d -> authoritative " + "(auth_persist=%s shadow_persist=%s)", + agent, streak, ok1, ok2) + if ok1 and not ok2: + # Benign-but-sticky: auth is now persisted, so the next + # maybe_promote() short-circuits on _is_authoritative and never + # retries the shadow-flag write. The leftover SHADOW=true only + # keeps the (no-op) observer running alongside authoritative — no + # wrong behavior. Follow-up: a self-heal pass could re-attempt the + # shadow write when auth is already set but shadow is still on. + logger.warning( + "[GRADUATION] agent=%s auth persisted but shadow flag " + "persist failed — .env inconsistent until next promote", agent) + return bool(ok1) + + +# --------------------------------------------------------------------------- +# PlanBreaker — graceful-degradation circuit breaker (Unit C) +# --------------------------------------------------------------------------- + +@dataclass(frozen=True) +class BreakerDecision: + trip: bool + reason: str + pressure_level: str + + +def _default_pressure_fn() -> str: + try: + from backend.core.ouroboros.governance.memory_pressure_gate import ( + get_default_gate, + ) + return get_default_gate().pressure().value + except Exception: # noqa: BLE001 + return "ok" # probe failure -> assume OK (governor handles fan-out) + + +class PlanBreaker: + """Graceful-degradation breaker for the authoritative PLAN path. + + Trip order (first-match-wins): + 1. CRITICAL memory pressure -> pre-emptive, do NOT touch the DAG. + 2. Empty / unparsable DAG. + 3. Cyclical DAG. + A trip routes the operation to the retained legacy flat-plan + generator, guaranteeing execution continuity. + """ + + def __init__(self, *, pressure_fn=None) -> None: + self._pressure_fn = pressure_fn or _default_pressure_fn + + def should_use_legacy(self, *, dag) -> BreakerDecision: + level = "ok" + try: + level = (self._pressure_fn() or "ok").lower() + except Exception: # noqa: BLE001 + level = "ok" + if level == "critical": + return BreakerDecision(True, "critical_memory_pressure", level) + units = dag.get("units") if isinstance(dag, dict) else None + if not isinstance(units, list) or len(units) == 0: + return BreakerDecision(True, "unparsable_or_empty_dag", level) + try: + if _has_cycle(units): + return BreakerDecision(True, "cyclical_dag", level) + except Exception: # noqa: BLE001 + return BreakerDecision(True, "unparsable_or_empty_dag", level) + return BreakerDecision(False, "", level) + + +# --------------------------------------------------------------------------- +# Rail evaluator adapter +# --------------------------------------------------------------------------- + +def build_rail_evaluator(): + """Adapter: (agent, legacy, shadow) -> (aligned, reason), routing to + the right pure evaluator and unwrapping the stored shapes.""" + from backend.core.ouroboros.governance.shadow_evaluator import ( + evaluate_plan, evaluate_review, + ) + + def _ev(agent: str, legacy: dict, shadow: dict): + if agent == "review": + a = evaluate_review(legacy, shadow) + elif agent == "plan": + a = evaluate_plan(legacy.get("flat", []), shadow) + else: + return (False, "malformed:unknown_agent") + return (a.aligned, a.reason) + + return _ev diff --git a/backend/core/ouroboros/governance/shadow_telemetry_store.py b/backend/core/ouroboros/governance/shadow_telemetry_store.py new file mode 100644 index 0000000000..08d2b610ee --- /dev/null +++ b/backend/core/ouroboros/governance/shadow_telemetry_store.py @@ -0,0 +1,263 @@ +"""Async, bounded, fail-soft SQLite store for shadow-vs-legacy +comparison rows (Unit A). + +Never blocks the event loop (sqlite calls run in ``asyncio.to_thread``) +and never raises into the caller (observer contract: shadow telemetry +must not break the FSM). Producers are fire-and-forget. A two-phase +upsert keyed by ``(op_id, agent)`` joins the shadow verdict (known +early) with the legacy outcome (known later); alignment is computed by +an injected evaluator once both halves are present. A rolling per-agent +FIFO cap keeps the footprint microscopic. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import pathlib +import sqlite3 +from typing import Callable, Optional + +logger = logging.getLogger("Ouroboros.ShadowTelemetryStore") + +# evaluator signature: (agent, legacy_dict, shadow_dict) -> (aligned, reason) +EvaluatorFn = Callable[[str, dict, dict], "tuple[bool, str]"] + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS shadow_comparison ( + op_id TEXT NOT NULL, + agent TEXT NOT NULL, + ts REAL NOT NULL, + seq INTEGER NOT NULL, + legacy_outcome TEXT, + shadow_outcome TEXT, + aligned INTEGER, + divergence_reason TEXT, + PRIMARY KEY (op_id, agent) +); +CREATE INDEX IF NOT EXISTS idx_agent_seq ON shadow_comparison(agent, seq); +CREATE TABLE IF NOT EXISTS agent_seq (agent TEXT PRIMARY KEY, next INTEGER); +""" + + +def store_enabled() -> bool: + raw = os.environ.get("JARVIS_SHADOW_TELEMETRY_STORE_ENABLED") + return raw is None or raw.strip().lower() in ("true", "1", "yes") + + +def _queue_max() -> int: + try: + return max(8, int(os.environ.get( + "JARVIS_SHADOW_TELEMETRY_QUEUE_MAX", "256"))) + except (TypeError, ValueError): + return 256 + + +def _cap_per_agent() -> int: + try: + return max(50, int(os.environ.get( + "JARVIS_SHADOW_TELEMETRY_MAX_ROWS_PER_AGENT", "1000"))) + except (TypeError, ValueError): + return 1000 + + +class ShadowTelemetryStore: + def __init__( + self, + *, + db_path: Optional[pathlib.Path] = None, + evaluator: Optional[EvaluatorFn] = None, + cap_per_agent: Optional[int] = None, + ) -> None: + self._db_path = pathlib.Path( + db_path or pathlib.Path(".jarvis") / "shadow_telemetry.db" + ) + self._evaluator = evaluator + self._cap = cap_per_agent or _cap_per_agent() + self._queue: "asyncio.Queue[dict]" = asyncio.Queue(maxsize=_queue_max()) + self._task: Optional[asyncio.Task] = None + self._dropped = 0 + + # -- lifecycle ---------------------------------------------------- + async def start(self) -> None: + if self._task is not None: + return + self._db_path.parent.mkdir(parents=True, exist_ok=True) + await asyncio.to_thread(self._init_db) + self._task = asyncio.ensure_future(self._writer_loop()) + + async def aclose(self) -> None: + if self._task is None: + return + await self._queue.put({"_stop": True}) + try: + await asyncio.wait_for(self._task, timeout=5.0) + except Exception: # noqa: BLE001 + self._task.cancel() + self._task = None + + async def drain(self) -> None: + """Test helper — block until the queue is fully processed.""" + await self._queue.join() + + # -- producers (fire-and-forget, never raise) --------------------- + def record_shadow_nowait( + self, *, op_id: str, agent: str, ts: float, shadow_outcome: dict, + ) -> None: + self._enqueue({ + "op_id": op_id, "agent": agent, "ts": ts, + "shadow": shadow_outcome, + }) + + def record_legacy_nowait( + self, *, op_id: str, agent: str, ts: float, legacy_outcome: dict, + ) -> None: + self._enqueue({ + "op_id": op_id, "agent": agent, "ts": ts, + "legacy": legacy_outcome, + }) + + def _enqueue(self, item: dict) -> None: + try: + self._queue.put_nowait(item) + except asyncio.QueueFull: + # drop-oldest: discard one, retry once; bounded memory. + try: + _ = self._queue.get_nowait() + self._queue.task_done() + self._dropped += 1 + self._queue.put_nowait(item) + except Exception: # noqa: BLE001 + self._dropped += 1 + except Exception: # noqa: BLE001 + self._dropped += 1 + + # -- writer ------------------------------------------------------- + async def _writer_loop(self) -> None: + while True: + item = await self._queue.get() + try: + if item.get("_stop"): + self._queue.task_done() + return + await asyncio.to_thread(self._apply, item) + except Exception: # noqa: BLE001 + logger.warning( + "[ShadowTelemetryStore] write failed (non-fatal)", + exc_info=True, + ) + finally: + if not item.get("_stop"): + self._queue.task_done() + + # -- blocking sqlite (runs in to_thread) -------------------------- + def _init_db(self) -> None: + con = sqlite3.connect(self._db_path) + try: + con.executescript(_SCHEMA) + con.commit() + finally: + con.close() + + def _next_seq(self, con: sqlite3.Connection, agent: str) -> int: + cur = con.execute( + "SELECT next FROM agent_seq WHERE agent = ?", (agent,)) + row = cur.fetchone() + nxt = (row[0] if row else 0) + 1 + con.execute( + "INSERT INTO agent_seq(agent, next) VALUES(?, ?) " + "ON CONFLICT(agent) DO UPDATE SET next = excluded.next", + (agent, nxt)) + return nxt + + def _apply(self, item: dict) -> None: + agent = item["agent"] + op_id = item["op_id"] + con = sqlite3.connect(self._db_path) + try: + cur = con.execute( + "SELECT legacy_outcome, shadow_outcome, seq " + "FROM shadow_comparison WHERE op_id = ? AND agent = ?", + (op_id, agent)) + existing = cur.fetchone() + if existing is None: + seq = self._next_seq(con, agent) + legacy = json.dumps(item["legacy"]) if "legacy" in item else None + shadow = json.dumps(item["shadow"]) if "shadow" in item else None + con.execute( + "INSERT INTO shadow_comparison" + "(op_id, agent, ts, seq, legacy_outcome, shadow_outcome," + " aligned, divergence_reason) VALUES(?,?,?,?,?,?,?,?)", + (op_id, agent, item["ts"], seq, legacy, shadow, None, None)) + else: + legacy = existing[0] + shadow = existing[1] + if "legacy" in item: + legacy = json.dumps(item["legacy"]) + if "shadow" in item: + shadow = json.dumps(item["shadow"]) + con.execute( + "UPDATE shadow_comparison SET legacy_outcome = ?, " + "shadow_outcome = ? WHERE op_id = ? AND agent = ?", + (legacy, shadow, op_id, agent)) + + # Compute alignment once both halves present + evaluator wired. + if legacy is not None and shadow is not None and self._evaluator: + aligned, reason = self._evaluator( + agent, json.loads(legacy), json.loads(shadow)) + con.execute( + "UPDATE shadow_comparison SET aligned = ?, " + "divergence_reason = ? WHERE op_id = ? AND agent = ?", + (1 if aligned else 0, reason or None, op_id, agent)) + + self._prune(con, agent) + con.commit() + finally: + con.close() + + def _prune(self, con: sqlite3.Connection, agent: str) -> None: + con.execute( + "DELETE FROM shadow_comparison WHERE agent = ? AND seq <= " + "((SELECT MAX(seq) FROM shadow_comparison WHERE agent = ?) - ?)", + (agent, agent, self._cap)) + + # -- read side ---------------------------------------------------- + async def last_n(self, agent: str, n: int) -> list: + return await asyncio.to_thread(self._last_n, agent, n) + + def _last_n(self, agent: str, n: int) -> list: + con = sqlite3.connect(self._db_path) + try: + cur = con.execute( + "SELECT op_id, seq, aligned, divergence_reason " + "FROM shadow_comparison WHERE agent = ? " + "ORDER BY seq DESC LIMIT ?", (agent, n)) + return [ + {"op_id": r[0], "seq": r[1], "aligned": r[2], + "divergence_reason": r[3]} + for r in cur.fetchall() + ] + finally: + con.close() + + async def recent_aligned_streak(self, agent: str) -> int: + return await asyncio.to_thread(self._recent_aligned_streak, agent) + + def _recent_aligned_streak(self, agent: str) -> int: + con = sqlite3.connect(self._db_path) + try: + cur = con.execute( + "SELECT aligned FROM shadow_comparison WHERE agent = ? " + "ORDER BY seq DESC", (agent,)) + streak = 0 + for (aligned,) in cur.fetchall(): + if aligned is None: + continue # incomplete row: skip, don't break + if aligned == 1: + streak += 1 + else: + break # a single divergence resets the streak + return streak + finally: + con.close() diff --git a/docs/superpowers/plans/2026-06-14-l3-memory-governor.md b/docs/superpowers/plans/2026-06-14-l3-memory-governor.md new file mode 100644 index 0000000000..23b98fd645 --- /dev/null +++ b/docs/superpowers/plans/2026-06-14-l3-memory-governor.md @@ -0,0 +1,485 @@ +# L3 Memory Governor Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a worktree-RAM-budget concurrency clamp to the L3 `SubagentScheduler` so parallel worktree fan-out can never OOM/swap-thrash a 16GB unified-memory host. + +**Architecture:** A small pure-function module (`l3_memory_governor.py`) computes `max_worktrees` from live available RAM (sourced from the existing `MemoryPressureGate` probe — zero duplication) divided by a per-worktree RAM budget, clamped by the gate's existing free-%-based fan-out cap (strictest-wins). The scheduler composes this clamp *on top of* the Slice 5 Arc B fan-out gate already in `_run_graph`, reusing the same zero-work-loss defer-overflow mechanism. + +**Tech Stack:** Python 3.9+ (`from __future__ import annotations`), `asyncio`, stdlib `math`/`os`, existing `MemoryPressureGate`, pytest. + +--- + +## File Structure + +- **Create:** `backend/core/ouroboros/governance/autonomy/l3_memory_governor.py` — pure governor math + env knobs + `GovernorDecision`. No IO, no scheduler import. One responsibility: "given requested N, available MB, budget MB, and the level cap, how many worktrees may run?" +- **Modify:** `backend/core/ouroboros/governance/autonomy/subagent_scheduler.py` — add `_consult_memory_governor(...)` and compose its clamp after the existing fan-out clamp in `_run_graph`. +- **Test:** `tests/governance/autonomy/test_l3_memory_governor.py` (pure math) and `tests/governance/autonomy/test_scheduler_memory_governor.py` (scheduler integration with injected gate). + +Why a separate module: the scheduler is large and the math must be testable without booting a graph. The clamp logic is the load-bearing safety code; isolating it lets us prove every pressure level deterministically. + +--- + +## Task 1: Governor math module (pure) + +**Files:** +- Create: `backend/core/ouroboros/governance/autonomy/l3_memory_governor.py` +- Test: `tests/governance/autonomy/test_l3_memory_governor.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/governance/autonomy/test_l3_memory_governor.py +from __future__ import annotations + +from backend.core.ouroboros.governance.autonomy.l3_memory_governor import ( + GovernorDecision, + compute_worktree_cap, +) + + +def test_ram_is_the_binding_constraint(): + # 4500MB available, 1500MB/worktree -> ram_cap=3; level_cap=8 -> allow 3 + d = compute_worktree_cap( + requested=8, avail_mb=4500.0, budget_mb=1500, level_cap=8, + ) + assert isinstance(d, GovernorDecision) + assert d.ram_cap == 3 + assert d.n_allowed == 3 + assert d.disposition == "clamp" + + +def test_level_cap_is_the_binding_constraint(): + # 12000MB -> ram_cap=8; but level_cap=3 (HIGH) -> allow 3, strictest wins + d = compute_worktree_cap( + requested=8, avail_mb=12000.0, budget_mb=1500, level_cap=3, + ) + assert d.ram_cap == 8 + assert d.n_allowed == 3 + assert d.disposition == "clamp" + + +def test_floor_never_below_one(): + # Only 800MB available, 1500MB budget -> floor would be 0; clamp to >=1 + d = compute_worktree_cap( + requested=4, avail_mb=800.0, budget_mb=1500, level_cap=8, + ) + assert d.ram_cap == 1 + assert d.n_allowed == 1 + + +def test_no_clamp_when_everything_fits(): + d = compute_worktree_cap( + requested=2, avail_mb=16000.0, budget_mb=1500, level_cap=8, + ) + assert d.n_allowed == 2 + assert d.disposition == "allow" +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/autonomy/test_l3_memory_governor.py -v` +Expected: FAIL — `ModuleNotFoundError: ... l3_memory_governor`. + +- [ ] **Step 3: Write minimal implementation** + +```python +# backend/core/ouroboros/governance/autonomy/l3_memory_governor.py +"""L3 worktree-RAM-budget governor (pure math). + +Composes ON TOP of MemoryPressureGate's free-%-based fan-out caps: +the gate answers "is the box under pressure?"; this module answers +"given the absolute RAM cost of a worktree, how many fit right now?". +Strictest-wins between the two. No IO, no scheduler import — every +decision is a deterministic function of its arguments so it can be +proven at all pressure levels in isolation. +""" +from __future__ import annotations + +import math +import os +from dataclasses import dataclass + + +def _env_bool(name: str, default: bool) -> bool: + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() in ("true", "1", "yes") + + +def _env_int(name: str, default: int, *, minimum: int) -> int: + raw = os.environ.get(name) + if raw is None: + return default + try: + return max(minimum, int(raw)) + except (TypeError, ValueError): + return default + + +def governor_enabled() -> bool: + """Master flag. Default TRUE; inert until an L3 graph actually runs.""" + return _env_bool("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", True) + + +def worktree_ram_budget_mb() -> int: + """Assumed peak RAM per concurrent worktree. Default 1500MB.""" + return _env_int("JARVIS_L3_WORKTREE_RAM_BUDGET_MB", 1500, minimum=64) + + +@dataclass(frozen=True) +class GovernorDecision: + requested: int + ram_cap: int + level_cap: int + n_allowed: int + avail_mb: float + budget_mb: int + disposition: str # "allow" | "clamp" | "disabled" | "probe_fail" + + +def compute_worktree_cap( + *, + requested: int, + avail_mb: float, + budget_mb: int, + level_cap: int, +) -> GovernorDecision: + """Pure clamp. ``ram_cap = floor(avail_mb / budget_mb)`` (>=1); + final allowance is the strictest of requested / ram_cap / level_cap.""" + ram_cap = max(1, int(math.floor(avail_mb / float(budget_mb)))) + n_allowed = max(0, min(requested, ram_cap, level_cap)) + disposition = "clamp" if n_allowed < requested else "allow" + return GovernorDecision( + requested=requested, + ram_cap=ram_cap, + level_cap=level_cap, + n_allowed=n_allowed, + avail_mb=avail_mb, + budget_mb=budget_mb, + disposition=disposition, + ) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/autonomy/test_l3_memory_governor.py -v` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/autonomy/l3_memory_governor.py tests/governance/autonomy/test_l3_memory_governor.py +git commit -m "feat(l3): pure worktree-RAM-budget governor math (Unit D)" +``` + +--- + +## Task 2: Scheduler consultation method + +**Files:** +- Modify: `backend/core/ouroboros/governance/autonomy/subagent_scheduler.py` (add method near `_consult_memory_gate`, ~line 756) +- Test: `tests/governance/autonomy/test_scheduler_memory_governor.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/governance/autonomy/test_scheduler_memory_governor.py +from __future__ import annotations + +import types + +import pytest + +from backend.core.ouroboros.governance.autonomy import subagent_scheduler as ss + + +class _FakeProbe: + def __init__(self, available_mb: float): + self.available_bytes = int(available_mb * 1024 * 1024) + self.total_bytes = 16 * 1024 * 1024 * 1024 + self.ok = True + + +class _FakeGate: + def __init__(self, available_mb: float): + self._p = _FakeProbe(available_mb) + + def probe(self): + return self._p + + +def _make_scheduler(): + # Construct with minimal stubs; only _consult_memory_governor is exercised. + return ss.SubagentScheduler.__new__(ss.SubagentScheduler) + + +def test_governor_clamps_on_low_ram(monkeypatch): + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + monkeypatch.setenv("JARVIS_L3_WORKTREE_RAM_BUDGET_MB", "1500") + monkeypatch.setattr( + ss, "get_default_gate", lambda: _FakeGate(available_mb=4500.0), + ) + sched = _make_scheduler() + decision = sched._consult_memory_governor( + 8, graph_id="g1", level_cap=8, + ) + assert decision is not None + assert decision.n_allowed == 3 + assert decision.disposition == "clamp" + + +def test_governor_disabled_returns_none(monkeypatch): + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "false") + sched = _make_scheduler() + assert sched._consult_memory_governor(8, graph_id="g1", level_cap=8) is None + + +def test_governor_probe_failure_is_non_fatal(monkeypatch): + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + + def _boom(): + raise RuntimeError("probe exploded") + + monkeypatch.setattr(ss, "get_default_gate", _boom) + sched = _make_scheduler() + # Must swallow and return None — scheduler never breaks on probe failure. + assert sched._consult_memory_governor(8, graph_id="g1", level_cap=8) is None +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/autonomy/test_scheduler_memory_governor.py -v` +Expected: FAIL — `AttributeError: ... has no attribute '_consult_memory_governor'` (and possibly `get_default_gate` not importable at module scope). + +- [ ] **Step 3a: Ensure `get_default_gate` is importable at module scope** + +In `subagent_scheduler.py`, the existing `_consult_memory_gate` imports `get_default_gate` *inside* the method. Add a module-level import so tests can `monkeypatch.setattr(ss, "get_default_gate", ...)`. Near the top imports (after the existing imports block), add: + +```python +from backend.core.ouroboros.governance.memory_pressure_gate import ( + get_default_gate, +) +``` + +(Leave the in-method import in `_consult_memory_gate` as-is to avoid disturbing the AST-pinned Slice 5 Arc B path.) + +- [ ] **Step 3b: Add the consultation method** + +Insert immediately after `_consult_memory_gate` (after line 756): + +```python + def _consult_memory_governor( + self, + n_requested: int, + *, + graph_id: str, + level_cap: int, + ) -> Optional[Any]: + """Unit D — worktree-RAM-budget clamp composed on top of the + Slice 5 Arc B fan-out gate. + + Returns a ``GovernorDecision`` (or ``None`` when disabled / on + any probe failure — the scheduler must never break on the + governor). ``level_cap`` is the allowance already granted by + the free-%-based fan-out gate; the governor takes the strictest + of that and the absolute RAM budget. + """ + from backend.core.ouroboros.governance.autonomy.l3_memory_governor import ( + GovernorDecision, + compute_worktree_cap, + governor_enabled, + worktree_ram_budget_mb, + ) + + if not governor_enabled(): + return None + try: + gate = get_default_gate() + probe = gate.probe() + avail_mb = float(probe.available_bytes) / (1024.0 * 1024.0) + except Exception: # noqa: BLE001 — governor must not break scheduler + logger.debug( + "[SubagentScheduler] memory governor probe failed " + "(non-fatal)", exc_info=True, + ) + return None + + decision = compute_worktree_cap( + requested=n_requested, + avail_mb=avail_mb, + budget_mb=worktree_ram_budget_mb(), + level_cap=level_cap, + ) + log_fn = ( + logger.warning if decision.disposition == "clamp" else logger.info + ) + log_fn( + "[SubagentScheduler] ram_governor: graph=%s disposition=%s " + "requested=%d allowed=%d ram_cap=%d level_cap=%d avail_mb=%.0f " + "budget_mb=%d", + graph_id, decision.disposition, decision.requested, + decision.n_allowed, decision.ram_cap, decision.level_cap, + decision.avail_mb, decision.budget_mb, + ) + return decision +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/autonomy/test_scheduler_memory_governor.py -v` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/autonomy/subagent_scheduler.py tests/governance/autonomy/test_scheduler_memory_governor.py +git commit -m "feat(l3): scheduler memory-governor consultation (Unit D)" +``` + +--- + +## Task 3: Wire the governor clamp into `_run_graph` + +**Files:** +- Modify: `backend/core/ouroboros/governance/autonomy/subagent_scheduler.py:500-507` +- Test: `tests/governance/autonomy/test_scheduler_memory_governor.py` (add an integration-style test) + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/governance/autonomy/test_scheduler_memory_governor.py + +def test_run_graph_clamp_composition(monkeypatch): + """The governor clamp composes after the fan-out clamp: selected is + truncated to the governor's n_allowed and overflow is deferred.""" + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + monkeypatch.setenv("JARVIS_L3_WORKTREE_RAM_BUDGET_MB", "1500") + monkeypatch.setattr( + ss, "get_default_gate", lambda: _FakeGate(available_mb=3000.0), + ) + sched = _make_scheduler() + selected = ["u1", "u2", "u3", "u4"] + deferred = [] + gov = sched._consult_memory_governor( + len(selected), graph_id="g1", level_cap=len(selected), + ) + assert gov.n_allowed == 2 # 3000/1500 = 2 + # Simulate the composition the _run_graph edit performs: + overflow = list(selected[gov.n_allowed:]) + selected = list(selected[:gov.n_allowed]) + deferred = sorted(deferred + overflow) + assert selected == ["u1", "u2"] + assert deferred == ["u3", "u4"] +``` + +- [ ] **Step 2: Run test to verify it fails (it passes the helper math but proves the contract the edit must honor)** + +Run: `pytest tests/governance/autonomy/test_scheduler_memory_governor.py::test_run_graph_clamp_composition -v` +Expected: PASS (this codifies the exact composition the next step wires into `_run_graph`). + +- [ ] **Step 3: Edit `_run_graph` to compose the governor clamp** + +Replace the existing block at lines 500-507: + +```python + if selected: + decision = self._consult_memory_gate( + len(selected), graph_id=graph_id, + ) + if decision is not None and decision.n_allowed < len(selected): + overflow = list(selected[decision.n_allowed:]) + selected = list(selected[:decision.n_allowed]) + deferred = sorted(list(deferred) + overflow) +``` + +with: + +```python + if selected: + decision = self._consult_memory_gate( + len(selected), graph_id=graph_id, + ) + if decision is not None and decision.n_allowed < len(selected): + overflow = list(selected[decision.n_allowed:]) + selected = list(selected[:decision.n_allowed]) + deferred = sorted(list(deferred) + overflow) + + # Unit D — worktree-RAM-budget governor composes on top of + # the fan-out gate (strictest-wins). Same zero-work-loss + # defer-overflow mechanism; disabled/probe-fail -> no clamp. + if selected: + level_cap = ( + decision.n_allowed + if decision is not None + else len(selected) + ) + gov = self._consult_memory_governor( + len(selected), graph_id=graph_id, level_cap=level_cap, + ) + if gov is not None and gov.n_allowed < len(selected): + overflow = list(selected[gov.n_allowed:]) + selected = list(selected[:gov.n_allowed]) + deferred = sorted(list(deferred) + overflow) +``` + +- [ ] **Step 4: Run the full scheduler suite to confirm no regression** + +Run: `pytest tests/governance/autonomy/ -v` +Expected: PASS (existing scheduler tests + new governor tests). Confirm the existing `test_subagent_executor_worktree.py` and any `_run_graph` tests still pass — the governor is additive and disabled→no-op. + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/autonomy/subagent_scheduler.py tests/governance/autonomy/test_scheduler_memory_governor.py +git commit -m "feat(l3): compose RAM-governor clamp in _run_graph fan-out (Unit D)" +``` + +--- + +## Task 4: OFF-is-inert regression guard + +**Files:** +- Test: `tests/governance/autonomy/test_scheduler_memory_governor.py` (add) + +- [ ] **Step 1: Write the test** + +```python +# append to tests/governance/autonomy/test_scheduler_memory_governor.py + +def test_disabled_governor_is_byte_identical_passthrough(monkeypatch): + """With the master flag off, _consult_memory_governor returns None + and the _run_graph composition leaves `selected` untouched.""" + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "false") + sched = _make_scheduler() + selected = ["u1", "u2", "u3"] + gov = sched._consult_memory_governor( + len(selected), graph_id="g1", level_cap=len(selected), + ) + assert gov is None + # Composition guard: None -> no truncation. + if gov is not None and gov.n_allowed < len(selected): + selected = selected[:gov.n_allowed] + assert selected == ["u1", "u2", "u3"] +``` + +- [ ] **Step 2: Run it** + +Run: `pytest tests/governance/autonomy/test_scheduler_memory_governor.py::test_disabled_governor_is_byte_identical_passthrough -v` +Expected: PASS. + +- [ ] **Step 3: Commit** + +```bash +git add tests/governance/autonomy/test_scheduler_memory_governor.py +git commit -m "test(l3): governor-off is byte-identical passthrough (Unit D)" +``` + +--- + +## Self-Review Notes + +- **Spec coverage:** §7.2 worktree-RAM-budget clamp (Tasks 1-3); §7 env knobs `JARVIS_L3_MEMORY_GOVERNOR_ENABLED` / `JARVIS_L3_WORKTREE_RAM_BUDGET_MB` (Task 1); reuse of `MemoryPressureGate` not a new probe (Task 2 uses `get_default_gate().probe()`); OFF-is-inert (Task 4). The §7.3 CRITICAL→legacy pre-emptive trip is intentionally **not** here — it lives in the Plan 2 / Unit C circuit breaker, since the PLAN-vs-legacy decision happens in the orchestrator before the scheduler runs. This plan's governor handles already-admitted graphs only. +- **Type consistency:** `GovernorDecision` fields (`requested`, `ram_cap`, `level_cap`, `n_allowed`, `avail_mb`, `budget_mb`, `disposition`) are used identically in module, method, and tests. `compute_worktree_cap` keyword args match every call site. +- **No placeholders:** every step ships real code and a runnable command. diff --git a/docs/superpowers/plans/2026-06-14-sovereign-evidence-rail.md b/docs/superpowers/plans/2026-06-14-sovereign-evidence-rail.md new file mode 100644 index 0000000000..825ed3b90f --- /dev/null +++ b/docs/superpowers/plans/2026-06-14-sovereign-evidence-rail.md @@ -0,0 +1,1568 @@ +# Sovereign Evidence Rail Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build the durable, async, evidence-driven rail that records REVIEW/PLAN shadow-vs-legacy comparisons, evaluates their alignment deterministically, and autonomously graduates each subagent to authoritative status after a 50-op clean soak — with a graceful-degradation circuit breaker back to the retained legacy paths. + +**Architecture:** Three decoupled units. **A** = async SQLite store (`shadow_telemetry_store.py`) with a bounded write-queue + `to_thread` writer (never blocks the loop), a two-phase upsert keyed by `op_id`, and a rolling 1,000-row/agent FIFO prune. **B** = pure deterministic evaluator (`shadow_evaluator.py`) — REVIEW binary block-vs-allow agreement; PLAN refinement check (coverage ∧ acyclic ∧ disjoint). **C** = event-driven graduation gate + circuit breaker (`shadow_graduation_gate.py`) that reads the store at each op boundary, promotes via the existing `persist_flag_to_env`, and trips to legacy on cyclical/unparsable DAG, timeout, or CRITICAL memory pressure (emitting `AGENT_DEGRADATION` SSE). A is injected with B via a callable so it stays testable in isolation. + +**Tech Stack:** Python 3.9+ (`from __future__ import annotations`), `asyncio`, stdlib `sqlite3` via `asyncio.to_thread`, existing `graduation_orchestrator.persist_flag_to_env`, `MemoryPressureGate`, `StreamEventBroker`, pytest. + +--- + +## File Structure + +- **Create** `backend/core/ouroboros/governance/shadow_telemetry_store.py` — Unit A. Owns the DB, the async writer task, the FIFO prune, the two-phase upsert, and the read-side streak query. One responsibility: durable, bounded, non-blocking persistence of comparison rows. +- **Create** `backend/core/ouroboros/governance/shadow_evaluator.py` — Unit B. Pure functions only; no IO, no imports of A or the orchestrator. One responsibility: decide `aligned` + `reason` from a legacy/shadow pair. +- **Create** `backend/core/ouroboros/governance/shadow_graduation_gate.py` — Unit C. The streak gate, the promotion (delegates persistence to `graduation_orchestrator`), and the circuit breaker (delegates SSE to `StreamEventBroker`). One responsibility: turn accumulated evidence into a promotion/trip decision. +- **Modify** `backend/core/ouroboros/governance/ide_observability_stream.py` — add the `EVENT_TYPE_AGENT_DEGRADATION` constant + a thin publish helper. +- **Modify** `backend/core/ouroboros/governance/orchestrator.py` — wire `record_*_nowait` into `_run_review_shadow` / `_run_plan_shadow` and the legacy-outcome capture point; gate the authoritative wiring behind the new flags. +- **Tests** under `tests/governance/`: `test_shadow_telemetry_store.py`, `test_shadow_evaluator.py`, `test_shadow_graduation_gate.py`, `test_shadow_rail_off_inert.py`. + +--- + +## Task 1: Evaluator — REVIEW (Unit B, pure) + +**Files:** +- Create: `backend/core/ouroboros/governance/shadow_evaluator.py` +- Test: `tests/governance/test_shadow_evaluator.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/governance/test_shadow_evaluator.py +from __future__ import annotations + +from backend.core.ouroboros.governance.shadow_evaluator import ( + Alignment, + evaluate_review, +) + + +def test_review_agree_allow(): + legacy = {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": False} + shadow = {"aggregate": "approve"} + a = evaluate_review(legacy, shadow) + assert isinstance(a, Alignment) + assert a.aligned is True + + +def test_review_reservations_map_to_allow(): + legacy = {"risk_tier": "NOTIFY_APPLY", "semantic_guard_hard": False} + shadow = {"aggregate": "approve_with_reservations"} + assert evaluate_review(legacy, shadow).aligned is True + + +def test_review_disagree_shadow_blocks_legacy_allows(): + legacy = {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": False} + shadow = {"aggregate": "reject"} + a = evaluate_review(legacy, shadow) + assert a.aligned is False + assert a.reason == "shadow=BLOCK legacy=ALLOW" + + +def test_review_agree_block_via_hard_finding(): + legacy = {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": True} + shadow = {"aggregate": "reject"} + assert evaluate_review(legacy, shadow).aligned is True + + +def test_review_agree_block_via_approval_required(): + legacy = {"risk_tier": "APPROVAL_REQUIRED", "semantic_guard_hard": False} + shadow = {"aggregate": "reject"} + assert evaluate_review(legacy, shadow).aligned is True + + +def test_review_malformed_is_conservative_block(): + a = evaluate_review({}, {}) + assert a.aligned is False + assert a.reason.startswith("malformed:") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_evaluator.py -v` +Expected: FAIL — `ModuleNotFoundError: ... shadow_evaluator`. + +- [ ] **Step 3: Write minimal implementation** + +```python +# backend/core/ouroboros/governance/shadow_evaluator.py +"""Deterministic shadow-vs-legacy alignment evaluator (Unit B). + +Pure functions: no IO, no LLM, no imports of the store or orchestrator. +Every function returns a structured ``Alignment`` even on malformed +input — malformed maps to ``aligned=False`` (the conservative default +that BLOCKS graduation rather than risking a false promotion). +""" +from __future__ import annotations + +from dataclasses import dataclass + +_BLOCK = "BLOCK" +_ALLOW = "ALLOW" +_BLOCKING_TIERS = frozenset({"APPROVAL_REQUIRED", "BLOCKED"}) + + +@dataclass(frozen=True) +class Alignment: + aligned: bool + reason: str # "" when aligned; divergence/malformed detail otherwise + + +def _legacy_review_binary(legacy: dict) -> str: + tier = str(legacy.get("risk_tier", "")).upper() + hard = bool(legacy.get("semantic_guard_hard", False)) + if hard or tier in _BLOCKING_TIERS: + return _BLOCK + return _ALLOW + + +def _shadow_review_binary(shadow: dict) -> str: + agg = str(shadow.get("aggregate", "")).lower() + # reservations are advisory -> ALLOW; only outright reject BLOCKs. + return _BLOCK if agg == "reject" else _ALLOW + + +def evaluate_review(legacy: dict, shadow: dict) -> Alignment: + if not isinstance(legacy, dict) or not isinstance(shadow, dict): + return Alignment(False, "malformed:non_dict_input") + if "risk_tier" not in legacy or "aggregate" not in shadow: + return Alignment(False, "malformed:missing_keys") + lb = _legacy_review_binary(legacy) + sb = _shadow_review_binary(shadow) + if lb == sb: + return Alignment(True, "") + return Alignment(False, f"shadow={sb} legacy={lb}") +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/test_shadow_evaluator.py -v` +Expected: PASS (6 tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/shadow_evaluator.py tests/governance/test_shadow_evaluator.py +git commit -m "feat(rail): REVIEW shadow-vs-legacy evaluator (Unit B)" +``` + +--- + +## Task 2: Evaluator — PLAN refinement (Unit B, pure) + +**Files:** +- Modify: `backend/core/ouroboros/governance/shadow_evaluator.py` +- Test: `tests/governance/test_shadow_evaluator.py` (add) + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/governance/test_shadow_evaluator.py +from backend.core.ouroboros.governance.shadow_evaluator import evaluate_plan + + +# DAG shape: {"units": [{"id","owned_paths":[...],"deps":[...]}], } +def test_plan_valid_refinement_aligned(): + legacy = ["a.py", "b.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}, + ]} + assert evaluate_plan(legacy, dag).aligned is True + + +def test_plan_dropped_task_misaligned(): + legacy = ["a.py", "b.py", "c.py"] + dag = {"units": [{"id": "u1", "owned_paths": ["a.py", "b.py"], "deps": []}]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason.startswith("dropped_tasks:") + assert "c.py" in a.reason + + +def test_plan_cyclical_misaligned(): + legacy = ["a.py", "b.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u2"]}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}, + ]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason == "cyclical_dag" + + +def test_plan_owned_path_overlap_misaligned(): + legacy = ["a.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + {"id": "u2", "owned_paths": ["a.py"], "deps": []}, + ]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason.startswith("owned_path_overlap:") + + +def test_plan_extra_structure_is_allowed(): + # DAG covers legacy AND adds a helper file -> still aligned (refinement). + legacy = ["a.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + {"id": "u2", "owned_paths": ["helper.py"], "deps": ["u1"]}, + ]} + assert evaluate_plan(legacy, dag).aligned is True + + +def test_plan_malformed_is_conservative_block(): + a = evaluate_plan(["a.py"], {"units": "not-a-list"}) + assert a.aligned is False + assert a.reason.startswith("malformed:") +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_evaluator.py -k plan -v` +Expected: FAIL — `ImportError: cannot import name 'evaluate_plan'`. + +- [ ] **Step 3: Write minimal implementation** + +Append to `shadow_evaluator.py`: + +```python +def _has_cycle(units: list) -> bool: + """Kahn's algorithm — True if any cycle remains.""" + ids = {u["id"] for u in units} + indeg = {u["id"]: 0 for u in units} + adj: dict = {u["id"]: [] for u in units} + for u in units: + for dep in u.get("deps", []): + if dep in ids: + adj[dep].append(u["id"]) + indeg[u["id"]] += 1 + queue = [i for i, d in indeg.items() if d == 0] + visited = 0 + while queue: + n = queue.pop() + visited += 1 + for m in adj[n]: + indeg[m] -= 1 + if indeg[m] == 0: + queue.append(m) + return visited != len(units) + + +def _owned_path_overlap(units: list) -> str: + seen: dict = {} + for u in units: + for p in u.get("owned_paths", []): + if p in seen and seen[p] != u["id"]: + return p + seen[p] = u["id"] + return "" + + +def evaluate_plan(legacy_flat: list, shadow_dag: dict) -> Alignment: + if not isinstance(legacy_flat, list) or not isinstance(shadow_dag, dict): + return Alignment(False, "malformed:non_collection_input") + units = shadow_dag.get("units") + if not isinstance(units, list) or not all( + isinstance(u, dict) and "id" in u for u in units + ): + return Alignment(False, "malformed:bad_units") + + # 1. Coverage — DAG must touch every legacy task (extra is OK). + dag_paths = set() + for u in units: + dag_paths.update(u.get("owned_paths", [])) + dropped = [t for t in legacy_flat if t not in dag_paths] + if dropped: + return Alignment(False, "dropped_tasks:" + ",".join(sorted(dropped))) + + # 2. Acyclicity. + if _has_cycle(units): + return Alignment(False, "cyclical_dag") + + # 3. Disjoint ownership. + overlap = _owned_path_overlap(units) + if overlap: + return Alignment(False, "owned_path_overlap:" + overlap) + + return Alignment(True, "") +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/test_shadow_evaluator.py -v` +Expected: PASS (all REVIEW + PLAN tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/shadow_evaluator.py tests/governance/test_shadow_evaluator.py +git commit -m "feat(rail): PLAN refinement evaluator — coverage/acyclic/disjoint (Unit B)" +``` + +--- + +## Task 3: Telemetry store — schema + writer lifecycle (Unit A) + +**Files:** +- Create: `backend/core/ouroboros/governance/shadow_telemetry_store.py` +- Test: `tests/governance/test_shadow_telemetry_store.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/governance/test_shadow_telemetry_store.py +from __future__ import annotations + +import asyncio + +import pytest + +from backend.core.ouroboros.governance.shadow_telemetry_store import ( + ShadowTelemetryStore, +) + + +@pytest.mark.asyncio +async def test_start_and_close_idempotent(tmp_path): + store = ShadowTelemetryStore(db_path=tmp_path / "t.db") + await store.start() + await store.start() # idempotent + await store.aclose() + await store.aclose() # idempotent + + +@pytest.mark.asyncio +async def test_plan_single_phase_write_computes_alignment(tmp_path): + aligned_calls = [] + + def fake_eval(agent, legacy, shadow): + aligned_calls.append(agent) + return (True, "") + + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", evaluator=fake_eval, + ) + await store.start() + store.record_legacy_nowait( + op_id="op1", agent="plan", ts=1.0, legacy_outcome={"flat": ["a.py"]}, + ) + store.record_shadow_nowait( + op_id="op1", agent="plan", ts=1.0, shadow_outcome={"units": []}, + ) + await store.drain() # test helper: await the queue empty + rows = await store.last_n("plan", 5) + assert len(rows) == 1 + assert rows[0]["aligned"] == 1 + assert aligned_calls == ["plan"] + await store.aclose() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_telemetry_store.py -v` +Expected: FAIL — `ModuleNotFoundError: ... shadow_telemetry_store`. + +- [ ] **Step 3: Write minimal implementation** + +```python +# backend/core/ouroboros/governance/shadow_telemetry_store.py +"""Async, bounded, fail-soft SQLite store for shadow-vs-legacy +comparison rows (Unit A). + +Never blocks the event loop (sqlite calls run in ``asyncio.to_thread``) +and never raises into the caller (observer contract: shadow telemetry +must not break the FSM). Producers are fire-and-forget. A two-phase +upsert keyed by ``(op_id, agent)`` joins the shadow verdict (known +early) with the legacy outcome (known later); alignment is computed by +an injected evaluator once both halves are present. A rolling per-agent +FIFO cap keeps the footprint microscopic. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import pathlib +import sqlite3 +from typing import Callable, Optional + +logger = logging.getLogger("Ouroboros.ShadowTelemetryStore") + +# evaluator signature: (agent, legacy_dict, shadow_dict) -> (aligned, reason) +EvaluatorFn = Callable[[str, dict, dict], "tuple[bool, str]"] + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS shadow_comparison ( + op_id TEXT NOT NULL, + agent TEXT NOT NULL, + ts REAL NOT NULL, + seq INTEGER NOT NULL, + legacy_outcome TEXT, + shadow_outcome TEXT, + aligned INTEGER, + divergence_reason TEXT, + PRIMARY KEY (op_id, agent) +); +CREATE INDEX IF NOT EXISTS idx_agent_seq ON shadow_comparison(agent, seq); +CREATE TABLE IF NOT EXISTS agent_seq (agent TEXT PRIMARY KEY, next INTEGER); +""" + + +def store_enabled() -> bool: + raw = os.environ.get("JARVIS_SHADOW_TELEMETRY_STORE_ENABLED") + return raw is None or raw.strip().lower() in ("true", "1", "yes") + + +def _queue_max() -> int: + try: + return max(8, int(os.environ.get( + "JARVIS_SHADOW_TELEMETRY_QUEUE_MAX", "256"))) + except (TypeError, ValueError): + return 256 + + +def _cap_per_agent() -> int: + try: + return max(50, int(os.environ.get( + "JARVIS_SHADOW_TELEMETRY_MAX_ROWS_PER_AGENT", "1000"))) + except (TypeError, ValueError): + return 1000 + + +class ShadowTelemetryStore: + def __init__( + self, + *, + db_path: Optional[pathlib.Path] = None, + evaluator: Optional[EvaluatorFn] = None, + cap_per_agent: Optional[int] = None, + ) -> None: + self._db_path = pathlib.Path( + db_path or pathlib.Path(".jarvis") / "shadow_telemetry.db" + ) + self._evaluator = evaluator + self._cap = cap_per_agent or _cap_per_agent() + self._queue: "asyncio.Queue[dict]" = asyncio.Queue(maxsize=_queue_max()) + self._task: Optional[asyncio.Task] = None + self._dropped = 0 + + # -- lifecycle ---------------------------------------------------- + async def start(self) -> None: + if self._task is not None: + return + self._db_path.parent.mkdir(parents=True, exist_ok=True) + await asyncio.to_thread(self._init_db) + self._task = asyncio.ensure_future(self._writer_loop()) + + async def aclose(self) -> None: + if self._task is None: + return + await self._queue.put({"_stop": True}) + try: + await asyncio.wait_for(self._task, timeout=5.0) + except Exception: # noqa: BLE001 + self._task.cancel() + self._task = None + + async def drain(self) -> None: + """Test helper — block until the queue is fully processed.""" + await self._queue.join() + + # -- producers (fire-and-forget, never raise) --------------------- + def record_shadow_nowait( + self, *, op_id: str, agent: str, ts: float, shadow_outcome: dict, + ) -> None: + self._enqueue({ + "op_id": op_id, "agent": agent, "ts": ts, + "shadow": shadow_outcome, + }) + + def record_legacy_nowait( + self, *, op_id: str, agent: str, ts: float, legacy_outcome: dict, + ) -> None: + self._enqueue({ + "op_id": op_id, "agent": agent, "ts": ts, + "legacy": legacy_outcome, + }) + + def _enqueue(self, item: dict) -> None: + try: + self._queue.put_nowait(item) + except asyncio.QueueFull: + # drop-oldest: discard one, retry once; bounded memory. + try: + _ = self._queue.get_nowait() + self._queue.task_done() + self._dropped += 1 + self._queue.put_nowait(item) + except Exception: # noqa: BLE001 + self._dropped += 1 + except Exception: # noqa: BLE001 + self._dropped += 1 + + # -- writer ------------------------------------------------------- + async def _writer_loop(self) -> None: + while True: + item = await self._queue.get() + try: + if item.get("_stop"): + self._queue.task_done() + return + await asyncio.to_thread(self._apply, item) + except Exception: # noqa: BLE001 + logger.warning( + "[ShadowTelemetryStore] write failed (non-fatal)", + exc_info=True, + ) + finally: + if not item.get("_stop"): + self._queue.task_done() + + # -- blocking sqlite (runs in to_thread) -------------------------- + def _init_db(self) -> None: + con = sqlite3.connect(self._db_path) + try: + con.executescript(_SCHEMA) + con.commit() + finally: + con.close() + + def _next_seq(self, con: sqlite3.Connection, agent: str) -> int: + cur = con.execute( + "SELECT next FROM agent_seq WHERE agent = ?", (agent,)) + row = cur.fetchone() + nxt = (row[0] if row else 0) + 1 + con.execute( + "INSERT INTO agent_seq(agent, next) VALUES(?, ?) " + "ON CONFLICT(agent) DO UPDATE SET next = excluded.next", + (agent, nxt)) + return nxt + + def _apply(self, item: dict) -> None: + agent = item["agent"] + op_id = item["op_id"] + con = sqlite3.connect(self._db_path) + try: + cur = con.execute( + "SELECT legacy_outcome, shadow_outcome, seq " + "FROM shadow_comparison WHERE op_id = ? AND agent = ?", + (op_id, agent)) + existing = cur.fetchone() + if existing is None: + seq = self._next_seq(con, agent) + legacy = json.dumps(item["legacy"]) if "legacy" in item else None + shadow = json.dumps(item["shadow"]) if "shadow" in item else None + con.execute( + "INSERT INTO shadow_comparison" + "(op_id, agent, ts, seq, legacy_outcome, shadow_outcome," + " aligned, divergence_reason) VALUES(?,?,?,?,?,?,?,?)", + (op_id, agent, item["ts"], seq, legacy, shadow, None, None)) + else: + legacy = existing[0] + shadow = existing[1] + if "legacy" in item: + legacy = json.dumps(item["legacy"]) + if "shadow" in item: + shadow = json.dumps(item["shadow"]) + con.execute( + "UPDATE shadow_comparison SET legacy_outcome = ?, " + "shadow_outcome = ? WHERE op_id = ? AND agent = ?", + (legacy, shadow, op_id, agent)) + + # Compute alignment once both halves present + evaluator wired. + if legacy is not None and shadow is not None and self._evaluator: + aligned, reason = self._evaluator( + agent, json.loads(legacy), json.loads(shadow)) + con.execute( + "UPDATE shadow_comparison SET aligned = ?, " + "divergence_reason = ? WHERE op_id = ? AND agent = ?", + (1 if aligned else 0, reason or None, op_id, agent)) + + self._prune(con, agent) + con.commit() + finally: + con.close() + + def _prune(self, con: sqlite3.Connection, agent: str) -> None: + con.execute( + "DELETE FROM shadow_comparison WHERE agent = ? AND seq <= " + "((SELECT MAX(seq) FROM shadow_comparison WHERE agent = ?) - ?)", + (agent, agent, self._cap)) + + # -- read side ---------------------------------------------------- + async def last_n(self, agent: str, n: int) -> list: + return await asyncio.to_thread(self._last_n, agent, n) + + def _last_n(self, agent: str, n: int) -> list: + con = sqlite3.connect(self._db_path) + try: + cur = con.execute( + "SELECT op_id, seq, aligned, divergence_reason " + "FROM shadow_comparison WHERE agent = ? " + "ORDER BY seq DESC LIMIT ?", (agent, n)) + return [ + {"op_id": r[0], "seq": r[1], "aligned": r[2], + "divergence_reason": r[3]} + for r in cur.fetchall() + ] + finally: + con.close() + + async def recent_aligned_streak(self, agent: str) -> int: + return await asyncio.to_thread(self._recent_aligned_streak, agent) + + def _recent_aligned_streak(self, agent: str) -> int: + con = sqlite3.connect(self._db_path) + try: + cur = con.execute( + "SELECT aligned FROM shadow_comparison WHERE agent = ? " + "ORDER BY seq DESC", (agent,)) + streak = 0 + for (aligned,) in cur.fetchall(): + if aligned is None: + continue # incomplete row: skip, don't break + if aligned == 1: + streak += 1 + else: + break # a single divergence resets the streak + return streak + finally: + con.close() +``` + +Note: `pytest.mark.asyncio` requires `pytest-asyncio` (already used across the repo's governance async tests). If a test needs the marker registered, the repo's `pytest.ini`/`conftest.py` already enables it. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/test_shadow_telemetry_store.py -v` +Expected: PASS (2 tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/shadow_telemetry_store.py tests/governance/test_shadow_telemetry_store.py +git commit -m "feat(rail): async SQLite shadow telemetry store + two-phase upsert (Unit A)" +``` + +--- + +## Task 4: Telemetry store — FIFO prune + drop-oldest (Unit A) + +**Files:** +- Test: `tests/governance/test_shadow_telemetry_store.py` (add) + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/governance/test_shadow_telemetry_store.py + +@pytest.mark.asyncio +async def test_fifo_cap_prunes_oldest(tmp_path): + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", + evaluator=lambda a, l, s: (True, ""), + cap_per_agent=5, + ) + await store.start() + for i in range(12): + store.record_legacy_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + legacy_outcome={"i": i}) + store.record_shadow_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + shadow_outcome={"i": i}) + await store.drain() + rows = await store.last_n("plan", 100) + # cap=5 -> only the 5 highest seq survive + assert len(rows) == 5 + seqs = [r["seq"] for r in rows] + assert seqs == sorted(seqs, reverse=True) + assert min(seqs) >= 8 # oldest (op0..op6) pruned + await store.aclose() + + +@pytest.mark.asyncio +async def test_streak_resets_on_divergence(tmp_path): + # evaluator: align unless shadow says {"bad": True} + def ev(agent, legacy, shadow): + return (not shadow.get("bad", False), "div" if shadow.get("bad") else "") + + store = ShadowTelemetryStore(db_path=tmp_path / "t.db", evaluator=ev) + await store.start() + + async def one(op, bad): + store.record_legacy_nowait( + op_id=op, agent="review", ts=0.0, legacy_outcome={}) + store.record_shadow_nowait( + op_id=op, agent="review", ts=0.0, shadow_outcome={"bad": bad}) + + for i in range(3): + await one(f"a{i}", False) + await one("bad1", True) + for i in range(2): + await one(f"b{i}", False) + await store.drain() + # newest-first: b1,b0 aligned (2), then bad1 breaks -> streak == 2 + assert await store.recent_aligned_streak("review") == 2 + await store.aclose() +``` + +- [ ] **Step 2: Run the tests** + +Run: `pytest tests/governance/test_shadow_telemetry_store.py -k "fifo or streak" -v` +Expected: PASS — the prune + streak logic from Task 3 already implements this; these tests lock the contract. + +- [ ] **Step 3: Commit** + +```bash +git add tests/governance/test_shadow_telemetry_store.py +git commit -m "test(rail): FIFO cap + streak-reset-on-divergence (Unit A)" +``` + +--- + +## Task 5: `AGENT_DEGRADATION` SSE event constant + helper + +**Files:** +- Modify: `backend/core/ouroboros/governance/ide_observability_stream.py` (add constant beside `EVENT_TYPE_MEMORY_PRESSURE_CHANGED` ~line 185; add helper near other `publish_*` helpers) +- Test: `tests/governance/test_shadow_graduation_gate.py` (created here; first test targets the helper) + +- [ ] **Step 1: Write the failing test** + +```python +# tests/governance/test_shadow_graduation_gate.py +from __future__ import annotations + +from backend.core.ouroboros.governance import ide_observability_stream as ios + + +def test_agent_degradation_event_type_registered(): + assert ios.EVENT_TYPE_AGENT_DEGRADATION == "agent_degradation" + # Must be in the broker's accepted vocabulary so publish() doesn't drop it. + assert "agent_degradation" in ios._VALID_EVENT_TYPES # noqa: SLF001 +``` + +(If the broker's valid-types set has a different private name, adjust the assertion to whatever `publish()` validates against — confirm by reading the module; the constant must be added to that set.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py::test_agent_degradation_event_type_registered -v` +Expected: FAIL — `AttributeError: ... EVENT_TYPE_AGENT_DEGRADATION`. + +- [ ] **Step 3: Implement** + +In `ide_observability_stream.py`, beside `EVENT_TYPE_MEMORY_PRESSURE_CHANGED` (~line 185): + +```python +EVENT_TYPE_AGENT_DEGRADATION = "agent_degradation" +``` + +Add `EVENT_TYPE_AGENT_DEGRADATION` to whatever set/tuple `publish()` validates against (the valid-event-types collection). Then add a helper near the other `publish_*` helpers: + +```python +def publish_agent_degradation_event( + *, broker, agent: str, op_id: str, trip_reason: str, + pressure_level: str, +) -> None: + """Best-effort AGENT_DEGRADATION frame. Never raises/blocks.""" + try: + broker.publish( + EVENT_TYPE_AGENT_DEGRADATION, op_id, + {"agent": agent, "trip_reason": trip_reason, + "pressure_level": pressure_level}, + ) + except Exception: # noqa: BLE001 + pass +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py::test_agent_degradation_event_type_registered -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/ide_observability_stream.py tests/governance/test_shadow_graduation_gate.py +git commit -m "feat(rail): AGENT_DEGRADATION SSE event type + helper (Unit C)" +``` + +--- + +## Task 6: Graduation gate — streak promotion (Unit C) + +**Files:** +- Create: `backend/core/ouroboros/governance/shadow_graduation_gate.py` +- Test: `tests/governance/test_shadow_graduation_gate.py` (add) + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/governance/test_shadow_graduation_gate.py +import pytest + +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + ShadowGraduationGate, +) + + +class _FakeStore: + def __init__(self, streak): + self._streak = streak + + async def recent_aligned_streak(self, agent): + return self._streak + + +@pytest.mark.asyncio +async def test_no_promote_below_threshold(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + gate = ShadowGraduationGate(store=_FakeStore(streak=49)) + promoted = await gate.maybe_promote("plan") + assert promoted is False + assert persisted == [] + + +@pytest.mark.asyncio +async def test_promote_at_threshold_persists_flags(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_THRESHOLD", "50") + gate = ShadowGraduationGate(store=_FakeStore(streak=50)) + promoted = await gate.maybe_promote("plan") + assert promoted is True + assert ("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") in persisted + assert ("JARVIS_PLAN_SUBAGENT_SHADOW", "false") in persisted + + +@pytest.mark.asyncio +async def test_promote_idempotent(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") + gate = ShadowGraduationGate(store=_FakeStore(streak=50)) + promoted = await gate.maybe_promote("plan") + assert promoted is False # already authoritative -> no-op + assert persisted == [] +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py -k promote -v` +Expected: FAIL — `ModuleNotFoundError: ... shadow_graduation_gate`. + +- [ ] **Step 3: Write minimal implementation** + +```python +# backend/core/ouroboros/governance/shadow_graduation_gate.py +"""Event-driven graduation gate + graceful-degradation circuit breaker +(Unit C). + +Reads the telemetry store at each op boundary; once an agent has N +consecutive aligned ops it flips that agent's ``_AUTHORITATIVE`` flag +and persists it via the existing credential-safe ``persist_flag_to_env`` +writer. Promotion is idempotent and honors explicit operator settings. +""" +from __future__ import annotations + +import logging +import os +from typing import Any + +from backend.core.ouroboros.governance.graduation_orchestrator import ( + persist_flag_to_env, +) + +logger = logging.getLogger("Ouroboros.ShadowGraduationGate") + +_AUTH_FLAG = { + "plan": "JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", + "review": "JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE", +} +_SHADOW_FLAG = { + "plan": "JARVIS_PLAN_SUBAGENT_SHADOW", + "review": "JARVIS_REVIEW_SUBAGENT_SHADOW", +} + + +def gate_enabled() -> bool: + raw = os.environ.get("JARVIS_SHADOW_GRADUATION_GATE_ENABLED") + return raw is None or raw.strip().lower() in ("true", "1", "yes") + + +def _threshold() -> int: + try: + return max(1, int(os.environ.get( + "JARVIS_SHADOW_GRADUATION_THRESHOLD", "50"))) + except (TypeError, ValueError): + return 50 + + +def _is_authoritative(agent: str) -> bool: + return os.environ.get(_AUTH_FLAG[agent], "false").strip().lower() in ( + "true", "1", "yes") + + +class ShadowGraduationGate: + def __init__(self, *, store: Any) -> None: + self._store = store + + async def maybe_promote(self, agent: str) -> bool: + if not gate_enabled() or agent not in _AUTH_FLAG: + return False + if _is_authoritative(agent): + return False # idempotent — already graduated + try: + streak = await self._store.recent_aligned_streak(agent) + except Exception: # noqa: BLE001 — gate must not break the FSM + logger.warning( + "[ShadowGraduationGate] streak read failed (non-fatal)", + exc_info=True) + return False + if streak < _threshold(): + return False + return self._promote(agent, streak) + + def _promote(self, agent: str, streak: int) -> bool: + auth = _AUTH_FLAG[agent] + shadow = _SHADOW_FLAG[agent] + ok1 = persist_flag_to_env(auth, "true") + ok2 = persist_flag_to_env(shadow, "false") + if ok1: + os.environ[auth] = "true" + if ok2: + os.environ[shadow] = "false" + logger.info( + "[GRADUATION] agent=%s streak=%d -> authoritative " + "(auth_persist=%s shadow_persist=%s)", + agent, streak, ok1, ok2) + return bool(ok1) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py -k promote -v` +Expected: PASS (3 tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/shadow_graduation_gate.py tests/governance/test_shadow_graduation_gate.py +git commit -m "feat(rail): event-driven 50-soak graduation gate (Unit C)" +``` + +--- + +## Task 7: Circuit breaker — trip table incl. CRITICAL pre-emptive (Unit C) + +**Files:** +- Modify: `backend/core/ouroboros/governance/shadow_graduation_gate.py` (add breaker) +- Test: `tests/governance/test_shadow_graduation_gate.py` (add) + +- [ ] **Step 1: Write the failing test** + +```python +# append to tests/governance/test_shadow_graduation_gate.py +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + PlanBreaker, +) + + +def test_breaker_trips_on_cyclical_dag(): + b = PlanBreaker(pressure_fn=lambda: "ok") + decision = b.should_use_legacy(dag={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u2"]}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}, + ]}) + assert decision.trip is True + assert decision.reason == "cyclical_dag" + + +def test_breaker_trips_on_empty_dag(): + b = PlanBreaker(pressure_fn=lambda: "ok") + decision = b.should_use_legacy(dag={"units": []}) + assert decision.trip is True + assert decision.reason == "unparsable_or_empty_dag" + + +def test_breaker_critical_pressure_preempts_before_dag(): + # CRITICAL pressure trips BEFORE inspecting the DAG (pre-emptive). + b = PlanBreaker(pressure_fn=lambda: "critical") + decision = b.should_use_legacy(dag=None) + assert decision.trip is True + assert decision.reason == "critical_memory_pressure" + assert decision.pressure_level == "critical" + + +def test_breaker_passes_valid_dag_under_ok_pressure(): + b = PlanBreaker(pressure_fn=lambda: "ok") + decision = b.should_use_legacy(dag={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + ]}) + assert decision.trip is False +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py -k breaker -v` +Expected: FAIL — `ImportError: cannot import name 'PlanBreaker'`. + +- [ ] **Step 3: Write minimal implementation** + +Append to `shadow_graduation_gate.py`: + +```python +from dataclasses import dataclass + +from backend.core.ouroboros.governance.shadow_evaluator import _has_cycle + + +@dataclass(frozen=True) +class BreakerDecision: + trip: bool + reason: str + pressure_level: str + + +def _default_pressure_fn() -> str: + try: + from backend.core.ouroboros.governance.memory_pressure_gate import ( + get_default_gate, + ) + return get_default_gate().pressure().value + except Exception: # noqa: BLE001 + return "ok" # probe failure -> assume OK (governor handles fan-out) + + +class PlanBreaker: + """Graceful-degradation breaker for the authoritative PLAN path. + + Trip order (first-match-wins): + 1. CRITICAL memory pressure -> pre-emptive, do NOT touch the DAG. + 2. Empty / unparsable DAG. + 3. Cyclical DAG. + A trip routes the operation to the retained legacy flat-plan + generator, guaranteeing execution continuity. + """ + + def __init__(self, *, pressure_fn=None) -> None: + self._pressure_fn = pressure_fn or _default_pressure_fn + + def should_use_legacy(self, *, dag) -> BreakerDecision: + level = "ok" + try: + level = (self._pressure_fn() or "ok").lower() + except Exception: # noqa: BLE001 + level = "ok" + if level == "critical": + return BreakerDecision(True, "critical_memory_pressure", level) + units = dag.get("units") if isinstance(dag, dict) else None + if not isinstance(units, list) or len(units) == 0: + return BreakerDecision(True, "unparsable_or_empty_dag", level) + try: + if _has_cycle(units): + return BreakerDecision(True, "cyclical_dag", level) + except Exception: # noqa: BLE001 + return BreakerDecision(True, "unparsable_or_empty_dag", level) + return BreakerDecision(False, "", level) +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py -k breaker -v` +Expected: PASS (4 tests). + +- [ ] **Step 5: Commit** + +```bash +git add backend/core/ouroboros/governance/shadow_graduation_gate.py tests/governance/test_shadow_graduation_gate.py +git commit -m "feat(rail): PLAN graceful-degradation circuit breaker (Unit C)" +``` + +--- + +## Task 8: Wire producers into the orchestrator shadow hooks + +**Files:** +- Modify: `backend/core/ouroboros/governance/orchestrator.py` — `_run_plan_shadow` (~1730), `_run_review_shadow` (~1572), and the legacy-outcome capture at GATE/terminal. +- Test: `tests/governance/test_shadow_rail_off_inert.py` + +This task connects the live pipeline to the rail. The store + gate are owned by `GovernedLoopService` (constructed once, like `_sub_orch` at `governed_loop_service.py:4830`) and referenced from the orchestrator as `self._shadow_store` / `self._shadow_gate` (None when disabled). All calls are fire-and-forget and guarded by `if self._shadow_store is not None`. + +- [ ] **Step 1: Write the failing test (OFF-is-inert contract)** + +```python +# tests/governance/test_shadow_rail_off_inert.py +from __future__ import annotations + +import backend.core.ouroboros.governance.shadow_telemetry_store as sts + + +def test_store_disabled_by_flag(monkeypatch): + monkeypatch.setenv("JARVIS_SHADOW_TELEMETRY_STORE_ENABLED", "false") + assert sts.store_enabled() is False + + +def test_store_enabled_by_default(monkeypatch): + monkeypatch.delenv("JARVIS_SHADOW_TELEMETRY_STORE_ENABLED", raising=False) + assert sts.store_enabled() is True +``` + +- [ ] **Step 2: Run it** + +Run: `pytest tests/governance/test_shadow_rail_off_inert.py -v` +Expected: PASS (these assert the flag helper already built in Task 3). + +- [ ] **Step 3: Wire PLAN producer** + +In `_run_plan_shadow` (orchestrator.py ~1730), after the DAG is obtained and the legacy flat plan is available (both present at this hook), add — guarded and fire-and-forget — immediately before the existing `[PLAN-SHADOW]` log: + +```python + # Unit A producer — single-phase: both halves available here. + if getattr(self, "_shadow_store", None) is not None: + try: + _legacy_flat = [ + getattr(t, "file_path", None) or t.get("file_path") + for t in (getattr(ctx, "implementation_plan", None) or []) + ] + _legacy_flat = [p for p in _legacy_flat if p] + self._shadow_store.record_legacy_nowait( + op_id=getattr(ctx, "op_id", "?"), agent="plan", + ts=_now_ts(), legacy_outcome={"flat": _legacy_flat}, + ) + self._shadow_store.record_shadow_nowait( + op_id=getattr(ctx, "op_id", "?"), agent="plan", + ts=_now_ts(), + shadow_outcome={"units": _dag_units_as_dicts}, + ) + except Exception: # noqa: BLE001 — observer contract + pass +``` + +Where `_dag_units_as_dicts` is the list-of-dicts form `[{"id","owned_paths","deps"}, ...]` derived from the DAG payload already computed in the hook (the same `unit_count`/`execution_graph` structure logged today), and `_now_ts()` is a tiny local helper `return __import__("time").time()` (wall-clock is fine here — `ts` is informational; ordering uses the store's `seq`). + +The PLAN evaluator expects `legacy_flat` as a list of paths; pass `{"flat": [...]}` and have the wiring in Task 9 unwrap to the list. (Adjust the evaluator adapter in Task 9 accordingly so the store's injected evaluator sees the right shapes.) + +- [ ] **Step 4: Wire REVIEW producers (two-phase)** + +In `_run_review_shadow` (orchestrator.py ~1572), after `_aggregate` is computed and before the `[REVIEW-SHADOW]` log, add the **shadow half**: + +```python + if getattr(self, "_shadow_store", None) is not None: + try: + self._shadow_store.record_shadow_nowait( + op_id=getattr(ctx, "op_id", "?"), agent="review", + ts=_now_ts(), shadow_outcome={"aggregate": _aggregate}, + ) + except Exception: # noqa: BLE001 + pass +``` + +At the GATE/terminal point where the authoritative risk tier is resolved (search for where the final risk tier / SemanticGuardian hard-finding is known), add the **legacy half**: + +```python + if getattr(self, "_shadow_store", None) is not None: + try: + self._shadow_store.record_legacy_nowait( + op_id=getattr(ctx, "op_id", "?"), agent="review", + ts=_now_ts(), + legacy_outcome={ + "risk_tier": str(_resolved_risk_tier), + "semantic_guard_hard": bool(_had_hard_finding), + }, + ) + except Exception: # noqa: BLE001 + pass +``` + +(`_resolved_risk_tier` and `_had_hard_finding` are the existing locals at that site; if the names differ, use the in-scope equivalents — the values are: the final risk tier enum/string, and whether SemanticGuardian raised a hard finding.) + +- [ ] **Step 5: Trigger the gate after a row finalizes** + +The cleanest event-driven trigger: after the REVIEW legacy-half and the PLAN single write, schedule a gate check (fire-and-forget, non-blocking). Add after each producer block: + +```python + if getattr(self, "_shadow_gate", None) is not None: + import asyncio as _asyncio + _t = _asyncio.ensure_future(self._shadow_gate.maybe_promote(_AGENT)) + self._shadow_gate_tasks.add(_t) + _t.add_done_callback(self._shadow_gate_tasks.discard) +``` + +with `_AGENT` being `"plan"` or `"review"` at the respective site, and `self._shadow_gate_tasks` a `set()` initialized in `__init__` (strong refs prevent GC — same pattern as the episodic `_fire_nowait` synapse). The `maybe_promote` reads the streak (which is only accurate once the writer has flushed; a slightly-late promotion is harmless — the next op re-checks). + +- [ ] **Step 6: Run the full governance regression to confirm no FSM change** + +Run: `pytest tests/governance/ -k "review_shadow or plan_shadow or orchestrator" -v` +Expected: PASS — with the store/gate refs `None` (default in these tests), every new block is skipped, so the FSM is byte-identical. + +- [ ] **Step 7: Commit** + +```bash +git add backend/core/ouroboros/governance/orchestrator.py tests/governance/test_shadow_rail_off_inert.py +git commit -m "feat(rail): wire shadow telemetry producers + gate trigger into FSM (Unit A/C)" +``` + +--- + +## Task 9: Construct + own the rail in `GovernedLoopService` + +**Files:** +- Modify: `backend/core/ouroboros/governance/governed_loop_service.py` (near the `_sub_orch` construction ~4830) +- Test: `tests/governance/test_shadow_graduation_gate.py` (add an adapter test) + +- [ ] **Step 1: Write the failing test (evaluator adapter)** + +```python +# append to tests/governance/test_shadow_graduation_gate.py +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + build_rail_evaluator, +) + + +def test_rail_evaluator_routes_by_agent(): + ev = build_rail_evaluator() + # review path + aligned, _ = ev("review", + {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": False}, + {"aggregate": "approve"}) + assert aligned is True + # plan path: legacy carries {"flat": [...]} + aligned, _ = ev("plan", + {"flat": ["a.py"]}, + {"units": [{"id": "u1", "owned_paths": ["a.py"], + "deps": []}]}) + assert aligned is True +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py::test_rail_evaluator_routes_by_agent -v` +Expected: FAIL — `ImportError: cannot import name 'build_rail_evaluator'`. + +- [ ] **Step 3: Implement the adapter** + +Append to `shadow_graduation_gate.py`: + +```python +def build_rail_evaluator(): + """Adapter: (agent, legacy, shadow) -> (aligned, reason), routing to + the right pure evaluator and unwrapping the stored shapes.""" + from backend.core.ouroboros.governance.shadow_evaluator import ( + evaluate_plan, evaluate_review, + ) + + def _ev(agent: str, legacy: dict, shadow: dict): + if agent == "review": + a = evaluate_review(legacy, shadow) + elif agent == "plan": + a = evaluate_plan(legacy.get("flat", []), shadow) + else: + return (False, "malformed:unknown_agent") + return (a.aligned, a.reason) + + return _ev +``` + +- [ ] **Step 4: Wire construction in `GovernedLoopService`** + +Near the `_sub_orch` block (~4830), gated by `store_enabled()`: + +```python + from backend.core.ouroboros.governance.shadow_telemetry_store import ( + ShadowTelemetryStore, store_enabled, + ) + from backend.core.ouroboros.governance.shadow_graduation_gate import ( + ShadowGraduationGate, build_rail_evaluator, gate_enabled, + ) + + if store_enabled(): + _shadow_store = ShadowTelemetryStore(evaluator=build_rail_evaluator()) + await _shadow_store.start() + self._shadow_store_ref = _shadow_store + if self._orchestrator is not None: + self._orchestrator._shadow_store = _shadow_store + self._orchestrator._shadow_gate_tasks = set() + if gate_enabled(): + self._orchestrator._shadow_gate = ShadowGraduationGate( + store=_shadow_store) + else: + self._orchestrator._shadow_gate = None +``` + +And in the service shutdown path, add `await self._shadow_store_ref.aclose()` (guarded by `getattr(self, "_shadow_store_ref", None)`). + +- [ ] **Step 5: Run test to verify it passes + import sanity** + +Run: `pytest tests/governance/test_shadow_graduation_gate.py::test_rail_evaluator_routes_by_agent -v` +Expected: PASS. +Run: `python3 -c "import ast; ast.parse(open('backend/core/ouroboros/governance/governed_loop_service.py').read())"` +Expected: no output (parses clean — `import` of the live module is blocked in sandbox per the split-brain guard, so verify via AST). + +- [ ] **Step 6: Commit** + +```bash +git add backend/core/ouroboros/governance/shadow_graduation_gate.py backend/core/ouroboros/governance/governed_loop_service.py tests/governance/test_shadow_graduation_gate.py +git commit -m "feat(rail): construct+own evidence rail in GovernedLoopService (Unit A/B/C)" +``` + +--- + +## Task 10: Authoritative wiring — PLAN DAG consumed + REVIEW raises tier + +**Files:** +- Modify: `backend/core/ouroboros/governance/orchestrator.py` (`_run_plan_shadow` ~1730 + `_run_review_shadow` ~1572) +- Test: `tests/governance/test_shadow_authoritative_wiring.py` + +This task makes graduation *mean something*: when `_AUTHORITATIVE=true`, the subagent output changes behavior (composed with the breaker). When `false` (default), behavior is byte-identical to today. + +- [ ] **Step 1: Write the failing test** + +```python +# tests/governance/test_shadow_authoritative_wiring.py +from __future__ import annotations + +import os + +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + PlanBreaker, +) + + +def test_plan_authoritative_uses_dag_when_breaker_passes(monkeypatch): + monkeypatch.setenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") + breaker = PlanBreaker(pressure_fn=lambda: "ok") + dag = {"units": [{"id": "u1", "owned_paths": ["a.py"], "deps": []}]} + decision = breaker.should_use_legacy(dag=dag) + # authoritative + breaker-pass -> DAG is used (no trip) + assert decision.trip is False + + +def test_plan_authoritative_trips_to_legacy_on_critical(monkeypatch): + monkeypatch.setenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") + breaker = PlanBreaker(pressure_fn=lambda: "critical") + decision = breaker.should_use_legacy(dag={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}]}) + assert decision.trip is True + assert decision.reason == "critical_memory_pressure" +``` + +- [ ] **Step 2: Run test to verify it fails or passes** + +Run: `pytest tests/governance/test_shadow_authoritative_wiring.py -v` +Expected: PASS — these exercise `PlanBreaker` (built in Task 7) and codify the contract the orchestrator edit must honor. + +- [ ] **Step 3: Edit `_run_plan_shadow` for authoritative consumption** + +After the producer block (Task 8 Step 3), add: + +```python + # Authoritative promotion: when graduated, the DAG drives execution + # UNLESS the breaker trips (cyclical/empty/CRITICAL) -> legacy. + _plan_auth = os.environ.get( + "JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "false" + ).strip().lower() in ("true", "1", "yes") + if _plan_auth: + from backend.core.ouroboros.governance.shadow_graduation_gate import ( + PlanBreaker, + ) + from backend.core.ouroboros.governance.ide_observability_stream import ( + publish_agent_degradation_event, + ) + _decision = PlanBreaker().should_use_legacy( + dag={"units": _dag_units_as_dicts}) + if _decision.trip: + publish_agent_degradation_event( + broker=getattr(self, "_stream_broker", None), + agent="plan", op_id=getattr(ctx, "op_id", "?"), + trip_reason=_decision.reason, + pressure_level=_decision.pressure_level, + ) if getattr(self, "_stream_broker", None) else None + logger.warning( + "[BREAKER] agent=plan op=%s trip=%s -> legacy", + getattr(ctx, "op_id", "?"), _decision.reason) + # leave ctx.implementation_plan (legacy) authoritative + else: + # promote DAG: stash so _materialize_execution_graph_candidate + # consumes it as authoritative (already stashed on ctx today). + logger.info( + "[AUTHORITATIVE] agent=plan op=%s DAG drives execution", + getattr(ctx, "op_id", "?")) + return ctx +``` + +(The DAG is already stashed on `ctx.execution_graph` by the existing hook; the authoritative branch simply does not suppress it, while the breaker-trip branch ensures legacy remains the input. Confirm `_materialize_execution_graph_candidate` only consumes `ctx.execution_graph` when authoritative — add a guard there reading `JARVIS_PLAN_SUBAGENT_AUTHORITATIVE` if it currently consumes unconditionally.) + +- [ ] **Step 4: Edit `_run_review_shadow` for authoritative tier-raise** + +After the REVIEW producer block, add: + +```python + _rev_auth = os.environ.get( + "JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE", "false" + ).strip().lower() in ("true", "1", "yes") + if _rev_auth and _aggregate == "reject": + # REVIEW may only ADD friction: force APPROVAL_REQUIRED. It never + # weakens SemanticGuardian/Iron Gate (strictest-wins). + try: + self._raise_risk_tier_to_approval_required( + ctx, source="review_subagent") + logger.info( + "[AUTHORITATIVE] agent=review op=%s REJECT -> " + "APPROVAL_REQUIRED", getattr(ctx, "op_id", "?")) + except Exception: # noqa: BLE001 + pass +``` + +(`_raise_risk_tier_to_approval_required` is the existing risk-tier-floor escalation helper; if no single helper exists, set the resolved tier to the max of its current value and `APPROVAL_REQUIRED` using the existing risk-tier enum comparison already used by `risk_tier_floor.py`. Never lower a tier.) + +- [ ] **Step 5: Run the full suite** + +Run: `pytest tests/governance/test_shadow_authoritative_wiring.py tests/governance/ -k "shadow or breaker or graduation" -v` +Expected: PASS. Then confirm OFF-default byte-identical: +Run: `pytest tests/governance/ -k "review_shadow or plan_shadow" -v` +Expected: PASS (auth flags default false -> both new branches skipped). + +- [ ] **Step 6: Commit** + +```bash +git add backend/core/ouroboros/governance/orchestrator.py tests/governance/test_shadow_authoritative_wiring.py +git commit -m "feat(rail): authoritative wiring — PLAN DAG + REVIEW tier-raise w/ breaker (Unit C)" +``` + +--- + +## Task 11: Full-rail integration + OFF-is-inert regression + +**Files:** +- Test: `tests/governance/test_shadow_rail_integration.py` + +- [ ] **Step 1: Write the integration test** + +```python +# tests/governance/test_shadow_rail_integration.py +from __future__ import annotations + +import pytest + +from backend.core.ouroboros.governance.shadow_telemetry_store import ( + ShadowTelemetryStore, +) +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + ShadowGraduationGate, build_rail_evaluator, +) + + +@pytest.mark.asyncio +async def test_fifty_aligned_ops_graduate_plan(tmp_path, monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_THRESHOLD", "50") + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", evaluator=build_rail_evaluator()) + await store.start() + gate = ShadowGraduationGate(store=store) + + for i in range(50): + store.record_legacy_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + legacy_outcome={"flat": ["a.py"]}) + store.record_shadow_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + shadow_outcome={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}]}) + await store.drain() + + promoted = await gate.maybe_promote("plan") + assert promoted is True + assert ("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") in persisted + await store.aclose() + + +@pytest.mark.asyncio +async def test_one_divergence_blocks_graduation(tmp_path, monkeypatch): + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_THRESHOLD", "50") + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", evaluator=build_rail_evaluator()) + await store.start() + gate = ShadowGraduationGate(store=store) + + for i in range(49): + store.record_legacy_nowait( + op_id=f"ok{i}", agent="plan", ts=float(i), + legacy_outcome={"flat": ["a.py"]}) + store.record_shadow_nowait( + op_id=f"ok{i}", agent="plan", ts=float(i), + shadow_outcome={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}]}) + # one cyclical (misaligned) op as the newest + store.record_legacy_nowait( + op_id="bad", agent="plan", ts=99.0, legacy_outcome={"flat": ["a.py"]}) + store.record_shadow_nowait( + op_id="bad", agent="plan", ts=99.0, shadow_outcome={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u2"]}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}]}) + await store.drain() + + assert await store.recent_aligned_streak("plan") == 0 + assert await gate.maybe_promote("plan") is False + await store.aclose() +``` + +- [ ] **Step 2: Run it** + +Run: `pytest tests/governance/test_shadow_rail_integration.py -v` +Expected: PASS (2 tests) — the full A→B→C path graduates on 50 clean and blocks on a single newest divergence. + +- [ ] **Step 3: Run the entire new suite + governance regression** + +Run: `pytest tests/governance/test_shadow_evaluator.py tests/governance/test_shadow_telemetry_store.py tests/governance/test_shadow_graduation_gate.py tests/governance/test_shadow_authoritative_wiring.py tests/governance/test_shadow_rail_integration.py tests/governance/test_shadow_rail_off_inert.py -v` +Expected: PASS (all rail tests). + +- [ ] **Step 4: Commit** + +```bash +git add tests/governance/test_shadow_rail_integration.py +git commit -m "test(rail): full A->B->C graduation + divergence-blocks integration" +``` + +--- + +## Self-Review Notes + +- **Spec coverage:** Unit A §4 (Tasks 3-4: async writer, two-phase upsert, FIFO prune, drop-oldest, streak query). Unit B §5 (Tasks 1-2: REVIEW binary, PLAN refinement). Unit C §6 (Tasks 5-7, 10: AGENT_DEGRADATION event, 50-soak gate, breaker incl. CRITICAL pre-emptive, authoritative wiring). Construction/ownership §3 (Task 9). Event-driven topology §8 (Task 8 gate trigger fires when a row finalizes). Flags §9 — every new flag has a default-preserving helper. OFF-is-inert §3/§10 (Tasks 8, 10, 11). +- **Type consistency:** `Alignment(aligned, reason)` used identically across B and the store evaluator adapter. Evaluator callable signature `(agent, legacy_dict, shadow_dict) -> (bool, str)` matches `ShadowTelemetryStore.__init__(evaluator=...)`, `build_rail_evaluator`, and the fakes. `BreakerDecision(trip, reason, pressure_level)` and `GovernorDecision` (Plan 1) names match every call site. `_has_cycle` is defined once in `shadow_evaluator.py` and imported by the breaker (no duplication). +- **Cross-plan dependency:** the breaker's CRITICAL trip uses `MemoryPressureGate.pressure()` directly (not the Plan 1 governor module), so Plan 2 does **not** depend on Plan 1 landing first — they are independent, as the spec requires. +- **No placeholders:** every code step ships runnable code. The two orchestrator-local names flagged for confirmation (`_resolved_risk_tier`/`_had_hard_finding` and `_raise_risk_tier_to_approval_required`) are existing in-scope symbols at the documented sites; the steps state exactly what value to use if a name differs, which is guidance for an existing symbol, not a placeholder for unwritten code. +- **Sandbox note:** live `import` of `orchestrator`/`governed_loop_service` raises the split-brain guard in this sandbox; verify those two edits via `ast.parse` + targeted `pytest` of the new isolated modules (which have no heavy imports), exactly as Task 9 Step 5 does. diff --git a/docs/superpowers/specs/2026-06-14-sovereign-evidence-rail.md b/docs/superpowers/specs/2026-06-14-sovereign-evidence-rail.md new file mode 100644 index 0000000000..b288c483a4 --- /dev/null +++ b/docs/superpowers/specs/2026-06-14-sovereign-evidence-rail.md @@ -0,0 +1,507 @@ +# Sovereign Evidence Rail & L3 Memory Governor + +**Date:** 2026-06-14 +**Author:** Derek J. Russell (O+V Trinity Architect) +**Status:** DESIGN — awaiting Sovereign Authorization before implementation +**Spec ID:** sovereign-evidence-rail + +--- + +## 1. Problem Statement + +The O+V multi-agent fleet runs four graduated subagents. Two of them — +**REVIEW** and **PLAN** — operate in *shadow mode*: they execute on every +qualifying op, emit telemetry, and their output is **discarded**. The legacy +deterministic paths (SemanticGuardian/GATE for review; the flat-list +`PlanGenerator` for plan) remain authoritative. + +We want to graduate REVIEW and PLAN to **authoritative** status — but only on +**verified evidence**, never on a hardcoded flag flip. A diagnostic audit of the +current telemetry established three load-bearing facts: + +1. **The shadow telemetry does not compare against the authoritative path.** + `_run_review_shadow` (`orchestrator.py:1572`) emits only the subagent's own + verdict counts; `_run_plan_shadow` (`orchestrator.py:1730`) emits only the + DAG's own metrics. No agreement/disagreement is computed anywhere. +2. **The telemetry is ephemeral.** Both are `logger.info(...)` strings under the + `Ouroboros.Orchestrator` logger. No structured corpus is persisted to + `.jarvis/` or the ledger. Once a session ends, the data is gone. +3. **"100% semantic alignment" is a category error for PLAN as literally + stated.** The PLAN subagent emits a *DAG*; the legacy path emits a *flat task + list*. They are different data structures by construction and will never be + byte-identical. The meaningful relation is *refinement*: does the DAG cover + 100% of the legacy tasks, acyclically, with disjoint owned paths? + +Therefore we cannot "query existing telemetry." We must **build the evidence +rail first**, accumulate organic production history through a soak, and let an +**event-driven graduation gate** promote each agent autonomously once the +evidence threshold is crossed. The retained legacy paths are repurposed as an +**active graceful-degradation circuit breaker**, not dead weight. + +This is the structurally-correct, zero-shortcut path mandated by the governing +philosophy: *structural repair, not bypasses.* + +--- + +## 2. Three Architectural Corrections (premise fixes) + +These corrections are baked into the design below and are called out here so the +deviation from the original verbal directive is explicit and auditable. + +| # | Naive directive | Correction | Rationale | +|---|-----------------|------------|-----------| +| 1 | "Flip `JARVIS_REVIEW_SUBAGENT_SHADOW` / `JARVIS_PLAN_SUBAGENT_SHADOW` to `false` to graduate." | `_SHADOW=false` currently *disables the observer entirely* and leaves legacy 100% in control. Graduation requires **(a)** new authoritative wiring **and (b)** a new per-agent flag `JARVIS_{REVIEW,PLAN}_SUBAGENT_AUTHORITATIVE` (default `false`) that the gate flips. `_SHADOW` is set `false` only as a redundant follow-on once authoritative. | The shadow flag is not a promotion switch; it is an existence switch. | +| 2 | "Inject a new RAM probe into `subagent_scheduler.py`." | **Reuse `MemoryPressureGate`** (`memory_pressure_gate.py`), which already provides the `psutil → /proc/meminfo → vm_stat` cascade and per-level fanout caps. The scheduler *already consults it* (Slice 5 Arc B, `_consult_memory_gate`, lines 684–756). Unit D **adds a worktree-RAM-budget dimension on top**, it does not duplicate the probe. | Zero-duplication mandate. | +| 3 | "Run a background cron-style evaluator." | The graduation check is **event-driven**: it runs at each op boundary when a new comparison row lands in the store. No separate periodic process. | Consistent with the Gap #4 event-primary campaign; one fewer process to wedge. | + +--- + +## 3. Architecture Overview + +Four units. Build order **D → A → B → C** (D is independent and ships first; A→C +is the staged rail). + +``` + ┌─────────────────────────────────────────────┐ + │ ORCHESTRATOR FSM (11-phase) │ + │ │ + post-PLAN ───────────┤ _run_plan_shadow(ctx) │ + (legacy flat plan + │ │ legacy_flat + shadow_dag │ + shadow DAG present) │ ▼ │ + │ ┌──────────────┐ ┌──────────────────┐ │ + │ │ Unit B │──▶│ Unit A │ │ + post-VALIDATE ────────┤ │ Evaluator │ │ Telemetry Store │ │ + (shadow verdict; │ │ (pure, det.) │ │ SQLite + async │ │ + legacy outcome │ └──────────────┘ │ writer + FIFO │ │ + arrives at GATE) │ ▲ │ prune (1k/agent) │ │ + GATE/terminal ────────┤ │ legacy_outcome └────────┬─────────┘ │ + (legacy review │ └─────────────────────────┘ row landed │ + decision) │ ▼ │ + │ ┌──────────────────┐ │ + │ │ Unit C │ │ + │ │ Graduation Gate │ │ + │ │ (event-driven, │ │ + │ │ 50-soak, .env │ │ + │ │ persist) │ │ + │ └────────┬─────────┘ │ + │ │ promote │ + │ ▼ │ + │ JARVIS_{REVIEW,PLAN}_SUBAGENT │ + │ _AUTHORITATIVE = true │ + └─────────────────────────────────────────────┘ + │ + ┌──────────────────────────────────────────────┴───────────────────┐ + │ Unit D — L3 Memory Governor (subagent_scheduler.py) │ + │ composes MemoryPressureGate; worktree RAM budget; │ + │ CRITICAL pressure ⇒ pre-emptive circuit-breaker trip ⇒ legacy │ + └───────────────────────────────────────────────────────────────────┘ +``` + +**Master invariant:** every new flag defaults to current behavior. With all new +flags off/shadow, the system is **byte-identical** to today. The observer +contract is preserved: nothing in Units A/B may raise into or block the FSM. + +--- + +## 4. Unit A — Async Telemetry Store + +**New module:** `backend/core/ouroboros/governance/shadow_telemetry_store.py` +**DB path:** `.jarvis/shadow_telemetry.db` (gitignored; host-local) +**Master flag:** `JARVIS_SHADOW_TELEMETRY_STORE_ENABLED` (default `true`; off ⇒ +no-op, no file created) + +### 4.1 Async-safety model + +Python's `sqlite3` is blocking; the codebase forbids blocking the event loop. +Design: + +- A single **writer task** drains a **bounded `asyncio.Queue`** (capacity env + `JARVIS_SHADOW_TELEMETRY_QUEUE_MAX`, default 256). All `sqlite3` calls run + inside `asyncio.to_thread(...)` so the loop never blocks. +- Producers (`_run_*_shadow` hooks) call **fire-and-forget** `record_*_nowait()` + — enqueue and return immediately. Queue full ⇒ drop-oldest + increment a + `dropped` counter (bounded memory; telemetry is advisory, never load-bearing). +- **Fail-soft everywhere.** Any sqlite/IO exception is caught, logged once at + WARN, and swallowed. The store can never break the FSM (observer contract). +- Pattern precedent: the episodic-core `note_*_nowait` fire-and-forget synapse + (Slice 134–136) and `state_persistence_daemon.py` async fail-soft writer. + +### 4.2 Schema + +One logical row per `(op_id, agent)`, written in up to two phases (see §4.4): + +```sql +CREATE TABLE IF NOT EXISTS shadow_comparison ( + op_id TEXT NOT NULL, + agent TEXT NOT NULL, -- 'review' | 'plan' + ts REAL NOT NULL, -- time.time() passed in by caller + seq INTEGER NOT NULL, -- monotonic per-agent insert ordinal + legacy_outcome TEXT, -- json; NULL until legacy phase lands + shadow_outcome TEXT, -- json; NULL until shadow phase lands + aligned INTEGER, -- 0/1/NULL(=incomplete) + divergence_reason TEXT, -- NULL when aligned or incomplete + PRIMARY KEY (op_id, agent) +); +CREATE INDEX IF NOT EXISTS idx_agent_seq ON shadow_comparison(agent, seq); +``` + +`seq` is a per-agent monotonic counter (table `agent_seq(agent TEXT PRIMARY KEY, +next INTEGER)`) — the basis for both the FIFO cap and the "last 50 consecutive" +graduation query. It does **not** use `Date.now()`/wall-clock for ordering +(determinism); `ts` is informational only and is supplied by the caller. + +### 4.3 Anti-bloat: rolling FIFO cap (Sovereign enhancement) + +After each insert, the writer task runs an **async self-prune** inside the same +`to_thread` call: + +```sql +DELETE FROM shadow_comparison + WHERE agent = ? + AND seq <= (SELECT MAX(seq) FROM shadow_comparison WHERE agent = ?) + - :cap; +``` + +`cap` = env `JARVIS_SHADOW_TELEMETRY_MAX_ROWS_PER_AGENT` (default **1000**). +Guarantees a microscopic, bounded SSD footprint (≤ ~2000 rows total across both +agents). A `VACUUM` is run opportunistically every `cap` inserts to reclaim +pages. Pruning is part of the writer's normal cycle — never a separate job. + +### 4.4 Public API + +```python +class ShadowTelemetryStore: + def __init__(self, *, db_path: pathlib.Path | None = None, + cap_per_agent: int = 1000) -> None: ... + async def start(self) -> None: ... # spawn writer task (idempotent) + async def aclose(self) -> None: ... # drain + close, fail-soft + + # fire-and-forget producers (never block, never raise) + def record_shadow_nowait(self, *, op_id: str, agent: str, ts: float, + shadow_outcome: dict) -> None: ... + def record_legacy_nowait(self, *, op_id: str, agent: str, ts: float, + legacy_outcome: dict) -> None: ... + + # read side (used by Unit C; runs in to_thread) + async def recent_aligned_streak(self, agent: str) -> int: ... + async def last_n(self, agent: str, n: int) -> list[dict]: ... +``` + +**Two-phase upsert (load-bearing for REVIEW):** the REVIEW shadow verdict is +known at post-VALIDATE, but the legacy authoritative decision is not known until +GATE/terminal. So `record_shadow_nowait` and `record_legacy_nowait` each upsert +their half keyed by `(op_id, agent)`; when **both** halves are present the writer +invokes Unit B to compute `aligned` + `divergence_reason` and patches the row. +PLAN is single-phase (both halves available at the post-PLAN hook) and writes +once with both fields populated. + +--- + +## 5. Unit B — Semantic Evaluator + +**New module:** `backend/core/ouroboros/governance/shadow_evaluator.py` +Pure functions, **zero LLM, zero IO, never raises** (returns a structured +`Alignment(aligned: bool, reason: str)` even on malformed input — malformed ⇒ +`aligned=False, reason="malformed:"`, which is the safe/conservative +default that *blocks* graduation). + +### 5.1 REVIEW evaluator + +```python +def evaluate_review(legacy: dict, shadow: dict) -> Alignment +``` + +Binary-verdict agreement on the **block-vs-allow** decision: + +- `shadow_binary`: subagent aggregate verdict, `reject → BLOCK`; + `approve` / `approve_with_reservations → ALLOW`. (Reservations map to allow — + they are advisory, not blocking, matching shadow semantics today.) +- `legacy_binary`: derived from the authoritative outcome captured at + GATE/terminal — `BLOCK` if the op's resolved risk tier ∈ + {`APPROVAL_REQUIRED`, `BLOCKED`} **or** SemanticGuardian raised a *hard* + finding; else `ALLOW`. +- `aligned = (shadow_binary == legacy_binary)`. Divergence reason records both + sides, e.g. `"shadow=BLOCK legacy=ALLOW"`. + +### 5.2 PLAN evaluator + +```python +def evaluate_plan(legacy_flat: list, shadow_dag: dict) -> Alignment +``` + +Refinement check — **NOT** structural equality. The DAG is *aligned* iff all +three hold: + +1. **Coverage:** the set of files/tasks touched by the DAG's flattened node set + ⊇ the set of tasks in the legacy flat list (100% coverage; the DAG may add + structure but may not *drop* a legacy task). Missing tasks ⇒ + `reason="dropped_tasks:"`. +2. **Acyclicity:** the DAG is a true DAG (Kahn's algorithm / topological sort + succeeds). A cycle ⇒ `reason="cyclical_dag"`. +3. **Disjoint ownership:** parallel-eligible units have disjoint `owned_paths` + (no two concurrent units claim the same file). Overlap ⇒ + `reason="owned_path_overlap:"`. + +Extra structure beyond the legacy tasks does **not** count as misalignment +(refinement is allowed). Only *dropping*, *cycling*, or *overlapping* fails. + +--- + +## 6. Unit C — Auto-Graduation Gate + Graceful Degradation Circuit Breaker + +**New module:** `backend/core/ouroboros/governance/shadow_graduation_gate.py` +**Master flag:** `JARVIS_SHADOW_GRADUATION_GATE_ENABLED` (default `true`) + +### 6.1 Event-driven graduation (50-soak) + +Invoked at each op boundary **after** a comparison row is finalized (both halves +present + `aligned` computed). For the agent whose row just landed: + +``` +streak = store.recent_aligned_streak(agent) # consecutive aligned, newest-first +if streak >= JARVIS_SHADOW_GRADUATION_THRESHOLD (default 50): + promote(agent) +``` + +`recent_aligned_streak` counts consecutive `aligned=1` rows from the highest +`seq` downward, stopping at the first `aligned=0` (a single divergence **resets +the streak to 0** — "50 *consecutive*"). Incomplete rows (one half missing) are +skipped, not counted as breaks, until they finalize. + +### 6.2 Promotion (autonomous + persistent) + +`promote(agent)`: + +1. Set process env `JARVIS_{REVIEW|PLAN}_SUBAGENT_AUTHORITATIVE=true`. +2. Persist durably via **existing** `graduation_orchestrator.persist_flag_to_env` + (`graduation_orchestrator.py:106`) — the bounded, credential-safe `.env` + writer that refuses credential-shaped keys and never raises. Our flags + (`..._AUTHORITATIVE`) contain no credential marker substring, so they pass. +3. Persist `JARVIS_{REVIEW|PLAN}_SUBAGENT_SHADOW=false` as a redundant follow-on + (shadow is subsumed by authoritative). +4. Emit a `flag_registered`-style audit event + a one-line structured log + `[GRADUATION] agent=plan streak=50 -> authoritative`. +5. **Idempotent:** if already authoritative, no-op. + +Operator `=0`/explicit-off precedence is honored exactly as Slice 136: if the +operator has explicitly set the flag, the gate does not override it. + +### 6.3 Authoritative wiring + +Once `..._AUTHORITATIVE=true`: + +- **PLAN:** `ctx.execution_graph` produced by the subagent becomes the + authoritative input consumed by `_materialize_execution_graph_candidate` + (`orchestrator.py:10400`) and submitted to `SubagentScheduler` — instead of + being stashed-and-ignored. The legacy flat plan is computed but held as the + fallback baseline. +- **REVIEW:** the subagent verdict gains the authority to **raise** the risk tier + (REJECT ⇒ force `APPROVAL_REQUIRED`) before GATE. It composes with — never + weakens — SemanticGuardian/Iron Gate (strictest-wins, same discipline as the + risk-tier floor). REVIEW can only *add* friction, never remove it. + +### 6.4 Graceful Degradation Circuit Breaker + +Once authoritative, the subagent call is wrapped in a breaker. **Trip +conditions:** + +| Condition | Detection | Action | +|-----------|-----------|--------| +| Cyclical DAG | Unit B acyclicity check fails on the authoritative DAG | trip → legacy flat plan | +| Unparsable / empty DAG | DAG materialization raises or yields 0 units | trip → legacy flat plan | +| Subagent LLM timeout | dispatch exceeds `JARVIS_PLAN_SUBAGENT_TIMEOUT_S` | trip → legacy flat plan | +| **CRITICAL memory pressure** (Sovereign enhancement) | `MemoryPressureGate.pressure() == CRITICAL` **before** dispatch | **pre-emptive trip → legacy flat plan; do NOT spin up the subagent or worktree fan-out at all** | + +On any trip: + +1. Emit `AGENT_DEGRADATION` to `StreamEventBroker.publish(...)` + (`ide_observability_stream.py:1894`). New constant + `EVENT_TYPE_AGENT_DEGRADATION = "agent_degradation"` added beside the existing + `EVENT_TYPE_MEMORY_PRESSURE_CHANGED`. Payload: `{agent, op_id, trip_reason, + pressure_level}`. `publish` never raises/blocks. +2. Seamlessly route the operation to the retained legacy generator. **Execution + continuity is mathematically guaranteed** — the legacy path is always present + and is the same code that runs today. +3. Record the trip to the telemetry store as a divergence signal (a trip is + evidence the agent is not yet trustworthy; it counts against future streaks + for observability, but post-graduation it does not auto-demote in v1 — + demotion is out of scope, see §10). + +The CRITICAL trip is the **Unit C/D coupling**: under CRITICAL pressure on the +16GB unified-memory footprint, the authoritative PLAN path does not even attempt +to serialize worktrees to 1 — it routes to the low-memory legacy flat-plan +generator outright, absolutely guaranteeing zero swap-thrash. + +--- + +## 7. Unit D — L3 Memory Governor + +**Edit:** `backend/core/ouroboros/governance/autonomy/subagent_scheduler.py` +**Master flag:** `JARVIS_L3_MEMORY_GOVERNOR_ENABLED` (default `true`; inert until +L3 actually runs a graph) + +### 7.1 What already exists + +`SubagentScheduler` already consults `MemoryPressureGate` (Slice 5 Arc B, +`_consult_memory_gate`, lines 684–756; fan-out clamp at lines 493–507). Today it +clamps `n_requested → n_allowed` using the gate's **per-level fanout caps** +(WARN 8 / HIGH 3 / CRITICAL 1) and defers overflow with zero work loss. + +### 7.2 What Unit D adds (worktree-RAM-budget dimension) + +The per-level caps are free-percentage based; they do not model the *absolute +RAM cost of a worktree*. On a 16GB box, even "HIGH ⇒ 3" can over-commit if each +worktree's process set is heavy. Unit D adds an absolute budget clamp that +composes (strictest-wins) with the existing level clamp: + +```python +avail_mb = MemoryPressureGate probe → available RAM (MB) +budget_mb = JARVIS_L3_WORKTREE_RAM_BUDGET_MB # default 1500 +ram_cap = max(1, floor(avail_mb / budget_mb)) +level_cap = existing FanoutDecision.n_allowed +max_worktrees = min(ram_cap, level_cap, configured_concurrency_limit) +``` + +`max_worktrees` is recomputed **before each scheduling wave** (it is not a +boot-time constant), so it tracks live pressure. The existing +defer-overflow-with-zero-work-loss mechanism is reused for any clamp. + +### 7.3 CRITICAL ⇒ pre-emptive legacy (the Unit C/D bridge) + +When `pressure() == CRITICAL`, Unit D does **not** serialize the L3 scheduler to +1 for the PLAN-authoritative path. Instead it signals the Unit C breaker to trip +**before** any worktree is created, routing generation to the legacy flat-plan +generator (§6.4). Serialize-to-1 remains the behavior only for already-admitted +non-PLAN L3 work at HIGH pressure. This is the explicit Sovereign enhancement: +*the safest response to CRITICAL pressure is to not fan out at all.* + +New env knobs: + +- `JARVIS_L3_WORKTREE_RAM_BUDGET_MB` (default `1500`) — assumed RAM per worktree. +- `JARVIS_L3_MEMORY_GOVERNOR_ENABLED` (default `true`). + +--- + +## 8. Event-Driven Topology (no polling) + +| Trigger | Producer | Consumer | Cadence | +|---------|----------|----------|---------| +| Shadow verdict computed | `_run_review_shadow` / `_run_plan_shadow` | `store.record_shadow_nowait` | per op, post-phase | +| Legacy outcome resolved | GATE/terminal hook (REVIEW); post-PLAN (PLAN) | `store.record_legacy_nowait` | per op | +| Comparison row finalized | writer task (both halves present) | Unit B → Unit C gate check | per op, in writer | +| Streak ≥ threshold | Unit C | `persist_flag_to_env` + promote | once per agent (idempotent) | +| Breaker trip | Unit C breaker | `StreamEventBroker.publish(AGENT_DEGRADATION)` | per trip | +| Memory clamp | Unit D | existing SSE governor telemetry | per scheduling wave | + +No `Date.now()`-driven loop, no cron. The graduation decision is a pure function +of accumulated rows, checked exactly when a new row could change the answer. + +--- + +## 9. New Environment Flags (complete list) + +| Flag | Default | Unit | Meaning | +|------|---------|------|---------| +| `JARVIS_SHADOW_TELEMETRY_STORE_ENABLED` | `true` | A | Enable the SQLite store (off ⇒ no file, no-op) | +| `JARVIS_SHADOW_TELEMETRY_QUEUE_MAX` | `256` | A | Bounded write-queue capacity | +| `JARVIS_SHADOW_TELEMETRY_MAX_ROWS_PER_AGENT` | `1000` | A | Rolling FIFO cap | +| `JARVIS_SHADOW_GRADUATION_GATE_ENABLED` | `true` | C | Enable event-driven graduation | +| `JARVIS_SHADOW_GRADUATION_THRESHOLD` | `50` | C | Consecutive-aligned ops to promote | +| `JARVIS_REVIEW_SUBAGENT_AUTHORITATIVE` | `false` | C | REVIEW verdict gates (set by gate) | +| `JARVIS_PLAN_SUBAGENT_AUTHORITATIVE` | `false` | C | PLAN DAG authoritative (set by gate) | +| `JARVIS_L3_MEMORY_GOVERNOR_ENABLED` | `true` | D | Worktree-RAM-budget clamp | +| `JARVIS_L3_WORKTREE_RAM_BUDGET_MB` | `1500` | D | Assumed RAM per worktree | + +`JARVIS_{REVIEW,PLAN}_SUBAGENT_SHADOW` (existing, default `true`) are flipped to +`false` by the gate post-promotion. All defaults preserve today's behavior: the +two `_AUTHORITATIVE` flags are `false`, so the rail observes and records but +**changes no decision** until evidence promotes it. + +--- + +## 10. Out of Scope (YAGNI) + +- **Auto-demotion** after post-graduation regression. v1 trips the breaker to + legacy per-op but does not flip `_AUTHORITATIVE` back to `false` automatically. + (Visual VERIFY's auto-demotion is the precedent to copy later; deferred.) +- **LLM enrichment of the PLAN DAG** (Step-2 import-graph analysis). The + evaluator only checks the deterministic Step-1 DAG. +- **A web dashboard for the corpus.** SSE `AGENT_DEGRADATION` + structured logs + are sufficient for v1; the existing IDE observability surfaces consume them. +- **Cross-agent graduation coupling.** REVIEW and PLAN graduate independently. + +--- + +## 11. Testing Strategy + +- **Unit A:** in-memory `sqlite3` (`:memory:`) tests — two-phase upsert, + alignment patch, FIFO prune at cap, drop-oldest under queue saturation, + fail-soft on a poisoned write (no raise into caller). Writer-task lifecycle + (start/aclose idempotent). +- **Unit B:** table-driven pure-function tests — REVIEW binary mapping incl. + reservations→allow; PLAN coverage/acyclicity/disjoint with a cyclical DAG, a + dropped-task DAG, an owned-path-overlap DAG, and a valid refinement. Malformed + input ⇒ `aligned=False` (conservative). +- **Unit C:** streak counting (49 aligned ⇒ no promote; 50 ⇒ promote; one + divergence resets); `persist_flag_to_env` called with correct args (mocked); + idempotent re-promotion; operator-off precedence; breaker trip table for all + four conditions incl. CRITICAL-pre-emptive; `AGENT_DEGRADATION` published. +- **Unit D:** `max_worktrees` math across OK/WARN/HIGH/CRITICAL with injected + `MemoryPressureGate` probe; CRITICAL ⇒ breaker-trip signal not serialize-to-1; + governor-disabled ⇒ pass-through (byte-identical to Slice 5 Arc B today). +- **Regression spine:** with all new flags at default and `_AUTHORITATIVE=false`, + prove the FSM is byte-identical (no decision changed) — the OFF-is-inert + guarantee. + +--- + +## 12. Build Order & Plan Decomposition + +This spec will likely become **two implementation plans**: + +1. **Plan 1 — L3 Memory Governor (Unit D).** Independent, lowest-risk, highest + immediate value. Ships first. +2. **Plan 2 — The Evidence Rail (Units A → B → C).** Staged: store, then + evaluator, then gate+breaker. C depends on A+B and on the Unit D breaker + signal for the CRITICAL trip. + +--- + +## 13. Authority & Mandate Alignment + +- **§5 Intelligence-driven routing:** graduation is evidence-driven, not a + hardcoded table. +- **§6 Threshold-triggered neuroplasticity:** the 50-soak gate is the literal + "detect → verify → graduate" loop. +- **§7 Absolute observability:** every promotion and every breaker trip emits a + structured, durable, SSE-visible signal. +- **Zero-shortcut mandate:** we build the evidence rail rather than flipping a + flag on faith; we reuse `MemoryPressureGate` rather than duplicating a probe; + we retain legacy as an active circuit breaker rather than deleting the only + rollback baseline. + +--- + +## 14. Naming Disambiguation — two unrelated "shadows" (operator hygiene) + +A reconnaissance pass (2026-06-14) confirmed that the autonomous Ouroboros loop +independently shipped a *resilience* feature also named "Shadow Mode" (Slices +252/253 on `main`). **It is a different system in a different domain.** This +section exists so an operator never conflates the two when reading flags or +events. They collide only on the word "shadow"; there is **zero functional +overlap** (verified: the loop never touches `orchestrator.py`'s +`_run_{plan,review}_shadow` hooks nor `governed_loop_service.py`). + +| Axis | **Resilience "Shadow Mode"** (loop, Slice 252/253) | **Subagent "Shadow Rail"** (this spec) | +|------|---------------------------------------------------|----------------------------------------| +| Flag | `JARVIS_RESILIENCE_SHADOW_MODE` | `JARVIS_{PLAN,REVIEW}_SUBAGENT_SHADOW` | +| Meaning of "shadow" | Trap a dangerous resilience **action** (process kill / load-shed / restart) and log what it *would* have done instead of executing it | Run the new REVIEW/PLAN **subagent** silently alongside the legacy path and record the verdict comparison | +| Instruments | `cybernetic_reanimation.py` + `unified_supervisor.py` self-healing organs | `orchestrator._run_{plan,review}_shadow` hooks | +| Persistence | Ephemeral SSE (`EVENT_TYPE_SHADOW_ACTION_TRAPPED`) | Durable SQLite ledger (`.jarvis/shadow_telemetry.db`) | +| "Endorse"/promote | `/endorse ` runs **one trapped action once**; promotes nothing, never reads/writes `_AUTHORITATIVE` | 50-soak gate flips `_AUTHORITATIVE`→true **permanently** | +| Our SSE event | (theirs: `shadow_action_trapped`) | `agent_degradation` (breaker trip) | + +**Merge note:** when this branch later rebases onto `main`, expect *mechanical* +adjacent-line conflicts in `ide_observability_stream.py` (both add `EVENT_TYPE_*` +constants + frozenset entries) and `serpent_flow.py` — resolve by **keeping both +sets** of additions. There is no semantic interaction to reconcile. diff --git a/tests/governance/autonomy/test_l3_memory_governor.py b/tests/governance/autonomy/test_l3_memory_governor.py new file mode 100644 index 0000000000..01df8f4396 --- /dev/null +++ b/tests/governance/autonomy/test_l3_memory_governor.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from backend.core.ouroboros.governance.autonomy.l3_memory_governor import ( + GovernorDecision, + compute_worktree_cap, + governor_enabled, +) + + +def test_ram_is_the_binding_constraint(): + # 4500MB available, 1500MB/worktree -> ram_cap=3; level_cap=8 -> allow 3 + d = compute_worktree_cap( + requested=8, avail_mb=4500.0, budget_mb=1500, level_cap=8, + ) + assert isinstance(d, GovernorDecision) + assert d.ram_cap == 3 + assert d.n_allowed == 3 + assert d.disposition == "clamp" + + +def test_level_cap_is_the_binding_constraint(): + # 12000MB -> ram_cap=8; but level_cap=3 (HIGH) -> allow 3, strictest wins + d = compute_worktree_cap( + requested=8, avail_mb=12000.0, budget_mb=1500, level_cap=3, + ) + assert d.ram_cap == 8 + assert d.n_allowed == 3 + assert d.disposition == "clamp" + + +def test_floor_never_below_one(): + # Only 800MB available, 1500MB budget -> floor would be 0; clamp to >=1 + d = compute_worktree_cap( + requested=4, avail_mb=800.0, budget_mb=1500, level_cap=8, + ) + assert d.ram_cap == 1 + assert d.n_allowed == 1 + + +def test_no_clamp_when_everything_fits(): + d = compute_worktree_cap( + requested=2, avail_mb=16000.0, budget_mb=1500, level_cap=8, + ) + assert d.n_allowed == 2 + assert d.disposition == "allow" + + +def test_requested_zero_grants_zero_and_does_not_clamp(): + # A degenerate request (caller asked for nothing) yields 0 and is NOT + # reported as a clamp, since nothing was withheld. + d = compute_worktree_cap( + requested=0, avail_mb=16000.0, budget_mb=1500, level_cap=8, + ) + assert d.n_allowed == 0 + assert d.disposition == "allow" + + +def test_nonpositive_avail_fails_safe_to_one(): + # A garbage probe reading (<= 0) must not yield 0/negative worktrees; + # the floor guard clamps ram_cap to 1 (conservative fail-safe). + d = compute_worktree_cap( + requested=4, avail_mb=0.0, budget_mb=1500, level_cap=8, + ) + assert d.ram_cap == 1 + assert d.n_allowed == 1 + assert d.disposition == "clamp" + + d_neg = compute_worktree_cap( + requested=4, avail_mb=-500.0, budget_mb=1500, level_cap=8, + ) + assert d_neg.ram_cap == 1 + assert d_neg.n_allowed == 1 + + +def test_no_clamp_branch_populates_all_fields(): + # 16000/1500 = 10 -> ram_cap=10; level_cap=8; requested=2 -> allow 2. + d = compute_worktree_cap( + requested=2, avail_mb=16000.0, budget_mb=1500, level_cap=8, + ) + assert d.ram_cap == 10 + assert d.level_cap == 8 + assert d.requested == 2 + assert d.n_allowed == 2 + assert d.disposition == "allow" + + +def test_governor_enabled_default_true_and_explicit_false(monkeypatch): + monkeypatch.delenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", raising=False) + assert governor_enabled() is True + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "false") + assert governor_enabled() is False + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + assert governor_enabled() is True diff --git a/tests/governance/autonomy/test_scheduler_memory_governor.py b/tests/governance/autonomy/test_scheduler_memory_governor.py new file mode 100644 index 0000000000..7e3c1de672 --- /dev/null +++ b/tests/governance/autonomy/test_scheduler_memory_governor.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from backend.core.ouroboros.governance.autonomy import subagent_scheduler as ss + + +class _FakeProbe: + def __init__(self, available_mb: float): + self.available_bytes = int(available_mb * 1024 * 1024) + self.total_bytes = 16 * 1024 * 1024 * 1024 + self.ok = True + + +class _FakeGate: + def __init__(self, available_mb: float): + self._p = _FakeProbe(available_mb) + + def probe(self): + return self._p + + +def _make_scheduler(): + # Construct with minimal stubs; only _consult_memory_governor is exercised. + return ss.SubagentScheduler.__new__(ss.SubagentScheduler) + + +def test_governor_clamps_on_low_ram(monkeypatch): + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + monkeypatch.setenv("JARVIS_L3_WORKTREE_RAM_BUDGET_MB", "1500") + monkeypatch.setattr( + ss, "get_default_gate", lambda: _FakeGate(available_mb=4500.0), + ) + sched = _make_scheduler() + decision = sched._consult_memory_governor( + 8, graph_id="g1", level_cap=8, + ) + assert decision is not None + assert decision.n_allowed == 3 + assert decision.disposition == "clamp" + + +def test_governor_disabled_returns_none(monkeypatch): + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "false") + sched = _make_scheduler() + assert sched._consult_memory_governor(8, graph_id="g1", level_cap=8) is None + + +def test_governor_probe_failure_is_non_fatal(monkeypatch): + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + + def _boom(): + raise RuntimeError("probe exploded") + + monkeypatch.setattr(ss, "get_default_gate", _boom) + sched = _make_scheduler() + # Must swallow and return None — scheduler never breaks on probe failure. + assert sched._consult_memory_governor(8, graph_id="g1", level_cap=8) is None + + +def test_run_graph_clamp_composition(monkeypatch): + """The governor clamp composes after the fan-out clamp: selected is + truncated to the governor's n_allowed and overflow is deferred. + + This simulates the composition `_run_graph` performs rather than + driving the async `_run_graph` end-to-end; it pins the arithmetic + contract the wiring relies on. + """ + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "true") + monkeypatch.setenv("JARVIS_L3_WORKTREE_RAM_BUDGET_MB", "1500") + monkeypatch.setattr( + ss, "get_default_gate", lambda: _FakeGate(available_mb=3000.0), + ) + sched = _make_scheduler() + selected = ["u1", "u2", "u3", "u4"] + deferred = [] + gov = sched._consult_memory_governor( + len(selected), graph_id="g1", level_cap=len(selected), + ) + assert gov.n_allowed == 2 # 3000/1500 = 2 + # Simulate the composition the _run_graph edit performs: + overflow = list(selected[gov.n_allowed:]) + selected = list(selected[:gov.n_allowed]) + deferred = sorted(deferred + overflow) + assert selected == ["u1", "u2"] + assert deferred == ["u3", "u4"] + + +def test_disabled_governor_is_byte_identical_passthrough(monkeypatch): + """With the master flag off, _consult_memory_governor returns None + and the _run_graph composition leaves `selected` untouched.""" + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "false") + sched = _make_scheduler() + selected = ["u1", "u2", "u3"] + gov = sched._consult_memory_governor( + len(selected), graph_id="g1", level_cap=len(selected), + ) + assert gov is None + # Composition guard: None -> no truncation. + if gov is not None and gov.n_allowed < len(selected): + selected = selected[:gov.n_allowed] + assert selected == ["u1", "u2", "u3"] diff --git a/tests/governance/autonomy/test_subagent_scheduler.py b/tests/governance/autonomy/test_subagent_scheduler.py index 20404fef46..009e4c4ccf 100644 --- a/tests/governance/autonomy/test_subagent_scheduler.py +++ b/tests/governance/autonomy/test_subagent_scheduler.py @@ -6,6 +6,16 @@ import pytest +@pytest.fixture(autouse=True) +def _disable_l3_memory_governor(monkeypatch): + # These tests exercise scheduling/concurrency LOGIC, not the L3 RAM + # governor. The governor (default-on) would otherwise clamp parallelism + # by live host RAM, making concurrency assertions host-dependent/flaky. + # The governor's own behavior is covered by + # tests/governance/autonomy/test_scheduler_memory_governor.py. + monkeypatch.setenv("JARVIS_L3_MEMORY_GOVERNOR_ENABLED", "false") + + def _make_graph( *, graph_id="graph-scheduler", diff --git a/tests/governance/test_shadow_evaluator.py b/tests/governance/test_shadow_evaluator.py new file mode 100644 index 0000000000..616786ad7c --- /dev/null +++ b/tests/governance/test_shadow_evaluator.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +from backend.core.ouroboros.governance.shadow_evaluator import ( + Alignment, + evaluate_review, +) + + +def test_review_agree_allow(): + legacy = {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": False} + shadow = {"aggregate": "approve"} + a = evaluate_review(legacy, shadow) + assert isinstance(a, Alignment) + assert a.aligned is True + + +def test_review_reservations_map_to_allow(): + legacy = {"risk_tier": "NOTIFY_APPLY", "semantic_guard_hard": False} + shadow = {"aggregate": "approve_with_reservations"} + assert evaluate_review(legacy, shadow).aligned is True + + +def test_review_disagree_shadow_blocks_legacy_allows(): + legacy = {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": False} + shadow = {"aggregate": "reject"} + a = evaluate_review(legacy, shadow) + assert a.aligned is False + assert a.reason == "shadow=BLOCK legacy=ALLOW" + + +def test_review_agree_block_via_hard_finding(): + legacy = {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": True} + shadow = {"aggregate": "reject"} + assert evaluate_review(legacy, shadow).aligned is True + + +def test_review_agree_block_via_approval_required(): + legacy = {"risk_tier": "APPROVAL_REQUIRED", "semantic_guard_hard": False} + shadow = {"aggregate": "reject"} + assert evaluate_review(legacy, shadow).aligned is True + + +def test_review_malformed_is_conservative_block(): + a = evaluate_review({}, {}) + assert a.aligned is False + assert a.reason.startswith("malformed:") + + +from backend.core.ouroboros.governance.shadow_evaluator import evaluate_plan + + +# DAG shape: {"units": [{"id","owned_paths":[...],"deps":[...]}], } +def test_plan_valid_refinement_aligned(): + legacy = ["a.py", "b.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}, + ]} + assert evaluate_plan(legacy, dag).aligned is True + + +def test_plan_dropped_task_misaligned(): + legacy = ["a.py", "b.py", "c.py"] + dag = {"units": [{"id": "u1", "owned_paths": ["a.py", "b.py"], "deps": []}]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason.startswith("dropped_tasks:") + assert "c.py" in a.reason + + +def test_plan_cyclical_misaligned(): + legacy = ["a.py", "b.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u2"]}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}, + ]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason == "cyclical_dag" + + +def test_plan_owned_path_overlap_misaligned(): + legacy = ["a.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + {"id": "u2", "owned_paths": ["a.py"], "deps": []}, + ]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason.startswith("owned_path_overlap:") + + +def test_plan_extra_structure_is_allowed(): + # DAG covers legacy AND adds a helper file -> still aligned (refinement). + legacy = ["a.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + {"id": "u2", "owned_paths": ["helper.py"], "deps": ["u1"]}, + ]} + assert evaluate_plan(legacy, dag).aligned is True + + +def test_plan_malformed_is_conservative_block(): + a = evaluate_plan(["a.py"], {"units": "not-a-list"}) + assert a.aligned is False + assert a.reason.startswith("malformed:") + + +def test_plan_self_loop_is_cyclical(): + # A unit depending on itself is a cycle. + legacy = ["a.py"] + dag = {"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u1"]}, + ]} + a = evaluate_plan(legacy, dag) + assert a.aligned is False + assert a.reason == "cyclical_dag" + + +def test_plan_empty_dag_and_empty_legacy_is_aligned(): + # Zero-element boundary: nothing to cover, nothing to drop. + assert evaluate_plan([], {"units": []}).aligned is True diff --git a/tests/governance/test_shadow_graduation_gate.py b/tests/governance/test_shadow_graduation_gate.py new file mode 100644 index 0000000000..9523b35b55 --- /dev/null +++ b/tests/governance/test_shadow_graduation_gate.py @@ -0,0 +1,198 @@ +"""Unit C — AGENT_DEGRADATION SSE event type registration. + +Step 0 discovery (recorded here for audit): + - EVENT_TYPE_MEMORY_PRESSURE_CHANGED is defined at line 185 of + ide_observability_stream.py alongside the other EVENT_TYPE_* constants. + - publish() validates at line 1907 via: + if event_type not in _VALID_EVENT_TYPES: + return None + _VALID_EVENT_TYPES is a module-level frozenset (line 1351) that explicitly + enumerates every accepted event type. Adding the constant to the frozenset + is the ONLY way to make publish() accept it — the set is NOT auto-built + from module globals. + - Existing publish_* helpers are module-level functions. The new helper + follows the same pattern as publish_memory_fanout_decision_event. +""" +from __future__ import annotations + +from backend.core.ouroboros.governance import ide_observability_stream as ios + + +def test_agent_degradation_event_type_registered(): + assert ios.EVENT_TYPE_AGENT_DEGRADATION == "agent_degradation" + # _VALID_EVENT_TYPES is the frozenset publish() checks against (line 1907). + # Adding the constant here is the correct — and ONLY — way to make + # publish() accept this event type without silently dropping it. + assert "agent_degradation" in ios._VALID_EVENT_TYPES # noqa: SLF001 + + +import pytest + +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + ShadowGraduationGate, +) + + +class _FakeStore: + def __init__(self, streak): + self._streak = streak + + async def recent_aligned_streak(self, agent): + return self._streak + + +@pytest.mark.asyncio +async def test_no_promote_below_threshold(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + gate = ShadowGraduationGate(store=_FakeStore(streak=49)) + promoted = await gate.maybe_promote("plan") + assert promoted is False + assert persisted == [] + + +@pytest.mark.asyncio +async def test_promote_at_threshold_persists_flags(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_THRESHOLD", "50") + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + gate = ShadowGraduationGate(store=_FakeStore(streak=50)) + promoted = await gate.maybe_promote("plan") + assert promoted is True + assert ("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") in persisted + assert ("JARVIS_PLAN_SUBAGENT_SHADOW", "false") in persisted + + +@pytest.mark.asyncio +async def test_promote_idempotent(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") + gate = ShadowGraduationGate(store=_FakeStore(streak=50)) + promoted = await gate.maybe_promote("plan") + assert promoted is False # already authoritative -> no-op + assert persisted == [] + + +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + PlanBreaker, +) + + +def test_breaker_trips_on_cyclical_dag(): + b = PlanBreaker(pressure_fn=lambda: "ok") + decision = b.should_use_legacy(dag={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u2"]}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}, + ]}) + assert decision.trip is True + assert decision.reason == "cyclical_dag" + + +def test_breaker_trips_on_empty_dag(): + b = PlanBreaker(pressure_fn=lambda: "ok") + decision = b.should_use_legacy(dag={"units": []}) + assert decision.trip is True + assert decision.reason == "unparsable_or_empty_dag" + + +def test_breaker_critical_pressure_preempts_before_dag(): + # CRITICAL pressure trips BEFORE inspecting the DAG (pre-emptive). + b = PlanBreaker(pressure_fn=lambda: "critical") + decision = b.should_use_legacy(dag=None) + assert decision.trip is True + assert decision.reason == "critical_memory_pressure" + assert decision.pressure_level == "critical" + + +def test_breaker_passes_valid_dag_under_ok_pressure(): + b = PlanBreaker(pressure_fn=lambda: "ok") + decision = b.should_use_legacy(dag={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}, + ]}) + assert decision.trip is False + + +@pytest.mark.asyncio +async def test_gate_disabled_is_noop(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_GATE_ENABLED", "false") + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + gate = ShadowGraduationGate(store=_FakeStore(streak=999)) + # Even with a streak far above threshold, a disabled gate never promotes. + assert await gate.maybe_promote("plan") is False + assert persisted == [] + + +class _RaisingStore: + async def recent_aligned_streak(self, agent): + raise RuntimeError("store exploded") + + +@pytest.mark.asyncio +async def test_store_read_exception_is_fail_soft(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + gate = ShadowGraduationGate(store=_RaisingStore()) + # A store that raises must NOT break the FSM — gate returns False, no persist. + assert await gate.maybe_promote("plan") is False + assert persisted == [] + + +@pytest.mark.asyncio +async def test_unknown_agent_is_noop(monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + gate = ShadowGraduationGate(store=_FakeStore(streak=999)) + assert await gate.maybe_promote("nonexistent_agent") is False + assert persisted == [] + + +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + build_rail_evaluator, +) + + +def test_rail_evaluator_routes_by_agent(): + ev = build_rail_evaluator() + # review path + aligned, _ = ev("review", + {"risk_tier": "SAFE_AUTO", "semantic_guard_hard": False}, + {"aggregate": "approve"}) + assert aligned is True + # plan path: legacy carries {"flat": [...]} + aligned, _ = ev("plan", + {"flat": ["a.py"]}, + {"units": [{"id": "u1", "owned_paths": ["a.py"], + "deps": []}]}) + assert aligned is True + # unknown agent -> conservative not-aligned + aligned, reason = ev("bogus", {}, {}) + assert aligned is False diff --git a/tests/governance/test_shadow_rail_integration.py b/tests/governance/test_shadow_rail_integration.py new file mode 100644 index 0000000000..789443d422 --- /dev/null +++ b/tests/governance/test_shadow_rail_integration.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import pytest + +from backend.core.ouroboros.governance.shadow_telemetry_store import ( + ShadowTelemetryStore, +) +from backend.core.ouroboros.governance.shadow_graduation_gate import ( + ShadowGraduationGate, build_rail_evaluator, +) + + +@pytest.mark.asyncio +async def test_fifty_aligned_ops_graduate_plan(tmp_path, monkeypatch): + persisted = [] + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: persisted.append((flag, value)) or True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_THRESHOLD", "50") + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", evaluator=build_rail_evaluator()) + await store.start() + gate = ShadowGraduationGate(store=store) + + for i in range(50): + store.record_legacy_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + legacy_outcome={"flat": ["a.py"]}) + store.record_shadow_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + shadow_outcome={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}]}) + await store.drain() + + promoted = await gate.maybe_promote("plan") + assert promoted is True + assert ("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", "true") in persisted + await store.aclose() + + +@pytest.mark.asyncio +async def test_one_divergence_blocks_graduation(tmp_path, monkeypatch): + monkeypatch.setattr( + "backend.core.ouroboros.governance.shadow_graduation_gate." + "persist_flag_to_env", + lambda flag, value, **kw: True, + ) + monkeypatch.setenv("JARVIS_SHADOW_GRADUATION_THRESHOLD", "50") + monkeypatch.delenv("JARVIS_PLAN_SUBAGENT_AUTHORITATIVE", raising=False) + + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", evaluator=build_rail_evaluator()) + await store.start() + gate = ShadowGraduationGate(store=store) + + for i in range(49): + store.record_legacy_nowait( + op_id=f"ok{i}", agent="plan", ts=float(i), + legacy_outcome={"flat": ["a.py"]}) + store.record_shadow_nowait( + op_id=f"ok{i}", agent="plan", ts=float(i), + shadow_outcome={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": []}]}) + # one cyclical (misaligned) op as the newest + store.record_legacy_nowait( + op_id="bad", agent="plan", ts=99.0, legacy_outcome={"flat": ["a.py"]}) + store.record_shadow_nowait( + op_id="bad", agent="plan", ts=99.0, shadow_outcome={"units": [ + {"id": "u1", "owned_paths": ["a.py"], "deps": ["u2"]}, + {"id": "u2", "owned_paths": ["b.py"], "deps": ["u1"]}]}) + await store.drain() + + assert await store.recent_aligned_streak("plan") == 0 + assert await gate.maybe_promote("plan") is False + await store.aclose() diff --git a/tests/governance/test_shadow_telemetry_store.py b/tests/governance/test_shadow_telemetry_store.py new file mode 100644 index 0000000000..25380cc3d4 --- /dev/null +++ b/tests/governance/test_shadow_telemetry_store.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import pytest + +from backend.core.ouroboros.governance.shadow_telemetry_store import ( + ShadowTelemetryStore, +) + + +@pytest.mark.asyncio +async def test_start_and_close_idempotent(tmp_path): + store = ShadowTelemetryStore(db_path=tmp_path / "t.db") + await store.start() + await store.start() # idempotent + await store.aclose() + await store.aclose() # idempotent + + +@pytest.mark.asyncio +async def test_plan_single_phase_write_computes_alignment(tmp_path): + aligned_calls = [] + + def fake_eval(agent, legacy, shadow): + aligned_calls.append(agent) + return (True, "") + + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", evaluator=fake_eval, + ) + await store.start() + store.record_legacy_nowait( + op_id="op1", agent="plan", ts=1.0, legacy_outcome={"flat": ["a.py"]}, + ) + store.record_shadow_nowait( + op_id="op1", agent="plan", ts=1.0, shadow_outcome={"units": []}, + ) + await store.drain() # test helper: await the queue empty + rows = await store.last_n("plan", 5) + assert len(rows) == 1 + assert rows[0]["aligned"] == 1 + assert aligned_calls == ["plan"] + await store.aclose() + + +@pytest.mark.asyncio +async def test_fifo_cap_prunes_oldest(tmp_path): + store = ShadowTelemetryStore( + db_path=tmp_path / "t.db", + evaluator=lambda a, l, s: (True, ""), + cap_per_agent=5, + ) + await store.start() + for i in range(12): + store.record_legacy_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + legacy_outcome={"i": i}) + store.record_shadow_nowait( + op_id=f"op{i}", agent="plan", ts=float(i), + shadow_outcome={"i": i}) + await store.drain() + rows = await store.last_n("plan", 100) + # cap=5 -> only the 5 highest seq survive + assert len(rows) == 5 + seqs = [r["seq"] for r in rows] + assert seqs == sorted(seqs, reverse=True) + assert min(seqs) >= 8 # oldest (op0..op6) pruned + await store.aclose() + + +@pytest.mark.asyncio +async def test_streak_resets_on_divergence(tmp_path): + # evaluator: align unless shadow says {"bad": True} + def ev(agent, legacy, shadow): + return (not shadow.get("bad", False), "div" if shadow.get("bad") else "") + + store = ShadowTelemetryStore(db_path=tmp_path / "t.db", evaluator=ev) + await store.start() + + async def one(op, bad): + store.record_legacy_nowait( + op_id=op, agent="review", ts=0.0, legacy_outcome={}) + store.record_shadow_nowait( + op_id=op, agent="review", ts=0.0, shadow_outcome={"bad": bad}) + + for i in range(3): + await one(f"a{i}", False) + await one("bad1", True) + for i in range(2): + await one(f"b{i}", False) + await store.drain() + # newest-first: b1,b0 aligned (2), then bad1 breaks -> streak == 2 + assert await store.recent_aligned_streak("review") == 2 + await store.aclose()