test(e2e): expose execution coverage matrix

.github/workflows/e2e.yaml

@@ -2698,6 +2698,9 @@ jobs:
       CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
       E2E_JOB: "1"
       INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }}
+      E2E_AGENT_RUNTIME: "openclaw"
+      E2E_OBSERVABLE_OUTCOME: "Exact staging image boots and completes the full E2E scenario"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference"
     steps:
       - name: Checkout trusted Launchable lane
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -3221,6 +3224,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "openshell-gateway-auth-contract"
+      E2E_AGENT_RUNTIME: "none"
+      E2E_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract
       NEMOCLAW_RUN_LIVE_E2E: "1"
       NEMOCLAW_NON_INTERACTIVE: "1"
@@ -3299,9 +3305,21 @@ jobs:
       fail-fast: false
       matrix:
         agent: [openclaw, hermes, deepagents]
+        include:
+          - agent: openclaw
+            agent_runtime: openclaw
+            coverage_variant: openclaw
+          - agent: hermes
+            agent_runtime: hermes
+            coverage_variant: hermes
+          - agent: deepagents
+            agent_runtime: langchain-deepagents-code
+            coverage_variant: deepagents
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "mcp-bridge"
+      E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_E2E_SHARD: ${{ matrix.agent }}
@@ -3626,6 +3644,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "openshell-credential-generation-window"
+      E2E_AGENT_RUNTIME: "openclaw"
+      E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_OPENSHELL_CHANNEL: stable
@@ -3784,9 +3805,21 @@ jobs:
       fail-fast: false
       matrix:
         agent: [openclaw, hermes, deepagents]
+        include:
+          - agent: openclaw
+            agent_runtime: openclaw
+            coverage_variant: openclaw
+          - agent: hermes
+            agent_runtime: hermes
+            coverage_variant: hermes
+          - agent: deepagents
+            agent_runtime: langchain-deepagents-code
+            coverage_variant: deepagents
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "mcp-bridge-dev"
+      E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_E2E_SHARD: ${{ matrix.agent }}
@@ -3985,13 +4018,19 @@ jobs:
           - platform: linux/amd64
             runner: ubuntu-24.04
             shard: linux-amd64
+            environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup"
+            coverage_variant: linux-amd64
           - platform: linux/arm64
             runner: ubuntu-24.04-arm
             shard: linux-arm64
+            environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup"
+            coverage_variant: linux-arm64
     env:
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}
       E2E_JOB: "1"
       E2E_TARGET_ID: "managed-image-multiarch-startup"
+      E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code"
+      E2E_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture"
       RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json
       NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
       NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }}
@@ -4415,6 +4454,10 @@ jobs:
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification
       E2E_JOB: "1"
       E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification"
+      E2E_AGENT_RUNTIME: "unresolved"
+      E2E_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA DGX Spark GB10; local llama.cpp inference"
+      E2E_UNRESOLVED_REASON: "The protected plan can enable or skip its OpenClaw subqualification"
       NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
       NEMOCLAW_E2E_SHARD: linux-arm64-gpu-dgx-spark-gb10
       NEMOCLAW_LLAMA_CPP_QUALIFICATION_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
@@ -4606,6 +4649,9 @@ jobs:
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime
       E2E_JOB: "1"
       E2E_TARGET_ID: "managed-image-protected-runtime"
+      E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code"
+      E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services"
       E2E_WORKLOAD_SOURCE: "managed-image"
       RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
@@ -4884,6 +4930,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "hermes-e2e"
+      E2E_AGENT_RUNTIME: "hermes"
+      E2E_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
@@ -4974,13 +5023,21 @@ jobs:
         include:
           - scenario: native
             sandbox_name: e2e-hgpu-native
+            observable_outcome: "Native GPU startup reaches the stable Ready route"
+            coverage_variant: native
           - scenario: fallback
             sandbox_name: e2e-hgpu-fallback
+            observable_outcome: "Fallback GPU startup reaches the stable Ready route"
+            coverage_variant: fallback
           - scenario: compatibility-only
             sandbox_name: e2e-hgpu-compat
+            observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route"
+            coverage_variant: compatibility-only
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "hermes-gpu-startup"
+      E2E_AGENT_RUNTIME: "hermes"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}
       E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
       NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }}
@@ -5283,6 +5340,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "cloud-onboard"
+      E2E_AGENT_RUNTIME: "openclaw"
+      E2E_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
@@ -5405,6 +5465,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "messaging-providers"
+      E2E_AGENT_RUNTIME: "openclaw"
+      E2E_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
@@ -5470,6 +5533,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release"
+      E2E_AGENT_RUNTIME: "openclaw"
+      E2E_OBSERVABLE_OUTCOME: "Release baseline plugin runtime survives the EXDEV boundary"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; release baseline package; no inference endpoint"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
@@ -5542,6 +5608,9 @@ jobs:
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "openclaw-plugin-runtime-exdev"
+      E2E_AGENT_RUNTIME: "openclaw"
+      E2E_OBSERVABLE_OUTCOME: "Current plugin lifecycle rebuild and dependencies survive the EXDEV boundary"
+      E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; current package; no inference endpoint"
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"

test/e2e-recommendations.test.ts

@@ -125,6 +125,7 @@ describe("E2E recommendation normalizer", () => {
         "tools/advisors/e2e-text.mts",
         "tools/advisors/json.mts",
         "tools/advisors/risk-plan.mts",
+        "tools/e2e/execution-coverage.mts",
         "tools/e2e/target-catalogue.mts",
         "scripts/checks/llama-cpp-dgx-spark-qualification-paths.mts",
         "scripts/checks/protected-managed-image-contract.ts",

test/e2e/README.md

@@ -318,7 +318,33 @@ npx tsx tools/e2e/workflow-plan.mts --summary >> "$GITHUB_STEP_SUMMARY"
 ```
 
 The workflow's `--ci-output` mode uses the same renderer for its job summary.
-The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, and retained workflow jobs.
+The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, retained workflow jobs, and staging Brev execution.
+
+Each execution row declares three coverage fields:
+
+- `agentRuntime` names the agent runtime that the execution asserts. Use `none` when the execution does not start an agent. Use `unresolved` only with an `unresolvedReason`.
+- `observableOutcome` names the behavior that produces the evidence. Catalogue targets use their outcome-oriented `displayName` as this value.
+- `environmentOrInferenceEndpoint` names the host boundary or inference endpoint that distinguishes the evidence.
+
+Keep coverage metadata with the execution owner:
+
+- Catalogue targets declare it in `tools/e2e/target-catalogue.mts`.
+- Executable typed targets declare it in `test/e2e/registry/definitions/baseline.ts`.
+- Shared credential-free tests declare it in `tools/e2e/credential-free-tests.mts`.
+- Retained workflow jobs and staging Brev declare it in `.github/workflows/e2e.yaml`.
+
+Single workflow jobs use the `E2E_AGENT_RUNTIME`, `E2E_OBSERVABLE_OUTCOME`,
+`E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT`, and optional `E2E_UNRESOLVED_REASON`
+environment entries. Matrix jobs put variant-specific values in the corresponding
+snake-case include entries and use `coverage_variant` when one job contributes
+multiple rows. `tools/e2e/workflow-plan.mts` composes and validates these sources.
+Do not add a separate hand-maintained execution list.
+
+The default coverage matrix excludes explicit-only jobs and inert typed-registry declarations. The rendered report lists those categories separately. The current inventory has one explicit-only execution and 22 inert typed declarations. The explicit-only row keeps its coverage dimensions but does not join the default release matrix. Inert declarations report unresolved coverage fields and the missing executable ownership.
+
+The inert declarations are combinatorial gaps, not supported matrix cells. #8285 owns the decision on the inert cross-runtime foundation. #8286 owns executable-only registry cleanup after that decision. Do not schedule other Cartesian-product cells without an accepted supported combination. This migration removes no execution, so it requires no duplicate-to-retained-evidence mapping. A documented gap does not schedule a new combination or change release judgment.
+
+The report also groups repeated observable outcomes. Those rows are retained only when agent runtime or environment provides distinct evidence. Validation rejects two rows with the same three coverage dimensions.
 
 ## Launch-readiness locked-image acceptance
 

test/e2e/registry/definitions/baseline.ts

@@ -12,6 +12,10 @@ import {
   wslRepoDocker,
 } from "../matrix.ts";
 import type { ExpectedFailureContract, TargetDefinition, TargetEnvironment } from "../types.ts";
+import {
+  type E2eExecutionMetadata,
+  validateE2eExecutionMetadata,
+} from "../../../../tools/e2e/execution-coverage.mts";
 
 interface CanonicalTargetInput {
   id: string;
@@ -21,6 +25,7 @@ interface CanonicalTargetInput {
   suiteIds: string[];
   onboardingAssertionIds?: string[];
   description?: string;
+  executionCoverage?: E2eExecutionMetadata;
   runnerRequirements?: string[];
   requiredSecrets?: string[];
   skippedCapabilities?: Array<Record<string, unknown>>;
@@ -48,7 +53,15 @@ function canonicalTarget(input: CanonicalTargetInput): TargetDefinition {
   if (input.expectedFailure) {
     builder = builder.expectedFailure(input.expectedFailure);
   }
-  return builder.build();
+  const definition = builder.build();
+  if (!input.executionCoverage) return definition;
+  return {
+    ...definition,
+    executionCoverage: validateE2eExecutionMetadata(
+      input.executionCoverage,
+      `Typed E2E target ${input.id}`,
+    ),
+  };
 }
 
 const macosDockerSkipped = [
@@ -68,6 +81,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [
     expectedStateId: "cloud-openclaw-ready",
     suiteIds: ["smoke", "inference", "credentials"],
     description: "Ubuntu repo checkout with Docker and cloud OpenClaw onboarding.",
+    executionCoverage: {
+      agentRuntime: "openclaw",
+      observableOutcome: "Repository install onboarding and hosted inference succeed",
+      environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference",
+      unresolvedReason: "",
+    },
     requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
   },
   {
@@ -88,6 +107,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [
     expectedStateId: "cloud-deepagents-code-ready",
     suiteIds: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"],
     description: "Ubuntu repo checkout with Docker and LangChain Deep Agents Code onboarding.",
+    executionCoverage: {
+      agentRuntime: "langchain-deepagents-code",
+      observableOutcome: "Repository install onboarding and hosted inference succeed",
+      environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference",
+      unresolvedReason: "",
+    },
     requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
   },
   {
@@ -179,6 +204,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [
     description:
       "Post-reboot recovery guard: the gateway must recover through the required user service " +
       "while preserving the local sandbox registry and container.",
+    executionCoverage: {
+      agentRuntime: "openclaw",
+      observableOutcome: "Docker-backed sandbox recovers after a simulated host reboot",
+      environmentOrInferenceEndpoint: "Ubuntu Docker host; local recovery fixture",
+      unresolvedReason: "",
+    },
   },
   {
     id: "ubuntu-repo-openai-compatible-openclaw",
@@ -327,6 +358,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [
     expectedStateId: "onboarding-failure-policy-presets-required",
     onboardingAssertionIds: ["base-installed", "preflight-passed"],
     suiteIds: [],
+    executionCoverage: {
+      agentRuntime: "openclaw",
+      observableOutcome: "Missing custom policy presets fail closed",
+      environmentOrInferenceEndpoint: "Ubuntu Docker host; local negative fixture",
+      unresolvedReason: "",
+    },
     requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"],
     expectedFailure: {
       phase: "onboarding",

test/e2e/registry/run.ts

@@ -4,6 +4,11 @@
 import { realpathSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 
+import {
+  type E2eAgentRuntime,
+  validateE2eExecutionMetadata,
+} from "../../../tools/e2e/execution-coverage.mts";
+
 import { listTargets, requireTargets } from "./registry.ts";
 import { resolveRunnerForTarget } from "./runner-routing.ts";
 import { type LiveTargetSupport, liveTargetSupport } from "./runtime-support.ts";
@@ -17,6 +22,10 @@ interface Args {
 
 export interface LiveTargetMatrixEntry {
   id: string;
+  agentRuntime: E2eAgentRuntime;
+  observableOutcome: string;
+  environmentOrInferenceEndpoint: string;
+  unresolvedReason: string;
   runner: string;
   label: string;
   platform: string;
@@ -92,8 +101,23 @@ function liveMatrixEntry(
   support: LiveTargetSupport,
 ): LiveTargetMatrixEntry {
   const { runner } = resolveRunnerForTarget(target);
+  if (support.supported && !target.executionCoverage) {
+    throw new Error(
+      `Executable typed E2E target ${target.id} requires execution coverage metadata`,
+    );
+  }
+  const executionCoverage = validateE2eExecutionMetadata(
+    target.executionCoverage ?? {
+      agentRuntime: "unresolved",
+      observableOutcome: "unresolved",
+      environmentOrInferenceEndpoint: "unresolved",
+      unresolvedReason: "This typed registry declaration has no executable owner",
+    },
+    `Typed E2E target ${target.id}`,
+  );
   return {
     id: target.id,
+    ...executionCoverage,
     runner,
     label: buildLabel(target),
     platform: target.environment?.platform ?? "unknown",

test/e2e/registry/types.ts

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import type { E2eExecutionMetadata } from "../../../tools/e2e/execution-coverage.mts";
+
 export type PhaseName = "environment" | "onboarding" | "state-validation" | "lifecycle" | "runtime";
 
 // Synthetic phase appended by the target runner when a target
@@ -194,6 +196,7 @@ export interface TargetEnvironment {
 export interface TargetDefinition {
   id: string;
   description?: string;
+  executionCoverage?: E2eExecutionMetadata;
   manifestPath?: string;
   environment?: TargetEnvironment;
   assertionGroups: AssertionGroup[];