diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2f0cdfca8a..6a5f444787 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2555,6 +2555,9 @@ jobs: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_JOB: "1" INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Exact staging image boots and completes the full E2E scenario" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference" steps: - name: Checkout trusted Launchable lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -3078,6 +3081,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openshell-gateway-auth-contract" + E2E_AGENT_RUNTIME: "none" + E2E_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" @@ -3156,9 +3162,21 @@ jobs: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] + include: + - agent: openclaw + agent_runtime: openclaw + coverage_variant: openclaw + - agent: hermes + agent_runtime: hermes + coverage_variant: hermes + - agent: deepagents + agent_runtime: langchain-deepagents-code + coverage_variant: deepagents env: E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge" + E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} @@ -3483,6 +3501,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openshell-credential-generation-window" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_OPENSHELL_CHANNEL: stable @@ -3641,9 +3662,21 @@ jobs: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] + include: + - agent: openclaw + agent_runtime: openclaw + coverage_variant: openclaw + - agent: hermes + agent_runtime: hermes + coverage_variant: hermes + - agent: deepagents + agent_runtime: langchain-deepagents-code + coverage_variant: deepagents env: E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge-dev" + E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} @@ -3842,13 +3875,19 @@ jobs: - platform: linux/amd64 runner: ubuntu-24.04 shard: linux-amd64 + environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup" + coverage_variant: linux-amd64 - platform: linux/arm64 runner: ubuntu-24.04-arm shard: linux-arm64 + environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup" + coverage_variant: linux-arm64 env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }} E2E_JOB: "1" E2E_TARGET_ID: "managed-image-multiarch-startup" + E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" + E2E_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }} @@ -4272,6 +4311,10 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification E2E_JOB: "1" E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification" + E2E_AGENT_RUNTIME: "unresolved" + E2E_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA DGX Spark GB10; local llama.cpp inference" + E2E_UNRESOLVED_REASON: "The protected plan can enable or skip its OpenClaw subqualification" NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_SHARD: linux-arm64-gpu-dgx-spark-gb10 NEMOCLAW_LLAMA_CPP_QUALIFICATION_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} @@ -4463,6 +4506,9 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime E2E_JOB: "1" E2E_TARGET_ID: "managed-image-protected-runtime" + E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" + E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services" E2E_WORKLOAD_SOURCE: "managed-image" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js @@ -4741,6 +4787,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "hermes-e2e" + E2E_AGENT_RUNTIME: "hermes" + E2E_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -4831,13 +4880,21 @@ jobs: include: - scenario: native sandbox_name: e2e-hgpu-native + observable_outcome: "Native GPU startup reaches the stable Ready route" + coverage_variant: native - scenario: fallback sandbox_name: e2e-hgpu-fallback + observable_outcome: "Fallback GPU startup reaches the stable Ready route" + coverage_variant: fallback - scenario: compatibility-only sandbox_name: e2e-hgpu-compat + observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route" + coverage_variant: compatibility-only env: E2E_JOB: "1" E2E_TARGET_ID: "hermes-gpu-startup" + E2E_AGENT_RUNTIME: "hermes" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }} E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }} @@ -5139,6 +5196,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "cloud-onboard" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5261,6 +5321,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "messaging-providers" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5326,6 +5389,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Release baseline plugin runtime survives the EXDEV boundary" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; release baseline package; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5398,6 +5464,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Current plugin lifecycle rebuild and dependencies survive the EXDEV boundary" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; current package; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" diff --git a/test/e2e-recommendations.test.ts b/test/e2e-recommendations.test.ts index 9dfb271141..514fc47fc4 100644 --- a/test/e2e-recommendations.test.ts +++ b/test/e2e-recommendations.test.ts @@ -54,6 +54,7 @@ const RUNTIME_INVENTORY_FILES = [ "tools/advisors/e2e-text.mts", "tools/advisors/json.mts", "tools/advisors/risk-plan.mts", + "tools/e2e/execution-coverage.mts", "tools/e2e/target-catalogue.mts", "scripts/checks/llama-cpp-dgx-spark-qualification-paths.mts", "scripts/checks/protected-managed-image-contract.ts", diff --git a/test/e2e/README.md b/test/e2e/README.md index cf87a04953..e57135d38a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -325,7 +325,36 @@ npx tsx tools/e2e/workflow-plan.mts --summary >> "$GITHUB_STEP_SUMMARY" ``` The workflow's `--ci-output` mode uses the same renderer for its job summary. -The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, and retained workflow jobs. +The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, retained workflow jobs, and staging Brev execution. + +Each execution row declares three coverage fields: + +- `agentRuntime` names the agent runtime that the execution asserts. Use `none` when the execution does not start an agent. Use `unresolved` only with an `unresolvedReason`. +- `observableOutcome` names the behavior that produces the evidence. Catalogue targets use their outcome-oriented `displayName` as this value. +- `environmentOrInferenceEndpoint` names the host boundary or inference endpoint that distinguishes the evidence. + +Keep coverage metadata with the execution owner: + +- Catalogue targets declare it in `tools/e2e/target-catalogue.mts`. +- Executable typed targets declare it in `test/e2e/registry/definitions/baseline.ts`. +- Shared credential-free tests declare it in `tools/e2e/credential-free-tests.mts`. +- Retained workflow jobs and staging Brev declare it in `.github/workflows/e2e.yaml`. + +Single workflow jobs use the `E2E_AGENT_RUNTIME`, `E2E_OBSERVABLE_OUTCOME`, +`E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT`, and optional `E2E_UNRESOLVED_REASON` +environment entries. Matrix jobs put variant-specific values in the corresponding +snake-case include entries and use `coverage_variant` when one job contributes +multiple rows. `tools/e2e/workflow-plan.mts` composes and validates these sources. +Do not add a separate hand-maintained execution list. + +The default coverage matrix excludes explicit-only jobs and inert typed-registry declarations. +The rendered report lists those categories separately. +Explicit-only rows keep their coverage dimensions but do not join the default release matrix. +Inert declarations report unresolved coverage fields and the missing executable ownership. + +The inert declarations are combinatorial gaps, not supported matrix cells. #8285 owns the decision on the inert cross-runtime foundation. #8286 owns executable-only registry cleanup after that decision. Do not schedule other Cartesian-product cells without an accepted supported combination. This migration removes no execution, so it requires no duplicate-to-retained-evidence mapping. A documented gap does not schedule a new combination or change release judgment. + +The report also groups repeated observable outcomes. Those rows are retained only when agent runtime or environment provides distinct evidence. Validation rejects two rows with the same three coverage dimensions. ## Launch-readiness locked-image acceptance diff --git a/test/e2e/registry/definitions/baseline.ts b/test/e2e/registry/definitions/baseline.ts index ea1dea6b24..8c86d461db 100644 --- a/test/e2e/registry/definitions/baseline.ts +++ b/test/e2e/registry/definitions/baseline.ts @@ -12,6 +12,10 @@ import { wslRepoDocker, } from "../matrix.ts"; import type { ExpectedFailureContract, TargetDefinition, TargetEnvironment } from "../types.ts"; +import { + type E2eExecutionMetadata, + validateE2eExecutionMetadata, +} from "../../../../tools/e2e/execution-coverage.mts"; interface CanonicalTargetInput { id: string; @@ -21,6 +25,7 @@ interface CanonicalTargetInput { suiteIds: string[]; onboardingAssertionIds?: string[]; description?: string; + executionCoverage?: E2eExecutionMetadata; runnerRequirements?: string[]; requiredSecrets?: string[]; skippedCapabilities?: Array>; @@ -48,7 +53,15 @@ function canonicalTarget(input: CanonicalTargetInput): TargetDefinition { if (input.expectedFailure) { builder = builder.expectedFailure(input.expectedFailure); } - return builder.build(); + const definition = builder.build(); + if (!input.executionCoverage) return definition; + return { + ...definition, + executionCoverage: validateE2eExecutionMetadata( + input.executionCoverage, + `Typed E2E target ${input.id}`, + ), + }; } const macosDockerSkipped = [ @@ -69,6 +82,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ suiteIds: ["smoke", "inference", "credentials", "personal-stock-fetch"], description: "Ubuntu repo checkout with Docker, Personal policy, and cloud OpenClaw stock-fetch proof.", + executionCoverage: { + agentRuntime: "openclaw", + observableOutcome: "Repository install onboarding and hosted inference succeed", + environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", + unresolvedReason: "", + }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, { @@ -89,6 +108,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "cloud-deepagents-code-ready", suiteIds: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], description: "Ubuntu repo checkout with Docker and LangChain Deep Agents Code onboarding.", + executionCoverage: { + agentRuntime: "langchain-deepagents-code", + observableOutcome: "Repository install onboarding and hosted inference succeed", + environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", + unresolvedReason: "", + }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, { @@ -180,6 +205,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ description: "Post-reboot recovery guard: the gateway must recover through the required user service " + "while preserving the local sandbox registry and container.", + executionCoverage: { + agentRuntime: "openclaw", + observableOutcome: "Docker-backed sandbox recovers after a simulated host reboot", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local recovery fixture", + unresolvedReason: "", + }, }, { id: "ubuntu-repo-openai-compatible-openclaw", @@ -328,6 +359,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "onboarding-failure-policy-presets-required", onboardingAssertionIds: ["base-installed", "preflight-passed"], suiteIds: [], + executionCoverage: { + agentRuntime: "openclaw", + observableOutcome: "Missing custom policy presets fail closed", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local negative fixture", + unresolvedReason: "", + }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], expectedFailure: { phase: "onboarding", diff --git a/test/e2e/registry/run.ts b/test/e2e/registry/run.ts index bd99288276..d73f6fa372 100644 --- a/test/e2e/registry/run.ts +++ b/test/e2e/registry/run.ts @@ -4,6 +4,11 @@ import { realpathSync } from "node:fs"; import { fileURLToPath } from "node:url"; +import { + type E2eAgentRuntime, + validateE2eExecutionMetadata, +} from "../../../tools/e2e/execution-coverage.mts"; + import { listTargets, requireTargets } from "./registry.ts"; import { resolveRunnerForTarget } from "./runner-routing.ts"; import { type LiveTargetSupport, liveTargetSupport } from "./runtime-support.ts"; @@ -17,6 +22,10 @@ interface Args { export interface LiveTargetMatrixEntry { id: string; + agentRuntime: E2eAgentRuntime; + observableOutcome: string; + environmentOrInferenceEndpoint: string; + unresolvedReason: string; runner: string; label: string; platform: string; @@ -92,8 +101,23 @@ function liveMatrixEntry( support: LiveTargetSupport, ): LiveTargetMatrixEntry { const { runner } = resolveRunnerForTarget(target); + if (support.supported && !target.executionCoverage) { + throw new Error( + `Executable typed E2E target ${target.id} requires execution coverage metadata`, + ); + } + const executionCoverage = validateE2eExecutionMetadata( + target.executionCoverage ?? { + agentRuntime: "unresolved", + observableOutcome: "unresolved", + environmentOrInferenceEndpoint: "unresolved", + unresolvedReason: "This typed registry declaration has no executable owner", + }, + `Typed E2E target ${target.id}`, + ); return { id: target.id, + ...executionCoverage, runner, label: buildLabel(target), platform: target.environment?.platform ?? "unknown", diff --git a/test/e2e/registry/types.ts b/test/e2e/registry/types.ts index 4e355621d0..4788d6e0f3 100644 --- a/test/e2e/registry/types.ts +++ b/test/e2e/registry/types.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { E2eExecutionMetadata } from "../../../tools/e2e/execution-coverage.mts"; + export type PhaseName = "environment" | "onboarding" | "state-validation" | "lifecycle" | "runtime"; // Synthetic phase appended by the target runner when a target @@ -195,6 +197,7 @@ export interface TargetEnvironment { export interface TargetDefinition { id: string; description?: string; + executionCoverage?: E2eExecutionMetadata; manifestPath?: string; environment?: TargetEnvironment; assertionGroups: AssertionGroup[]; diff --git a/test/e2e/support/e2e-matrix.test.ts b/test/e2e/support/e2e-matrix.test.ts index f0384856b1..ed5184c6b0 100644 --- a/test/e2e/support/e2e-matrix.test.ts +++ b/test/e2e/support/e2e-matrix.test.ts @@ -29,6 +29,15 @@ function requireUnsupportedTarget() { return unsupported!; } +function expectExecutableTypedTargetCoverage(): void { + for (const row of buildLiveTargetMatrix()) { + expect(row.agentRuntime).not.toBe("unresolved"); + expect(row.observableOutcome).not.toBe("unresolved"); + expect(row.environmentOrInferenceEndpoint).not.toBe("unresolved"); + expect(row.unresolvedReason).toBe(""); + } +} + describe("live E2E target matrix", () => { it("honors an explicit runs-on: