From dbfdf910699ac19d874ffb00031cf3c10e2874b8 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:35:07 -0700 Subject: [PATCH 1/7] test(e2e): expose semantic coverage matrix --- .github/workflows/e2e.yaml | 69 ++++ test/e2e/README.md | 23 +- test/e2e/registry/definitions/baseline.ts | 39 +- test/e2e/registry/run.ts | 22 ++ test/e2e/registry/types.ts | 3 + test/e2e/support/e2e-matrix.test.ts | 14 + test/e2e/support/e2e-workflow.test.ts | 91 +++++ test/e2e/support/workflow-plan.test.ts | 164 ++++++++- tools/e2e/credential-free-tests.mts | 25 ++ .../hermes-gpu-startup-workflow-boundary.mts | 40 +- ...aged-image-multiarch-workflow-boundary.mts | 4 + .../e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- tools/e2e/mcp-workflow-boundary.mts | 19 +- tools/e2e/semantic-coverage.mts | 118 ++++++ tools/e2e/target-catalogue.mts | 159 +++++++- tools/e2e/workflow-boundary.mts | 83 ++++- tools/e2e/workflow-plan.mts | 342 +++++++++++++++--- 17 files changed, 1123 insertions(+), 94 deletions(-) create mode 100644 tools/e2e/semantic-coverage.mts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4f037e4e2ab..1d6acedbfa6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2698,6 +2698,9 @@ jobs: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_JOB: "1" INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Exact staging image boots and completes the full E2E scenario" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference" steps: - name: Checkout trusted Launchable lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -3221,6 +3224,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openshell-gateway-auth-contract" + E2E_SEMANTIC_AGENT_RUNTIME: "none" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" @@ -3299,9 +3305,21 @@ jobs: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] + include: + - agent: openclaw + semantic_agent_runtime: openclaw + semantic_variant: openclaw + - agent: hermes + semantic_agent_runtime: hermes + semantic_variant: hermes + - agent: deepagents + semantic_agent_runtime: langchain-deepagents-code + semantic_variant: deepagents env: E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} @@ -3626,6 +3644,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openshell-credential-generation-window" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_OPENSHELL_CHANNEL: stable @@ -3784,9 +3805,21 @@ jobs: fail-fast: false matrix: agent: [openclaw, hermes, deepagents] + include: + - agent: openclaw + semantic_agent_runtime: openclaw + semantic_variant: openclaw + - agent: hermes + semantic_agent_runtime: hermes + semantic_variant: hermes + - agent: deepagents + semantic_agent_runtime: langchain-deepagents-code + semantic_variant: deepagents env: E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge-dev" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} @@ -3985,13 +4018,19 @@ jobs: - platform: linux/amd64 runner: ubuntu-24.04 shard: linux-amd64 + semantic_environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup" + semantic_variant: linux-amd64 - platform: linux/arm64 runner: ubuntu-24.04-arm shard: linux-arm64 + semantic_environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup" + semantic_variant: linux-arm64 env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }} E2E_JOB: "1" E2E_TARGET_ID: "managed-image-multiarch-startup" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }} @@ -4415,6 +4454,10 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification E2E_JOB: "1" E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification" + E2E_SEMANTIC_AGENT_RUNTIME: "unresolved" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA DGX Spark GB10; local llama.cpp inference" + E2E_SEMANTIC_UNRESOLVED_REASON: "The protected plan can enable or skip its OpenClaw subqualification" NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_SHARD: linux-arm64-gpu-dgx-spark-gb10 NEMOCLAW_LLAMA_CPP_QUALIFICATION_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} @@ -4606,6 +4649,9 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime E2E_JOB: "1" E2E_TARGET_ID: "managed-image-protected-runtime" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services" E2E_WORKLOAD_SOURCE: "managed-image" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js @@ -4884,6 +4930,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "hermes-e2e" + E2E_SEMANTIC_AGENT_RUNTIME: "hermes" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -4974,13 +5023,21 @@ jobs: include: - scenario: native sandbox_name: e2e-hgpu-native + semantic_observable_outcome: "Native GPU startup reaches the stable Ready route" + semantic_variant: native - scenario: fallback sandbox_name: e2e-hgpu-fallback + semantic_observable_outcome: "Fallback GPU startup reaches the stable Ready route" + semantic_variant: fallback - scenario: compatibility-only sandbox_name: e2e-hgpu-compat + semantic_observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route" + semantic_variant: compatibility-only env: E2E_JOB: "1" E2E_TARGET_ID: "hermes-gpu-startup" + E2E_SEMANTIC_AGENT_RUNTIME: "hermes" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }} E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }} @@ -5283,6 +5340,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "cloud-onboard" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5405,6 +5465,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "messaging-providers" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5470,6 +5533,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Release baseline plugin runtime survives the EXDEV boundary" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; release baseline package; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5542,6 +5608,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev" + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Current plugin lifecycle rebuild and dependencies survive the EXDEV boundary" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; current package; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" diff --git a/test/e2e/README.md b/test/e2e/README.md index ed0f54e6116..19428b95709 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -318,7 +318,28 @@ npx tsx tools/e2e/workflow-plan.mts --summary >> "$GITHUB_STEP_SUMMARY" ``` The workflow's `--ci-output` mode uses the same renderer for its job summary. -The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, and retained workflow jobs. +The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, retained workflow jobs, and staging Brev execution. + +Each execution row declares three semantic fields: + +- `agentRuntime` names the agent runtime that the execution asserts. Use `none` when the execution does not start an agent. Use `unresolved` only with an `unresolvedReason`. +- `observableOutcome` names the behavior that produces the evidence. Catalogue targets use their outcome-oriented `displayName` as this value. +- `environmentOrInferenceEndpoint` names the host boundary or inference endpoint that distinguishes the evidence. + +Keep semantic metadata with the execution owner: + +- Catalogue targets declare it in `tools/e2e/target-catalogue.mts`. +- Executable typed targets declare it in `test/e2e/registry/definitions/baseline.ts`. +- Shared credential-free tests declare it in `tools/e2e/credential-free-tests.mts`. +- Retained workflow jobs and staging Brev declare it in `.github/workflows/e2e.yaml`. + +Single workflow jobs use the `E2E_SEMANTIC_*` environment entries. Matrix jobs put variant-specific values in `semantic_*` include entries. `tools/e2e/workflow-plan.mts` composes and validates these sources. Do not add a separate hand-maintained execution list. + +The default semantic matrix excludes explicit-only jobs and inert typed-registry declarations. The rendered report lists those categories separately. The current inventory has one explicit-only execution and 22 inert typed declarations. The explicit-only row keeps its semantic dimensions but does not join the default release matrix. Inert declarations report unresolved semantic fields and the missing executable ownership. + +The inert declarations are combinatorial gaps, not supported matrix cells. #8285 owns the decision on the inert cross-runtime foundation. #8286 owns executable-only registry cleanup after that decision. Do not schedule other Cartesian-product cells without an accepted supported combination. This migration removes no execution, so it requires no duplicate-to-retained-evidence mapping. A documented gap does not schedule a new combination or change release judgment. + +The report also groups repeated observable outcomes. Those rows are retained only when agent runtime or environment provides distinct evidence. Validation rejects two rows with the same three semantic dimensions. ## Launch-readiness locked-image acceptance diff --git a/test/e2e/registry/definitions/baseline.ts b/test/e2e/registry/definitions/baseline.ts index 123ae7b68dd..3684df59744 100644 --- a/test/e2e/registry/definitions/baseline.ts +++ b/test/e2e/registry/definitions/baseline.ts @@ -12,6 +12,10 @@ import { wslRepoDocker, } from "../matrix.ts"; import type { ExpectedFailureContract, TargetDefinition, TargetEnvironment } from "../types.ts"; +import { + type E2eSemanticMetadata, + validateE2eSemanticMetadata, +} from "../../../../tools/e2e/semantic-coverage.mts"; interface CanonicalTargetInput { id: string; @@ -21,6 +25,7 @@ interface CanonicalTargetInput { suiteIds: string[]; onboardingAssertionIds?: string[]; description?: string; + semanticCoverage?: E2eSemanticMetadata; runnerRequirements?: string[]; requiredSecrets?: string[]; skippedCapabilities?: Array>; @@ -48,7 +53,15 @@ function canonicalTarget(input: CanonicalTargetInput): TargetDefinition { if (input.expectedFailure) { builder = builder.expectedFailure(input.expectedFailure); } - return builder.build(); + const definition = builder.build(); + if (!input.semanticCoverage) return definition; + return { + ...definition, + semanticCoverage: validateE2eSemanticMetadata( + input.semanticCoverage, + `Typed E2E target ${input.id}`, + ), + }; } const macosDockerSkipped = [ @@ -68,6 +81,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "cloud-openclaw-ready", suiteIds: ["smoke", "inference", "credentials"], description: "Ubuntu repo checkout with Docker and cloud OpenClaw onboarding.", + semanticCoverage: { + agentRuntime: "openclaw", + observableOutcome: "Repository install onboarding and hosted inference succeed", + environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", + unresolvedReason: "", + }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, { @@ -88,6 +107,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "cloud-deepagents-code-ready", suiteIds: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], description: "Ubuntu repo checkout with Docker and LangChain Deep Agents Code onboarding.", + semanticCoverage: { + agentRuntime: "langchain-deepagents-code", + observableOutcome: "Repository install onboarding and hosted inference succeed", + environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", + unresolvedReason: "", + }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], }, { @@ -179,6 +204,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ description: "Post-reboot recovery guard: the gateway must recover through the required user service " + "while preserving the local sandbox registry and container.", + semanticCoverage: { + agentRuntime: "openclaw", + observableOutcome: "Docker-backed sandbox recovers after a simulated host reboot", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local recovery fixture", + unresolvedReason: "", + }, }, { id: "ubuntu-repo-openai-compatible-openclaw", @@ -327,6 +358,12 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "onboarding-failure-policy-presets-required", onboardingAssertionIds: ["base-installed", "preflight-passed"], suiteIds: [], + semanticCoverage: { + agentRuntime: "openclaw", + observableOutcome: "Missing custom policy presets fail closed", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local negative fixture", + unresolvedReason: "", + }, requiredSecrets: ["NVIDIA_INFERENCE_API_KEY"], expectedFailure: { phase: "onboarding", diff --git a/test/e2e/registry/run.ts b/test/e2e/registry/run.ts index bd992882763..8cecb52ae68 100644 --- a/test/e2e/registry/run.ts +++ b/test/e2e/registry/run.ts @@ -4,6 +4,11 @@ import { realpathSync } from "node:fs"; import { fileURLToPath } from "node:url"; +import { + type E2eAgentRuntime, + validateE2eSemanticMetadata, +} from "../../../tools/e2e/semantic-coverage.mts"; + import { listTargets, requireTargets } from "./registry.ts"; import { resolveRunnerForTarget } from "./runner-routing.ts"; import { type LiveTargetSupport, liveTargetSupport } from "./runtime-support.ts"; @@ -17,6 +22,10 @@ interface Args { export interface LiveTargetMatrixEntry { id: string; + agentRuntime: E2eAgentRuntime; + observableOutcome: string; + environmentOrInferenceEndpoint: string; + unresolvedReason: string; runner: string; label: string; platform: string; @@ -92,8 +101,21 @@ function liveMatrixEntry( support: LiveTargetSupport, ): LiveTargetMatrixEntry { const { runner } = resolveRunnerForTarget(target); + if (support.supported && !target.semanticCoverage) { + throw new Error(`Executable typed E2E target ${target.id} requires semantic coverage metadata`); + } + const semanticCoverage = validateE2eSemanticMetadata( + target.semanticCoverage ?? { + agentRuntime: "unresolved", + observableOutcome: "unresolved", + environmentOrInferenceEndpoint: "unresolved", + unresolvedReason: "This typed registry declaration has no executable owner", + }, + `Typed E2E target ${target.id}`, + ); return { id: target.id, + ...semanticCoverage, runner, label: buildLabel(target), platform: target.environment?.platform ?? "unknown", diff --git a/test/e2e/registry/types.ts b/test/e2e/registry/types.ts index 4761ec3f393..133bab17f5f 100644 --- a/test/e2e/registry/types.ts +++ b/test/e2e/registry/types.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import type { E2eSemanticMetadata } from "../../../tools/e2e/semantic-coverage.mts"; + export type PhaseName = "environment" | "onboarding" | "state-validation" | "lifecycle" | "runtime"; // Synthetic phase appended by the target runner when a target @@ -194,6 +196,7 @@ export interface TargetEnvironment { export interface TargetDefinition { id: string; description?: string; + semanticCoverage?: E2eSemanticMetadata; manifestPath?: string; environment?: TargetEnvironment; assertionGroups: AssertionGroup[]; diff --git a/test/e2e/support/e2e-matrix.test.ts b/test/e2e/support/e2e-matrix.test.ts index f0384856b1a..a2e4e91c7f5 100644 --- a/test/e2e/support/e2e-matrix.test.ts +++ b/test/e2e/support/e2e-matrix.test.ts @@ -90,12 +90,26 @@ describe("live E2E target matrix", () => { expect(buildLiveTargetMatrix([unsupported.id])).toEqual([ expect.objectContaining({ id: unsupported.id, + agentRuntime: "unresolved", + observableOutcome: "unresolved", + environmentOrInferenceEndpoint: "unresolved", + unresolvedReason: "This typed registry declaration has no executable owner", supported: false, supportReasons: support.reasons, }), ]); }); + it("exposes semantic coverage for every executable typed target (#9167)", () => { + expect(buildLiveTargetMatrix()).toHaveLength(4); + for (const row of buildLiveTargetMatrix()) { + expect(row.agentRuntime).not.toBe("unresolved"); + expect(row.observableOutcome).not.toBe("unresolved"); + expect(row.environmentOrInferenceEndpoint).not.toBe("unresolved"); + expect(row.unresolvedReason).toBe(""); + } + }); + it("prints a single-line JSON array of supported live E2E targets for --emit-live-matrix", () => { const result = runEmitLiveMatrix(); expect(result.status, result.stderr).toBe(0); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 4389b510b64..04a483e1a71 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -622,9 +622,100 @@ describe("e2e workflow boundary", () => { }); }); + it("derives semantic coverage from retained workflow jobs without counting wrapper jobs", () => { + const inventory = readFreeStandingJobsInventory(); + + expect(inventory.semanticRows).toHaveLength(21); + expect(inventory.semanticRows.filter((row) => row.source === "retained-workflow")).toHaveLength( + 20, + ); + expect(inventory.semanticRows.filter((row) => row.source === "staging")).toHaveLength(1); + expect(inventory.semanticRows.filter((row) => row.id === "mcp-bridge")).toEqual([ + expect.objectContaining({ variant: "openclaw", agentRuntime: "openclaw" }), + expect.objectContaining({ variant: "hermes", agentRuntime: "hermes" }), + expect.objectContaining({ + variant: "deepagents", + agentRuntime: "langchain-deepagents-code", + }), + ]); + expect( + inventory.semanticRows.filter((row) => row.id === "llama-cpp-dgx-spark-qualification"), + ).toEqual([ + expect.objectContaining({ + agentRuntime: "unresolved", + unresolvedReason: "The protected plan can enable or skip its OpenClaw subqualification", + }), + ]); + }); + it.each([ { body: ` +jobs: + first-evidence: + env: + E2E_JOB: "1" + E2E_TARGET_ID: first-evidence + E2E_SEMANTIC_AGENT_RUNTIME: openclaw + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Same behavior succeeds" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; no inference endpoint" + second-evidence: + env: + E2E_JOB: "1" + E2E_TARGET_ID: second-evidence + E2E_SEMANTIC_AGENT_RUNTIME: openclaw + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Same behavior succeeds" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; no inference endpoint" +`, + error: "E2E semantic coverage duplicates evidence between first-evidence and second-evidence", + }, + { + body: ` +jobs: + fixture-version-check: + env: + E2E_JOB: "1" + E2E_TARGET_ID: fixture-version-check +`, + error: "fixture-version-check job requires semantic coverage metadata", + }, + { + body: ` +jobs: + fixture-version-check: + env: + E2E_JOB: "1" + E2E_TARGET_ID: fixture-version-check + E2E_SEMANTIC_AGENT_RUNTIME: openclaw + E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Injected | Markdown row" + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; no inference endpoint" +`, + error: "E2E workflow job fixture-version-check has an invalid observable outcome", + }, + { + body: ` +jobs: + fixture-version-check: + strategy: + matrix: + include: + - semantic_agent_runtime: openclaw + semantic_observable_outcome: "First valid outcome" + semantic_environment_or_inference_endpoint: "Ubuntu; no inference endpoint" + semantic_variant: duplicate + - semantic_agent_runtime: openclaw + semantic_observable_outcome: "Second valid outcome" + semantic_environment_or_inference_endpoint: "Ubuntu; no inference endpoint" + semantic_variant: duplicate + env: + E2E_JOB: "1" + E2E_TARGET_ID: fixture-version-check +`, + error: + "E2E semantic coverage contains a duplicate row: retained-workflow:fixture-version-check:duplicate", + }, + { + body: ` jobs: fixture-version-check: env: diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 9a0a143a086..fb0c8bf45b5 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -68,14 +68,23 @@ function expectedCiOutput(plan: ReturnType): string function prCandidatePlan( plan: ReturnType, ): ReturnType { + const catalogueMatrices = Object.fromEntries( + Object.entries(plan.catalogueMatrices).map(([profile, rows]) => [ + profile, + rows.filter((row) => isPrCandidateCatalogueTarget(catalogueTarget(row.id))), + ]), + ) as ReturnType["catalogueMatrices"]; + const catalogueIds = new Set( + Object.values(catalogueMatrices) + .flat() + .map((row) => row.id), + ); return { ...plan, - catalogueMatrices: Object.fromEntries( - Object.entries(plan.catalogueMatrices).map(([profile, rows]) => [ - profile, - rows.filter((row) => isPrCandidateCatalogueTarget(catalogueTarget(row.id))), - ]), - ) as ReturnType["catalogueMatrices"], + catalogueMatrices, + semanticMatrix: plan.semanticMatrix.filter( + (row) => row.source !== "catalogue" || catalogueIds.has(row.id), + ), }; } @@ -86,6 +95,25 @@ describe("E2E workflow plan", () => { expect(plan.matrix).toEqual(buildLiveTargetMatrix()); expect(plan.testMatrix).toEqual(discoverCredentialFreeTests()); expect(Object.values(plan.catalogueMatrices).flat()).toHaveLength(E2E_TARGET_CATALOGUE.length); + expect(plan.semanticMatrix).toHaveLength(90); + expect( + plan.semanticMatrix.reduce>((counts, row) => { + counts[row.source] = (counts[row.source] ?? 0) + 1; + return counts; + }, {}), + ).toEqual({ + catalogue: 64, + "typed-registry": 4, + "shared-e2e": 2, + "retained-workflow": 19, + staging: 1, + }); + expect(plan.semanticMatrix.filter((row) => row.unresolvedReason !== "")).toEqual([ + expect.objectContaining({ + id: "spark-install", + agentRuntime: "unresolved", + }), + ]); expect(plan.hermesSelected).toBe(true); expect(plan.explicitOnlyJobs).toEqual(["llama-cpp-dgx-spark-qualification"]); expect(releaseRequiredWorkflowJobs()).toContain("live"); @@ -93,6 +121,43 @@ describe("E2E workflow plan", () => { expect(releaseRequiredWorkflowJobs()).not.toContain("llama-cpp-dgx-spark-qualification"); }); + it("keeps multiple inert declarations visibly unresolved without treating them as evidence (#9167)", () => { + const plan = buildE2eWorkflowPlan({ + targets: "ubuntu-repo-cloud-hermes,ubuntu-repo-cloud-hermes-slack", + }); + + expect(plan.matrix).toHaveLength(2); + expect(plan.matrix.every((row) => !row.supported)).toBe(true); + expect(plan.semanticMatrix).toEqual([ + expect.objectContaining({ id: "ubuntu-repo-cloud-hermes", agentRuntime: "unresolved" }), + expect.objectContaining({ + id: "ubuntu-repo-cloud-hermes-slack", + agentRuntime: "unresolved", + }), + ]); + expect(() => validateE2eWorkflowPlan(plan)).not.toThrow(); + }); + + it("includes staging only when the execution plan selects it (#9167)", () => { + const stagingPlan = buildE2eWorkflowPlan({ jobs: "staging-brev-launchable" }); + + expect(stagingPlan.selectedJobs).toEqual(["staging-brev-launchable"]); + expect(stagingPlan.semanticMatrix).toEqual([ + expect.objectContaining({ id: "staging-brev-launchable", source: "staging" }), + ]); + + const hermesPlan = buildE2eWorkflowPlan({ jobs: "hermes-e2e" }); + const stagingRow = buildE2eWorkflowPlan().semanticMatrix.find( + (row) => row.id === "staging-brev-launchable", + )!; + expect(() => + validateE2eWorkflowPlan({ + ...hermesPlan, + semanticMatrix: [stagingRow, ...hermesPlan.semanticMatrix], + }), + ).toThrow("semantic coverage that does not match its execution plan"); + }); + it("waives only named release-required E2E jobs", () => { const defaultJobs = releaseRequiredWorkflowJobs(); const requestedWaivers = ["live", "staging-brev-launchable"]; @@ -393,6 +458,23 @@ describe("E2E workflow plan", () => { expect(catalogueTarget(id)).toMatchObject(contract); }); + it("requires explicit semantic coverage for every catalogue target (#9167)", () => { + expect(E2E_TARGET_CATALOGUE).toHaveLength(64); + for (const target of E2E_TARGET_CATALOGUE) { + expect(target.observableOutcome).toBe(target.displayName); + expect(target.agentRuntime).not.toBe(""); + expect(target.environmentOrInferenceEndpoint).not.toBe(""); + } + + const target = catalogueTarget("cloud-inference"); + expect(() => + validateE2eTargetCatalogue([{ ...target, observableOutcome: "Injected | Markdown row" }]), + ).toThrow("invalid observable outcome"); + expect(() => + validateE2eTargetCatalogue([{ ...target, agentRuntime: "unresolved", unresolvedReason: "" }]), + ).toThrow("must declare an unresolved reason"); + }); + it.each([ [ "bedrock-runtime-compatible-anthropic", @@ -455,11 +537,7 @@ describe("E2E workflow plan", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-pr-")); const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); - const plan = buildE2eWorkflowPlan(); - plan.catalogueMatrices["nvidia-api"] = []; - plan.catalogueMatrices["nvidia-inference"] = []; - plan.catalogueMatrices["github-read"] = []; - plan.catalogueMatrices["brave-nvidia-inference"] = []; + const plan = prCandidatePlan(buildE2eWorkflowPlan()); try { writeE2eWorkflowPlanCiOutput( @@ -740,6 +818,7 @@ describe("E2E workflow plan", () => { "github-read": [], "brave-nvidia-inference": [], }, + semanticMatrix: [], selectedJobs: [], hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, @@ -782,6 +861,7 @@ describe("E2E workflow plan", () => { "github-read": [], "brave-nvidia-inference": [], }, + semanticMatrix: [], selectedJobs: ["jetson-nvmap-gpu"], hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, @@ -803,6 +883,7 @@ describe("E2E workflow plan", () => { "github-read": [], "brave-nvidia-inference": [], }, + semanticMatrix: [], selectedJobs: [], hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, @@ -884,8 +965,10 @@ describe("E2E workflow plan", () => { const validPlan = buildE2eWorkflowPlan(); const [registryRow] = validPlan.matrix; const [testRow] = validPlan.testMatrix; + const [semanticRow] = validPlan.semanticMatrix; expect(registryRow).toBeDefined(); expect(testRow).toBeDefined(); + expect(semanticRow).toBeDefined(); const { explicitOnlyJobs: _omitted, ...missingField } = validPlan; const malformedPlans = [ missingField, @@ -896,6 +979,14 @@ describe("E2E workflow plan", () => { testMatrix: [{ ...testRow, project: "e2e-live", file: "test/e2e/live/../secret.test.ts" }], }, { ...validPlan, testMatrix: [{ ...testRow, id: registryRow.id }] }, + { ...validPlan, semanticMatrix: [...validPlan.semanticMatrix, { ...semanticRow }] }, + { + ...validPlan, + semanticMatrix: [ + { ...semanticRow, observableOutcome: "Injected | Markdown row" }, + ...validPlan.semanticMatrix.slice(1), + ], + }, { ...validPlan, hermesSelected: "false" }, ]; @@ -906,6 +997,17 @@ describe("E2E workflow plan", () => { } }); + it("rejects semantic coverage that differs from its execution owner (#9167)", () => { + const plan = buildE2eWorkflowPlan({ jobs: "cloud-inference" }); + const semanticMatrix = plan.semanticMatrix.map((row) => + row.id === "cloud-inference" ? { ...row, observableOutcome: "Different valid outcome" } : row, + ); + + expect(() => validateE2eWorkflowPlan({ ...plan, semanticMatrix })).toThrow( + "semantic coverage that does not match its execution plan", + ); + }); + it("writes byte-compatible GitHub outputs and the execution-plan summary", () => { const testId = firstId(discoverCredentialFreeTests(), "credential-free test"); const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-")); @@ -986,6 +1088,7 @@ describe("E2E workflow plan", () => { "selectedJobs", "hermesSelected", "explicitOnlyJobs", + "semanticMatrix", ]); expect(output).toBe(`${JSON.stringify(parsed)}\n`); }); @@ -1000,9 +1103,9 @@ describe("E2E workflow plan", () => { expect(filtered.status, filtered.stderr).toBe(0); expect(filtered.stdout).toBe(`## E2E Execution Plan -| Target or job | Execution | Runner | -| --- | --- | --- | -| \`hermes-e2e\` | retained workflow job | declared by job | +| Target or job | Agent runtime | Observable outcome | Environment or inference endpoint | Source | Unresolved reason | +| --- | --- | --- | --- | --- | --- | +| \`hermes-e2e\` | hermes | Install onboarding health inference lifecycle dashboard and security succeed | Ubuntu; mock or NVIDIA hosted inference | retained-workflow | | `); const complete = spawnSync(TSX, [PLANNER_CLI, "--summary"], { @@ -1013,17 +1116,38 @@ describe("E2E workflow plan", () => { expect(complete.status, complete.stderr).toBe(0); expect(complete.stdout).toContain( - "| `cloud-onboard` | retained workflow job | declared by job |", + "| `cloud-onboard` | openclaw | Public install onboarding hosted inference and security checks succeed | Ubuntu; NVIDIA hosted inference | retained-workflow | |", + ); + expect(complete.stdout).toContain( + "| `ubuntu-repo-cloud-openclaw` | openclaw | Repository install onboarding and hosted inference succeed | Ubuntu Docker host; NVIDIA hosted inference | typed-registry | |", + ); + expect(complete.stdout).toContain( + "| `vllm-docker-storage` | none | vLLM storage gate accepts and rejects the intended host states | Native Linux Docker host; no inference endpoint | shared-e2e | |", + ); + expect(complete.stdout).toContain( + "| `channels-add-remove` | openclaw | Messaging: adds and removes Telegram configuration | Ubuntu; no inference endpoint | catalogue | |", + ); + expect(complete.stdout).toContain( + "| `model-router-provider-routed-inference` | openclaw | Inference: Model Router returns a provider-routed response | Ubuntu; NVIDIA API and Model Router | catalogue | |", + ); + expect(complete.stdout).toContain( + "| `spark-install` | unresolved | Install: leaves NemoClaw and OpenShell usable after standard installation | Ubuntu; NVIDIA hosted inference | catalogue | The test asserts CLI usability but does not assert an agent runtime |", + ); + expect(complete.stdout).toContain("### Repeated outcomes with distinct evidence"); + expect(complete.stdout).toContain( + "| Repository install onboarding and hosted inference succeed | `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw` | agent runtime |", ); + expect(complete.stdout).toContain("### Intentional exclusions"); expect(complete.stdout).toContain( - "| `ubuntu-repo-cloud-openclaw` | typed registry | `ubuntu-latest` |", + "| `llama-cpp-dgx-spark-qualification` | unresolved | Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence | NVIDIA DGX Spark GB10; local llama.cpp inference | Explicit dispatch only; excluded from the default release matrix | The protected plan can enable or skip its OpenClaw subqualification |", ); - expect(complete.stdout).toContain("| shared E2E job | `ubuntu-latest` |"); - expect(complete.stdout).toContain("| `channels-add-remove` | `standard` profile |"); + expect(complete.stdout).toContain("### Unsupported or unresolved typed declarations"); expect(complete.stdout).toContain( - "| `model-router-provider-routed-inference` | `nvidia-api` profile |", + "| `ubuntu-repo-cloud-hermes` | unresolved | unresolved | unresolved | onboarding 'cloud-hermes' is not wired for live fixtures |", ); - expect(complete.stdout).toContain("| `cloud-inference` | `nvidia-inference` profile |"); + expect(complete.stdout).toContain("The 22 inert typed declarations above"); + expect(complete.stdout).toContain("#8285"); + expect(complete.stdout).toContain("#8286"); }); it("keeps CI and readable summary output modes separate", () => { diff --git a/tools/e2e/credential-free-tests.mts b/tools/e2e/credential-free-tests.mts index bd3d2e9b2e9..0513108b5f8 100644 --- a/tools/e2e/credential-free-tests.mts +++ b/tools/e2e/credential-free-tests.mts @@ -7,6 +7,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { moduleTagDeclarations, stripModuleTagDeclarations } from "./module-tags.mts"; +import { type E2eSemanticMetadata, validateE2eSemanticMetadata } from "./semantic-coverage.mts"; export const CREDENTIAL_FREE_TEST_TAG = "e2e/credential-free"; export const SHARED_E2E_JOB_ID = "shared-e2e"; @@ -38,6 +39,30 @@ const E2E_LIVE_CREDENTIAL_FREE_TEST_PATTERN = const INTEGRATION_CREDENTIAL_FREE_TEST_PATTERN = /^test\/(?!e2e\/)(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+\.test\.(?:js|ts)$/; const SUPPORTED_PROJECTS = new Set(["e2e-live", "integration"]); +const CREDENTIAL_FREE_TEST_SEMANTICS = { + "onboard-managed-image-buildless-e2e": { + agentRuntime: "none", + observableOutcome: "Buildless onboarding selects exact managed images for every agent", + environmentOrInferenceEndpoint: "Mocked integration environment; no inference endpoint", + unresolvedReason: "", + }, + "vllm-docker-storage": { + agentRuntime: "none", + observableOutcome: "vLLM storage gate accepts and rejects the intended host states", + environmentOrInferenceEndpoint: "Native Linux Docker host; no inference endpoint", + unresolvedReason: "", + }, +} as const satisfies Readonly>; + +export function credentialFreeTestSemantics(id: string): E2eSemanticMetadata { + const metadata = ( + CREDENTIAL_FREE_TEST_SEMANTICS as Readonly> + )[id]; + if (!metadata) { + throw new Error(`Credential-free test ${id} requires semantic coverage metadata`); + } + return validateE2eSemanticMetadata(metadata, `Credential-free test ${id}`); +} export function credentialFreeTestProjectForFile( file: string, diff --git a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts index 2bbaf777a17..b7f1b015877 100644 --- a/tools/e2e/hermes-gpu-startup-workflow-boundary.mts +++ b/tools/e2e/hermes-gpu-startup-workflow-boundary.mts @@ -60,7 +60,23 @@ function stringValue(value: unknown): string { return typeof value === "string" ? value : ""; } -const TOKENS = { "@bash": '/bin/bash "$trusted_fixture" "$@"', "@bin": "/usr/bin", "@daemon": '"$daemon_json"', "@docker": "/etc/docker/daemon.json", "@env": "/usr/bin/sudo -n /usr/bin/env -i", "@fixture": '"$trusted_fixture"', "@gpu": "hermes-gpu-fallback-docker-runtime", "@install": "/usr/bin/sudo /usr/bin/install", "@root": '"$trusted_state_root"', "@run": "run_trusted_fixture", "@sha": '"$TRUSTED_FIXTURE_SHA256"', "@source": '"$trusted_source"', "@state": '"$state_dir"', "@sudo": "/usr/bin/sudo", "@workflow": '"$TRUSTED_WORKFLOW_SHA"' } as const; +const TOKENS = { + "@bash": '/bin/bash "$trusted_fixture" "$@"', + "@bin": "/usr/bin", + "@daemon": '"$daemon_json"', + "@docker": "/etc/docker/daemon.json", + "@env": "/usr/bin/sudo -n /usr/bin/env -i", + "@fixture": '"$trusted_fixture"', + "@gpu": "hermes-gpu-fallback-docker-runtime", + "@install": "/usr/bin/sudo /usr/bin/install", + "@root": '"$trusted_state_root"', + "@run": "run_trusted_fixture", + "@sha": '"$TRUSTED_FIXTURE_SHA256"', + "@source": '"$trusted_source"', + "@state": '"$state_dir"', + "@sudo": "/usr/bin/sudo", + "@workflow": '"$TRUSTED_WORKFLOW_SHA"', +} as const; function proof(spec: string): string[] { return spec @@ -119,9 +135,25 @@ export function validateHermesGpuStartupWorkflow( strategy["max-parallel"] !== 1 || JSON.stringify(matrix.include) !== JSON.stringify([ - { scenario: "native", sandbox_name: "e2e-hgpu-native" }, - { scenario: "fallback", sandbox_name: "e2e-hgpu-fallback" }, - { scenario: "compatibility-only", sandbox_name: "e2e-hgpu-compat" }, + { + scenario: "native", + sandbox_name: "e2e-hgpu-native", + semantic_observable_outcome: "Native GPU startup reaches the stable Ready route", + semantic_variant: "native", + }, + { + scenario: "fallback", + sandbox_name: "e2e-hgpu-fallback", + semantic_observable_outcome: "Fallback GPU startup reaches the stable Ready route", + semantic_variant: "fallback", + }, + { + scenario: "compatibility-only", + sandbox_name: "e2e-hgpu-compat", + semantic_observable_outcome: + "Compatibility-only GPU startup reaches the stable Ready route", + semantic_variant: "compatibility-only", + }, ]) ) { errors.push(`${JOB_NAME} must serialize GPU scenarios`); diff --git a/tools/e2e/managed-image-multiarch-workflow-boundary.mts b/tools/e2e/managed-image-multiarch-workflow-boundary.mts index 69b5831a198..0bf4c67156f 100644 --- a/tools/e2e/managed-image-multiarch-workflow-boundary.mts +++ b/tools/e2e/managed-image-multiarch-workflow-boundary.mts @@ -128,11 +128,15 @@ export function validateManagedImageMultiarchWorkflow(workflow: WorkflowRecord): platform: "linux/amd64", runner: "ubuntu-24.04", shard: "linux-amd64", + semantic_environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup", + semantic_variant: "linux-amd64", }, { platform: "linux/arm64", runner: "ubuntu-24.04-arm", shard: "linux-arm64", + semantic_environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup", + semantic_variant: "linux-arm64", }, ], }, diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index 452d8523b4d..8dbc41c4f89 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -6,7 +6,7 @@ import { createHash } from "node:crypto"; export const MCP_DEV_WORKFLOW_EXECUTION_CONTEXT_SHA256 = "052c49d5e8688266dbf38fa911733132d33e4470a29a61deb6e7a11067737559"; export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = - "9f9983804a29816d7e1b35e9e791f453f4e9e83f4ec41b906953e976d372353e"; + "1a8952e270c4c0c163e5cd20c2a39835c8cbfe87d4b3dd87eee7b9478f96e045"; export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = diff --git a/tools/e2e/mcp-workflow-boundary.mts b/tools/e2e/mcp-workflow-boundary.mts index 44312a34e89..3a714d74472 100644 --- a/tools/e2e/mcp-workflow-boundary.mts +++ b/tools/e2e/mcp-workflow-boundary.mts @@ -43,8 +43,7 @@ const DEV_ARTIFACT_TRUSTED_CHECKOUT_NAME = "Checkout trusted OpenShell dev tooli const DEV_ARTIFACT_TRUSTED_CHECKOUT = ".trusted-openshell-dev-artifact"; const DEV_ARTIFACT_TRUSTED_PATHS = "scripts/install-openshell.sh\ntools/e2e/openshell-dev-artifact.mts\n"; -const DEV_ARTIFACT_SHARD_TRUSTED_PATHS = - `.github/scripts/docker-auth-cleanup.sh\n${DEV_ARTIFACT_TRUSTED_PATHS}`; +const DEV_ARTIFACT_SHARD_TRUSTED_PATHS = `.github/scripts/docker-auth-cleanup.sh\n${DEV_ARTIFACT_TRUSTED_PATHS}`; const DEV_ARTIFACT_TRUSTED_TOOL = `\${{ github.workspace }}/${DEV_ARTIFACT_TRUSTED_CHECKOUT}/${DEV_ARTIFACT_TOOL}`; const DEV_ARTIFACT_TRUSTED_INSTALLER = `\${{ github.workspace }}/${DEV_ARTIFACT_TRUSTED_CHECKOUT}/scripts/install-openshell.sh`; const DEV_ARTIFACT_SOURCE_OUTPUT = "${{ needs.openshell-dev-artifact.outputs.source_commit }}"; @@ -584,9 +583,7 @@ function validateJobExecution( trustedCheckoutIndex !== dockerAuthIndex + 1 || prepareIndex !== installIndex + 1 || restoreCliIndex !== prepareIndex + 1 || - trustedInstallSequence.some( - (step, offset) => steps[trustedCheckoutIndex + offset] !== step, - ) + trustedInstallSequence.some((step, offset) => steps[trustedCheckoutIndex + offset] !== step) ) { errors.push( "mcp-bridge-dev must complete trusted Node.js setup, Docker auth, artifact verification, credential revocation, and installation before candidate dependency preparation and CLI restore", @@ -594,12 +591,9 @@ function validateJobExecution( } if ( installIndex < 0 || - contentSha256(steps.slice(0, installIndex + 1)) !== - MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 + contentSha256(steps.slice(0, installIndex + 1)) !== MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 ) { - errors.push( - "mcp-bridge-dev must preserve every reviewed step through trusted installation", - ); + errors.push("mcp-bridge-dev must preserve every reviewed step through trusted installation"); } if ( prepareIndex < 0 || @@ -891,6 +885,11 @@ function validateCredentialWindowJob( const expectedEnv = { E2E_JOB: "1", E2E_TARGET_ID: CREDENTIAL_WINDOW_JOB, + E2E_SEMANTIC_AGENT_RUNTIME: "openclaw", + E2E_SEMANTIC_OBSERVABLE_OUTCOME: + "Credential expiry rotation detach and rebuild preserve the intended access window", + E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: + "Ubuntu Docker host; local compatible inference and MCP endpoint", E2E_ARTIFACT_DIR: `\${{ github.workspace }}/${CREDENTIAL_WINDOW_ARTIFACT_DIR}`, NEMOCLAW_CLI_BIN: "${{ github.workspace }}/bin/nemoclaw.js", NEMOCLAW_OPENSHELL_CHANNEL: "stable", diff --git a/tools/e2e/semantic-coverage.mts b/tools/e2e/semantic-coverage.mts new file mode 100644 index 00000000000..a94e4991452 --- /dev/null +++ b/tools/e2e/semantic-coverage.mts @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const E2E_AGENT_RUNTIMES = [ + "openclaw", + "hermes", + "langchain-deepagents-code", + "none", + "openclaw + hermes", + "openclaw + langchain-deepagents-code", + "openclaw + hermes + langchain-deepagents-code", + "unresolved", +] as const; + +export type E2eAgentRuntime = (typeof E2E_AGENT_RUNTIMES)[number]; + +export interface E2eSemanticMetadata { + agentRuntime: E2eAgentRuntime; + observableOutcome: string; + environmentOrInferenceEndpoint: string; + unresolvedReason: string; +} + +export const E2E_SEMANTIC_EXECUTION_SOURCES = [ + "catalogue", + "typed-registry", + "shared-e2e", + "retained-workflow", + "staging", +] as const; + +export type E2eSemanticExecutionSource = (typeof E2E_SEMANTIC_EXECUTION_SOURCES)[number]; + +export interface E2eSemanticExecutionRow extends E2eSemanticMetadata { + id: string; + variant: string; + source: E2eSemanticExecutionSource; +} + +const SELECTOR_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; +const SEMANTIC_TEXT_PATTERN = /^[A-Za-z0-9][A-Za-z0-9 .,+()/:;_-]{0,199}$/u; + +export function validateE2eSemanticMetadata( + metadata: E2eSemanticMetadata, + context: string, +): E2eSemanticMetadata { + if (!E2E_AGENT_RUNTIMES.includes(metadata.agentRuntime)) { + throw new Error(`${context} has an invalid agent runtime`); + } + for (const [field, value] of [ + ["observable outcome", metadata.observableOutcome], + ["environment or inference endpoint", metadata.environmentOrInferenceEndpoint], + ] as const) { + if (!SEMANTIC_TEXT_PATTERN.test(value)) { + throw new Error(`${context} has an invalid ${field}`); + } + } + const unresolved = + metadata.agentRuntime === "unresolved" || + metadata.observableOutcome === "unresolved" || + metadata.environmentOrInferenceEndpoint === "unresolved"; + if (unresolved !== (metadata.unresolvedReason !== "")) { + throw new Error( + `${context} must declare an unresolved reason exactly when a semantic field is unresolved`, + ); + } + if (metadata.unresolvedReason !== "" && !SEMANTIC_TEXT_PATTERN.test(metadata.unresolvedReason)) { + throw new Error(`${context} has an invalid unresolved reason`); + } + return metadata; +} + +export function validateE2eSemanticExecutionRows( + rows: readonly E2eSemanticExecutionRow[], +): readonly E2eSemanticExecutionRow[] { + const keys = new Set(); + const semanticEvidence = new Map(); + for (const row of rows) { + if (!SELECTOR_ID_PATTERN.test(row.id)) { + throw new Error(`E2E semantic coverage contains an invalid ID: ${row.id}`); + } + if (row.variant !== "" && !SELECTOR_ID_PATTERN.test(row.variant)) { + throw new Error(`E2E semantic coverage ${row.id} has an invalid variant`); + } + if (!E2E_SEMANTIC_EXECUTION_SOURCES.includes(row.source)) { + throw new Error(`E2E semantic coverage ${row.id} has an invalid source`); + } + validateE2eSemanticMetadata(row, `E2E semantic coverage ${row.id}`); + const key = `${row.source}:${row.id}:${row.variant}`; + if (keys.has(key)) { + throw new Error(`E2E semantic coverage contains a duplicate row: ${key}`); + } + keys.add(key); + if ( + row.agentRuntime !== "unresolved" && + row.observableOutcome !== "unresolved" && + row.environmentOrInferenceEndpoint !== "unresolved" + ) { + const evidenceKey = [ + row.agentRuntime, + row.observableOutcome, + row.environmentOrInferenceEndpoint, + ].join("\u0000"); + const previous = semanticEvidence.get(evidenceKey); + if (previous) { + throw new Error( + `E2E semantic coverage duplicates evidence between ${previous} and ${e2eSemanticExecutionLabel(row)}`, + ); + } + semanticEvidence.set(evidenceKey, e2eSemanticExecutionLabel(row)); + } + } + return rows; +} + +export function e2eSemanticExecutionLabel(row: E2eSemanticExecutionRow): string { + return row.variant === "" ? row.id : `${row.id} / ${row.variant}`; +} diff --git a/tools/e2e/target-catalogue.mts b/tools/e2e/target-catalogue.mts index c46b8675496..227d5f6f8ef 100644 --- a/tools/e2e/target-catalogue.mts +++ b/tools/e2e/target-catalogue.mts @@ -7,6 +7,8 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import { isDeepStrictEqual } from "node:util"; +import { type E2eAgentRuntime, validateE2eSemanticMetadata } from "./semantic-coverage.mts"; + export const E2E_EXECUTION_PROFILES = [ "standard", "nvidia-api", @@ -43,6 +45,10 @@ export interface E2eCatalogueTarget { id: string; targetId: string; displayName: string; + agentRuntime: E2eAgentRuntime; + observableOutcome: string; + environmentOrInferenceEndpoint: string; + unresolvedReason: string; testFile: string; profile: E2eExecutionProfile; runner: string; @@ -71,6 +77,10 @@ export interface E2eCatalogueMatrixRow { id: string; target_id: string; display_name: string; + agent_runtime: E2eAgentRuntime; + observable_outcome: string; + environment_or_inference_endpoint: string; + unresolved_reason: string; runner: string; runner_key: string; test_file: string; @@ -94,6 +104,10 @@ type TargetOptions = Omit< | "testFile" | "owningPaths" | "releaseRequired" + | "agentRuntime" + | "observableOutcome" + | "environmentOrInferenceEndpoint" + | "unresolvedReason" | "environment" | "hostPackages" | "cloudflared" @@ -109,6 +123,9 @@ type TargetOptions = Omit< | "shard" | "artifactLayout" > & { + agentRuntime: E2eAgentRuntime; + environmentOrInferenceEndpoint: string; + unresolvedReason?: string; owningPaths?: readonly string[]; environment?: Readonly>; hostPackages?: readonly E2eHostPackage[]; @@ -130,6 +147,9 @@ type TargetOptions = Omit< function target(id: string, options: TargetOptions): E2eCatalogueTarget { const { displayName, + agentRuntime, + environmentOrInferenceEndpoint, + unresolvedReason = "", owningPaths = [], environment = {}, hostPackages = [], @@ -151,6 +171,10 @@ function target(id: string, options: TargetOptions): E2eCatalogueTarget { return { id, displayName, + agentRuntime, + observableOutcome: displayName, + environmentOrInferenceEndpoint, + unresolvedReason, testFile, owningPaths: [testFile, ...owningPaths], releaseRequired: true, @@ -183,6 +207,7 @@ const nonInteractive = { function commonEgressTarget(options: { displayName: string; + environmentOrInferenceEndpoint: string; hermes?: boolean; selector: string; shard: string; @@ -190,6 +215,8 @@ function commonEgressTarget(options: { return target(`common-egress-agent-${options.shard}`, { targetId: "common-egress-agent", displayName: options.displayName, + agentRuntime: options.hermes ? "hermes" : "openclaw", + environmentOrInferenceEndpoint: options.environmentOrInferenceEndpoint, profile: "brave-nvidia-inference", testFile: "test/e2e/live/common-egress-agent.test.ts", timeoutMinutes: 60, @@ -230,6 +257,11 @@ function gatewayUpgradeTarget(options: GatewayUpgradeTargetOptions): E2eCatalogu return target(`openshell-gateway-upgrade-${options.shard}`, { targetId: "openshell-gateway-upgrade", displayName: options.displayName, + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: + options.runner === "ubuntu-24.04-arm" + ? "Arm64 Ubuntu; GitHub release artifacts; no inference endpoint" + : "x86-64 Ubuntu; GitHub release artifacts; no inference endpoint", profile: "github-read", runner: options.runner ?? "ubuntu-latest", testFile: "test/e2e/live/openshell-gateway-upgrade.test.ts", @@ -340,6 +372,8 @@ export function catalogueExclusionReason(id: string): string | undefined { export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("agent-turn-latency", { displayName: "Performance: bounds hosted inference turns for OpenClaw and Hermes", + agentRuntime: "openclaw + hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, timeoutMinutes: 110, @@ -365,6 +399,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("bedrock-runtime-compatible-anthropic-openclaw", { targetId: "bedrock-runtime-compatible-anthropic", displayName: "Inference: OpenClaw routes an Anthropic request through Amazon Bedrock", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; Amazon Bedrock Anthropic-compatible endpoint", profile: "standard", testFile: "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", timeoutMinutes: 60, @@ -383,6 +419,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("bedrock-runtime-compatible-anthropic-hermes", { targetId: "bedrock-runtime-compatible-anthropic", displayName: "Inference: Hermes routes an Anthropic request through Amazon Bedrock", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; Amazon Bedrock Anthropic-compatible endpoint", profile: "standard", testFile: "test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts", timeoutMinutes: 60, @@ -401,6 +439,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("bootstrap-install-smoke", { displayName: "Install: bootstraps NemoClaw and completes hosted inference", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, timeoutMinutes: 30, @@ -422,6 +462,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("brave-search", { displayName: "Search: OpenClaw returns a Brave result without exposing its key", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Brave Search", profile: "brave-nvidia-inference", timeoutMinutes: 45, installMode: "authenticated", @@ -437,6 +479,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("channels-add-remove", { displayName: "Messaging: adds and removes Telegram configuration", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; no inference endpoint", profile: "standard", timeoutMinutes: 75, installMode: "credential-free", @@ -454,6 +498,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("channels-stop-start-openclaw", { targetId: "channels-stop-start", displayName: "Messaging: OpenClaw preserves channels across stop and start", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, testFile: "test/e2e/live/channels-stop-start.test.ts", @@ -480,6 +526,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("channels-stop-start-hermes", { targetId: "channels-stop-start", displayName: "Messaging: Hermes preserves channels across stop and start", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, testFile: "test/e2e/live/channels-stop-start.test.ts", @@ -508,6 +556,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("cloud-inference", { displayName: "Inference: OpenClaw uses hosted inference", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 50, installMode: "none", @@ -521,22 +571,27 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), commonEgressTarget({ displayName: "Networking: OpenClaw answers through balanced egress", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and public weather endpoint", shard: "openclaw-balanced-weather", selector: "^common-egress.+C1.+$", }), commonEgressTarget({ displayName: "Networking: OpenClaw reaches a public reference through open egress", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and public reference endpoint", shard: "openclaw-open-reference", selector: "^common-egress.+C2.+$", }), commonEgressTarget({ displayName: "Networking: Hermes reaches a public reference through open egress", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and public reference endpoint", hermes: true, shard: "hermes-open-reference", selector: "^common-egress.+C3.+$", }), target("concurrent-gateway-ports", { displayName: "Gateway: isolates ports for concurrent sandboxes", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local gateway; no inference endpoint", profile: "standard", timeoutMinutes: 90, installMode: "authenticated", @@ -546,6 +601,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("cron-preflight-inference-local", { displayName: "Preflight: reaches managed inference without DNS failure", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local managed inference", profile: "nvidia-inference", timeoutMinutes: 45, installMode: "authenticated", @@ -561,6 +618,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("dashboard-remote-bind", { displayName: "Dashboard: retains audit findings when bound remotely", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 65, installMode: "none", @@ -576,6 +635,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("device-auth-health", { displayName: "Health: treats a 401 authentication response as reachable", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; local authentication fixture; no inference endpoint", profile: "standard", timeoutMinutes: 40, installMode: "authenticated", @@ -590,6 +651,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("double-onboard", { displayName: "Onboarding: reuses the gateway and preserves sibling sandboxes", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local gateway fixtures", profile: "standard", timeoutMinutes: 90, installMode: "authenticated", @@ -599,6 +662,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("gpu-double-onboard", { displayName: "Onboarding: preserves Ollama authentication after GPU re-onboarding", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "NVIDIA GPU runner; local Ollama", profile: "standard", runner: "linux-amd64-gpu-rtxpro6000-latest-1", timeoutMinutes: 100, @@ -615,6 +680,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("gpu-e2e", { displayName: "Inference: routes an agent turn through GPU Ollama", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "NVIDIA GPU runner; local Ollama", profile: "standard", runner: "linux-amd64-gpu-rtxpro6000-latest-1", timeoutMinutes: 90, @@ -633,6 +700,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("full-e2e", { displayName: "OpenClaw: installs, onboards, and completes an agent turn", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 75, installMode: "authenticated", @@ -651,6 +720,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("gateway-guard-recovery", { displayName: "Gateway: restores the guard chain after recreation", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 45, installMode: "authenticated", @@ -666,6 +737,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("hermes-discord", { displayName: "Messaging: Hermes preserves Discord configuration across rebuild", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Discord", profile: "nvidia-inference", prAdvisorSelectable: true, timeoutMinutes: 90, @@ -689,6 +762,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("hermes-inference-switch", { displayName: "Inference: Hermes switches to an Anthropic-compatible endpoint", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; Anthropic-compatible inference fixture", profile: "standard", timeoutMinutes: 55, installMode: "authenticated", @@ -712,6 +787,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("hermes-shields-config", { displayName: "Shields: restores stopped Hermes across posture changes", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu Docker host; no inference endpoint", profile: "standard", timeoutMinutes: 60, installMode: "none", @@ -729,6 +806,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("hermes-slack", { displayName: "Messaging: isolates Hermes Slack credentials and reaches Slack APIs", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Slack", profile: "nvidia-inference", runner: "linux-amd64-cpu4", testFile: "test/e2e/live/hermes-slack-e2e.test.ts", @@ -751,6 +830,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("issue-2478-crash-loop-recovery", { displayName: "Gateway: recovers after process termination and remains stable", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local gateway; no inference endpoint", profile: "standard", timeoutMinutes: 30, installMode: "authenticated", @@ -764,6 +845,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("issue-4462-scope-upgrade-approval", { displayName: "Authorization: approves a write-scope upgrade without operator.admin", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 90, installMode: "authenticated", @@ -777,6 +860,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("inference-routing", { displayName: "Inference: rejects unsafe routes and proves runtime identities", + agentRuntime: "openclaw + langchain-deepagents-code", + environmentOrInferenceEndpoint: "Ubuntu; local compatible and HTTPS inference fixtures", profile: "standard", timeoutMinutes: 45, installMode: "none", @@ -786,6 +871,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("kimi-inference-compat", { displayName: "Inference: configures a Kimi-compatible endpoint", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; Kimi-compatible inference fixture", profile: "standard", timeoutMinutes: 50, installMode: "authenticated", @@ -800,6 +887,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("llama-cpp-generic-gpu", { displayName: "Inference: completes an agent turn with llama.cpp on a generic NVIDIA GPU", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "NVIDIA GPU runner; local llama.cpp", profile: "standard", runner: "linux-amd64-gpu-rtxpro6000-latest-1", timeoutMinutes: 120, @@ -816,6 +905,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("messaging-compatible-endpoint", { displayName: "Messaging: routes Telegram through a compatible endpoint", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; compatible inference and Telegram fixtures", profile: "standard", timeoutMinutes: 45, installMode: "none", @@ -831,6 +922,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("model-router-provider-routed-inference", { displayName: "Inference: Model Router returns a provider-routed response", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA API and Model Router", profile: "nvidia-api", timeoutMinutes: 45, installMode: "none", @@ -840,6 +933,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("network-policy", { displayName: "Network policy: enforces restricted allow and deny rules", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and network probes", profile: "nvidia-inference", timeoutMinutes: 90, installMode: "credential-free", @@ -866,6 +961,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("ollama-auth-proxy", { displayName: "Inference: Ollama proxy enforces and preserves authentication", + agentRuntime: "none", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local Ollama proxy", profile: "standard", timeoutMinutes: 45, installMode: "none", @@ -878,6 +975,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("onboard-repair", { displayName: "Onboarding: repairs a missing sandbox and rejects conflicting resume input", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local onboarding fixtures", profile: "standard", timeoutMinutes: 75, installMode: "authenticated", @@ -887,6 +986,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("onboard-policy-preset-sequencing", { displayName: "Onboarding: preserves policy preset step order", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; no inference endpoint", profile: "standard", timeoutMinutes: 60, installMode: "authenticated", @@ -897,6 +998,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("onboard-resume", { displayName: "Onboarding: resumes interrupted setup from recorded progress", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; local onboarding fixtures", profile: "standard", timeoutMinutes: 45, installMode: "credential-free", @@ -906,6 +1009,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("openclaw-discord-pairing", { displayName: "Messaging: shares OpenClaw Discord pairing approval", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Discord", profile: "nvidia-inference", timeoutMinutes: 60, installMode: "credential-free", @@ -921,6 +1026,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("openclaw-skill-cli", { displayName: "Skills: OpenClaw installs and inspects workspace skills", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 60, installMode: "none", @@ -934,6 +1041,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("openclaw-inference-switch", { displayName: "Inference: OpenClaw switches providers and remains responsive", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; compatible inference fixtures", profile: "standard", timeoutMinutes: 90, installMode: "none", @@ -954,6 +1063,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("openclaw-tui-chat-correlation", { displayName: "TUI: keeps rapid OpenClaw turns correlated", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 75, installMode: "none", @@ -977,6 +1088,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("openclaw-slack-pairing", { displayName: "Messaging: shares OpenClaw Slack pairing approval", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Slack", profile: "nvidia-inference", timeoutMinutes: 60, installMode: "credential-free", @@ -994,6 +1107,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ ...GATEWAY_UPGRADE_TARGETS, target("rebuild-openclaw", { displayName: "Rebuild: preserves OpenClaw state and rotates the gateway token", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 130, installMode: "credential-free", @@ -1007,6 +1122,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("rebuild-hermes", { displayName: "Rebuild: preserves Hermes state and recovers cron dispatch", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, owningPaths: ["test/e2e/live/rebuild-hermes-cron-restore.ts"], @@ -1034,6 +1151,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("rebuild-hermes-stale-base", { displayName: "Rebuild: refreshes a stale Hermes base and restores state", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, testFile: "test/e2e/live/rebuild-hermes.test.ts", @@ -1063,6 +1182,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("sandbox-survival", { displayName: "Lifecycle: preserves sandbox state after an OpenShell gateway restart", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 30, installMode: "none", @@ -1077,6 +1198,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("sandbox-operations", { displayName: "Sandbox: preserves lifecycle and multi-sandbox operations", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 60, installMode: "credential-free", @@ -1095,6 +1218,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("security-posture-openclaw", { targetId: "security-posture", displayName: "Security: OpenClaw retains the required sandbox posture", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, testFile: "test/e2e/live/full-e2e.test.ts", @@ -1121,6 +1246,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ target("security-posture-hermes", { targetId: "security-posture", displayName: "Security: Hermes retains the required sandbox posture", + agentRuntime: "hermes", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, testFile: "test/e2e/live/hermes-e2e.test.ts", @@ -1149,6 +1276,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("sessions-agents-cli", { displayName: "CLI: routes sessions and agents to OpenClaw", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 70, installMode: "credential-free", @@ -1163,6 +1292,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("shields-config", { displayName: "Shields: restores stopped OpenClaw across posture changes", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 45, installMode: "none", @@ -1178,6 +1309,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("snapshot-commands", { displayName: "Snapshot: restores selected sandbox state without credential leaks", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu Docker host; no inference endpoint", profile: "standard", timeoutMinutes: 40, installMode: "none", @@ -1197,6 +1330,9 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("spark-install", { displayName: "Install: leaves NemoClaw and OpenShell usable after standard installation", + agentRuntime: "unresolved", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", + unresolvedReason: "The test asserts CLI usability but does not assert an agent runtime", profile: "nvidia-inference", timeoutMinutes: 45, installMode: "none", @@ -1213,6 +1349,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("skill-agent", { displayName: "Skills: OpenClaw reads an injected sandbox skill", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", prAdvisorSelectable: true, timeoutMinutes: 30, @@ -1223,6 +1361,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("state-backup-restore", { displayName: "Backup: restores workspace files and memory", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference", profile: "nvidia-inference", timeoutMinutes: 60, installMode: "credential-free", @@ -1237,6 +1377,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("telegram-injection", { displayName: "Messaging: treats Telegram shell metacharacters as data", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Telegram fixture", profile: "nvidia-inference", timeoutMinutes: 45, installMode: "credential-free", @@ -1251,6 +1393,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("token-rotation", { displayName: "Messaging: rotates one provider token without rebuilding siblings", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; no inference endpoint", profile: "github-read", timeoutMinutes: 45, installMode: "none", @@ -1269,6 +1413,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("tunnel-lifecycle", { displayName: "Tunnel: starts, probes, and stops a public dashboard tunnel", + agentRuntime: "openclaw", + environmentOrInferenceEndpoint: "Ubuntu; NVIDIA hosted inference and Cloudflare tunnel", profile: "nvidia-inference", timeoutMinutes: 75, installMode: "none", @@ -1284,6 +1430,8 @@ export const E2E_TARGET_CATALOGUE: readonly E2eCatalogueTarget[] = [ }), target("whatsapp-qr-compact", { displayName: "Messaging: renders a compact WhatsApp pairing QR code", + agentRuntime: "none", + environmentOrInferenceEndpoint: "Ubuntu; no sandbox or inference endpoint", profile: "standard", timeoutMinutes: 15, installMode: "none", @@ -1356,8 +1504,7 @@ export function validateE2eTargetCatalogue( } if ( entry.runnerKey !== "" && - (!ID_PATTERN.test(entry.runnerKey) || - !E2E_CATALOGUE_RUNNER_KEY_SET.has(entry.runnerKey)) + (!ID_PATTERN.test(entry.runnerKey) || !E2E_CATALOGUE_RUNNER_KEY_SET.has(entry.runnerKey)) ) { throw new Error(`E2E target ${entry.id} has an invalid runner routing key`); } @@ -1435,6 +1582,10 @@ export function validateE2eTargetCatalogue( throw new Error(`E2E target ${entry.id} has an invalid environment entry`); } } + validateE2eSemanticMetadata(entry, `E2E target ${entry.id}`); + if (entry.observableOutcome !== entry.displayName) { + throw new Error(`E2E target ${entry.id} must use its display name as its observable outcome`); + } } return targets; } @@ -1487,6 +1638,10 @@ export function catalogueMatrix( id: entry.id, target_id: entry.targetId, display_name: entry.displayName, + agent_runtime: entry.agentRuntime, + observable_outcome: entry.observableOutcome, + environment_or_inference_endpoint: entry.environmentOrInferenceEndpoint, + unresolved_reason: entry.unresolvedReason, runner: entry.runner, runner_key: entry.runnerKey, test_file: entry.testFile, diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f894bfd4400..c85b8e5651f 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -39,6 +39,11 @@ import { } from "./operations-workflow-boundary.mts"; import { validateRunnerComparisonWorkflowBoundary } from "./runner-comparison-workflow-boundary.mts"; import { normalizeE2eSelectorIds } from "./selector-aliases.mts"; +import { + type E2eSemanticExecutionRow, + validateE2eSemanticExecutionRows, + validateE2eSemanticMetadata, +} from "./semantic-coverage.mts"; import { validateStandardProfileWorkflowBoundary } from "./standard-profile-workflow-boundary.mts"; import { validateTrustedHermesSwapHelperSource, @@ -114,6 +119,7 @@ export interface FreeStandingJobsInventory { freeStandingTargets: string[]; targetToJob: Map; liveTestToJobs: Map; + semanticRows: E2eSemanticExecutionRow[]; } export interface FocusedE2eJob { @@ -154,6 +160,11 @@ const LIVE_TEST_FILE_PATTERN = /test\/e2e\/live\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0- const FREE_STANDING_JOB_MARKER = "E2E_JOB"; const FREE_STANDING_TARGET_MARKER = "E2E_TARGET_ID"; const FREE_STANDING_DEFAULT_ENABLED_MARKER = "E2E_DEFAULT_ENABLED"; +const SEMANTIC_AGENT_RUNTIME_MARKER = "E2E_SEMANTIC_AGENT_RUNTIME"; +const SEMANTIC_OUTCOME_MARKER = "E2E_SEMANTIC_OBSERVABLE_OUTCOME"; +const SEMANTIC_ENVIRONMENT_MARKER = "E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT"; +const SEMANTIC_UNRESOLVED_MARKER = "E2E_SEMANTIC_UNRESOLVED_REASON"; +const STAGING_BREV_JOB_ID = "staging-brev-launchable"; const COMMON_SECRET_ENV_NAMES = [ "NVIDIA_API_KEY", "NVIDIA_INFERENCE_API_KEY", @@ -449,6 +460,50 @@ function findDuplicates(values: readonly string[]): string[] { return [...duplicates].sort(); } +function workflowSemanticRows(jobId: string, job: WorkflowRecord): E2eSemanticExecutionRow[] { + const env = asRecord(job.env); + const matrix = asRecord(asRecord(job.strategy).matrix); + const includes = Array.isArray(matrix.include) + ? matrix.include + .map(asRecord) + .filter((entry) => Object.keys(entry).some((key) => key.startsWith("semantic_"))) + : []; + const hasEnvironmentMetadata = [ + SEMANTIC_AGENT_RUNTIME_MARKER, + SEMANTIC_OUTCOME_MARKER, + SEMANTIC_ENVIRONMENT_MARKER, + SEMANTIC_UNRESOLVED_MARKER, + ].some((key) => Object.hasOwn(env, key)); + if (!hasEnvironmentMetadata && includes.length === 0) return []; + + const candidates = includes.length > 0 ? includes : [{}]; + return candidates.map((entry) => { + const metadata = validateE2eSemanticMetadata( + { + agentRuntime: stringValue( + entry.semantic_agent_runtime || env[SEMANTIC_AGENT_RUNTIME_MARKER], + ), + observableOutcome: stringValue( + entry.semantic_observable_outcome || env[SEMANTIC_OUTCOME_MARKER], + ), + environmentOrInferenceEndpoint: stringValue( + entry.semantic_environment_or_inference_endpoint || env[SEMANTIC_ENVIRONMENT_MARKER], + ), + unresolvedReason: stringValue( + entry.semantic_unresolved_reason || env[SEMANTIC_UNRESOLVED_MARKER], + ), + } as Parameters[0], + `E2E workflow job ${jobId}`, + ); + return { + id: jobId, + variant: stringValue(entry.semantic_variant), + source: jobId === STAGING_BREV_JOB_ID ? "staging" : "retained-workflow", + ...metadata, + }; + }); +} + function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { errors: string[]; inventory: FreeStandingJobsInventory; @@ -460,10 +515,16 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { const freeStandingTargets: string[] = []; const targetToJob = new Map(); const liveTestToJobs = new Map(); + const semanticRows: E2eSemanticExecutionRow[] = []; for (const [jobId, rawJob] of Object.entries(jobs)) { const job = asRecord(rawJob); const env = asRecord(job.env); + try { + semanticRows.push(...workflowSemanticRows(jobId, job)); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } if (jobId === SHARED_E2E_JOB_ID) continue; const hasJobMarker = Object.hasOwn(env, FREE_STANDING_JOB_MARKER); const hasTargetMarker = Object.hasOwn(env, FREE_STANDING_TARGET_MARKER); @@ -532,6 +593,22 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { for (const duplicate of findDuplicates(freeStandingTargets)) { errors.push(`free-standing workflow metadata repeats target id: ${duplicate}`); } + for (const jobId of workflowJobs) { + if (jobId !== SHARED_E2E_JOB_ID && !semanticRows.some((row) => row.id === jobId)) { + errors.push(`${jobId} job requires semantic coverage metadata`); + } + } + if ( + Object.hasOwn(jobs, STAGING_BREV_JOB_ID) && + !semanticRows.some((row) => row.source === "staging") + ) { + errors.push(`${STAGING_BREV_JOB_ID} job requires semantic coverage metadata`); + } + try { + validateE2eSemanticExecutionRows(semanticRows); + } catch (error) { + errors.push(error instanceof Error ? error.message : String(error)); + } return { errors, @@ -541,6 +618,7 @@ function deriveFreeStandingJobsInventoryFromJobs(jobs: WorkflowRecord): { explicitOnlyJobs, freeStandingTargets, targetToJob, + semanticRows, liveTestToJobs: new Map( [...liveTestToJobs] .sort(([left], [right]) => left.localeCompare(right)) @@ -568,6 +646,7 @@ function cloneFreeStandingJobsInventory( explicitOnlyJobs: [...inventory.explicitOnlyJobs], freeStandingTargets: [...inventory.freeStandingTargets], targetToJob: new Map(inventory.targetToJob), + semanticRows: inventory.semanticRows.map((row) => ({ ...row })), liveTestToJobs: cloneStringArrayMap(inventory.liveTestToJobs), }; } @@ -2488,9 +2567,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (liveTargets["runs-on"] !== "${{ matrix.runner }}") { errors.push("live job must run on the matrix runner"); } - if ( - !isDeepStrictEqual(liveTargets.needs, ["base-image-publication", "generate-matrix"]) - ) { + if (!isDeepStrictEqual(liveTargets.needs, ["base-image-publication", "generate-matrix"])) { errors.push("live job must depend on base-image-publication and generate-matrix"); } if (liveTargets.if !== "${{ needs.generate-matrix.outputs.matrix != '[]' }}") { diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index b7e05b8f313..7f649515981 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -4,11 +4,14 @@ import { appendFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { isDeepStrictEqual } from "node:util"; import { buildLiveTargetMatrix, type LiveTargetMatrixEntry } from "../../test/e2e/registry/run.ts"; +import { listTargets } from "../../test/e2e/registry/registry.ts"; import { buildRiskPlan } from "../advisors/risk-plan.mts"; import { type CredentialFreeTestMatrixRow, + credentialFreeTestSemantics, discoverCredentialFreeTests, SHARED_E2E_JOB_ID, } from "./credential-free-tests.mts"; @@ -32,6 +35,12 @@ import { focusedE2eJobsForChangedFiles, readFreeStandingJobsInventory, } from "./workflow-boundary.mts"; +import { + e2eSemanticExecutionLabel, + type E2eSemanticExecutionRow, + validateE2eSemanticExecutionRows, + validateE2eSemanticMetadata, +} from "./semantic-coverage.mts"; export type WorkflowPlanSelectors = { jobs?: string; @@ -42,6 +51,7 @@ export type E2eWorkflowPlan = { matrix: LiveTargetMatrixEntry[]; testMatrix: CredentialFreeTestMatrixRow[]; catalogueMatrices: Record; + semanticMatrix: E2eSemanticExecutionRow[]; selectedJobs: string[]; hermesSelected: boolean; explicitOnlyJobs: string[]; @@ -105,15 +115,43 @@ function isStringArray(value: unknown): value is string[] { return Array.isArray(value) && value.every((entry) => typeof entry === "string"); } +function hasValidSemanticMetadata(value: Record): boolean { + if ( + typeof value.agentRuntime !== "string" || + typeof value.observableOutcome !== "string" || + typeof value.environmentOrInferenceEndpoint !== "string" || + typeof value.unresolvedReason !== "string" + ) { + return false; + } + try { + validateE2eSemanticMetadata( + { + agentRuntime: value.agentRuntime as E2eSemanticExecutionRow["agentRuntime"], + observableOutcome: value.observableOutcome, + environmentOrInferenceEndpoint: value.environmentOrInferenceEndpoint, + unresolvedReason: value.unresolvedReason, + }, + "E2E workflow plan row", + ); + return true; + } catch { + return false; + } +} + function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry { if (!isRecord(value)) return false; if ( !hasExactKeys(value, [ + "agentRuntime", + "environmentOrInferenceEndpoint", "expectedStateId", "id", "install", "label", "onboarding", + "observableOutcome", "pendingRuntimeSuites", "platform", "requiredSecrets", @@ -122,6 +160,7 @@ function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry "suites", "supportReasons", "supported", + "unresolvedReason", ]) ) { return false; @@ -137,11 +176,16 @@ function isLiveTargetMatrixEntry(value: unknown): value is LiveTargetMatrixEntry typeof value.runtime === "string" && typeof value.onboarding === "string" && typeof value.expectedStateId === "string" && + typeof value.agentRuntime === "string" && + typeof value.observableOutcome === "string" && + typeof value.environmentOrInferenceEndpoint === "string" && + typeof value.unresolvedReason === "string" && typeof value.supported === "boolean" && isStringArray(value.suites) && isStringArray(value.requiredSecrets) && isStringArray(value.supportReasons) && - isStringArray(value.pendingRuntimeSuites) + isStringArray(value.pendingRuntimeSuites) && + hasValidSemanticMetadata(value) ); } @@ -174,15 +218,18 @@ function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { isRecord(value) && hasExactKeys(value, [ "artifact_layout", + "agent_runtime", "cloudflared", "compatible_api_key", "id", "display_name", + "environment_or_inference_endpoint", "host_preparation", "host_packages", "install_mode", "install_non_interactive", "restore_cli", + "observable_outcome", "runner", "runner_comparison", "runner_key", @@ -191,11 +238,16 @@ function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { "target_id", "test_file", "timeout_minutes", + "unresolved_reason", ]) && typeof value.id === "string" && /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(value.id) && typeof value.display_name === "string" && /^[A-Z][A-Za-z0-9 .'+()-]+: [^/\r\n]{1,72}$/u.test(value.display_name) && + typeof value.agent_runtime === "string" && + typeof value.observable_outcome === "string" && + typeof value.environment_or_inference_endpoint === "string" && + typeof value.unresolved_reason === "string" && typeof value.runner === "string" && /^[A-Za-z0-9._-]+$/u.test(value.runner) && typeof value.runner_key === "string" && @@ -223,10 +275,53 @@ function isCatalogueMatrixRow(value: unknown): value is E2eCatalogueMatrixRow { (value.install_mode === "none" || value.install_mode === "authenticated" || value.install_mode === "credential-free") && - typeof value.restore_cli === "boolean" + typeof value.restore_cli === "boolean" && + hasValidSemanticMetadata({ + agentRuntime: value.agent_runtime, + observableOutcome: value.observable_outcome, + environmentOrInferenceEndpoint: value.environment_or_inference_endpoint, + unresolvedReason: value.unresolved_reason, + }) ); } +function isSemanticExecutionRow(value: unknown): value is E2eSemanticExecutionRow { + if ( + !isRecord(value) || + !hasExactKeys(value, [ + "agentRuntime", + "environmentOrInferenceEndpoint", + "id", + "observableOutcome", + "source", + "unresolvedReason", + "variant", + ]) || + typeof value.id !== "string" || + typeof value.variant !== "string" || + typeof value.source !== "string" || + !hasValidSemanticMetadata(value) + ) { + return false; + } + try { + validateE2eSemanticExecutionRows([value as unknown as E2eSemanticExecutionRow]); + return true; + } catch { + return false; + } +} + +function isSemanticExecutionRows(value: unknown): value is E2eSemanticExecutionRow[] { + if (!Array.isArray(value) || !value.every(isSemanticExecutionRow)) return false; + try { + validateE2eSemanticExecutionRows(value); + return true; + } catch { + return false; + } +} + function isCatalogueMatrixRowForProfile( value: unknown, profile: E2eExecutionProfile, @@ -237,6 +332,10 @@ function isCatalogueMatrixRowForProfile( target?.profile === profile && target.targetId === value.target_id && target.displayName === value.display_name && + target.agentRuntime === value.agent_runtime && + target.observableOutcome === value.observable_outcome && + target.environmentOrInferenceEndpoint === value.environment_or_inference_endpoint && + target.unresolvedReason === value.unresolved_reason && target.runner === value.runner && target.runnerKey === value.runner_key && target.testFile === value.test_file && @@ -374,12 +473,62 @@ function emptyE2eWorkflowPlan(): E2eWorkflowPlan { matrix: [], testMatrix: [], catalogueMatrices: emptyCatalogueMatrices(), + semanticMatrix: [], selectedJobs: [], hermesSelected: false, explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }; } +type E2eWorkflowPlanWithoutSemantics = Omit; + +function semanticMatrixForPlan( + plan: E2eWorkflowPlanWithoutSemantics, + inventory: ReturnType, +): E2eSemanticExecutionRow[] { + const catalogueRows = E2E_EXECUTION_PROFILES.flatMap((profile) => + plan.catalogueMatrices[profile].map((row) => ({ + id: row.id, + variant: "", + source: "catalogue" as const, + agentRuntime: row.agent_runtime, + observableOutcome: row.observable_outcome, + environmentOrInferenceEndpoint: row.environment_or_inference_endpoint, + unresolvedReason: row.unresolved_reason, + })), + ); + const registryRows = plan.matrix.map((row) => ({ + id: row.id, + variant: "", + source: "typed-registry" as const, + agentRuntime: row.agentRuntime, + observableOutcome: row.observableOutcome, + environmentOrInferenceEndpoint: row.environmentOrInferenceEndpoint, + unresolvedReason: row.unresolvedReason, + })); + const sharedRows = plan.testMatrix.map((row) => ({ + id: row.id, + variant: "", + source: "shared-e2e" as const, + ...credentialFreeTestSemantics(row.id), + })); + const selectedJobs = new Set(plan.selectedJobs); + const workflowRows = inventory.semanticRows.filter((row) => selectedJobs.has(row.id)); + const rows = [...catalogueRows, ...registryRows, ...sharedRows, ...workflowRows]; + validateE2eSemanticExecutionRows(rows); + return rows; +} + +function withSemanticMatrix( + plan: E2eWorkflowPlanWithoutSemantics, + inventory: ReturnType, +): E2eWorkflowPlan { + return { + ...plan, + semanticMatrix: semanticMatrixForPlan(plan, inventory), + }; +} + export function releaseRequiredWorkflowJobs(options?: { waivedJobs?: readonly string[]; }): string[] { @@ -454,14 +603,17 @@ export function buildE2eWorkflowPlan( (jobs.length === 1 && jobs[0] === JETSON_DISPATCH_TARGET && targets.length === 0) || (targets.length === 1 && targets[0] === JETSON_DISPATCH_TARGET && jobs.length === 0); if (jetsonDispatchSelected) { - return { - matrix: [], - testMatrix: [], - catalogueMatrices: emptyCatalogueMatrices(), - selectedJobs: [JETSON_DISPATCH_TARGET], - hermesSelected: false, - explicitOnlyJobs: [...inventory.explicitOnlyJobs], - }; + return withSemanticMatrix( + { + matrix: [], + testMatrix: [], + catalogueMatrices: emptyCatalogueMatrices(), + selectedJobs: [JETSON_DISPATCH_TARGET], + hermesSelected: false, + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }, + inventory, + ); } const credentialFreeTests = discoverCredentialFreeTests(); const catalogueIds = new Set( @@ -496,14 +648,17 @@ export function buildE2eWorkflowPlan( selectedJobSet.add("openshell-credential-generation-window"); } const selectedJobs = [...selectedJobSet]; - return { - matrix: registryTargets.length > 0 ? buildLiveTargetMatrix(registryTargets) : [], - testMatrix: selectTestRows(credentialFreeTests, [...jobs, ...targets]), - catalogueMatrices: catalogueMatrices(selectedCatalogueTargets), - selectedJobs, - hermesSelected: selectedJobs.includes(HERMES_JOB_ID), - explicitOnlyJobs: [...inventory.explicitOnlyJobs], - }; + return withSemanticMatrix( + { + matrix: registryTargets.length > 0 ? buildLiveTargetMatrix(registryTargets) : [], + testMatrix: selectTestRows(credentialFreeTests, [...jobs, ...targets]), + catalogueMatrices: catalogueMatrices(selectedCatalogueTargets), + selectedJobs, + hermesSelected: selectedJobs.includes(HERMES_JOB_ID), + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }, + inventory, + ); } if (options.changedFiles) { @@ -559,24 +714,32 @@ export function buildE2eWorkflowPlan( ...registryTargetsForChangedFiles(changedFiles), ...(riskTargetIds.length > 0 ? buildLiveTargetMatrix(riskTargetIds) : []), ].filter((entry, index, rows) => rows.findIndex((row) => row.id === entry.id) === index); - return { - matrix: registryMatrix, - testMatrix: selectedTests, - catalogueMatrices: catalogueMatrices(selectedCatalogueTargets), - selectedJobs, - hermesSelected: selectedJobs.includes(HERMES_JOB_ID), - explicitOnlyJobs: [...inventory.explicitOnlyJobs], - }; + return withSemanticMatrix( + { + matrix: registryMatrix, + testMatrix: selectedTests, + catalogueMatrices: catalogueMatrices(selectedCatalogueTargets), + selectedJobs, + hermesSelected: selectedJobs.includes(HERMES_JOB_ID), + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }, + inventory, + ); } - return { - matrix: buildLiveTargetMatrix(), - testMatrix: credentialFreeTests, - catalogueMatrices: catalogueMatrices(E2E_TARGET_CATALOGUE), - selectedJobs: inventory.workflowJobs.filter((job) => !inventory.explicitOnlyJobs.includes(job)), - hermesSelected: true, - explicitOnlyJobs: [...inventory.explicitOnlyJobs], - }; + return withSemanticMatrix( + { + matrix: buildLiveTargetMatrix(), + testMatrix: credentialFreeTests, + catalogueMatrices: catalogueMatrices(E2E_TARGET_CATALOGUE), + selectedJobs: inventory.workflowJobs.filter( + (job) => !inventory.explicitOnlyJobs.includes(job), + ), + hermesSelected: true, + explicitOnlyJobs: [...inventory.explicitOnlyJobs], + }, + inventory, + ); } export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { @@ -587,6 +750,7 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { "explicitOnlyJobs", "hermesSelected", "matrix", + "semanticMatrix", "selectedJobs", "testMatrix", ]) @@ -605,6 +769,7 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { !plan.matrix.every(isLiveTargetMatrixEntry) || !Array.isArray(plan.testMatrix) || !plan.testMatrix.every(isCredentialFreeTestMatrixRow) || + !isSemanticExecutionRows(plan.semanticMatrix) || !hasUniqueIds([...plan.matrix, ...plan.testMatrix, ...catalogueMatrixRows]) || !isStringArray(plan.selectedJobs) || !plan.selectedJobs.every((job) => /^[A-Za-z0-9_-]+$/u.test(job)) || @@ -616,6 +781,16 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { ) { throw new Error("E2E planner returned an invalid output schema"); } + const { semanticMatrix, ...planWithoutSemantics } = plan as E2eWorkflowPlan; + const expectedSemanticMatrix = semanticMatrixForPlan( + planWithoutSemantics, + readFreeStandingJobsInventory(), + ); + if (!isDeepStrictEqual(semanticMatrix, expectedSemanticMatrix)) { + throw new Error( + "E2E planner returned semantic coverage that does not match its execution plan", + ); + } return plan as E2eWorkflowPlan; } @@ -633,14 +808,21 @@ function expectedHermesSelection( function withoutCredentialedCatalogueProfiles(plan: E2eWorkflowPlan): E2eWorkflowPlan { const eligibleRows = (rows: E2eCatalogueMatrixRow[]) => rows.filter((row) => isPrCandidateCatalogueTarget(catalogueTarget(row.id))); + const catalogueMatrices = Object.fromEntries( + E2E_EXECUTION_PROFILES.map((profile) => [ + profile, + eligibleRows(plan.catalogueMatrices[profile]), + ]), + ) as Record; + const eligibleCatalogueIds = new Set( + E2E_EXECUTION_PROFILES.flatMap((profile) => catalogueMatrices[profile].map((row) => row.id)), + ); return { ...plan, - catalogueMatrices: Object.fromEntries( - E2E_EXECUTION_PROFILES.map((profile) => [ - profile, - eligibleRows(plan.catalogueMatrices[profile]), - ]), - ) as Record, + catalogueMatrices, + semanticMatrix: plan.semanticMatrix.filter( + (row) => row.source !== "catalogue" || eligibleCatalogueIds.has(row.id), + ), }; } @@ -648,23 +830,79 @@ export function renderE2eWorkflowPlanSummary(plan: E2eWorkflowPlan): string { const lines = [ "## E2E Execution Plan", "", - "| Target or job | Execution | Runner |", - "| --- | --- | --- |", + "| Target or job | Agent runtime | Observable outcome | Environment or inference endpoint | Source | Unresolved reason |", + "| --- | --- | --- | --- | --- | --- |", ]; - for (const job of plan.selectedJobs) { - lines.push(`| \`${job}\` | retained workflow job | declared by job |`); + for (const row of plan.semanticMatrix) { + lines.push( + `| \`${e2eSemanticExecutionLabel(row)}\` | ${row.agentRuntime} | ${row.observableOutcome} | ${row.environmentOrInferenceEndpoint} | ${row.source} | ${row.unresolvedReason} |`, + ); } - for (const row of plan.matrix) { - lines.push(`| \`${row.id}\` | typed registry | \`${row.runner}\` |`); + if (!plan.semanticMatrix.some((row) => row.source === "staging")) { + return `${lines.join("\n")}\n`; } - for (const row of plan.testMatrix) { - lines.push(`| \`${row.id}\` | shared E2E job | \`ubuntu-latest\` |`); + const inventory = readFreeStandingJobsInventory(); + const explicitOnlyRows = inventory.semanticRows.filter((row) => + plan.explicitOnlyJobs.includes(row.id), + ); + const unsupportedDeclarations = buildLiveTargetMatrix( + listTargets().map((target) => target.id), + ).filter((row) => !row.supported); + const outcomeRows = new Map(); + for (const row of plan.semanticMatrix) { + const rows = outcomeRows.get(row.observableOutcome) ?? []; + rows.push(row); + outcomeRows.set(row.observableOutcome, rows); } - for (const profile of E2E_EXECUTION_PROFILES) { - for (const row of plan.catalogueMatrices[profile]) { - lines.push(`| \`${row.id}\` | \`${profile}\` profile | \`${row.runner}\` |`); - } + const repeatedOutcomes = [...outcomeRows].filter(([, rows]) => rows.length > 1); + lines.push( + "", + "### Repeated outcomes with distinct evidence", + "", + "| Observable outcome | Rows | Distinguishing dimensions |", + "| --- | --- | --- |", + ); + for (const [outcome, rows] of repeatedOutcomes) { + const dimensions = [ + new Set(rows.map((row) => row.agentRuntime)).size > 1 ? "agent runtime" : "", + new Set(rows.map((row) => row.environmentOrInferenceEndpoint)).size > 1 + ? "environment or inference endpoint" + : "", + ].filter(Boolean); + lines.push( + `| ${outcome} | ${rows.map((row) => `\`${e2eSemanticExecutionLabel(row)}\``).join(", ")} | ${dimensions.join(" and ")} |`, + ); } + lines.push( + "", + "### Intentional exclusions", + "", + "| Target or job | Agent runtime | Observable outcome | Environment or inference endpoint | Exclusion | Unresolved reason |", + "| --- | --- | --- | --- | --- | --- |", + ); + for (const row of explicitOnlyRows) { + lines.push( + `| \`${e2eSemanticExecutionLabel(row)}\` | ${row.agentRuntime} | ${row.observableOutcome} | ${row.environmentOrInferenceEndpoint} | Explicit dispatch only; excluded from the default release matrix | ${row.unresolvedReason} |`, + ); + } + lines.push( + "", + "### Unsupported or unresolved typed declarations", + "", + "| Declaration | Agent runtime | Observable outcome | Environment or inference endpoint | Missing executable ownership |", + "| --- | --- | --- | --- | --- |", + ); + for (const row of unsupportedDeclarations) { + lines.push( + `| \`${row.id}\` | ${row.agentRuntime} | ${row.observableOutcome} | ${row.environmentOrInferenceEndpoint} | ${row.supportReasons.join("; ")} |`, + ); + } + lines.push( + "", + "### Combinatorial gaps", + "", + `The ${unsupportedDeclarations.length} inert typed declarations above are not executable matrix cells. #8285 owns the decision on the inert cross-runtime foundation, and #8286 owns executable-only registry cleanup after that decision. Unlisted Cartesian-product cells are not required without an accepted supported combination. This migration removes no execution, so no duplicate-to-retained-evidence mapping is required.`, + ); return `${lines.join("\n")}\n`; } From 8c056623188d090d6080fb1abaf573f7fcf35968 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Mon, 17 Aug 2026 15:54:01 -0700 Subject: [PATCH 2/7] refactor(e2e): name execution coverage directly --- .github/workflows/e2e.yaml | 122 +++++++++--------- test/e2e/README.md | 15 ++- test/e2e/registry/definitions/baseline.ts | 22 ++-- test/e2e/registry/run.ts | 16 ++- test/e2e/registry/types.ts | 4 +- test/e2e/support/e2e-matrix.test.ts | 18 ++- test/e2e/support/e2e-workflow.test.ts | 53 ++++---- test/e2e/support/workflow-plan.test.ts | 62 ++++----- tools/e2e/credential-free-tests.mts | 14 +- ...ic-coverage.mts => execution-coverage.mts} | 52 ++++---- .../hermes-gpu-startup-workflow-boundary.mts | 13 +- ...aged-image-multiarch-workflow-boundary.mts | 8 +- .../e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- tools/e2e/mcp-workflow-boundary.mts | 6 +- tools/e2e/target-catalogue.mts | 4 +- tools/e2e/workflow-boundary.mts | 81 ++++++------ tools/e2e/workflow-plan.mts | 98 +++++++------- 17 files changed, 304 insertions(+), 286 deletions(-) rename tools/e2e/{semantic-coverage.mts => execution-coverage.mts} (61%) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1d6acedbfa6..ff54459dd0a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2698,9 +2698,9 @@ jobs: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} E2E_JOB: "1" INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Exact staging image boots and completes the full E2E scenario" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Exact staging image boots and completes the full E2E scenario" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Brev Launchable Docker host; NVIDIA hosted inference" steps: - name: Checkout trusted Launchable lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -3224,9 +3224,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openshell-gateway-auth-contract" - E2E_SEMANTIC_AGENT_RUNTIME: "none" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint" + E2E_AGENT_RUNTIME: "none" + E2E_OBSERVABLE_OUTCOME: "Gateway mTLS and sandbox JWT authentication boundaries hold" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract NEMOCLAW_RUN_LIVE_E2E: "1" NEMOCLAW_NON_INTERACTIVE: "1" @@ -3307,19 +3307,19 @@ jobs: agent: [openclaw, hermes, deepagents] include: - agent: openclaw - semantic_agent_runtime: openclaw - semantic_variant: openclaw + agent_runtime: openclaw + coverage_variant: openclaw - agent: hermes - semantic_agent_runtime: hermes - semantic_variant: hermes + agent_runtime: hermes + coverage_variant: hermes - agent: deepagents - semantic_agent_runtime: langchain-deepagents-code - semantic_variant: deepagents + agent_runtime: langchain-deepagents-code + coverage_variant: deepagents env: E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" + E2E_OBSERVABLE_OUTCOME: "Stable OpenShell MCP bridge reaches tools and inference" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} @@ -3644,9 +3644,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openshell-credential-generation-window" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Credential expiry rotation detach and rebuild preserve the intended access window" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_OPENSHELL_CHANNEL: stable @@ -3807,19 +3807,19 @@ jobs: agent: [openclaw, hermes, deepagents] include: - agent: openclaw - semantic_agent_runtime: openclaw - semantic_variant: openclaw + agent_runtime: openclaw + coverage_variant: openclaw - agent: hermes - semantic_agent_runtime: hermes - semantic_variant: hermes + agent_runtime: hermes + coverage_variant: hermes - agent: deepagents - semantic_agent_runtime: langchain-deepagents-code - semantic_variant: deepagents + agent_runtime: langchain-deepagents-code + coverage_variant: deepagents env: E2E_JOB: "1" E2E_TARGET_ID: "mcp-bridge-dev" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" + E2E_OBSERVABLE_OUTCOME: "Development OpenShell MCP bridge reaches tools and inference" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu Docker host; local compatible inference and MCP endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_E2E_SHARD: ${{ matrix.agent }} @@ -4018,19 +4018,19 @@ jobs: - platform: linux/amd64 runner: ubuntu-24.04 shard: linux-amd64 - semantic_environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup" - semantic_variant: linux-amd64 + environment_or_inference_endpoint: "AMD64 Ubuntu; exact managed image startup" + coverage_variant: linux-amd64 - platform: linux/arm64 runner: ubuntu-24.04-arm shard: linux-arm64 - semantic_environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup" - semantic_variant: linux-arm64 + environment_or_inference_endpoint: "Arm64 Ubuntu; exact managed image startup" + coverage_variant: linux-arm64 env: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }} E2E_JOB: "1" E2E_TARGET_ID: "managed-image-multiarch-startup" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture" + E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" + E2E_OBSERVABLE_OUTCOME: "Exact managed images start directly on the native architecture" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }} @@ -4454,10 +4454,10 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification E2E_JOB: "1" E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification" - E2E_SEMANTIC_AGENT_RUNTIME: "unresolved" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA DGX Spark GB10; local llama.cpp inference" - E2E_SEMANTIC_UNRESOLVED_REASON: "The protected plan can enable or skip its OpenClaw subqualification" + E2E_AGENT_RUNTIME: "unresolved" + E2E_OBSERVABLE_OUTCOME: "Exact NemoClaw-built llama.cpp image produces protected DGX Spark evidence" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA DGX Spark GB10; local llama.cpp inference" + E2E_UNRESOLVED_REASON: "The protected plan can enable or skip its OpenClaw subqualification" NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} NEMOCLAW_E2E_SHARD: linux-arm64-gpu-dgx-spark-gb10 NEMOCLAW_LLAMA_CPP_QUALIFICATION_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }} @@ -4649,9 +4649,9 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime E2E_JOB: "1" E2E_TARGET_ID: "managed-image-protected-runtime" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services" + E2E_AGENT_RUNTIME: "openclaw + hermes + langchain-deepagents-code" + E2E_OBSERVABLE_OUTCOME: "Protected GPU runtime supports Ollama vLLM NIM rollback and cleanup" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local and hosted inference services" E2E_WORKLOAD_SOURCE: "managed-image" RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js @@ -4930,9 +4930,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "hermes-e2e" - E2E_SEMANTIC_AGENT_RUNTIME: "hermes" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference" + E2E_AGENT_RUNTIME: "hermes" + E2E_OBSERVABLE_OUTCOME: "Install onboarding health inference lifecycle dashboard and security succeed" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; mock or NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5023,21 +5023,21 @@ jobs: include: - scenario: native sandbox_name: e2e-hgpu-native - semantic_observable_outcome: "Native GPU startup reaches the stable Ready route" - semantic_variant: native + observable_outcome: "Native GPU startup reaches the stable Ready route" + coverage_variant: native - scenario: fallback sandbox_name: e2e-hgpu-fallback - semantic_observable_outcome: "Fallback GPU startup reaches the stable Ready route" - semantic_variant: fallback + observable_outcome: "Fallback GPU startup reaches the stable Ready route" + coverage_variant: fallback - scenario: compatibility-only sandbox_name: e2e-hgpu-compat - semantic_observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route" - semantic_variant: compatibility-only + observable_outcome: "Compatibility-only GPU startup reaches the stable Ready route" + coverage_variant: compatibility-only env: E2E_JOB: "1" E2E_TARGET_ID: "hermes-gpu-startup" - E2E_SEMANTIC_AGENT_RUNTIME: "hermes" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference" + E2E_AGENT_RUNTIME: "hermes" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "NVIDIA GPU runner; local GPU inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }} E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }} NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }} @@ -5340,9 +5340,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "cloud-onboard" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Public install onboarding hosted inference and security checks succeed" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5465,9 +5465,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "messaging-providers" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Provider configuration redaction and optional real sends succeed" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; NVIDIA hosted inference and messaging providers" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5533,9 +5533,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Release baseline plugin runtime survives the EXDEV boundary" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; release baseline package; no inference endpoint" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Release baseline plugin runtime survives the EXDEV boundary" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; release baseline package; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" @@ -5608,9 +5608,9 @@ jobs: env: E2E_JOB: "1" E2E_TARGET_ID: "openclaw-plugin-runtime-exdev" - E2E_SEMANTIC_AGENT_RUNTIME: "openclaw" - E2E_SEMANTIC_OBSERVABLE_OUTCOME: "Current plugin lifecycle rebuild and dependencies survive the EXDEV boundary" - E2E_SEMANTIC_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; current package; no inference endpoint" + E2E_AGENT_RUNTIME: "openclaw" + E2E_OBSERVABLE_OUTCOME: "Current plugin lifecycle rebuild and dependencies survive the EXDEV boundary" + E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT: "Ubuntu; current package; no inference endpoint" E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" diff --git a/test/e2e/README.md b/test/e2e/README.md index 19428b95709..b5dfa9bb840 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -320,26 +320,31 @@ npx tsx tools/e2e/workflow-plan.mts --summary >> "$GITHUB_STEP_SUMMARY" The workflow's `--ci-output` mode uses the same renderer for its job summary. The table includes the typed registry matrix, shared test matrix, catalogue profile matrices, retained workflow jobs, and staging Brev execution. -Each execution row declares three semantic fields: +Each execution row declares three coverage fields: - `agentRuntime` names the agent runtime that the execution asserts. Use `none` when the execution does not start an agent. Use `unresolved` only with an `unresolvedReason`. - `observableOutcome` names the behavior that produces the evidence. Catalogue targets use their outcome-oriented `displayName` as this value. - `environmentOrInferenceEndpoint` names the host boundary or inference endpoint that distinguishes the evidence. -Keep semantic metadata with the execution owner: +Keep coverage metadata with the execution owner: - Catalogue targets declare it in `tools/e2e/target-catalogue.mts`. - Executable typed targets declare it in `test/e2e/registry/definitions/baseline.ts`. - Shared credential-free tests declare it in `tools/e2e/credential-free-tests.mts`. - Retained workflow jobs and staging Brev declare it in `.github/workflows/e2e.yaml`. -Single workflow jobs use the `E2E_SEMANTIC_*` environment entries. Matrix jobs put variant-specific values in `semantic_*` include entries. `tools/e2e/workflow-plan.mts` composes and validates these sources. Do not add a separate hand-maintained execution list. +Single workflow jobs use the `E2E_AGENT_RUNTIME`, `E2E_OBSERVABLE_OUTCOME`, +`E2E_ENVIRONMENT_OR_INFERENCE_ENDPOINT`, and optional `E2E_UNRESOLVED_REASON` +environment entries. Matrix jobs put variant-specific values in the corresponding +snake-case include entries and use `coverage_variant` when one job contributes +multiple rows. `tools/e2e/workflow-plan.mts` composes and validates these sources. +Do not add a separate hand-maintained execution list. -The default semantic matrix excludes explicit-only jobs and inert typed-registry declarations. The rendered report lists those categories separately. The current inventory has one explicit-only execution and 22 inert typed declarations. The explicit-only row keeps its semantic dimensions but does not join the default release matrix. Inert declarations report unresolved semantic fields and the missing executable ownership. +The default coverage matrix excludes explicit-only jobs and inert typed-registry declarations. The rendered report lists those categories separately. The current inventory has one explicit-only execution and 22 inert typed declarations. The explicit-only row keeps its coverage dimensions but does not join the default release matrix. Inert declarations report unresolved coverage fields and the missing executable ownership. The inert declarations are combinatorial gaps, not supported matrix cells. #8285 owns the decision on the inert cross-runtime foundation. #8286 owns executable-only registry cleanup after that decision. Do not schedule other Cartesian-product cells without an accepted supported combination. This migration removes no execution, so it requires no duplicate-to-retained-evidence mapping. A documented gap does not schedule a new combination or change release judgment. -The report also groups repeated observable outcomes. Those rows are retained only when agent runtime or environment provides distinct evidence. Validation rejects two rows with the same three semantic dimensions. +The report also groups repeated observable outcomes. Those rows are retained only when agent runtime or environment provides distinct evidence. Validation rejects two rows with the same three coverage dimensions. ## Launch-readiness locked-image acceptance diff --git a/test/e2e/registry/definitions/baseline.ts b/test/e2e/registry/definitions/baseline.ts index 3684df59744..06f85f11d8e 100644 --- a/test/e2e/registry/definitions/baseline.ts +++ b/test/e2e/registry/definitions/baseline.ts @@ -13,9 +13,9 @@ import { } from "../matrix.ts"; import type { ExpectedFailureContract, TargetDefinition, TargetEnvironment } from "../types.ts"; import { - type E2eSemanticMetadata, - validateE2eSemanticMetadata, -} from "../../../../tools/e2e/semantic-coverage.mts"; + type E2eExecutionMetadata, + validateE2eExecutionMetadata, +} from "../../../../tools/e2e/execution-coverage.mts"; interface CanonicalTargetInput { id: string; @@ -25,7 +25,7 @@ interface CanonicalTargetInput { suiteIds: string[]; onboardingAssertionIds?: string[]; description?: string; - semanticCoverage?: E2eSemanticMetadata; + executionCoverage?: E2eExecutionMetadata; runnerRequirements?: string[]; requiredSecrets?: string[]; skippedCapabilities?: Array>; @@ -54,11 +54,11 @@ function canonicalTarget(input: CanonicalTargetInput): TargetDefinition { builder = builder.expectedFailure(input.expectedFailure); } const definition = builder.build(); - if (!input.semanticCoverage) return definition; + if (!input.executionCoverage) return definition; return { ...definition, - semanticCoverage: validateE2eSemanticMetadata( - input.semanticCoverage, + executionCoverage: validateE2eExecutionMetadata( + input.executionCoverage, `Typed E2E target ${input.id}`, ), }; @@ -81,7 +81,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "cloud-openclaw-ready", suiteIds: ["smoke", "inference", "credentials"], description: "Ubuntu repo checkout with Docker and cloud OpenClaw onboarding.", - semanticCoverage: { + executionCoverage: { agentRuntime: "openclaw", observableOutcome: "Repository install onboarding and hosted inference succeed", environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", @@ -107,7 +107,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "cloud-deepagents-code-ready", suiteIds: ["smoke", "inference", "terminal-agent", "deepagents-code-policy"], description: "Ubuntu repo checkout with Docker and LangChain Deep Agents Code onboarding.", - semanticCoverage: { + executionCoverage: { agentRuntime: "langchain-deepagents-code", observableOutcome: "Repository install onboarding and hosted inference succeed", environmentOrInferenceEndpoint: "Ubuntu Docker host; NVIDIA hosted inference", @@ -204,7 +204,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ description: "Post-reboot recovery guard: the gateway must recover through the required user service " + "while preserving the local sandbox registry and container.", - semanticCoverage: { + executionCoverage: { agentRuntime: "openclaw", observableOutcome: "Docker-backed sandbox recovers after a simulated host reboot", environmentOrInferenceEndpoint: "Ubuntu Docker host; local recovery fixture", @@ -358,7 +358,7 @@ const canonicalTargetInputs: CanonicalTargetInput[] = [ expectedStateId: "onboarding-failure-policy-presets-required", onboardingAssertionIds: ["base-installed", "preflight-passed"], suiteIds: [], - semanticCoverage: { + executionCoverage: { agentRuntime: "openclaw", observableOutcome: "Missing custom policy presets fail closed", environmentOrInferenceEndpoint: "Ubuntu Docker host; local negative fixture", diff --git a/test/e2e/registry/run.ts b/test/e2e/registry/run.ts index 8cecb52ae68..d73f6fa372c 100644 --- a/test/e2e/registry/run.ts +++ b/test/e2e/registry/run.ts @@ -6,8 +6,8 @@ import { fileURLToPath } from "node:url"; import { type E2eAgentRuntime, - validateE2eSemanticMetadata, -} from "../../../tools/e2e/semantic-coverage.mts"; + validateE2eExecutionMetadata, +} from "../../../tools/e2e/execution-coverage.mts"; import { listTargets, requireTargets } from "./registry.ts"; import { resolveRunnerForTarget } from "./runner-routing.ts"; @@ -101,11 +101,13 @@ function liveMatrixEntry( support: LiveTargetSupport, ): LiveTargetMatrixEntry { const { runner } = resolveRunnerForTarget(target); - if (support.supported && !target.semanticCoverage) { - throw new Error(`Executable typed E2E target ${target.id} requires semantic coverage metadata`); + if (support.supported && !target.executionCoverage) { + throw new Error( + `Executable typed E2E target ${target.id} requires execution coverage metadata`, + ); } - const semanticCoverage = validateE2eSemanticMetadata( - target.semanticCoverage ?? { + const executionCoverage = validateE2eExecutionMetadata( + target.executionCoverage ?? { agentRuntime: "unresolved", observableOutcome: "unresolved", environmentOrInferenceEndpoint: "unresolved", @@ -115,7 +117,7 @@ function liveMatrixEntry( ); return { id: target.id, - ...semanticCoverage, + ...executionCoverage, runner, label: buildLabel(target), platform: target.environment?.platform ?? "unknown", diff --git a/test/e2e/registry/types.ts b/test/e2e/registry/types.ts index 133bab17f5f..e4663c95af5 100644 --- a/test/e2e/registry/types.ts +++ b/test/e2e/registry/types.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import type { E2eSemanticMetadata } from "../../../tools/e2e/semantic-coverage.mts"; +import type { E2eExecutionMetadata } from "../../../tools/e2e/execution-coverage.mts"; export type PhaseName = "environment" | "onboarding" | "state-validation" | "lifecycle" | "runtime"; @@ -196,7 +196,7 @@ export interface TargetEnvironment { export interface TargetDefinition { id: string; description?: string; - semanticCoverage?: E2eSemanticMetadata; + executionCoverage?: E2eExecutionMetadata; manifestPath?: string; environment?: TargetEnvironment; assertionGroups: AssertionGroup[]; diff --git a/test/e2e/support/e2e-matrix.test.ts b/test/e2e/support/e2e-matrix.test.ts index a2e4e91c7f5..ed5184c6b06 100644 --- a/test/e2e/support/e2e-matrix.test.ts +++ b/test/e2e/support/e2e-matrix.test.ts @@ -29,6 +29,15 @@ function requireUnsupportedTarget() { return unsupported!; } +function expectExecutableTypedTargetCoverage(): void { + for (const row of buildLiveTargetMatrix()) { + expect(row.agentRuntime).not.toBe("unresolved"); + expect(row.observableOutcome).not.toBe("unresolved"); + expect(row.environmentOrInferenceEndpoint).not.toBe("unresolved"); + expect(row.unresolvedReason).toBe(""); + } +} + describe("live E2E target matrix", () => { it("honors an explicit runs-on: