From 5e8b8f765239647d3eb8edf33c5a93200a6a83c7 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Mon, 17 Aug 2026 16:18:29 -0400 Subject: [PATCH] ci(e2e): restore staging launchable validation --- .../SKILL.md | 21 +---- .../skills/nemoclaw-maintainer-e2e/SKILL.md | 57 ++++++------ .../references/release-train.md | 21 +---- .../SKILL.md | 33 +++---- .github/workflows/e2e.yaml | 35 +++++--- test/e2e/README.md | 65 +++++++------- test/e2e/docs/README.md | 9 +- test/e2e/support/e2e-workflow.test.ts | 35 ++++---- test/maintainer-e2e-skill.test.ts | 6 +- test/maintainer-launchable-skill.test.ts | 41 +++------ ...upload-e2e-artifacts-workflow-boundary.mts | 4 +- tools/e2e/workflow-boundary.mts | 87 +++++++++++-------- 12 files changed, 201 insertions(+), 213 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index 045c48e4475..e6d68b01bd9 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -125,7 +125,7 @@ Read the generated `plan.json` and show the maintainer: For the plan's full `origin/main` SHA, require a completed, successful `Release qualification` check from a pre-tag `.github/workflows/e2e.yaml` run. The workflow planner derives the required jobs from the workflow's E2E metadata. -By default, the check requires every release-required execution result, including `Publish staging Brev Launchable image`, to succeed. +By default, the check requires every release-required E2E execution result, including `Exact staging Brev Launchable`, to succeed. A repository administrator may waive one or more release-required E2E execution jobs for a documented release exception. The waiver requires a comma-separated `release_qualification_waived_jobs` list and a `release_qualification_waiver_reason`. The reason must begin with an ASCII letter or digit and contain 10-500 characters chosen from ASCII letters, digits, spaces, and `.,:;/_()'-`. @@ -167,20 +167,6 @@ Before showing the confirmation prompt, present the candidate SHA, workflow URL, For a waived run, also present the waived jobs, their outcomes, the waiver reason, and both recorded actor identities. No release-note-only delta exception is currently defined. -After image publication succeeds, present this advisory manual validation: - -- State that the image-publication job built and published the candidate image to the staging family used by the [NemoClaw staging Launchable](https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo). -- Encourage the maintainer to deploy one instance and hand its Brev environment URL to a Codex session that invokes `nemoclaw-maintainer-validate-launchable`. -- Require the manual validation to compare the deployed concrete image with `launchable-image.json`; do not assume that the mutable family still points to the candidate. -- State that browser-control capability is required for Codex to click and verify the web interface. -- State that a securely supplied inference credential is required to complete hosted and sandbox inference validation. Never ask the maintainer to paste the credential into chat. -- Record the manual result as `complete pass`, `partially blocked`, `failed`, or `not run` when the maintainer provides it. - -This manual validation is advisory while the automated Launchable path is blocked by issue #8924. -Its absence, partial result, or failure does not block the signing preflight, confirmation prompt, or release tag. -Do not describe successful image publication as successful Launchable, runtime, or inference validation. -Apply the temporary policy in [Pre-Tag E2E Evidence](../nemoclaw-maintainer-policies/references/release-train.md#temporary-staging-launchable-qualification-policy): NemoClaw maintainers own it while #8924 remains open, the successful exact image-publication job and artifact remain required release evidence under normal Actions retention, and the full automated lane returns only after a checksum-pinned Brev release passes deployment through verified cleanup on trusted `main`. - Run the release script's signing preflight before asking for confirmation: ```bash @@ -337,11 +323,10 @@ If the Announcement is valid, return its URL with the release artifacts and mark - Plan generation fails: fix the named precondition, then regenerate the plan. - Documentation workflow state is incomplete: return to `nemoclaw-maintainer-evening`, then repeat Step 1 after the documentation PR merges. -- Full-mode E2E waits in the Launchable concurrency queue: keep the run pending until the earlier Launchable image-publication job finishes. +- Full-mode E2E waits in the Launchable concurrency queue: keep the run pending until the earlier Launchable E2E job finishes. - Full-mode E2E ran for another SHA: reject the run and dispatch full mode for the plan candidate SHA. - No qualifying `Release qualification` exists: inspect the GitHub result and run pre-tag E2E for the planned SHA only when no qualifying run already exists. Use a job waiver only with explicit repository administrator authorization. Do not release until the release script accepts the canonical check. -- Launchable image publication fails: inspect `launchable-image.json` and the producer run, correct the failure, and rerun the affected work. Do not infer image publication from manual Launchable validation. -- Advisory Launchable validation is blocked or fails: record the exact partial result and continue the release flow. Do not convert the result into a release gate or an automated E2E pass. +- Launchable E2E or cleanup fails: inspect the diagnostic artifacts, correct the failure, and rerun the affected E2E work. Do not infer Launchable success from another workflow result. - `origin/main` moved after plan generation: regenerate the plan and ask for the new confirmation phrase. - Remote semver tag already exists: stop; do not retag unless the maintainer explicitly starts protected-tag remediation. - Signing preflight fails: fix the reported Git signer or signing-key failure. Run the preflight again before requesting confirmation. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index d4f9317e351..6f0404a296f 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-e2e -description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the latest PR commit and staging Launchable image publication. Use for requests such as run E2E for PR #123, run the E2E suite, publish the Launchable image, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. +description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the latest PR commit. Use for requests such as run E2E for PR #123, run the E2E suite, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. --- @@ -34,14 +34,17 @@ The workflow does not rotate or revoke these API keys or messaging credentials. Live targets can create external resources. After a failure, inspect the artifacts and remove resources that target cleanup did not remove. -`Publish staging Brev Launchable image` reads this credential from repository Actions secrets: +`Exact staging Brev Launchable` reads these credentials from repository Actions secrets: +- `BREV_API_KEY` authenticates the trusted host-side Brev CLI for workspace operations in the organization identified by `BREV_ORG_ID`. Candidate code does not receive this API key. - `NEMOCLAW_IMAGE_DISPATCH_TOKEN` is exposed as `GH_TOKEN` only to the trusted host script. It grants Actions read/write access to `brevdev/nemoclaw-image`, which the script uses to dispatch the image workflow, inspect its run, and download its handoff artifact. +- `NVIDIA_INFERENCE_API_KEY` is exported into the Brev guest for the full E2E process. Code in the baked candidate checkout can read and use it. -This credential remains valid until it expires or an administrator revokes it in GitHub. Rotate or revoke it to remove later access. -The job does not receive `BREV_API_KEY`, `BREV_ORG_ID`, or `NVIDIA_INFERENCE_API_KEY`. -It does not install or authenticate the Brev CLI, create a workspace, or run inference. -This image-publication credential boundary applies only to trusted Launchable or full manual dispatches against `main`. It does not apply to `main` pushes or manual PR runs. +`brev login` writes `BREV_API_KEY` and `BREV_ORG_ID` to `$HOME/.brev/credentials.json` on the GitHub-hosted runner. Later trusted steps and processes in the same job can read that file. The workflow does not delete it explicitly; it remains on the ephemeral runner filesystem until runner teardown discards that filesystem. +These credentials remain valid until they expire or an administrator revokes them in their issuing services. If cleanup fails, remove the recorded Brev workspace. Rotate or revoke each credential to remove later access. +This Brev credential boundary applies only to trusted Launchable or full manual dispatches against `main`. It does not apply to `main` pushes or manual PR runs. + +The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the standing Launchable. Keep its value equal to the Launchable ID in the default URL owned by [`nemoclaw-maintainer-validate-launchable`](../nemoclaw-maintainer-validate-launchable/SKILL.md). For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. The live fixture removes the temporary NIM container only if its exact ID, name, requested image, immutable image ID, cohort owner, and provider kind match the recorded authority. The test fails if evidence is missing or ambiguous, a name is reused, authority drifts, removal is indeterminate, or the exact ID or name remains. A cleanup refusal can leave the container and its API key in place until runner teardown. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. @@ -67,7 +70,7 @@ Require a review reason containing 10 to 500 printable characters. Choose exactly one mode: - For a PR revision run, leave `E2E_JOBS` empty. The run selects: - - every default-selected free-standing workflow E2E except `Publish staging Brev Launchable image`; + - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. @@ -158,8 +161,7 @@ A changed head repository, head SHA, or base SHA invalidates the evidence and re | Request | Mode | `jobs` | `include_staging_brev_launchable` | |---|---|---|---| | “Run the E2E suite” | Ordinary | empty | `false` | -| “Publish the Launchable image” | Launchable image | `staging-brev-launchable` | `false` | -| “Run the Launchable E2E” | Clarify before dispatch | not applicable | not applicable | +| “Run the Launchable E2E” | Launchable | `staging-brev-launchable` | `false` | | “Run the full E2E suite” | Full | empty | `true` | | “deploy pre-release full E2E” | Full | empty | `true` | | “run pre-tag full E2E” | Full | empty | `true` | @@ -167,16 +169,12 @@ A changed head repository, head SHA, or base SHA invalidates the evidence and re | “run pre-tag E2E with an administrator job waiver” | Administrator-waived full | empty | `true` | A generic E2E request must not authorize the Brev Launchable path. -For “Run the Launchable E2E,” explain that issue #8924 blocks automated deployment, runtime, and inference validation. -Ask whether the maintainer wants image publication or advisory validation through `nemoclaw-maintainer-validate-launchable` against one deployed instance. -Do not dispatch until the maintainer selects one of those operations. Do not infer full mode from words such as “all” or “complete.” -Ask for clarification when the request uses the legacy Launchable E2E phrase or contains conflicting mode phrases. +Ask for clarification only when the request contains conflicting mode phrases. -Ordinary mode selects every default-selected workflow E2E except `Publish staging Brev Launchable image`. -Launchable image mode runs only `Publish staging Brev Launchable image`. -Full mode adds `Publish staging Brev Launchable image` to the default E2E selection in the same workflow run. -The Launchable image job stops after exact image-publication evidence and does not deploy a workspace or run inference. +Ordinary mode selects every default-selected workflow E2E except `Exact staging Brev Launchable`. +Launchable mode runs only `Exact staging Brev Launchable`. +Full mode adds `Exact staging Brev Launchable` to the default E2E selection in the same workflow run. Administrator-waived full mode runs the full suite but omits the approved execution jobs from release qualification. Every waived job still runs. Use this mode only when a repository administrator explicitly authorizes the job IDs and supplies the reason. @@ -222,7 +220,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f "correlation_id=${CORRELATION_ID}" ``` -For Launchable image mode: +For Launchable mode: ```bash gh workflow run .github/workflows/e2e.yaml \ @@ -273,8 +271,8 @@ gh workflow run .github/workflows/e2e.yaml \ ``` Do not set `jobs=staging-brev-launchable` for full mode. -Empty `jobs` and `targets` select every default-selected workflow E2E except `Publish staging Brev Launchable image`. -The `include_staging_brev_launchable` input adds the Launchable image-publication job to that same run. +Empty `jobs` and `targets` select every default-selected workflow E2E except `Exact staging Brev Launchable`. +The `include_staging_brev_launchable` input adds the Launchable E2E job to that same run. The trusted `main` workflow verifies that the dispatching and rerunning actors have repository `maintain` or `admin` permission before the Launchable path's source checkout. That role check is the authorization. @@ -300,7 +298,7 @@ empty-selector manual run or enable explicit qualification selection. Set it only after a repository administrator confirms an online DGX Spark runner in the authoritative runner inventory. If GitHub pauses the qualification job for the `approve-dgx-spark-image-qualification` environment, an authorized environment reviewer must approve it before qualification starts. -`Publish staging Brev Launchable image` does not require environment approval. +`Exact staging Brev Launchable` does not require environment approval. Find the run by its unique title: @@ -334,7 +332,7 @@ Wait for completion: gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw ``` -Launchable image and full modes can wait in the non-cancelling Launchable concurrency queue. +Launchable and full modes can wait in the non-cancelling Launchable concurrency queue. Queued, waiting, or accepted dispatch state is not success. Classify the completed workflow and `Release qualification` job with the checks below. @@ -356,21 +354,20 @@ Require `run-$RUN_ID.json` to report: - `head_sha` equal to `CANDIDATE_SHA`; - `status` equal to `completed`. -For ordinary, Launchable image, and unwaived full modes, require `conclusion` equal to `success`. +For ordinary, Launchable, and unwaived full modes, require `conclusion` equal to `success`. For administrator-waived full mode, permit `conclusion` equal to `success` or `failure`. A `failure` conclusion is acceptable only when one completed, successful `Release qualification` job and a valid exact-run waiver artifact with at least one canonical waived job failure both exist. -For Launchable image mode, also require `jobs-latest-$RUN_ID.json` to contain one completed, successful -`Publish staging Brev Launchable image` job. Return the workflow and job URLs. -Require its artifact to contain `launchable-image.json` for the selected candidate SHA and concrete staging image URI. +For Launchable mode, also require `jobs-latest-$RUN_ID.json` to contain one completed, successful +`Exact staging Brev Launchable` job. Return the workflow and job URLs. For a full run, with or without a job waiver, require `jobs-latest-$RUN_ID.json` to contain one completed, successful `Release qualification` job. Return its job URL with the workflow URL. -In full mode, that job waits for every default-required result, including `Publish staging Brev Launchable image`. -The Launchable image job verifies only the exact candidate image producer receipt and staging-family publication. -Its `launchable-image.json` artifact records Launchable, runtime, and inference validation as not run. +In full mode, that job waits for every default-required E2E result, including `Exact staging Brev Launchable`. +The Launchable job directly verifies the candidate checkout, in-guest full E2E result, and workspace cleanup before it succeeds. +Its `launchable-e2e.json`, `full-e2e.log`, and `cleanup.json` artifacts remain available for diagnosis. A skipped, cancelled, queued, or failed `Release qualification` job is not evidence. -A Launchable image-only run is not full-mode or pre-tag release evidence. +A Launchable-only run is not full-mode or pre-tag release evidence. For administrator-waived full mode, the job waits for every unwaived release-required result. A waived execution job may fail without failing `Release qualification`. diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index adabae8d954..3bce66b0d90 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -52,25 +52,9 @@ The release candidate is the full `origin/main` commit SHA captured by the gener Before asking for the release confirmation phrase, require a completed, successful `Release qualification` check from a pre-tag manual run at that SHA. -### Temporary Staging Launchable Qualification Policy - -Issue #8924 temporarily limits the trusted staging Launchable job to exact image publication. -NemoClaw maintainers own this policy while that issue remains open. -For each release candidate, the required automated evidence is a successful `Publish staging Brev Launchable image` job and its `launchable-image.json` artifact, bound to the exact candidate SHA through the successful `Release qualification` check. -GitHub retains the workflow logs and artifact under the repository's normal Actions retention policy. -An image-publication failure still blocks release qualification unless a repository administrator uses the existing documented job-waiver mechanism. - -The temporary risk acceptance permits a release tag without automated or manual proof of the staging Launchable web deployment, environment access, exact booted image, baked runtime, inference, or workspace cleanup. -Manual validation remains advisory, and a missing, partial, or failed result needs no per-release waiver. -It must not be reported as an automated E2E pass. - -Restore the automated deployment lane when a published Brev CLI release contains the Launchable image-forwarding fix and the host-route fix tracked by #8924. -NemoClaw must checksum-pin that release and complete a trusted `main` run that verifies deployment, environment access, exact image and runtime identity, hosted and sandbox inference, and workspace cleanup. -That successful run is the reactivation evidence; closing #8924 records the end of this temporary policy. - - `.github/workflows/e2e.yaml` derives the release-required jobs from its E2E metadata. Do not copy them into a second release test list. - Push runs publish `Relevant E2E`; only full manual runs dispatched against `main` with empty selectors publish `Release qualification`. -- By default, the check requires every default-required workflow result to succeed, including `Publish staging Brev Launchable image`. +- By default, the check requires every default-required workflow E2E result to succeed, including `Exact staging Brev Launchable`. - A repository administrator may waive one or more release-required E2E execution jobs with `release_qualification_waived_jobs` and `release_qualification_waiver_reason`. - `release_qualification_waived_jobs` is a comma-separated list of requested job IDs. - The reason must begin with an ASCII letter or digit and contain 10-500 characters chosen from ASCII letters, digits, spaces, and `.,:;/_()'-`. @@ -85,8 +69,7 @@ That successful run is the reactivation evidence; closing #8924 records the end - A normal full run must conclude with `success`. - An administrator-waived full run may conclude with `failure` when a waived execution job fails, `Release qualification` succeeds, and the waiver artifact binds that failure to the candidate, run, actors, reason, and canonical waived job IDs. - `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` remain separate opt-in work and do not block this check. -- A successful Launchable image job proves that the producer published the exact candidate image to the staging family. Its `launchable-image.json` artifact records Launchable, runtime, and inference validation as not run. -- Manual staging Launchable validation is advisory while issue #8924 blocks the automated deployment path. A missing, partial, or failed manual result does not block the release tag and must not be reported as an automated E2E pass. +- A successful Launchable job proves the candidate checkout, in-guest full E2E result, and cleanup. Its artifacts are diagnostic evidence, not a second status ledger. - A skipped, queued, in-progress, cancelled, or failed `Release qualification` check is not release evidence. - A check from another commit SHA is not release evidence. - Use an existing qualifying pre-tag run for the candidate SHA; run `nemoclaw-maintainer-e2e` in full mode when none exists, with an administrator-authorized job waiver when required. diff --git a/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md b/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md index 0b3d4aac3e9..52fab60658e 100644 --- a/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-validate-launchable -description: Validate the user-facing staging Brev Launchable deployment, exact NemoClaw image and runtime identity, onboarding, CLI behavior, and inference. Use when a maintainer asks to test the staging Launchable in the Brev web interface, provides a deployed Brev environment URL, hands a Launchable instance to Codex, or needs advisory manual validation while the automated Launchable E2E is blocked. +description: Validate the user-facing staging Brev Launchable deployment, exact NemoClaw image and runtime identity, onboarding, CLI behavior, and inference. Use when a maintainer asks to test the staging Launchable in the Brev web interface, provides a deployed Brev environment URL, hands a Launchable instance to Codex, or needs advisory web validation separate from automated Launchable E2E. --- @@ -25,14 +25,15 @@ Use the current checkout and workflow artifacts as the source of truth for image - Redact credentials from captured output and delete temporary raw logs after producing redacted evidence. Use this staging Launchable unless the maintainer supplies another accepted target: -[Deploy the NemoClaw staging Launchable](https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo). +[Deploy the NemoClaw staging Launchable](https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3I2w334slP4GKSce9kKK0hGerjJ). +Keep the Launchable ID in this default URL equal to the `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable used by the automated job. ## Define the Result Boundary Require all of these results for a complete pass: 1. The Launchable web page and deployment flow work in an authenticated browser session. -2. The deployed environment boots the exact concrete image recorded by the selected image-publication workflow artifact. +2. The deployed environment boots the exact concrete image recorded by the selected automated Launchable workflow artifact. 3. The baked provision receipt and source checkout without tracked or untracked changes identify the selected NemoClaw commit. 4. The preinstalled user journey completes onboarding, CLI checks, sandbox inference, recovery, logs, and cleanup. 5. A hosted inference request and the sandbox inference request succeed with a securely supplied credential. @@ -50,17 +51,17 @@ When one or more checks ran without failure but another required check did not r ## Resolve the Candidate and Image Evidence Record the expected NemoClaw commit SHA before deployment. -Use the latest successful `Publish staging Brev Launchable image` job for that exact SHA. -Record the selected publication workflow and job URLs and the producer run ID selected in that job's log. -Download its private artifact and require `launchable-image.json` to report: +Use the latest successful `Exact staging Brev Launchable` job for that exact SHA. +Record the selected workflow and job URLs and the producer run ID selected in that job's log. +Download its private artifact and require `launchable-e2e.json` to report: -- `schemaVersion` equal to `1`; -- `kind` equal to `nemoclaw-staging-launchable-image-v1`; - `candidateSha` equal to the selected commit SHA; -- `producer.repository` equal to `brevdev/nemoclaw-image`, `producer.status` equal to `success`, and `producer.runId` equal to the producer run ID selected by the publication job; -- `image.family` equal to `nemoclaw-brev-staging-cpu`; -- a concrete `image.uri` and a lowercase 40-character `image.imageRepositorySha`; and -- `validation.launchable`, `validation.runtime`, and `validation.inference` equal to `not-run`. +- `producer.status` equal to `success` and `producer.runId` equal to the producer run ID selected by the automated job; +- a concrete `boot.bootImage` URI; +- `boot.schemaVersion` equal to `1`, `boot.sourceRepository` equal to `NVIDIA/NemoClaw`, and `boot.sourcePath` equal to `/opt/nemoclaw-image/NemoClaw`; +- `boot.repoSha` and `boot.provisionSha` equal to the selected commit SHA; +- a lowercase 40-character `boot.imageRepositorySha`, `boot.repoClean` equal to `true`, and `boot.runtimeOverrides` equal to `false`; and +- `fullE2e` equal to `passed`. Stop when the artifact is absent, malformed, or belongs to another commit. Classify that evidence as `failed` when GitHub is available and the selected job or artifact can be inspected. @@ -102,7 +103,7 @@ Perform these checks without changing the instance: 1. Use the supplied environment ID as the authoritative identity. If a name is also supplied, require it to match that environment. Use an instance-name lookup only when no environment ID is available, and require exactly one match. 2. Require the environment to report its successful running state. 3. Establish the user-facing SSH or terminal access path shown by Brev. -4. Read the GCE instance image metadata and require exact equality with the concrete image URI from `launchable-image.json`. +4. Read the GCE instance image metadata and require exact equality with `boot.bootImage` from `launchable-e2e.json`. 5. Read `/etc/nemoclaw/provision.json` with the privileges provided by the image. 6. Require the provision receipt, source repository, source path, image-repository commit SHA, and NemoClaw commit SHA to match the selected artifact and candidate. 7. Require the baked source checkout to have no tracked or untracked changes and no runtime override receipt. @@ -123,7 +124,7 @@ Run the validation from a short-lived local process that receives the key throug The local validation process and its SSH child can read the key; the remote shell exports it to the baked full E2E process, so candidate code can read and use it. Before exposing the key, record the authorized candidate repository and commit SHA, require the repository to be `NVIDIA/NemoClaw`, and reject a candidate from a fork pull request. Explain that the selected candidate code can read and use the key, then obtain explicit maintainer approval immediately before starting the credential-bearing process. -If the issuing service cannot rotate or revoke the inference API key after the run, require a maintainer-approved waiver tied to the exact candidate commit SHA and selected image-publication run ID before starting validation. +If the issuing service cannot rotate or revoke the inference API key after the run, require a maintainer-approved waiver tied to the exact candidate commit SHA and selected automated Launchable run ID before starting validation. Do not persist the key in shell startup files, temporary files, SSH configuration, or the Brev environment after the test process exits. If it is unavailable: @@ -137,7 +138,7 @@ Require the baked full E2E success sentinel and retain only redacted logs. The test must remove its `e2e-` sandbox and verify the expected cleanup result even after a test failure. After the local and remote test processes exit, unset any shell variable created for the run and verify that no temporary credential file remains. Unless the approved waiver applies, rotate or revoke the inference API key in the issuing NVIDIA service after the run and record non-sensitive confirmation. -When the waiver applies, record its approver, exact candidate commit SHA, selected image-publication run ID, and the accepted period of later API-key access without recording the key. +When the waiver applies, record its approver, exact candidate commit SHA, selected automated Launchable run ID, and the accepted period of later API-key access without recording the key. ## Finish the Instance Handoff @@ -157,7 +158,7 @@ Return this structure: - Evidence mode: advisory manual validation; not automated E2E evidence - Candidate repository and commit SHA: -- Image-publication workflow and job URL: +- Automated Launchable workflow and job URL: - Expected concrete image URI: - Launchable URL: - Environment URL, ID, and name: diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ce0ed3dd5dd..1bd378b36b4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -15,12 +15,12 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for staging Brev Launchable image publication. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix." + description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix." required: false default: "" type: string include_staging_brev_launchable: - description: "Include staging Brev Launchable image publication in a full E2E run when jobs and targets are empty." + description: "Include Exact staging Brev Launchable in a full E2E run when jobs and targets are empty." required: false default: false type: boolean @@ -490,7 +490,7 @@ jobs: name: e2e-dispatch-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json - - name: Authorize Launchable image publication + - name: Authorize Launchable E2E maintainer dispatch if: ${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} env: ACTOR: ${{ github.actor }} @@ -503,7 +503,7 @@ jobs: require_maintainer() { local maintainer="$1" if [[ ! "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ || "$maintainer" == -* || "$maintainer" == *- ]]; then - echo "::error::Launchable image publication actor is invalid" >&2 + echo "::error::Launchable E2E actor is invalid" >&2 exit 1 fi @@ -514,13 +514,13 @@ jobs: --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission")" if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then - echo "::error::Launchable image publication permission response did not match the actor" >&2 + echo "::error::Launchable E2E permission response did not match the actor" >&2 exit 1 fi case "$(jq -r '.role_name // ""' <<< "$permission_json")" in maintain | admin) ;; *) - echo "::error::Launchable image publication requires a repository maintainer or administrator" >&2 + echo "::error::Launchable E2E requires a repository maintainer or administrator" >&2 exit 1 ;; esac @@ -2523,7 +2523,7 @@ jobs: path: e2e-artifacts/live/retired-selector-compatibility/ staging-brev-launchable: - name: Publish staging Brev Launchable image + name: Exact staging Brev Launchable needs: generate-matrix if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} runs-on: ubuntu-latest @@ -2536,6 +2536,8 @@ jobs: cancel-in-progress: false env: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} + E2E_JOB: "1" + INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Checkout trusted Launchable lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -2548,16 +2550,27 @@ jobs: - id: workspace name: Prepare the trusted lane + env: + BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_API_KEY || '' }} + BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32 + BREV_CLI_VERSION: 0.6.334 + BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_ORG_ID || '' }} run: | set -euo pipefail work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")" chmod 700 "$work_dir" + archive="${RUNNER_TEMP}/brev-cli.tar.gz" + curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" + printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - + tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev + brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - - name: Build and verify the staging Launchable image + - name: Build, deploy, verify, test, and clean up env: + BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} - NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY: "1" + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-e2e.sh @@ -2568,7 +2581,9 @@ jobs: name: staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ steps.workspace.outputs.work_dir }}/lane.log - ${{ steps.workspace.outputs.work_dir }}/launchable-image.json + ${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json + ${{ steps.workspace.outputs.work_dir }}/full-e2e.log + ${{ steps.workspace.outputs.work_dir }}/cleanup.json live: needs: [base-image-publication, generate-matrix] diff --git a/test/e2e/README.md b/test/e2e/README.md index b80f1d8e93e..ed0f54e6116 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -21,9 +21,8 @@ before those targets run; local runners must provide it themselves. - `.github/workflows/e2e-main-retry.yaml` evaluates eligible `E2E main` push attempts and uploads attempt evidence. It never authorizes a broad failed-job rerun; retry decisions belong to bounded operation-level policies. -- The `staging-brev-launchable` job in `.github/workflows/e2e.yaml` publishes - the exact candidate image to the staging family and records the concrete image URI. - It does not deploy or validate a Brev environment while issue #8924 blocks the automated path. +- The `staging-brev-launchable` job in `.github/workflows/e2e.yaml` validates + the baked candidate without installing or copying NemoClaw source. - `.github/workflows/platform-vitest-main.yaml` publishes `CI / Platform Evidence` for Ubuntu 26.04, macOS, and WSL. On shard 1, its macOS and WSL live E2E run only when the workflow tests `main` and Docker is available. This workflow does not publish or satisfy `Release qualification`. @@ -170,13 +169,12 @@ GitHub invalidates `GITHUB_TOKEN` after the job. ## Retired Brev source-install coverage -Issue #7490 retired the generic Brev source-install lane. Current validation uses -the unified workflow, staging image-publication job, and advisory manual -Launchable validation: +Issue #7490 retired the generic Brev source-install lane. The unified workflow +and exact-staging Launchable job own its product coverage: | Legacy suite | Disposition | Current owner | |---|---|---| -| `full` | Manual Launchable validation | `staging-brev-launchable` publishes the exact candidate image. Use `nemoclaw-maintainer-validate-launchable` to validate a deployed instance while issue #8924 blocks automation. | +| `full` | Launchable E2E | `staging-brev-launchable` runs `full-e2e` in preinstalled mode against the exact baked candidate. | | `credential-sanitization` | Unified E2E | `credential-sanitization` | | `telegram-injection` | Unified E2E | `telegram-injection` | | `messaging-providers` | Unified E2E | `messaging-providers` | @@ -715,11 +713,11 @@ rm -rf -- "$evidence_dir" test ! -e "$evidence_dir" ``` -A manual run with `jobs=staging-brev-launchable` runs only `Publish staging Brev Launchable image`. +A manual run with `jobs=staging-brev-launchable` runs only `Exact staging Brev Launchable`. Push runs do not select this job. A manual run with `include_staging_brev_launchable=true` and empty `jobs` and -`targets` selectors runs the default workflow E2E selection plus the Launchable image-publication job. +`targets` selectors runs the default workflow E2E selection plus the Launchable E2E job. This selection is the full manual `main` run for pre-tag release evidence. Each full dispatch uses `github.run_id` in its workflow concurrency identity, so another full dispatch @@ -728,11 +726,11 @@ verifies that the dispatching and rerunning actors have repository `maintain` or `admin` permission before the Launchable path's source checkout. That automatic role check authorizes `staging-brev-launchable`; the job does not use GitHub environment approval. The job uses the non-cancelling -`staging-brev-launchable-cpu` group with `queue: max`, so pending Launchable image +`staging-brev-launchable-cpu` group with `queue: max`, so pending Launchable E2E runs remain queued instead of replacing one another. For a full manual run dispatched against `main`, `Release qualification` waits for every E2E job that does not require a separate opt-in. -The check requires each of those jobs to pass, including `Publish staging Brev Launchable image`. +The check requires each of those jobs to pass, including `Exact staging Brev Launchable`. A passing check at the candidate commit SHA is the pre-tag release E2E evidence. Ensure that each candidate commit SHA has a qualifying full manual `main` run. Dispatch another full run only when no qualifying run exists. @@ -744,16 +742,11 @@ Local fixture remotes skip the canonical repository gate only when tests set the Canonical-equivalent `NVIDIA/NemoClaw` remotes always run the gate, even when that override is set. A local fixture cannot authorize a production release. Maintainers do not build a local evidence ledger or infer GitHub job status from an artifact. -The Launchable image job retains `launchable-image.json` with the candidate SHA, producer run, concrete image URI, staging family, and explicit not-run validation fields. -Manual web, runtime, and inference validation is advisory while issue #8924 remains open. -It does not block the release tag and must not be reported as an automated E2E pass. - -This is a temporary NemoClaw maintainer policy owned while #8924 remains open. -Each release candidate still requires the successful exact image-publication job and `launchable-image.json` through `Release qualification`; GitHub keeps its logs and artifact under the repository's normal Actions retention policy. -The accepted temporary risk is that a tag can proceed without automated or manual proof of the Launchable web deployment, environment access, exact booted image, baked runtime, inference, or workspace cleanup. -A missing, partial, or failed manual validation needs no per-release waiver, but an image-publication failure remains release-blocking unless an administrator uses the existing job-waiver mechanism. -Restore automated validation only after NemoClaw checksum-pins a published Brev CLI release containing both required fixes and a trusted `main` run passes deployment, access, exact image and runtime identity, hosted and sandbox inference, and verified workspace cleanup. -Closing #8924 records the end of the temporary policy. +After preparation succeeds, the Launchable upload retains `lane.log` and each +phase artifact created before exit. A preparation failure can produce no +artifact. A later early failure can retain only `lane.log`. A successful job +contains `launchable-e2e.json`, `full-e2e.log`, and `cleanup.json`; +`cleanup.json` exists only after the job confirms workspace absence. Manual ordinary and full runs exclude the Jetson nvmap and DGX Spark llama.cpp jobs unless their independent opt-in flags are `true`. @@ -1083,18 +1076,32 @@ A main push can queue repository-owned GPU runners or create external resources The main-run observer records attempt evidence but does not request broad failed-job reruns. Each E2E test owns any bounded operation-level retry policy. -`Publish staging Brev Launchable image` runs only for a trusted manual dispatch against `main`. -The job reads this credential from repository Actions secrets: +`Exact staging Brev Launchable` runs only for a trusted manual dispatch against `main`. +The job reads these credentials from repository Actions secrets: +- `BREV_API_KEY` authenticates the trusted host-side Brev CLI for workspace + operations in the organization identified by `BREV_ORG_ID`. Candidate code + does not receive this API key. - `NEMOCLAW_IMAGE_DISPATCH_TOKEN` is exposed as `GH_TOKEN` only to the trusted host script. It grants Actions read/write access to `brevdev/nemoclaw-image`, which the script uses to dispatch the image workflow, inspect its run, and download its handoff artifact. - -The credential remains valid until it expires or an administrator revokes it in GitHub. -Rotate or revoke it to remove later access. -The job does not receive `BREV_API_KEY`, `BREV_ORG_ID`, or `NVIDIA_INFERENCE_API_KEY`. -It does not install or authenticate the Brev CLI, create a workspace, or run inference. +- `NVIDIA_INFERENCE_API_KEY` is exported into the Brev guest for the full E2E + process. Code in the baked candidate checkout can read and use it. + +`brev login` writes `BREV_API_KEY` and `BREV_ORG_ID` to +`$HOME/.brev/credentials.json` on the GitHub-hosted runner. Later trusted steps +and processes in the same job can read that file. The workflow does not delete +it explicitly; it remains on the ephemeral runner filesystem until runner +teardown discards that filesystem. +These credentials remain valid until they expire or an administrator revokes +them in their issuing services. If cleanup fails, remove the recorded Brev +workspace. Rotate or revoke each credential to remove later access. + +The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the +standing Launchable. Keep its value equal to the Launchable ID in the default +URL owned by +[`nemoclaw-maintainer-validate-launchable`](../../.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md). When an eligible `E2E main` push workflow completes, `E2E / Main Retry` records its conclusion and the available source-attempt evidence. It does not request a broad failed-job or workflow rerun. @@ -1107,7 +1114,7 @@ The observer ignores manual PR runs and a run superseded by a newer `main` push. For a PR revision run, a repository maintainer or administrator leaves `jobs` and `targets` empty. The run selects: -- every default-selected free-standing workflow E2E except `Publish staging Brev Launchable image`; +- every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; - every catalogue target in the `standard` profile; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index a8d5de979dc..82f786b0307 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -301,7 +301,7 @@ test/e2e/ For a PR revision run, leave `jobs` and `targets` empty. The run selects every default-selected free-standing workflow - E2E except `Publish staging Brev Launchable image`, every catalogue target in the + E2E except `Exact staging Brev Launchable`, every catalogue target in the `standard` profile, all shared credential-free tests, and these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, @@ -363,10 +363,9 @@ test/e2e/ - `.github/workflows/podman-cpu-proof.yaml` provides PR-only experimental runtime evidence with Docker disabled. - `.github/workflows/sandbox-images-and-e2e.yaml` provides reusable image build and test evidence through manual dispatch and `workflow_call`. `.github/workflows/e2e.yaml` selects free-standing jobs, including `whatsapp-qr-compact` and `ollama-auth-proxy`. -- The `staging-brev-launchable` job verifies the exact image-producer receipt, - records the concrete staging image in `launchable-image.json`, and stops before deployment. - Use `nemoclaw-maintainer-validate-launchable` for advisory deployment, runtime, - inference, and cleanup validation while issue #8924 blocks automation. +- The `staging-brev-launchable` job validates the exact baked candidate in + preinstalled mode. Generic Brev VMs with source overlays are not a + qualification boundary. - `vitest.config.ts` contains `e2e-support` for fast fixture/support tests and `e2e-live` for opt-in live target execution. The PR and `main` CLI coverage shards include `e2e-support` for code changes; they never opt into live diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 2add2e7ca82..a1e5f16fb06 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -104,7 +104,7 @@ describe("e2e workflow boundary", () => { () => expect(validateE2eWorkflowBoundary()).toEqual([]), ); - it("rejects a Launchable environment gate, authorization drift, and credential expansion", () => { + it("rejects a Launchable environment gate, authorization drift, and credential boundary drift", () => { const workflow = readWorkflow() as { jobs: Record< string, @@ -122,17 +122,19 @@ describe("e2e workflow boundary", () => { }; const job = workflow.jobs["staging-brev-launchable"]!; job.environment = { name: "unprotected" }; + (job as { env?: Record }).env!.BREV_API_KEY = "${{ secrets.BREV_API_KEY }}"; const prepare = job.steps!.find((step) => step.name === "Prepare the trusted lane")!; - prepare.env ??= {}; prepare.env!.BREV_API_KEY = "${{ secrets.BREV_API_KEY }}"; - const publish = job.steps!.find( - (step) => step.name === "Build and verify the staging Launchable image", + prepare.env!.BREV_CLI_SHA256 = "latest"; + const run = job.steps!.find( + (step) => step.name === "Build, deploy, verify, test, and clean up", )!; - publish.env!.GH_TOKEN = "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}"; - publish.env!.NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY = "0"; + run.env!.GH_TOKEN = "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}"; + run.env!.BREV_LAUNCHABLE_ID = "env-hardcoded"; + run.env!.NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY = "1"; const generateSteps = workflow.jobs["generate-matrix"]!.steps!; const authorization = generateSteps.find( - (step) => step.name === "Authorize Launchable image publication", + (step) => step.name === "Authorize Launchable E2E maintainer dispatch", )!; delete authorization.env!.TRIGGERING_ACTOR; authorization.run = authorization.run!.replace("maintain | admin", "write"); @@ -141,12 +143,15 @@ describe("e2e workflow boundary", () => { expect(validateE2eWorkflow(workflow)).toEqual( expect.arrayContaining([ "staging-brev-launchable must not use a GitHub environment", - "Launchable image publication authorization must bind TRIGGERING_ACTOR", - "step 'Authorize Launchable image publication' run script must include maintain | admin", - "Launchable image publication authorization must run before generate-matrix checkout", - "staging-brev-launchable preparation step must not receive BREV_API_KEY", + "Launchable E2E maintainer authorization must bind TRIGGERING_ACTOR", + "step 'Authorize Launchable E2E maintainer dispatch' run script must include maintain | admin", + "Launchable E2E maintainer authorization must run before generate-matrix checkout", + "staging-brev-launchable BREV_API_KEY must use the trusted-run secret guard", "staging-brev-launchable GH_TOKEN must use the trusted-run secret guard", - "staging-brev-launchable must stop after verified image publication", + "staging-brev-launchable must read the repository Launchable ID variable", + "staging-brev-launchable must not stop after image publication", + "staging-brev-launchable job must not receive BREV_API_KEY", + "staging-brev-launchable must pin the Brev CLI version and SHA-256 checksum", ]), ); }); @@ -245,7 +250,7 @@ describe("e2e workflow boundary", () => { ); }); - it("selects Launchable image publication only for trusted manual dispatches (#7487)", () => { + it("selects Launchable E2E only for trusted manual dispatches (#7487)", () => { expect( evaluateStagingBrevLaunchableDispatch({ eventName: "workflow_dispatch", @@ -335,7 +340,7 @@ describe("e2e workflow boundary", () => { ); }); - it("rejects superseding full-dispatch and Launchable publication concurrency drift (#7487)", () => { + it("rejects superseding full-dispatch and Launchable E2E concurrency drift (#7487)", () => { const workflow = readWorkflow() as { concurrency: Record; jobs: Record }>; @@ -349,7 +354,7 @@ describe("e2e workflow boundary", () => { expect.arrayContaining([ "workflow concurrency must isolate each full dispatch with github.run_id", "workflow concurrency must not cancel an active Jetson dispatch", - "staging-brev-launchable concurrency must queue all pending image publications without cancellation", + "staging-brev-launchable concurrency must queue all pending Launchable E2E runs without cancellation", ]), ); }); diff --git a/test/maintainer-e2e-skill.test.ts b/test/maintainer-e2e-skill.test.ts index d3bddbf0f75..14d8ea88ede 100644 --- a/test/maintainer-e2e-skill.test.ts +++ b/test/maintainer-e2e-skill.test.ts @@ -35,10 +35,10 @@ describe("nemoclaw-maintainer-e2e workflow routing", () => { expect(skill).toContain("git rev-parse origin/main"); expect(skill).toContain("correlation_id=${CORRELATION_ID}"); expect(skill).toContain("head_sha"); - expect(skill).toContain("Publish staging Brev Launchable image"); + expect(skill).toContain("Exact staging Brev Launchable"); expect(skill).toContain("Release qualification"); - expect(skill).toContain("launchable-image.json"); - expect(skill).toContain("records Launchable, runtime, and inference validation as not run"); + expect(skill).toContain("launchable-e2e.json"); + expect(skill).toContain("cleanup.json"); expect(skill).toContain("provisional release evidence"); expect(skill).toContain("If the release candidate SHA changes"); expect(skill).toContain("nemoclaw-maintainer-cut-release-tag"); diff --git a/test/maintainer-launchable-skill.test.ts b/test/maintainer-launchable-skill.test.ts index a33db46129e..e6b42d99caf 100644 --- a/test/maintainer-launchable-skill.test.ts +++ b/test/maintainer-launchable-skill.test.ts @@ -11,20 +11,12 @@ const launchable = fs.readFileSync( path.join(skillsRoot, "nemoclaw-maintainer-validate-launchable", "SKILL.md"), "utf8", ); -const release = fs.readFileSync( - path.join(skillsRoot, "nemoclaw-maintainer-cut-release-tag", "SKILL.md"), - "utf8", -); const guide = fs.readFileSync(path.join(skillsRoot, "nemoclaw-skills-guide", "SKILL.md"), "utf8"); -const releasePolicy = fs.readFileSync( - path.join(skillsRoot, "nemoclaw-maintainer-policies", "references", "release-train.md"), - "utf8", -); describe("staging Launchable maintainer guidance", () => { it("keeps browser and inference gaps visible as partial validation (#8924)", () => { expect(launchable).toContain( - "https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo", + "https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3I2w334slP4GKSce9kKK0hGerjJ", ); expect(launchable).toContain("When authenticated browser-control tools are available"); expect(launchable).toContain("When browser-control tools are unavailable"); @@ -39,13 +31,13 @@ describe("staging Launchable maintainer guidance", () => { "Require a short-lived inference API key scoped only to the required validation", ); expect(launchable).toContain( - "require a maintainer-approved waiver tied to the exact candidate commit SHA and selected image-publication run ID", + "require a maintainer-approved waiver tied to the exact candidate commit SHA and selected automated Launchable run ID", ); expect(launchable).toContain( "rotate or revoke the inference API key in the issuing NVIDIA service after the run", ); expect(launchable).toContain( - "record its approver, exact candidate commit SHA, selected image-publication run ID, and the accepted period of later API-key access", + "record its approver, exact candidate commit SHA, selected automated Launchable run ID, and the accepted period of later API-key access", ); expect(launchable).toContain( "obtain explicit maintainer approval immediately before starting the credential-bearing process", @@ -75,8 +67,10 @@ describe("staging Launchable maintainer guidance", () => { it("binds image and environment identity before manual validation (#8924)", () => { expect(launchable).toContain( - "`producer.runId` equal to the producer run ID selected by the publication job", + "`producer.runId` equal to the producer run ID selected by the automated job", ); + expect(launchable).toContain("`fullE2e` equal to `passed`"); + expect(launchable).toContain("`boot.bootImage` from `launchable-e2e.json`"); expect(launchable).toContain("Use the supplied environment ID as the authoritative identity"); expect(launchable).toContain( "Use an instance-name lookup only when no environment ID is available", @@ -86,25 +80,10 @@ describe("staging Launchable maintainer guidance", () => { expect(launchable).toContain("`not run` only when no required validation check started"); }); - it("keeps manual Launchable validation advisory during release tagging (#8924)", () => { - expect(release).toContain("nemoclaw-maintainer-validate-launchable"); - expect(release).toContain( - "https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo", - ); - expect(release).toContain("Its absence, partial result, or failure does not block"); - expect(release).toContain( - "Do not describe successful image publication as successful Launchable, runtime, or inference validation", - ); - expect(release).toContain( - "do not assume that the mutable family still points to the candidate", - ); - expect(release).toContain("temporary-staging-launchable-qualification-policy"); - expect(releasePolicy).toContain("NemoClaw maintainers own this policy"); - expect(releasePolicy).toContain("GitHub retains the workflow logs and artifact"); - expect(releasePolicy).toContain( - "missing, partial, or failed result needs no per-release waiver", - ); - expect(releasePolicy).toContain("That successful run is the reactivation evidence"); + it("keeps manual Launchable validation separate from automated E2E evidence (#8924)", () => { + expect(launchable).toContain("This report is advisory manual validation"); + expect(launchable).toContain("Do not use it as automated E2E evidence"); + expect(launchable).toContain("Automated Launchable workflow and job URL"); expect(guide).toContain("`nemoclaw-maintainer-validate-launchable`"); }); }); diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index db30bb8a04c..c5b02b050f1 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -154,7 +154,9 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ name: "staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}", path: [ "${{ steps.workspace.outputs.work_dir }}/lane.log", - "${{ steps.workspace.outputs.work_dir }}/launchable-image.json", + "${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json", + "${{ steps.workspace.outputs.work_dir }}/full-e2e.log", + "${{ steps.workspace.outputs.work_dir }}/cleanup.json", "", ].join("\n"), }, diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 132f548c83b..f894bfd4400 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1786,8 +1786,8 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord): void { const job = asRecord(jobs["staging-brev-launchable"]); - if (job.name !== "Publish staging Brev Launchable image") { - errors.push("staging-brev-launchable must identify image publication without claiming E2E"); + if (job.name !== "Exact staging Brev Launchable") { + errors.push("staging-brev-launchable must identify the exact Launchable E2E contract"); } if (Object.hasOwn(job, "environment")) { errors.push("staging-brev-launchable must not use a GitHub environment"); @@ -1810,15 +1810,15 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord errors, "generate-matrix", generateSteps, - "Authorize Launchable image publication", + "Authorize Launchable E2E maintainer dispatch", ); const expectedAuthorizationSelector = "${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; if (authorization?.if !== expectedAuthorizationSelector) { - errors.push("Launchable image publication authorization must cover exact and full dispatches"); + errors.push("Launchable E2E maintainer authorization must cover exact and full dispatches"); } if (authorization?.shell !== "bash") { - errors.push("Launchable image publication authorization must use bash"); + errors.push("Launchable E2E maintainer authorization must use bash"); } const authorizationEnv = asRecord(authorization?.env); for (const [key, expected] of [ @@ -1827,7 +1827,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord ["TRIGGERING_ACTOR", "${{ github.triggering_actor }}"], ] as const) { if (authorizationEnv[key] !== expected) { - errors.push(`Launchable image publication authorization must bind ${key}`); + errors.push(`Launchable E2E maintainer authorization must bind ${key}`); } } for (const required of [ @@ -1849,9 +1849,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord generateCheckout && generateSteps.indexOf(authorization) >= generateSteps.indexOf(generateCheckout) ) { - errors.push( - "Launchable image publication authorization must run before generate-matrix checkout", - ); + errors.push("Launchable E2E maintainer authorization must run before generate-matrix checkout"); } const concurrency = asRecord(job.concurrency); if ( @@ -1860,52 +1858,69 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord concurrency["cancel-in-progress"] !== false ) { errors.push( - "staging-brev-launchable concurrency must queue all pending image publications without cancellation", + "staging-brev-launchable concurrency must queue all pending Launchable E2E runs without cancellation", ); } const steps = asSteps(job.steps); + const checkout = requireStep(errors, steps, "Checkout trusted Launchable lane"); + const checkoutWith = asRecord(checkout?.with); + if ( + checkoutWith.ref !== "${{ github.workflow_sha }}" || + checkoutWith["persist-credentials"] !== false + ) { + errors.push("staging-brev-launchable must check out trusted workflow source without credentials"); + } const prepare = requireStep(errors, steps, "Prepare the trusted lane"); const prepareEnv = asRecord(prepare?.env); - const run = requireStep(errors, steps, "Build and verify the staging Launchable image"); + const run = requireStep(errors, steps, "Build, deploy, verify, test, and clean up"); if (prepare && run && steps.indexOf(prepare) >= steps.indexOf(run)) { - errors.push("staging-brev-launchable must prepare the workspace before image publication"); + errors.push("staging-brev-launchable must prepare the workspace before the Launchable E2E run"); } const runEnv = asRecord(run?.env); - const expectedImageToken = `\${{ ${trustedRun} && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }}`; - if (runEnv.GH_TOKEN !== expectedImageToken) { - errors.push("staging-brev-launchable GH_TOKEN must use the trusted-run secret guard"); + for (const [env, key, secret] of [ + [prepareEnv, "BREV_API_KEY", "BREV_API_KEY"], + [prepareEnv, "BREV_ORG_ID", "BREV_ORG_ID"], + [runEnv, "GH_TOKEN", "NEMOCLAW_IMAGE_DISPATCH_TOKEN"], + [runEnv, "NVIDIA_INFERENCE_API_KEY", "NVIDIA_INFERENCE_API_KEY"], + ] as const) { + const expected = `\${{ ${trustedRun} && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.${secret} || '' }}`; + if (env[key] !== expected) { + errors.push(`staging-brev-launchable ${key} must use the trusted-run secret guard`); + } } - if (runEnv.NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY !== "1") { - errors.push("staging-brev-launchable must stop after verified image publication"); + if (runEnv.BREV_LAUNCHABLE_ID !== "${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}") { + errors.push("staging-brev-launchable must read the repository Launchable ID variable"); } if (runEnv.WORK_DIR !== "${{ steps.workspace.outputs.work_dir }}") { errors.push("staging-brev-launchable must pass its private evidence directory to the lane"); } - for (const [env, scope] of [ - [asRecord(job.env), "job"], - [prepareEnv, "preparation step"], - [runEnv, "image publication step"], + if (Object.hasOwn(runEnv, "NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY")) { + errors.push("staging-brev-launchable must not stop after image publication"); + } + const jobEnv = asRecord(job.env); + for (const [env, scope, forbidden] of [ + [jobEnv, "job", ["BREV_API_KEY", "BREV_ORG_ID", "GH_TOKEN", "NVIDIA_INFERENCE_API_KEY"]], + [prepareEnv, "preparation step", ["GH_TOKEN", "NVIDIA_INFERENCE_API_KEY"]], + [runEnv, "execution step", ["BREV_API_KEY", "BREV_ORG_ID"]], ] as const) { - for (const key of [ - "BREV_API_KEY", - "BREV_ORG_ID", - "BREV_LAUNCHABLE_ID", - "NVIDIA_INFERENCE_API_KEY", - ]) { + for (const key of forbidden) { if (Object.hasOwn(env, key)) { errors.push(`staging-brev-launchable ${scope} must not receive ${key}`); } } } + if ( + !/^0\.\d+\.\d+$/u.test(stringValue(prepareEnv.BREV_CLI_VERSION)) || + !/^[0-9a-f]{64}$/u.test(stringValue(prepareEnv.BREV_CLI_SHA256)) + ) { + errors.push("staging-brev-launchable must pin the Brev CLI version and SHA-256 checksum"); + } const prepareRun = stringValue(prepare?.run); if ( - prepareRun.includes("brev login") || - prepareRun.includes("BREV_CLI_VERSION") || - prepareRun.includes("BREV_CLI_SHA256") + !prepareRun.includes("sha256sum -c -") || + !prepareRun.includes("brev login --api-key \"$BREV_API_KEY\" --org-id \"$BREV_ORG_ID\"") ) { - errors.push( - "staging-brev-launchable preparation must not install or authenticate the Brev CLI", - ); + errors.push("staging-brev-launchable must verify and authenticate the pinned Brev CLI"); } } @@ -1921,12 +1936,12 @@ function validateStagingBrevLaunchableInput( } const description = stringValue(input.description); if ( - !description.includes("staging Brev Launchable image publication") || + !description.includes("Exact staging Brev Launchable") || !description.includes("jobs and targets are empty") || !description.includes("full E2E run") ) { errors.push( - "workflow_dispatch include_staging_brev_launchable input must document full-run Launchable image publication scope", + "workflow_dispatch include_staging_brev_launchable input must document full-run Launchable E2E scope", ); } } @@ -2253,7 +2268,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { const jobsDescription = stringValue(jobsInput.description); if (!jobsDescription.includes("include_staging_brev_launchable")) { errors.push( - "workflow_dispatch jobs input description must identify how to include staging Brev Launchable image publication", + "workflow_dispatch jobs input description must identify how to include Exact staging Brev Launchable", ); } if (Object.hasOwn(dispatchInputs, "test_filter")) {