diff --git a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md index e6d68b01bd..045c48e447 100644 --- a/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-cut-release-tag/SKILL.md @@ -125,7 +125,7 @@ Read the generated `plan.json` and show the maintainer: For the plan's full `origin/main` SHA, require a completed, successful `Release qualification` check from a pre-tag `.github/workflows/e2e.yaml` run. The workflow planner derives the required jobs from the workflow's E2E metadata. -By default, the check requires every release-required E2E execution result, including `Exact staging Brev Launchable`, to succeed. +By default, the check requires every release-required execution result, including `Publish staging Brev Launchable image`, to succeed. A repository administrator may waive one or more release-required E2E execution jobs for a documented release exception. The waiver requires a comma-separated `release_qualification_waived_jobs` list and a `release_qualification_waiver_reason`. The reason must begin with an ASCII letter or digit and contain 10-500 characters chosen from ASCII letters, digits, spaces, and `.,:;/_()'-`. @@ -167,6 +167,20 @@ Before showing the confirmation prompt, present the candidate SHA, workflow URL, For a waived run, also present the waived jobs, their outcomes, the waiver reason, and both recorded actor identities. No release-note-only delta exception is currently defined. +After image publication succeeds, present this advisory manual validation: + +- State that the image-publication job built and published the candidate image to the staging family used by the [NemoClaw staging Launchable](https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo). +- Encourage the maintainer to deploy one instance and hand its Brev environment URL to a Codex session that invokes `nemoclaw-maintainer-validate-launchable`. +- Require the manual validation to compare the deployed concrete image with `launchable-image.json`; do not assume that the mutable family still points to the candidate. +- State that browser-control capability is required for Codex to click and verify the web interface. +- State that a securely supplied inference credential is required to complete hosted and sandbox inference validation. Never ask the maintainer to paste the credential into chat. +- Record the manual result as `complete pass`, `partially blocked`, `failed`, or `not run` when the maintainer provides it. + +This manual validation is advisory while the automated Launchable path is blocked by issue #8924. +Its absence, partial result, or failure does not block the signing preflight, confirmation prompt, or release tag. +Do not describe successful image publication as successful Launchable, runtime, or inference validation. +Apply the temporary policy in [Pre-Tag E2E Evidence](../nemoclaw-maintainer-policies/references/release-train.md#temporary-staging-launchable-qualification-policy): NemoClaw maintainers own it while #8924 remains open, the successful exact image-publication job and artifact remain required release evidence under normal Actions retention, and the full automated lane returns only after a checksum-pinned Brev release passes deployment through verified cleanup on trusted `main`. + Run the release script's signing preflight before asking for confirmation: ```bash @@ -323,10 +337,11 @@ If the Announcement is valid, return its URL with the release artifacts and mark - Plan generation fails: fix the named precondition, then regenerate the plan. - Documentation workflow state is incomplete: return to `nemoclaw-maintainer-evening`, then repeat Step 1 after the documentation PR merges. -- Full-mode E2E waits in the Launchable concurrency queue: keep the run pending until the earlier Launchable E2E job finishes. +- Full-mode E2E waits in the Launchable concurrency queue: keep the run pending until the earlier Launchable image-publication job finishes. - Full-mode E2E ran for another SHA: reject the run and dispatch full mode for the plan candidate SHA. - No qualifying `Release qualification` exists: inspect the GitHub result and run pre-tag E2E for the planned SHA only when no qualifying run already exists. Use a job waiver only with explicit repository administrator authorization. Do not release until the release script accepts the canonical check. -- Launchable E2E or cleanup fails: inspect the diagnostic artifacts, correct the failure, and rerun the affected E2E work. Do not infer Launchable success from another workflow result. +- Launchable image publication fails: inspect `launchable-image.json` and the producer run, correct the failure, and rerun the affected work. Do not infer image publication from manual Launchable validation. +- Advisory Launchable validation is blocked or fails: record the exact partial result and continue the release flow. Do not convert the result into a release gate or an automated E2E pass. - `origin/main` moved after plan generation: regenerate the plan and ask for the new confirmation phrase. - Remote semver tag already exists: stop; do not retag unless the maintainer explicitly starts protected-tag remediation. - Signing preflight fails: fix the reported Git signer or signing-key failure. Run the preflight again before requesting confirmation. diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 6f0404a296..d4f9317e35 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-e2e -description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the latest PR commit. Use for requests such as run E2E for PR #123, run the E2E suite, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. +description: Dispatches and verifies trusted GitHub Actions E2E for NemoClaw maintainers, including manual PR E2E for the latest PR commit and staging Launchable image publication. Use for requests such as run E2E for PR #123, run the E2E suite, publish the Launchable image, run the Launchable E2E, run the full E2E suite, deploy pre-release full E2E, run pre-tag full E2E, or run release-candidate E2E. --- @@ -34,17 +34,14 @@ The workflow does not rotate or revoke these API keys or messaging credentials. Live targets can create external resources. After a failure, inspect the artifacts and remove resources that target cleanup did not remove. -`Exact staging Brev Launchable` reads these credentials from repository Actions secrets: +`Publish staging Brev Launchable image` reads this credential from repository Actions secrets: -- `BREV_API_KEY` authenticates the trusted host-side Brev CLI for workspace operations in the organization identified by `BREV_ORG_ID`. Candidate code does not receive this API key. - `NEMOCLAW_IMAGE_DISPATCH_TOKEN` is exposed as `GH_TOKEN` only to the trusted host script. It grants Actions read/write access to `brevdev/nemoclaw-image`, which the script uses to dispatch the image workflow, inspect its run, and download its handoff artifact. -- `NVIDIA_INFERENCE_API_KEY` is exported into the Brev guest for the full E2E process. Code in the baked candidate checkout can read and use it. -`brev login` writes `BREV_API_KEY` and `BREV_ORG_ID` to `$HOME/.brev/credentials.json` on the GitHub-hosted runner. Later trusted steps and processes in the same job can read that file. The workflow does not delete it explicitly; it remains on the ephemeral runner filesystem until runner teardown discards that filesystem. -These credentials remain valid until they expire or an administrator revokes them in their issuing services. If cleanup fails, remove the recorded Brev workspace. Rotate or revoke each credential to remove later access. -This Brev credential boundary applies only to trusted Launchable or full manual dispatches against `main`. It does not apply to `main` pushes or manual PR runs. - -The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the standing Launchable. Keep its value equal to the Launchable ID in the default URL owned by [`nemoclaw-maintainer-validate-launchable`](../nemoclaw-maintainer-validate-launchable/SKILL.md). +This credential remains valid until it expires or an administrator revokes it in GitHub. Rotate or revoke it to remove later access. +The job does not receive `BREV_API_KEY`, `BREV_ORG_ID`, or `NVIDIA_INFERENCE_API_KEY`. +It does not install or authenticate the Brev CLI, create a workspace, or run inference. +This image-publication credential boundary applies only to trusted Launchable or full manual dispatches against `main`. It does not apply to `main` pushes or manual PR runs. For `managed-image-protected-runtime`, the workflow supplies the long-lived `NVIDIA_API_KEY` repository secret only to the trusted qualification step. Trusted host code uses it for NGC login and passes it as `NGC_API_KEY` and `NIM_NGC_API_KEY` to the temporary NIM container. Candidate managed sandboxes receive generated local route tokens instead of this key. The live fixture removes the temporary NIM container only if its exact ID, name, requested image, immutable image ID, cohort owner, and provider kind match the recorded authority. The test fails if evidence is missing or ambiguous, a name is reused, authority drifts, removal is indeterminate, or the exact ID or name remains. A cleanup refusal can leave the container and its API key in place until runner teardown. The final workflow step removes the job's isolated Docker credential directory and fails if that removal does not complete. The workflow does not revoke the NVIDIA API key. Revoke it, or rotate it and disable the old value, in the issuing NVIDIA service. Verify that the exposed key is no longer valid. @@ -70,7 +67,7 @@ Require a review reason containing 10 to 500 printable characters. Choose exactly one mode: - For a PR revision run, leave `E2E_JOBS` empty. The run selects: - - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; + - every default-selected free-standing workflow E2E except `Publish staging Brev Launchable image`; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. @@ -161,7 +158,8 @@ A changed head repository, head SHA, or base SHA invalidates the evidence and re | Request | Mode | `jobs` | `include_staging_brev_launchable` | |---|---|---|---| | “Run the E2E suite” | Ordinary | empty | `false` | -| “Run the Launchable E2E” | Launchable | `staging-brev-launchable` | `false` | +| “Publish the Launchable image” | Launchable image | `staging-brev-launchable` | `false` | +| “Run the Launchable E2E” | Clarify before dispatch | not applicable | not applicable | | “Run the full E2E suite” | Full | empty | `true` | | “deploy pre-release full E2E” | Full | empty | `true` | | “run pre-tag full E2E” | Full | empty | `true` | @@ -169,12 +167,16 @@ A changed head repository, head SHA, or base SHA invalidates the evidence and re | “run pre-tag E2E with an administrator job waiver” | Administrator-waived full | empty | `true` | A generic E2E request must not authorize the Brev Launchable path. +For “Run the Launchable E2E,” explain that issue #8924 blocks automated deployment, runtime, and inference validation. +Ask whether the maintainer wants image publication or advisory validation through `nemoclaw-maintainer-validate-launchable` against one deployed instance. +Do not dispatch until the maintainer selects one of those operations. Do not infer full mode from words such as “all” or “complete.” -Ask for clarification only when the request contains conflicting mode phrases. +Ask for clarification when the request uses the legacy Launchable E2E phrase or contains conflicting mode phrases. -Ordinary mode selects every default-selected workflow E2E except `Exact staging Brev Launchable`. -Launchable mode runs only `Exact staging Brev Launchable`. -Full mode adds `Exact staging Brev Launchable` to the default E2E selection in the same workflow run. +Ordinary mode selects every default-selected workflow E2E except `Publish staging Brev Launchable image`. +Launchable image mode runs only `Publish staging Brev Launchable image`. +Full mode adds `Publish staging Brev Launchable image` to the default E2E selection in the same workflow run. +The Launchable image job stops after exact image-publication evidence and does not deploy a workspace or run inference. Administrator-waived full mode runs the full suite but omits the approved execution jobs from release qualification. Every waived job still runs. Use this mode only when a repository administrator explicitly authorizes the job IDs and supplies the reason. @@ -220,7 +222,7 @@ gh workflow run .github/workflows/e2e.yaml \ -f "correlation_id=${CORRELATION_ID}" ``` -For Launchable mode: +For Launchable image mode: ```bash gh workflow run .github/workflows/e2e.yaml \ @@ -271,8 +273,8 @@ gh workflow run .github/workflows/e2e.yaml \ ``` Do not set `jobs=staging-brev-launchable` for full mode. -Empty `jobs` and `targets` select every default-selected workflow E2E except `Exact staging Brev Launchable`. -The `include_staging_brev_launchable` input adds the Launchable E2E job to that same run. +Empty `jobs` and `targets` select every default-selected workflow E2E except `Publish staging Brev Launchable image`. +The `include_staging_brev_launchable` input adds the Launchable image-publication job to that same run. The trusted `main` workflow verifies that the dispatching and rerunning actors have repository `maintain` or `admin` permission before the Launchable path's source checkout. That role check is the authorization. @@ -298,7 +300,7 @@ empty-selector manual run or enable explicit qualification selection. Set it only after a repository administrator confirms an online DGX Spark runner in the authoritative runner inventory. If GitHub pauses the qualification job for the `approve-dgx-spark-image-qualification` environment, an authorized environment reviewer must approve it before qualification starts. -`Exact staging Brev Launchable` does not require environment approval. +`Publish staging Brev Launchable image` does not require environment approval. Find the run by its unique title: @@ -332,7 +334,7 @@ Wait for completion: gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw ``` -Launchable and full modes can wait in the non-cancelling Launchable concurrency queue. +Launchable image and full modes can wait in the non-cancelling Launchable concurrency queue. Queued, waiting, or accepted dispatch state is not success. Classify the completed workflow and `Release qualification` job with the checks below. @@ -354,20 +356,21 @@ Require `run-$RUN_ID.json` to report: - `head_sha` equal to `CANDIDATE_SHA`; - `status` equal to `completed`. -For ordinary, Launchable, and unwaived full modes, require `conclusion` equal to `success`. +For ordinary, Launchable image, and unwaived full modes, require `conclusion` equal to `success`. For administrator-waived full mode, permit `conclusion` equal to `success` or `failure`. A `failure` conclusion is acceptable only when one completed, successful `Release qualification` job and a valid exact-run waiver artifact with at least one canonical waived job failure both exist. -For Launchable mode, also require `jobs-latest-$RUN_ID.json` to contain one completed, successful -`Exact staging Brev Launchable` job. Return the workflow and job URLs. +For Launchable image mode, also require `jobs-latest-$RUN_ID.json` to contain one completed, successful +`Publish staging Brev Launchable image` job. Return the workflow and job URLs. +Require its artifact to contain `launchable-image.json` for the selected candidate SHA and concrete staging image URI. For a full run, with or without a job waiver, require `jobs-latest-$RUN_ID.json` to contain one completed, successful `Release qualification` job. Return its job URL with the workflow URL. -In full mode, that job waits for every default-required E2E result, including `Exact staging Brev Launchable`. -The Launchable job directly verifies the candidate checkout, in-guest full E2E result, and workspace cleanup before it succeeds. -Its `launchable-e2e.json`, `full-e2e.log`, and `cleanup.json` artifacts remain available for diagnosis. +In full mode, that job waits for every default-required result, including `Publish staging Brev Launchable image`. +The Launchable image job verifies only the exact candidate image producer receipt and staging-family publication. +Its `launchable-image.json` artifact records Launchable, runtime, and inference validation as not run. A skipped, cancelled, queued, or failed `Release qualification` job is not evidence. -A Launchable-only run is not full-mode or pre-tag release evidence. +A Launchable image-only run is not full-mode or pre-tag release evidence. For administrator-waived full mode, the job waits for every unwaived release-required result. A waived execution job may fail without failing `Release qualification`. diff --git a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md index 3bce66b0d9..adabae8d95 100644 --- a/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md +++ b/.agents/skills/nemoclaw-maintainer-policies/references/release-train.md @@ -52,9 +52,25 @@ The release candidate is the full `origin/main` commit SHA captured by the gener Before asking for the release confirmation phrase, require a completed, successful `Release qualification` check from a pre-tag manual run at that SHA. +### Temporary Staging Launchable Qualification Policy + +Issue #8924 temporarily limits the trusted staging Launchable job to exact image publication. +NemoClaw maintainers own this policy while that issue remains open. +For each release candidate, the required automated evidence is a successful `Publish staging Brev Launchable image` job and its `launchable-image.json` artifact, bound to the exact candidate SHA through the successful `Release qualification` check. +GitHub retains the workflow logs and artifact under the repository's normal Actions retention policy. +An image-publication failure still blocks release qualification unless a repository administrator uses the existing documented job-waiver mechanism. + +The temporary risk acceptance permits a release tag without automated or manual proof of the staging Launchable web deployment, environment access, exact booted image, baked runtime, inference, or workspace cleanup. +Manual validation remains advisory, and a missing, partial, or failed result needs no per-release waiver. +It must not be reported as an automated E2E pass. + +Restore the automated deployment lane when a published Brev CLI release contains the Launchable image-forwarding fix and the host-route fix tracked by #8924. +NemoClaw must checksum-pin that release and complete a trusted `main` run that verifies deployment, environment access, exact image and runtime identity, hosted and sandbox inference, and workspace cleanup. +That successful run is the reactivation evidence; closing #8924 records the end of this temporary policy. + - `.github/workflows/e2e.yaml` derives the release-required jobs from its E2E metadata. Do not copy them into a second release test list. - Push runs publish `Relevant E2E`; only full manual runs dispatched against `main` with empty selectors publish `Release qualification`. -- By default, the check requires every default-required workflow E2E result to succeed, including `Exact staging Brev Launchable`. +- By default, the check requires every default-required workflow result to succeed, including `Publish staging Brev Launchable image`. - A repository administrator may waive one or more release-required E2E execution jobs with `release_qualification_waived_jobs` and `release_qualification_waiver_reason`. - `release_qualification_waived_jobs` is a comma-separated list of requested job IDs. - The reason must begin with an ASCII letter or digit and contain 10-500 characters chosen from ASCII letters, digits, spaces, and `.,:;/_()'-`. @@ -69,7 +85,8 @@ Before asking for the release confirmation phrase, require a completed, successf - A normal full run must conclude with `success`. - An administrator-waived full run may conclude with `failure` when a waived execution job fails, `Release qualification` succeeds, and the waiver artifact binds that failure to the candidate, run, actors, reason, and canonical waived job IDs. - `jetson-nvmap-gpu`, `llama-cpp-dgx-spark-plan`, and `llama-cpp-dgx-spark-qualification` remain separate opt-in work and do not block this check. -- A successful Launchable job proves the candidate checkout, in-guest full E2E result, and cleanup. Its artifacts are diagnostic evidence, not a second status ledger. +- A successful Launchable image job proves that the producer published the exact candidate image to the staging family. Its `launchable-image.json` artifact records Launchable, runtime, and inference validation as not run. +- Manual staging Launchable validation is advisory while issue #8924 blocks the automated deployment path. A missing, partial, or failed manual result does not block the release tag and must not be reported as an automated E2E pass. - A skipped, queued, in-progress, cancelled, or failed `Release qualification` check is not release evidence. - A check from another commit SHA is not release evidence. - Use an existing qualifying pre-tag run for the candidate SHA; run `nemoclaw-maintainer-e2e` in full mode when none exists, with an administrator-authorized job waiver when required. diff --git a/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md b/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md index 52fab60658..0b3d4aac3e 100644 --- a/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md @@ -1,6 +1,6 @@ --- name: nemoclaw-maintainer-validate-launchable -description: Validate the user-facing staging Brev Launchable deployment, exact NemoClaw image and runtime identity, onboarding, CLI behavior, and inference. Use when a maintainer asks to test the staging Launchable in the Brev web interface, provides a deployed Brev environment URL, hands a Launchable instance to Codex, or needs advisory web validation separate from automated Launchable E2E. +description: Validate the user-facing staging Brev Launchable deployment, exact NemoClaw image and runtime identity, onboarding, CLI behavior, and inference. Use when a maintainer asks to test the staging Launchable in the Brev web interface, provides a deployed Brev environment URL, hands a Launchable instance to Codex, or needs advisory manual validation while the automated Launchable E2E is blocked. --- @@ -25,15 +25,14 @@ Use the current checkout and workflow artifacts as the source of truth for image - Redact credentials from captured output and delete temporary raw logs after producing redacted evidence. Use this staging Launchable unless the maintainer supplies another accepted target: -[Deploy the NemoClaw staging Launchable](https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3I2w334slP4GKSce9kKK0hGerjJ). -Keep the Launchable ID in this default URL equal to the `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable used by the automated job. +[Deploy the NemoClaw staging Launchable](https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo). ## Define the Result Boundary Require all of these results for a complete pass: 1. The Launchable web page and deployment flow work in an authenticated browser session. -2. The deployed environment boots the exact concrete image recorded by the selected automated Launchable workflow artifact. +2. The deployed environment boots the exact concrete image recorded by the selected image-publication workflow artifact. 3. The baked provision receipt and source checkout without tracked or untracked changes identify the selected NemoClaw commit. 4. The preinstalled user journey completes onboarding, CLI checks, sandbox inference, recovery, logs, and cleanup. 5. A hosted inference request and the sandbox inference request succeed with a securely supplied credential. @@ -51,17 +50,17 @@ When one or more checks ran without failure but another required check did not r ## Resolve the Candidate and Image Evidence Record the expected NemoClaw commit SHA before deployment. -Use the latest successful `Exact staging Brev Launchable` job for that exact SHA. -Record the selected workflow and job URLs and the producer run ID selected in that job's log. -Download its private artifact and require `launchable-e2e.json` to report: +Use the latest successful `Publish staging Brev Launchable image` job for that exact SHA. +Record the selected publication workflow and job URLs and the producer run ID selected in that job's log. +Download its private artifact and require `launchable-image.json` to report: +- `schemaVersion` equal to `1`; +- `kind` equal to `nemoclaw-staging-launchable-image-v1`; - `candidateSha` equal to the selected commit SHA; -- `producer.status` equal to `success` and `producer.runId` equal to the producer run ID selected by the automated job; -- a concrete `boot.bootImage` URI; -- `boot.schemaVersion` equal to `1`, `boot.sourceRepository` equal to `NVIDIA/NemoClaw`, and `boot.sourcePath` equal to `/opt/nemoclaw-image/NemoClaw`; -- `boot.repoSha` and `boot.provisionSha` equal to the selected commit SHA; -- a lowercase 40-character `boot.imageRepositorySha`, `boot.repoClean` equal to `true`, and `boot.runtimeOverrides` equal to `false`; and -- `fullE2e` equal to `passed`. +- `producer.repository` equal to `brevdev/nemoclaw-image`, `producer.status` equal to `success`, and `producer.runId` equal to the producer run ID selected by the publication job; +- `image.family` equal to `nemoclaw-brev-staging-cpu`; +- a concrete `image.uri` and a lowercase 40-character `image.imageRepositorySha`; and +- `validation.launchable`, `validation.runtime`, and `validation.inference` equal to `not-run`. Stop when the artifact is absent, malformed, or belongs to another commit. Classify that evidence as `failed` when GitHub is available and the selected job or artifact can be inspected. @@ -103,7 +102,7 @@ Perform these checks without changing the instance: 1. Use the supplied environment ID as the authoritative identity. If a name is also supplied, require it to match that environment. Use an instance-name lookup only when no environment ID is available, and require exactly one match. 2. Require the environment to report its successful running state. 3. Establish the user-facing SSH or terminal access path shown by Brev. -4. Read the GCE instance image metadata and require exact equality with `boot.bootImage` from `launchable-e2e.json`. +4. Read the GCE instance image metadata and require exact equality with the concrete image URI from `launchable-image.json`. 5. Read `/etc/nemoclaw/provision.json` with the privileges provided by the image. 6. Require the provision receipt, source repository, source path, image-repository commit SHA, and NemoClaw commit SHA to match the selected artifact and candidate. 7. Require the baked source checkout to have no tracked or untracked changes and no runtime override receipt. @@ -124,7 +123,7 @@ Run the validation from a short-lived local process that receives the key throug The local validation process and its SSH child can read the key; the remote shell exports it to the baked full E2E process, so candidate code can read and use it. Before exposing the key, record the authorized candidate repository and commit SHA, require the repository to be `NVIDIA/NemoClaw`, and reject a candidate from a fork pull request. Explain that the selected candidate code can read and use the key, then obtain explicit maintainer approval immediately before starting the credential-bearing process. -If the issuing service cannot rotate or revoke the inference API key after the run, require a maintainer-approved waiver tied to the exact candidate commit SHA and selected automated Launchable run ID before starting validation. +If the issuing service cannot rotate or revoke the inference API key after the run, require a maintainer-approved waiver tied to the exact candidate commit SHA and selected image-publication run ID before starting validation. Do not persist the key in shell startup files, temporary files, SSH configuration, or the Brev environment after the test process exits. If it is unavailable: @@ -138,7 +137,7 @@ Require the baked full E2E success sentinel and retain only redacted logs. The test must remove its `e2e-` sandbox and verify the expected cleanup result even after a test failure. After the local and remote test processes exit, unset any shell variable created for the run and verify that no temporary credential file remains. Unless the approved waiver applies, rotate or revoke the inference API key in the issuing NVIDIA service after the run and record non-sensitive confirmation. -When the waiver applies, record its approver, exact candidate commit SHA, selected automated Launchable run ID, and the accepted period of later API-key access without recording the key. +When the waiver applies, record its approver, exact candidate commit SHA, selected image-publication run ID, and the accepted period of later API-key access without recording the key. ## Finish the Instance Handoff @@ -158,7 +157,7 @@ Return this structure: - Evidence mode: advisory manual validation; not automated E2E evidence - Candidate repository and commit SHA: -- Automated Launchable workflow and job URL: +- Image-publication workflow and job URL: - Expected concrete image URI: - Launchable URL: - Environment URL, ID, and name: diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4f037e4e2a..ebd8efd743 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -15,12 +15,12 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix." + description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for staging Brev Launchable image publication. Use staging-brev-launchable-e2e-once only on current main for one staging deployment and full E2E. brev login stores Brev credentials in runner-local $HOME/.brev/credentials.json for later trusted job processes until hosted-runner teardown. Only the trusted host step receives the image token; baked candidate code receives the NVIDIA inference key. Issuing services retain credentials until expiry or revocation. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix." required: false default: "" type: string include_staging_brev_launchable: - description: "Include Exact staging Brev Launchable in a full E2E run when jobs and targets are empty." + description: "Include staging Brev Launchable image publication in a full E2E run when jobs and targets are empty." required: false default: false type: boolean @@ -598,8 +598,8 @@ jobs: name: e2e-dispatch-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json - - name: Authorize Launchable E2E maintainer dispatch - if: ${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} + - name: Authorize Launchable workflow dispatch + if: ${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} env: ACTOR: ${{ github.actor }} GITHUB_TOKEN: ${{ github.token }} @@ -667,20 +667,20 @@ jobs: require_maintainer() { local maintainer="$1" if [[ ! "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ || "$maintainer" == -* || "$maintainer" == *- ]]; then - echo "::error::Launchable E2E actor is invalid" >&2 + echo "::error::Launchable workflow dispatch actor is invalid" >&2 exit 1 fi local permission_json permission_json="$(read_collaborator_permission "$maintainer")" if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then - echo "::error::Launchable E2E permission response did not match the actor" >&2 + echo "::error::Launchable workflow dispatch permission response did not match the actor" >&2 exit 1 fi case "$(jq -r '.role_name // ""' <<< "$permission_json")" in maintain | admin) ;; *) - echo "::error::Launchable E2E requires a repository maintainer or administrator" >&2 + echo "::error::Launchable workflow dispatch requires a repository maintainer or administrator" >&2 exit 1 ;; esac @@ -2683,7 +2683,7 @@ jobs: path: e2e-artifacts/live/retired-selector-compatibility/ staging-brev-launchable: - name: Exact staging Brev Launchable + name: Publish staging Brev Launchable image needs: generate-matrix if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} runs-on: ubuntu-latest @@ -2696,8 +2696,6 @@ jobs: cancel-in-progress: false env: CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }} - E2E_JOB: "1" - INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} steps: - name: Checkout trusted Launchable lane uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -2710,35 +2708,91 @@ jobs: - id: workspace name: Prepare the trusted lane + run: | + set -euo pipefail + work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")" + chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + + - name: Build and verify the staging Launchable image env: - BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_API_KEY || '' }} - BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32 - BREV_CLI_VERSION: 0.6.334 - BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_ORG_ID || '' }} + GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} + NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY: "1" + WORK_DIR: ${{ steps.workspace.outputs.work_dir }} + run: tools/e2e/brev-launchable-e2e.sh + + - name: Upload Launchable evidence + if: ${{ always() && steps.workspace.outputs.work_dir != '' }} + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + with: + name: staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ steps.workspace.outputs.work_dir }}/lane.log + ${{ steps.workspace.outputs.work_dir }}/launchable-image.json + + staging-brev-launchable-e2e-once: + name: Run one-off staging Brev Launchable E2E + needs: generate-matrix + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '' }} + runs-on: ubuntu-latest + timeout-minutes: 180 + permissions: + contents: read + concurrency: + group: staging-brev-launchable-cpu + queue: max + cancel-in-progress: false + env: + CANDIDATE_SHA: ${{ github.sha }} + E2E_DEFAULT_ENABLED: "0" + E2E_JOB: "1" + INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }} + steps: + - name: Checkout trusted one-off Launchable lane + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + persist-credentials: false + sparse-checkout: | + tools/e2e/brev-launchable-e2e.sh + sparse-checkout-cone-mode: false + + - id: workspace + name: Prepare the one-off Launchable lane run: | set -euo pipefail work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")" chmod 700 "$work_dir" + printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" + + - name: Install Brev CLI for one-off Launchable E2E + env: + BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '' && secrets.BREV_API_KEY || '' }} + BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32 + BREV_CLI_VERSION: 0.6.334 + BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '' && secrets.BREV_ORG_ID || '' }} + run: | + set -euo pipefail archive="${RUNNER_TEMP}/brev-cli.tar.gz" curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz" printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c - - tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev + tar -xzf "$archive" -C "${RUNNER_TEMP}" brev + sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID" - printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT" - - name: Build, deploy, verify, test, and clean up + - name: Build, deploy, verify, test, and clean up once env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} - GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} - NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '' && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-e2e.sh - - name: Upload Launchable evidence + - name: Upload one-off Launchable evidence if: ${{ always() && steps.workspace.outputs.work_dir != '' }} uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 with: - name: staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} + name: staging-brev-launchable-e2e-once-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ steps.workspace.outputs.work_dir }}/lane.log ${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json @@ -5612,6 +5666,7 @@ jobs: generate-matrix, retired-selector-compatibility, staging-brev-launchable, + staging-brev-launchable-e2e-once, live, shared-e2e, catalogue-standard, diff --git a/test/e2e/README.md b/test/e2e/README.md index ed0f54e611..b80f1d8e93 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -21,8 +21,9 @@ before those targets run; local runners must provide it themselves. - `.github/workflows/e2e-main-retry.yaml` evaluates eligible `E2E main` push attempts and uploads attempt evidence. It never authorizes a broad failed-job rerun; retry decisions belong to bounded operation-level policies. -- The `staging-brev-launchable` job in `.github/workflows/e2e.yaml` validates - the baked candidate without installing or copying NemoClaw source. +- The `staging-brev-launchable` job in `.github/workflows/e2e.yaml` publishes + the exact candidate image to the staging family and records the concrete image URI. + It does not deploy or validate a Brev environment while issue #8924 blocks the automated path. - `.github/workflows/platform-vitest-main.yaml` publishes `CI / Platform Evidence` for Ubuntu 26.04, macOS, and WSL. On shard 1, its macOS and WSL live E2E run only when the workflow tests `main` and Docker is available. This workflow does not publish or satisfy `Release qualification`. @@ -169,12 +170,13 @@ GitHub invalidates `GITHUB_TOKEN` after the job. ## Retired Brev source-install coverage -Issue #7490 retired the generic Brev source-install lane. The unified workflow -and exact-staging Launchable job own its product coverage: +Issue #7490 retired the generic Brev source-install lane. Current validation uses +the unified workflow, staging image-publication job, and advisory manual +Launchable validation: | Legacy suite | Disposition | Current owner | |---|---|---| -| `full` | Launchable E2E | `staging-brev-launchable` runs `full-e2e` in preinstalled mode against the exact baked candidate. | +| `full` | Manual Launchable validation | `staging-brev-launchable` publishes the exact candidate image. Use `nemoclaw-maintainer-validate-launchable` to validate a deployed instance while issue #8924 blocks automation. | | `credential-sanitization` | Unified E2E | `credential-sanitization` | | `telegram-injection` | Unified E2E | `telegram-injection` | | `messaging-providers` | Unified E2E | `messaging-providers` | @@ -713,11 +715,11 @@ rm -rf -- "$evidence_dir" test ! -e "$evidence_dir" ``` -A manual run with `jobs=staging-brev-launchable` runs only `Exact staging Brev Launchable`. +A manual run with `jobs=staging-brev-launchable` runs only `Publish staging Brev Launchable image`. Push runs do not select this job. A manual run with `include_staging_brev_launchable=true` and empty `jobs` and -`targets` selectors runs the default workflow E2E selection plus the Launchable E2E job. +`targets` selectors runs the default workflow E2E selection plus the Launchable image-publication job. This selection is the full manual `main` run for pre-tag release evidence. Each full dispatch uses `github.run_id` in its workflow concurrency identity, so another full dispatch @@ -726,11 +728,11 @@ verifies that the dispatching and rerunning actors have repository `maintain` or `admin` permission before the Launchable path's source checkout. That automatic role check authorizes `staging-brev-launchable`; the job does not use GitHub environment approval. The job uses the non-cancelling -`staging-brev-launchable-cpu` group with `queue: max`, so pending Launchable E2E +`staging-brev-launchable-cpu` group with `queue: max`, so pending Launchable image runs remain queued instead of replacing one another. For a full manual run dispatched against `main`, `Release qualification` waits for every E2E job that does not require a separate opt-in. -The check requires each of those jobs to pass, including `Exact staging Brev Launchable`. +The check requires each of those jobs to pass, including `Publish staging Brev Launchable image`. A passing check at the candidate commit SHA is the pre-tag release E2E evidence. Ensure that each candidate commit SHA has a qualifying full manual `main` run. Dispatch another full run only when no qualifying run exists. @@ -742,11 +744,16 @@ Local fixture remotes skip the canonical repository gate only when tests set the Canonical-equivalent `NVIDIA/NemoClaw` remotes always run the gate, even when that override is set. A local fixture cannot authorize a production release. Maintainers do not build a local evidence ledger or infer GitHub job status from an artifact. -After preparation succeeds, the Launchable upload retains `lane.log` and each -phase artifact created before exit. A preparation failure can produce no -artifact. A later early failure can retain only `lane.log`. A successful job -contains `launchable-e2e.json`, `full-e2e.log`, and `cleanup.json`; -`cleanup.json` exists only after the job confirms workspace absence. +The Launchable image job retains `launchable-image.json` with the candidate SHA, producer run, concrete image URI, staging family, and explicit not-run validation fields. +Manual web, runtime, and inference validation is advisory while issue #8924 remains open. +It does not block the release tag and must not be reported as an automated E2E pass. + +This is a temporary NemoClaw maintainer policy owned while #8924 remains open. +Each release candidate still requires the successful exact image-publication job and `launchable-image.json` through `Release qualification`; GitHub keeps its logs and artifact under the repository's normal Actions retention policy. +The accepted temporary risk is that a tag can proceed without automated or manual proof of the Launchable web deployment, environment access, exact booted image, baked runtime, inference, or workspace cleanup. +A missing, partial, or failed manual validation needs no per-release waiver, but an image-publication failure remains release-blocking unless an administrator uses the existing job-waiver mechanism. +Restore automated validation only after NemoClaw checksum-pins a published Brev CLI release containing both required fixes and a trusted `main` run passes deployment, access, exact image and runtime identity, hosted and sandbox inference, and verified workspace cleanup. +Closing #8924 records the end of the temporary policy. Manual ordinary and full runs exclude the Jetson nvmap and DGX Spark llama.cpp jobs unless their independent opt-in flags are `true`. @@ -1076,32 +1083,18 @@ A main push can queue repository-owned GPU runners or create external resources The main-run observer records attempt evidence but does not request broad failed-job reruns. Each E2E test owns any bounded operation-level retry policy. -`Exact staging Brev Launchable` runs only for a trusted manual dispatch against `main`. -The job reads these credentials from repository Actions secrets: +`Publish staging Brev Launchable image` runs only for a trusted manual dispatch against `main`. +The job reads this credential from repository Actions secrets: -- `BREV_API_KEY` authenticates the trusted host-side Brev CLI for workspace - operations in the organization identified by `BREV_ORG_ID`. Candidate code - does not receive this API key. - `NEMOCLAW_IMAGE_DISPATCH_TOKEN` is exposed as `GH_TOKEN` only to the trusted host script. It grants Actions read/write access to `brevdev/nemoclaw-image`, which the script uses to dispatch the image workflow, inspect its run, and download its handoff artifact. -- `NVIDIA_INFERENCE_API_KEY` is exported into the Brev guest for the full E2E - process. Code in the baked candidate checkout can read and use it. - -`brev login` writes `BREV_API_KEY` and `BREV_ORG_ID` to -`$HOME/.brev/credentials.json` on the GitHub-hosted runner. Later trusted steps -and processes in the same job can read that file. The workflow does not delete -it explicitly; it remains on the ephemeral runner filesystem until runner -teardown discards that filesystem. -These credentials remain valid until they expire or an administrator revokes -them in their issuing services. If cleanup fails, remove the recorded Brev -workspace. Rotate or revoke each credential to remove later access. - -The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the -standing Launchable. Keep its value equal to the Launchable ID in the default -URL owned by -[`nemoclaw-maintainer-validate-launchable`](../../.agents/skills/nemoclaw-maintainer-validate-launchable/SKILL.md). + +The credential remains valid until it expires or an administrator revokes it in GitHub. +Rotate or revoke it to remove later access. +The job does not receive `BREV_API_KEY`, `BREV_ORG_ID`, or `NVIDIA_INFERENCE_API_KEY`. +It does not install or authenticate the Brev CLI, create a workspace, or run inference. When an eligible `E2E main` push workflow completes, `E2E / Main Retry` records its conclusion and the available source-attempt evidence. It does not request a broad failed-job or workflow rerun. @@ -1114,7 +1107,7 @@ The observer ignores manual PR runs and a run superseded by a newer `main` push. For a PR revision run, a repository maintainer or administrator leaves `jobs` and `targets` empty. The run selects: -- every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; +- every default-selected free-standing workflow E2E except `Publish staging Brev Launchable image`; - every catalogue target in the `standard` profile; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. diff --git a/test/e2e/docs/README.md b/test/e2e/docs/README.md index 82f786b030..a8d5de979d 100644 --- a/test/e2e/docs/README.md +++ b/test/e2e/docs/README.md @@ -301,7 +301,7 @@ test/e2e/ For a PR revision run, leave `jobs` and `targets` empty. The run selects every default-selected free-standing workflow - E2E except `Exact staging Brev Launchable`, every catalogue target in the + E2E except `Publish staging Brev Launchable image`, every catalogue target in the `standard` profile, all shared credential-free tests, and these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, @@ -363,9 +363,10 @@ test/e2e/ - `.github/workflows/podman-cpu-proof.yaml` provides PR-only experimental runtime evidence with Docker disabled. - `.github/workflows/sandbox-images-and-e2e.yaml` provides reusable image build and test evidence through manual dispatch and `workflow_call`. `.github/workflows/e2e.yaml` selects free-standing jobs, including `whatsapp-qr-compact` and `ollama-auth-proxy`. -- The `staging-brev-launchable` job validates the exact baked candidate in - preinstalled mode. Generic Brev VMs with source overlays are not a - qualification boundary. +- The `staging-brev-launchable` job verifies the exact image-producer receipt, + records the concrete staging image in `launchable-image.json`, and stops before deployment. + Use `nemoclaw-maintainer-validate-launchable` for advisory deployment, runtime, + inference, and cleanup validation while issue #8924 blocks automation. - `vitest.config.ts` contains `e2e-support` for fast fixture/support tests and `e2e-live` for opt-in live target execution. The PR and `main` CLI coverage shards include `e2e-support` for code changes; they never opt into live diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index 4d3d48d0b7..316d8e9568 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -18,7 +18,11 @@ import { import { readWorkflow } from "../../helpers/e2e-workflow-contract"; import { testTimeout } from "../../helpers/timeouts"; -const NO_IMAGE_E2E_JOBS = ["staging-brev-launchable", "shared-e2e"] as const; +const NO_IMAGE_E2E_JOBS = [ + "staging-brev-launchable", + "staging-brev-launchable-e2e-once", + "shared-e2e", +] as const; const AUTH_STEP_NAME = "Authenticate to Docker Hub"; const CLEANUP_STEP_NAME = "Clean up Docker auth"; const CLEANUP_HELPER_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; diff --git a/test/e2e/support/e2e-collaborator-permission-retry.test.ts b/test/e2e/support/e2e-collaborator-permission-retry.test.ts index 635d785e2a..d5081374ac 100644 --- a/test/e2e/support/e2e-collaborator-permission-retry.test.ts +++ b/test/e2e/support/e2e-collaborator-permission-retry.test.ts @@ -36,9 +36,9 @@ const AUTHORIZATION_STEPS: AuthorizationStep[] = [ name: "Authorize release qualification waiver", }, { - deniedMessage: "Launchable image publication requires a repository maintainer or administrator", - mismatchMessage: "Launchable image publication permission response did not match the actor", - name: "Authorize Launchable image publication", + deniedMessage: "Launchable workflow dispatch requires a repository maintainer or administrator", + mismatchMessage: "Launchable workflow dispatch permission response did not match the actor", + name: "Authorize Launchable workflow dispatch", }, ]; diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 4389b510b6..47b1f76c56 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -112,7 +112,7 @@ describe("e2e workflow boundary", () => { () => expect(validateE2eWorkflowBoundary()).toEqual([]), ); - it("rejects a Launchable environment gate, authorization drift, and credential boundary drift", () => { + it("rejects a Launchable environment gate, authorization drift, and credential expansion", () => { const workflow = readWorkflow() as { jobs: Record< string, @@ -130,19 +130,17 @@ describe("e2e workflow boundary", () => { }; const job = workflow.jobs["staging-brev-launchable"]!; job.environment = { name: "unprotected" }; - (job as { env?: Record }).env!.BREV_API_KEY = "${{ secrets.BREV_API_KEY }}"; const prepare = job.steps!.find((step) => step.name === "Prepare the trusted lane")!; + prepare.env ??= {}; prepare.env!.BREV_API_KEY = "${{ secrets.BREV_API_KEY }}"; - prepare.env!.BREV_CLI_SHA256 = "latest"; - const run = job.steps!.find( - (step) => step.name === "Build, deploy, verify, test, and clean up", + const publish = job.steps!.find( + (step) => step.name === "Build and verify the staging Launchable image", )!; - run.env!.GH_TOKEN = "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}"; - run.env!.BREV_LAUNCHABLE_ID = "env-hardcoded"; - run.env!.NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY = "1"; + publish.env!.GH_TOKEN = "${{ secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN }}"; + publish.env!.NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY = "0"; const generateSteps = workflow.jobs["generate-matrix"]!.steps!; const authorization = generateSteps.find( - (step) => step.name === "Authorize Launchable E2E maintainer dispatch", + (step) => step.name === "Authorize Launchable workflow dispatch", )!; delete authorization.env!.TRIGGERING_ACTOR; authorization.run = authorization.run!.replace("maintain | admin", "write"); @@ -151,15 +149,60 @@ describe("e2e workflow boundary", () => { expect(validateE2eWorkflow(workflow)).toEqual( expect.arrayContaining([ "staging-brev-launchable must not use a GitHub environment", - "Launchable E2E maintainer authorization must bind TRIGGERING_ACTOR", - "step 'Authorize Launchable E2E maintainer dispatch' run script must include maintain | admin", - "Launchable E2E maintainer authorization must run before generate-matrix checkout", - "staging-brev-launchable BREV_API_KEY must use the trusted-run secret guard", + "Launchable workflow authorization must bind TRIGGERING_ACTOR", + "step 'Authorize Launchable workflow dispatch' run script must include maintain | admin", + "Launchable workflow authorization must run before generate-matrix checkout", + "staging-brev-launchable preparation step must not receive BREV_API_KEY", "staging-brev-launchable GH_TOKEN must use the trusted-run secret guard", - "staging-brev-launchable must read the repository Launchable ID variable", - "staging-brev-launchable must not stop after image publication", - "staging-brev-launchable job must not receive BREV_API_KEY", - "staging-brev-launchable must pin the Brev CLI version and SHA-256 checksum", + "staging-brev-launchable must stop after verified image publication", + ]), + ); + }); + + it("keeps the one-off Launchable E2E explicit, trusted, pinned, and credential-scoped", () => { + expect(evaluateE2eWorkflowDispatchSelectors({}).selectedFreeStandingJobs).not.toContain( + "staging-brev-launchable-e2e-once", + ); + expect( + evaluateE2eWorkflowDispatchSelectors({ jobs: "staging-brev-launchable-e2e-once" }), + ).toMatchObject({ + valid: true, + selectedFreeStandingJobs: ["staging-brev-launchable-e2e-once"], + }); + expect(buildE2eWorkflowPlan({ jobs: "staging-brev-launchable-e2e-once" })).toMatchObject({ + matrix: [], + selectedJobs: ["staging-brev-launchable-e2e-once"], + }); + + const workflow = readWorkflow() as { + jobs: Record< + string, + { + env?: Record; + if?: string; + steps?: Array<{ env?: Record; name?: string }>; + } + >; + }; + const job = workflow.jobs["staging-brev-launchable-e2e-once"]!; + const install = job.steps!.find( + (step) => step.name === "Install Brev CLI for one-off Launchable E2E", + )!; + const run = job.steps!.find( + (step) => step.name === "Build, deploy, verify, test, and clean up once", + )!; + job.if = "${{ github.event_name == 'workflow_dispatch' }}"; + job.env!.E2E_DEFAULT_ENABLED = "1"; + job.env!.CANDIDATE_SHA = "${{ inputs.checkout_sha || github.sha }}"; + install.env!.BREV_CLI_SHA256 = "latest"; + run.env!.BREV_LAUNCHABLE_ID = "env-hardcoded"; + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "staging-brev-launchable-e2e-once must require its exact trusted-main manual selector", + "staging-brev-launchable-e2e-once must be explicit-only and use an isolated workflow instance", + "staging-brev-launchable-e2e-once must scope Brev credentials and pin the reviewed CLI archive", + "staging-brev-launchable-e2e-once must scope its Launchable, image, inference, and evidence inputs", ]), ); }); @@ -258,7 +301,7 @@ describe("e2e workflow boundary", () => { ); }); - it("selects Launchable E2E only for trusted manual dispatches (#7487)", () => { + it("selects Launchable image publication only for trusted manual dispatches (#7487)", () => { expect( evaluateStagingBrevLaunchableDispatch({ eventName: "workflow_dispatch", @@ -348,7 +391,7 @@ describe("e2e workflow boundary", () => { ); }); - it("rejects superseding full-dispatch and Launchable E2E concurrency drift (#7487)", () => { + it("rejects superseding full-dispatch and Launchable publication concurrency drift (#7487)", () => { const workflow = readWorkflow() as { concurrency: Record; jobs: Record }>; @@ -362,7 +405,7 @@ describe("e2e workflow boundary", () => { expect.arrayContaining([ "workflow concurrency must isolate each full dispatch with github.run_id", "workflow concurrency must not cancel an active Jetson dispatch", - "staging-brev-launchable concurrency must queue all pending Launchable E2E runs without cancellation", + "staging-brev-launchable concurrency must queue all pending image publications without cancellation", ]), ); }); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 9a0a143a08..618971c27a 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -87,9 +87,13 @@ describe("E2E workflow plan", () => { expect(plan.testMatrix).toEqual(discoverCredentialFreeTests()); expect(Object.values(plan.catalogueMatrices).flat()).toHaveLength(E2E_TARGET_CATALOGUE.length); expect(plan.hermesSelected).toBe(true); - expect(plan.explicitOnlyJobs).toEqual(["llama-cpp-dgx-spark-qualification"]); + expect(plan.explicitOnlyJobs).toEqual([ + "staging-brev-launchable-e2e-once", + "llama-cpp-dgx-spark-qualification", + ]); expect(releaseRequiredWorkflowJobs()).toContain("live"); expect(releaseRequiredWorkflowJobs()).toContain("staging-brev-launchable"); + expect(releaseRequiredWorkflowJobs()).not.toContain("staging-brev-launchable-e2e-once"); expect(releaseRequiredWorkflowJobs()).not.toContain("llama-cpp-dgx-spark-qualification"); }); diff --git a/test/maintainer-e2e-skill.test.ts b/test/maintainer-e2e-skill.test.ts index 14d8ea88ed..d3bddbf0f7 100644 --- a/test/maintainer-e2e-skill.test.ts +++ b/test/maintainer-e2e-skill.test.ts @@ -35,10 +35,10 @@ describe("nemoclaw-maintainer-e2e workflow routing", () => { expect(skill).toContain("git rev-parse origin/main"); expect(skill).toContain("correlation_id=${CORRELATION_ID}"); expect(skill).toContain("head_sha"); - expect(skill).toContain("Exact staging Brev Launchable"); + expect(skill).toContain("Publish staging Brev Launchable image"); expect(skill).toContain("Release qualification"); - expect(skill).toContain("launchable-e2e.json"); - expect(skill).toContain("cleanup.json"); + expect(skill).toContain("launchable-image.json"); + expect(skill).toContain("records Launchable, runtime, and inference validation as not run"); expect(skill).toContain("provisional release evidence"); expect(skill).toContain("If the release candidate SHA changes"); expect(skill).toContain("nemoclaw-maintainer-cut-release-tag"); diff --git a/test/maintainer-launchable-skill.test.ts b/test/maintainer-launchable-skill.test.ts index e6b42d99ca..a33db46129 100644 --- a/test/maintainer-launchable-skill.test.ts +++ b/test/maintainer-launchable-skill.test.ts @@ -11,12 +11,20 @@ const launchable = fs.readFileSync( path.join(skillsRoot, "nemoclaw-maintainer-validate-launchable", "SKILL.md"), "utf8", ); +const release = fs.readFileSync( + path.join(skillsRoot, "nemoclaw-maintainer-cut-release-tag", "SKILL.md"), + "utf8", +); const guide = fs.readFileSync(path.join(skillsRoot, "nemoclaw-skills-guide", "SKILL.md"), "utf8"); +const releasePolicy = fs.readFileSync( + path.join(skillsRoot, "nemoclaw-maintainer-policies", "references", "release-train.md"), + "utf8", +); describe("staging Launchable maintainer guidance", () => { it("keeps browser and inference gaps visible as partial validation (#8924)", () => { expect(launchable).toContain( - "https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3I2w334slP4GKSce9kKK0hGerjJ", + "https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo", ); expect(launchable).toContain("When authenticated browser-control tools are available"); expect(launchable).toContain("When browser-control tools are unavailable"); @@ -31,13 +39,13 @@ describe("staging Launchable maintainer guidance", () => { "Require a short-lived inference API key scoped only to the required validation", ); expect(launchable).toContain( - "require a maintainer-approved waiver tied to the exact candidate commit SHA and selected automated Launchable run ID", + "require a maintainer-approved waiver tied to the exact candidate commit SHA and selected image-publication run ID", ); expect(launchable).toContain( "rotate or revoke the inference API key in the issuing NVIDIA service after the run", ); expect(launchable).toContain( - "record its approver, exact candidate commit SHA, selected automated Launchable run ID, and the accepted period of later API-key access", + "record its approver, exact candidate commit SHA, selected image-publication run ID, and the accepted period of later API-key access", ); expect(launchable).toContain( "obtain explicit maintainer approval immediately before starting the credential-bearing process", @@ -67,10 +75,8 @@ describe("staging Launchable maintainer guidance", () => { it("binds image and environment identity before manual validation (#8924)", () => { expect(launchable).toContain( - "`producer.runId` equal to the producer run ID selected by the automated job", + "`producer.runId` equal to the producer run ID selected by the publication job", ); - expect(launchable).toContain("`fullE2e` equal to `passed`"); - expect(launchable).toContain("`boot.bootImage` from `launchable-e2e.json`"); expect(launchable).toContain("Use the supplied environment ID as the authoritative identity"); expect(launchable).toContain( "Use an instance-name lookup only when no environment ID is available", @@ -80,10 +86,25 @@ describe("staging Launchable maintainer guidance", () => { expect(launchable).toContain("`not run` only when no required validation check started"); }); - it("keeps manual Launchable validation separate from automated E2E evidence (#8924)", () => { - expect(launchable).toContain("This report is advisory manual validation"); - expect(launchable).toContain("Do not use it as automated E2E evidence"); - expect(launchable).toContain("Automated Launchable workflow and job URL"); + it("keeps manual Launchable validation advisory during release tagging (#8924)", () => { + expect(release).toContain("nemoclaw-maintainer-validate-launchable"); + expect(release).toContain( + "https://brev.nvidia.com/launchable/deploy/now?launchableID=env-3GdbIjswX4fs3VJ6cYRHr5zoQXo", + ); + expect(release).toContain("Its absence, partial result, or failure does not block"); + expect(release).toContain( + "Do not describe successful image publication as successful Launchable, runtime, or inference validation", + ); + expect(release).toContain( + "do not assume that the mutable family still points to the candidate", + ); + expect(release).toContain("temporary-staging-launchable-qualification-policy"); + expect(releasePolicy).toContain("NemoClaw maintainers own this policy"); + expect(releasePolicy).toContain("GitHub retains the workflow logs and artifact"); + expect(releasePolicy).toContain( + "missing, partial, or failed result needs no per-release waiver", + ); + expect(releasePolicy).toContain("That successful run is the reactivation evidence"); expect(guide).toContain("`nemoclaw-maintainer-validate-launchable`"); }); }); diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index f89dbf28db..eccf91c56f 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -458,8 +458,10 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.name === "Check out the E2E result evaluator" && step.with?.ref === "${{ github.workflow_sha }}"; const trustedLaunchableLaneCheckout = - jobName === "staging-brev-launchable" && - step.name === "Checkout trusted Launchable lane" && + ((jobName === "staging-brev-launchable" && + step.name === "Checkout trusted Launchable lane") || + (jobName === "staging-brev-launchable-e2e-once" && + step.name === "Checkout trusted one-off Launchable lane")) && step.with?.ref === "${{ github.workflow_sha }}"; const trustedPublicationCheckout = jobName === "base-image-publication" && diff --git a/tools/e2e/prepare-e2e-workflow-boundary.mts b/tools/e2e/prepare-e2e-workflow-boundary.mts index 8e7cd7a9d0..71c3ca96da 100644 --- a/tools/e2e/prepare-e2e-workflow-boundary.mts +++ b/tools/e2e/prepare-e2e-workflow-boundary.mts @@ -20,7 +20,10 @@ export const PREPARE_E2E_STEP = "Prepare E2E workspace"; const CHECKOUT_LOCAL_PREPARE_E2E_ACTION = "./.github/actions/prepare-e2e"; export const CLI_ARTIFACT_PRODUCER_JOB = E2E_JOB_POLICY.cliArtifactProducer; -const PREINSTALLED_E2E_JOBS = new Set(["staging-brev-launchable"]); +const PREINSTALLED_E2E_JOBS = new Set([ + "staging-brev-launchable", + "staging-brev-launchable-e2e-once", +]); const NATIVE_RUNTIME_QUALIFICATION_PRODUCER_PREPARE_CONDITION = "${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}"; const RETIRED_SELECTOR_COMPATIBILITY_JOB = "retired-selector-compatibility"; diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index c5b02b050f..447ba92bf2 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -152,6 +152,17 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ "staging-brev-launchable", { name: "staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}", + path: [ + "${{ steps.workspace.outputs.work_dir }}/lane.log", + "${{ steps.workspace.outputs.work_dir }}/launchable-image.json", + "", + ].join("\n"), + }, + ], + [ + "staging-brev-launchable-e2e-once", + { + name: "staging-brev-launchable-e2e-once-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}", path: [ "${{ steps.workspace.outputs.work_dir }}/lane.log", "${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json", @@ -267,6 +278,7 @@ const EXPLICIT_CALLER_CONDITIONS = new Map([ ["native-runtime-qualification-podman-toolchain", "success()"], ["native-runtime-qualification-producer", "success()"], ["staging-brev-launchable", "${{ always() && steps.workspace.outputs.work_dir != '' }}"], + ["staging-brev-launchable-e2e-once", "${{ always() && steps.workspace.outputs.work_dir != '' }}"], ["mcp-bridge", MCP_SCANNED_UPLOAD_CONDITION], ["mcp-bridge-dev", MCP_SCANNED_UPLOAD_CONDITION], ["openshell-dev-artifact", "${{ always() }}"], @@ -397,6 +409,7 @@ export function validateUploadE2eArtifactsInvocations(workflow: WorkflowRecord): const env = record(job.env); return ( jobName === "staging-brev-launchable" || + jobName === "staging-brev-launchable-e2e-once" || jobName === "generate-matrix" || jobName === "jetson-nvmap-gpu" || jobName === "live" || diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f894bfd440..2df07f4e6f 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -170,13 +170,18 @@ const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ "managed-image-protected-runtime", "openshell-credential-generation-window", "staging-brev-launchable", + "staging-brev-launchable-e2e-once", ]); const ADAPTER_MANAGED_INFERENCE_JOBS = new Set(["hermes-e2e"]); const PUBLIC_NVIDIA_ENDPOINT_KEY_JOBS = new Set([ "device-auth-health", "model-router-provider-routed-inference", ]); -const NO_IMAGE_E2E_JOBS = new Set(["staging-brev-launchable", SHARED_E2E_JOB_ID]); +const NO_IMAGE_E2E_JOBS = new Set([ + "staging-brev-launchable", + "staging-brev-launchable-e2e-once", + SHARED_E2E_JOB_ID, +]); const DOCKER_HUB_AUTH_STEP = "Authenticate to Docker Hub"; const DOCKER_HUB_CLEANUP_STEP = "Clean up Docker auth"; const DOCKER_HUB_CLEANUP_RUN = "bash .github/scripts/docker-auth-cleanup.sh"; @@ -1786,8 +1791,8 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord): void { const job = asRecord(jobs["staging-brev-launchable"]); - if (job.name !== "Exact staging Brev Launchable") { - errors.push("staging-brev-launchable must identify the exact Launchable E2E contract"); + if (job.name !== "Publish staging Brev Launchable image") { + errors.push("staging-brev-launchable must identify image publication without claiming E2E"); } if (Object.hasOwn(job, "environment")) { errors.push("staging-brev-launchable must not use a GitHub environment"); @@ -1810,15 +1815,15 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord errors, "generate-matrix", generateSteps, - "Authorize Launchable E2E maintainer dispatch", + "Authorize Launchable workflow dispatch", ); const expectedAuthorizationSelector = - "${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; + "${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; if (authorization?.if !== expectedAuthorizationSelector) { - errors.push("Launchable E2E maintainer authorization must cover exact and full dispatches"); + errors.push("Launchable workflow authorization must cover image and one-off E2E dispatches"); } if (authorization?.shell !== "bash") { - errors.push("Launchable E2E maintainer authorization must use bash"); + errors.push("Launchable workflow authorization must use bash"); } const authorizationEnv = asRecord(authorization?.env); for (const [key, expected] of [ @@ -1827,7 +1832,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord ["TRIGGERING_ACTOR", "${{ github.triggering_actor }}"], ] as const) { if (authorizationEnv[key] !== expected) { - errors.push(`Launchable E2E maintainer authorization must bind ${key}`); + errors.push(`Launchable workflow authorization must bind ${key}`); } } for (const required of [ @@ -1849,7 +1854,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord generateCheckout && generateSteps.indexOf(authorization) >= generateSteps.indexOf(generateCheckout) ) { - errors.push("Launchable E2E maintainer authorization must run before generate-matrix checkout"); + errors.push("Launchable workflow authorization must run before generate-matrix checkout"); } const concurrency = asRecord(job.concurrency); if ( @@ -1858,69 +1863,153 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord concurrency["cancel-in-progress"] !== false ) { errors.push( - "staging-brev-launchable concurrency must queue all pending Launchable E2E runs without cancellation", + "staging-brev-launchable concurrency must queue all pending image publications without cancellation", ); } const steps = asSteps(job.steps); - const checkout = requireStep(errors, steps, "Checkout trusted Launchable lane"); - const checkoutWith = asRecord(checkout?.with); - if ( - checkoutWith.ref !== "${{ github.workflow_sha }}" || - checkoutWith["persist-credentials"] !== false - ) { - errors.push("staging-brev-launchable must check out trusted workflow source without credentials"); - } const prepare = requireStep(errors, steps, "Prepare the trusted lane"); const prepareEnv = asRecord(prepare?.env); - const run = requireStep(errors, steps, "Build, deploy, verify, test, and clean up"); + const run = requireStep(errors, steps, "Build and verify the staging Launchable image"); if (prepare && run && steps.indexOf(prepare) >= steps.indexOf(run)) { - errors.push("staging-brev-launchable must prepare the workspace before the Launchable E2E run"); + errors.push("staging-brev-launchable must prepare the workspace before image publication"); } const runEnv = asRecord(run?.env); - for (const [env, key, secret] of [ - [prepareEnv, "BREV_API_KEY", "BREV_API_KEY"], - [prepareEnv, "BREV_ORG_ID", "BREV_ORG_ID"], - [runEnv, "GH_TOKEN", "NEMOCLAW_IMAGE_DISPATCH_TOKEN"], - [runEnv, "NVIDIA_INFERENCE_API_KEY", "NVIDIA_INFERENCE_API_KEY"], - ] as const) { - const expected = `\${{ ${trustedRun} && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.${secret} || '' }}`; - if (env[key] !== expected) { - errors.push(`staging-brev-launchable ${key} must use the trusted-run secret guard`); - } + const expectedImageToken = `\${{ ${trustedRun} && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }}`; + if (runEnv.GH_TOKEN !== expectedImageToken) { + errors.push("staging-brev-launchable GH_TOKEN must use the trusted-run secret guard"); } - if (runEnv.BREV_LAUNCHABLE_ID !== "${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}") { - errors.push("staging-brev-launchable must read the repository Launchable ID variable"); + if (runEnv.NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY !== "1") { + errors.push("staging-brev-launchable must stop after verified image publication"); } if (runEnv.WORK_DIR !== "${{ steps.workspace.outputs.work_dir }}") { errors.push("staging-brev-launchable must pass its private evidence directory to the lane"); } - if (Object.hasOwn(runEnv, "NEMOCLAW_BREV_LAUNCHABLE_IMAGE_ONLY")) { - errors.push("staging-brev-launchable must not stop after image publication"); - } - const jobEnv = asRecord(job.env); - for (const [env, scope, forbidden] of [ - [jobEnv, "job", ["BREV_API_KEY", "BREV_ORG_ID", "GH_TOKEN", "NVIDIA_INFERENCE_API_KEY"]], - [prepareEnv, "preparation step", ["GH_TOKEN", "NVIDIA_INFERENCE_API_KEY"]], - [runEnv, "execution step", ["BREV_API_KEY", "BREV_ORG_ID"]], + for (const [env, scope] of [ + [asRecord(job.env), "job"], + [prepareEnv, "preparation step"], + [runEnv, "image publication step"], ] as const) { - for (const key of forbidden) { + for (const key of [ + "BREV_API_KEY", + "BREV_ORG_ID", + "BREV_LAUNCHABLE_ID", + "NVIDIA_INFERENCE_API_KEY", + ]) { if (Object.hasOwn(env, key)) { errors.push(`staging-brev-launchable ${scope} must not receive ${key}`); } } } + const prepareRun = stringValue(prepare?.run); if ( - !/^0\.\d+\.\d+$/u.test(stringValue(prepareEnv.BREV_CLI_VERSION)) || - !/^[0-9a-f]{64}$/u.test(stringValue(prepareEnv.BREV_CLI_SHA256)) + prepareRun.includes("brev login") || + prepareRun.includes("BREV_CLI_VERSION") || + prepareRun.includes("BREV_CLI_SHA256") ) { - errors.push("staging-brev-launchable must pin the Brev CLI version and SHA-256 checksum"); + errors.push( + "staging-brev-launchable preparation must not install or authenticate the Brev CLI", + ); } - const prepareRun = stringValue(prepare?.run); +} + +function validateOneOffStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "staging-brev-launchable-e2e-once"; + const job = asRecord(jobs[jobName]); + if (job.name !== "Run one-off staging Brev Launchable E2E") { + errors.push(`${jobName} must identify its one-off full E2E scope`); + } + if (Object.hasOwn(job, "environment")) { + errors.push(`${jobName} must not use a GitHub environment`); + } + const trustedRun = "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main'"; + const expectedSelector = + "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == '' }}"; + if (job.if !== expectedSelector) { + errors.push(`${jobName} must require its exact trusted-main manual selector`); + } + if ( + job.needs !== "generate-matrix" || + job["runs-on"] !== "ubuntu-latest" || + job["timeout-minutes"] !== 180 || + asRecord(job.permissions).contents !== "read" + ) { + errors.push(`${jobName} must retain its trusted runner, dependency, permission, and timeout`); + } + const concurrency = asRecord(job.concurrency); + if ( + concurrency.group !== "staging-brev-launchable-cpu" || + concurrency.queue !== "max" || + concurrency["cancel-in-progress"] !== false + ) { + errors.push(`${jobName} must queue with Launchable image publication without cancellation`); + } + const expectedJobEnv = { + CANDIDATE_SHA: "${{ github.sha }}", + E2E_DEFAULT_ENABLED: "0", + E2E_JOB: "1", + INSTANCE_NAME: "nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }}", + }; + if (!isDeepStrictEqual(asRecord(job.env), expectedJobEnv)) { + errors.push(`${jobName} must be explicit-only and use an isolated workflow instance`); + } + + const steps = asSteps(job.steps); + const checkout = requireStep(errors, steps, "Checkout trusted one-off Launchable lane"); + const checkoutWith = asRecord(checkout?.with); + if ( + checkoutWith.ref !== "${{ github.workflow_sha }}" || + checkoutWith["persist-credentials"] !== false || + checkoutWith["sparse-checkout"] !== "tools/e2e/brev-launchable-e2e.sh\n" || + checkoutWith["sparse-checkout-cone-mode"] !== false + ) { + errors.push(`${jobName} must check out only the trusted Launchable lane without credentials`); + } + const prepare = requireStep(errors, steps, "Prepare the one-off Launchable lane"); + const install = requireStep(errors, steps, "Install Brev CLI for one-off Launchable E2E"); + const run = requireStep(errors, steps, "Build, deploy, verify, test, and clean up once"); + if ( + !prepare || + !install || + !run || + steps.indexOf(prepare) >= steps.indexOf(install) || + steps.indexOf(install) >= steps.indexOf(run) + ) { + errors.push(`${jobName} must prepare, authenticate, and run in that order`); + } + const secretGuard = `${trustedRun} && github.event_name == 'workflow_dispatch' && inputs.jobs == 'staging-brev-launchable-e2e-once' && inputs.targets == '' && inputs.checkout_sha == ''`; + const expectedSecret = (secret: string): string => + `\${{ ${secretGuard} && secrets.${secret} || '' }}`; + const installEnv = asRecord(install?.env); + if ( + !isDeepStrictEqual(installEnv, { + BREV_API_KEY: expectedSecret("BREV_API_KEY"), + BREV_CLI_SHA256: "d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32", + BREV_CLI_VERSION: "0.6.334", + BREV_ORG_ID: expectedSecret("BREV_ORG_ID"), + }) + ) { + errors.push(`${jobName} must scope Brev credentials and pin the reviewed CLI archive`); + } + const installRun = stringValue(install?.run); if ( - !prepareRun.includes("sha256sum -c -") || - !prepareRun.includes("brev login --api-key \"$BREV_API_KEY\" --org-id \"$BREV_ORG_ID\"") + !installRun.includes("sha256sum -c -") || + !installRun.includes('brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID"') ) { - errors.push("staging-brev-launchable must verify and authenticate the pinned Brev CLI"); + errors.push(`${jobName} must verify and authenticate the pinned Brev CLI`); + } + const runEnv = asRecord(run?.env); + if ( + !isDeepStrictEqual(runEnv, { + BREV_LAUNCHABLE_ID: "${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}", + GH_TOKEN: expectedSecret("NEMOCLAW_IMAGE_DISPATCH_TOKEN"), + NVIDIA_INFERENCE_API_KEY: expectedSecret("NVIDIA_INFERENCE_API_KEY"), + WORK_DIR: "${{ steps.workspace.outputs.work_dir }}", + }) + ) { + errors.push(`${jobName} must scope its Launchable, image, inference, and evidence inputs`); + } + if (run?.run !== "tools/e2e/brev-launchable-e2e.sh") { + errors.push(`${jobName} must run only the trusted Launchable lane script`); } } @@ -1936,12 +2025,12 @@ function validateStagingBrevLaunchableInput( } const description = stringValue(input.description); if ( - !description.includes("Exact staging Brev Launchable") || + !description.includes("staging Brev Launchable image publication") || !description.includes("jobs and targets are empty") || !description.includes("full E2E run") ) { errors.push( - "workflow_dispatch include_staging_brev_launchable input must document full-run Launchable E2E scope", + "workflow_dispatch include_staging_brev_launchable input must document full-run Launchable image publication scope", ); } } @@ -2268,9 +2357,30 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { const jobsDescription = stringValue(jobsInput.description); if (!jobsDescription.includes("include_staging_brev_launchable")) { errors.push( - "workflow_dispatch jobs input description must identify how to include Exact staging Brev Launchable", + "workflow_dispatch jobs input description must identify how to include staging Brev Launchable image publication", + ); + } + if (!jobsDescription.includes("staging-brev-launchable-e2e-once")) { + errors.push( + "workflow_dispatch jobs input description must identify the one-off staging Launchable E2E selector", ); } + for (const required of [ + "only on current main", + "$HOME/.brev/credentials.json", + "later trusted job processes", + "hosted-runner teardown", + "trusted host step receives the image token", + "baked candidate code receives the NVIDIA inference key", + "expiry or revocation", + ]) { + if (!jobsDescription.includes(required)) { + errors.push( + "workflow_dispatch jobs input description must document one-off Launchable credential custody", + ); + break; + } + } if (Object.hasOwn(dispatchInputs, "test_filter")) { errors.push("workflow_dispatch must not expose legacy test_filter input"); } @@ -2488,9 +2598,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (liveTargets["runs-on"] !== "${{ matrix.runner }}") { errors.push("live job must run on the matrix runner"); } - if ( - !isDeepStrictEqual(liveTargets.needs, ["base-image-publication", "generate-matrix"]) - ) { + if (!isDeepStrictEqual(liveTargets.needs, ["base-image-publication", "generate-matrix"])) { errors.push("live job must depend on base-image-publication and generate-matrix"); } if (liveTargets.if !== "${{ needs.generate-matrix.outputs.matrix != '[]' }}") { @@ -2823,6 +2931,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateSharedE2eJob(errors, jobs); validateStagingBrevLaunchableJob(errors, jobs); + validateOneOffStagingBrevLaunchableJob(errors, jobs); validateCatalogueOwnedJobs(errors, jobs); validateHermesE2EJob(errors, jobs); validateHermesTimeoutHeadroom(errors, jobs);