From 236d5b4418ad3756c5c6f05babff8ca84e5b4d9f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Mon, 17 Aug 2026 16:40:22 -0500 Subject: [PATCH 01/16] ci(e2e): authorize NVIDIA-owned PR revisions Signed-off-by: Aaron Erickson --- .../skills/nemoclaw-maintainer-e2e/SKILL.md | 52 ++-- .github/workflows/e2e.yaml | 291 +++++++----------- test/e2e/README.md | 49 +-- test/e2e/RETRY_INVENTORY.md | 2 +- .../dockerhub-auth-workflow-boundary.test.ts | 2 +- .../e2e-collaborator-permission-retry.test.ts | 15 +- .../e2e-operations-workflow-boundary.test.ts | 180 ++++++++--- ...e2e-report-to-pr-workflow-boundary.test.ts | 2 + test/e2e/support/e2e-workflow.test.ts | 27 +- .../support/hermes-workflow-boundary.test.ts | 4 +- ...d-image-protected-runtime-workflow.test.ts | 17 + ...standard-profile-workflow-boundary.test.ts | 12 + test/e2e/support/workflow-plan.test.ts | 66 ++-- ...ge-protected-runtime-workflow-boundary.mts | 4 +- .../e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- tools/e2e/operations-workflow-boundary.mts | 62 ++-- .../standard-profile-workflow-boundary.mts | 6 +- tools/e2e/workflow-boundary.mts | 117 +++++-- tools/e2e/workflow-plan.mts | 4 +- 19 files changed, 552 insertions(+), 362 deletions(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index 6f0404a296f..ed6bd8a430f 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -19,18 +19,20 @@ Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly ## Manual PR E2E -Use this mode when the maintainer requests E2E for a pull request. -It runs an authorized E2E selection against the current PR head commit while the workflow definition remains on `main`. +Use this mode when a repository-authorized dispatcher requests E2E for a pull request. +GitHub's permission to dispatch the workflow is the actor authorization; do not add a second repository-role check. +The run executes an E2E selection against the current PR head commit while the workflow definition remains on `main`. It is advisory and does not create a required PR check. -An empty-selector manual run exposes these values to candidate-controlled job processes: +An empty-selector NVIDIA-owned PR run exposes these values to candidate-controlled job processes: - Long-lived API keys from repository secrets: `NVIDIA_INFERENCE_API_KEY`, `NVIDIA_API_KEY`, and `BRAVE_API_KEY`. +- Docker Hub credentials from `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN`, available to candidate processes through the job's temporary Docker configuration until cleanup. - Long-lived messaging credentials from repository secrets: `TELEGRAM_BOT_TOKEN_REAL`, `DISCORD_BOT_TOKEN_REAL`, `SLACK_BOT_TOKEN_REAL`, and `SLACK_APP_TOKEN_REAL`. -- The job-scoped `GITHUB_TOKEN` in the `token-rotation` and `openshell-gateway-upgrade` jobs. It has `checks: read`, `contents: read`, and `pull-requests: read` access. Candidate code can use it while either job runs. GitHub Actions invalidates it after the job. +- The job-scoped `GITHUB_TOKEN` in the `token-rotation` and `openshell-gateway-upgrade` jobs. It has `contents: read` access. Candidate code can use it while either job runs. GitHub Actions invalidates it after the job. - Messaging account and channel identifiers from repository secrets: `TELEGRAM_ALLOWED_IDS`, `TELEGRAM_AUTHORIZED_CHAT_IDS`, `TELEGRAM_CHAT_ID`, `TELEGRAM_CHAT_ID_E2E`, `DISCORD_CHANNEL_ID_E2E`, and `SLACK_CHANNEL_ID_E2E`. -The workflow does not rotate or revoke these API keys or messaging credentials. To remove later access, rotate or revoke every listed credential in the external service that issued it. The workflow cannot erase identifiers copied by candidate code. Review the complete candidate diff before dispatch. +The workflow does not rotate or revoke these API keys, Docker Hub credentials, or messaging credentials. To remove later access, rotate or revoke every listed credential in the external service that issued it. The workflow cannot erase identifiers copied by candidate code. Review the complete candidate diff before dispatch. Live targets can create external resources. After a failure, inspect the artifacts and remove resources that target cleanup did not remove. @@ -42,7 +44,7 @@ After a failure, inspect the artifacts and remove resources that target cleanup `brev login` writes `BREV_API_KEY` and `BREV_ORG_ID` to `$HOME/.brev/credentials.json` on the GitHub-hosted runner. Later trusted steps and processes in the same job can read that file. The workflow does not delete it explicitly; it remains on the ephemeral runner filesystem until runner teardown discards that filesystem. These credentials remain valid until they expire or an administrator revokes them in their issuing services. If cleanup fails, remove the recorded Brev workspace. Rotate or revoke each credential to remove later access. -This Brev credential boundary applies only to trusted Launchable or full manual dispatches against `main`. It does not apply to `main` pushes or manual PR runs. +This Brev credential boundary applies to trusted manual dispatches against `main` that select Launchable, including NVIDIA-owned PR revision runs. It does not apply to `main` pushes or non-Launchable PR runs. The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the standing Launchable. Keep its value equal to the Launchable ID in the default URL owned by [`nemoclaw-maintainer-validate-launchable`](../nemoclaw-maintainer-validate-launchable/SKILL.md). @@ -57,36 +59,37 @@ git fetch --prune origin main WORKFLOW_SHA="$(git rev-parse origin/main)" PR_JSON="$(gh api "repos/NVIDIA/NemoClaw/pulls/${PR_NUMBER}")" test "$(jq -r .state <<<"$PR_JSON")" = open +test "$(jq -r .base.repo.full_name <<<"$PR_JSON")" = NVIDIA/NemoClaw +test "$(jq -r .base.ref <<<"$PR_JSON")" = main HEAD_SHA="$(jq -r .head.sha <<<"$PR_JSON")" BASE_SHA="$(jq -r .base.sha <<<"$PR_JSON")" HEAD_REPOSITORY="$(jq -r .head.repo.full_name <<<"$PR_JSON")" +HEAD_OWNER="$(jq -r .head.repo.owner.login <<<"$PR_JSON")" +HEAD_OWNER_TYPE="$(jq -r .head.repo.owner.type <<<"$PR_JSON")" [[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$BASE_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]] ``` -Require a review reason containing 10 to 500 printable characters. - Choose exactly one mode: -- For a PR revision run, leave `E2E_JOBS` empty. The run selects: +- For an NVIDIA-owned PR revision, require `HEAD_OWNER=NVIDIA` and `HEAD_OWNER_TYPE=Organization`. Empty `jobs` and `targets` select every default-enabled E2E, including every catalogue credential profile. Any supported E2E job or target selector is allowed. +- For an external PR revision, the existing bounded controller selection remains in effect: - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; - every shared credential-free test; and - these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. - The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. - It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. + +Both modes skip `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. +Jetson and Launchable dispatch additionally require the PR branch to be in `NVIDIA/NemoClaw`; their operator and image-producer backends do not accept a sibling-repository candidate. +Both modes skip `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. + - For protected managed-image runtime qualification, set `E2E_JOBS=managed-image-protected-runtime`. The exact candidate must contain `ci/protected-managed-image-multiarch-activation-v1.json` and `ci/protected-managed-image-runtime-activation-v1.json`. - For native-runtime qualification evidence, set `E2E_JOBS=native-runtime-qualification-producer`. Use a same-repository open PR and the first workflow attempt. The trusted workflow runs each case under a credential-free candidate account on a reviewed ephemeral runner. The candidate must contain `test/e2e/live/native-runtime-qualification-case.test.ts` before the selector can pass. -Leave `targets` empty and keep Launchable disabled: +The ordinary PR example leaves `targets` empty and keeps Launchable disabled: ```bash E2E_JOBS="${E2E_JOBS:-}" -case "$E2E_JOBS" in - "" | managed-image-protected-runtime | native-runtime-qualification-producer) ;; - *) echo "Unsupported manual PR E2E job selector" >&2; exit 1 ;; -esac -REVIEW_REASON='Reviewed the commit under review and selected E2E boundary.' CORRELATION_ID="$(python3 -c 'import uuid; print(uuid.uuid4())')" INFERENCE_MODE=mock ALLOW_JETSON_DISPATCH=false @@ -105,13 +108,14 @@ gh workflow run .github/workflows/e2e.yaml \ -f "checkout_repository=${HEAD_REPOSITORY}" \ -f "base_sha=${BASE_SHA}" \ -f "workflow_sha=${WORKFLOW_SHA}" \ - -f "review_reason=${REVIEW_REASON}" \ -f "correlation_id=${CORRELATION_ID}" ``` -The trusted pre-checkout step requires current `maintain` or `admin` permission. -It validates the actor, open PR, repository, head SHA, base SHA, workflow SHA, review reason, and allowed jobs, targets, and Launchable combination. -A second validation after checkout rejects a changed PR identity before preparation. +The trusted pre-checkout step validates the open PR, target repository and branch, source repository, source owner, head SHA, base SHA, and workflow SHA. +An API-confirmed `NVIDIA` organization owner unlocks the full candidate plan and credential profiles. +External PRs retain the bounded controller matrix and receive no repository credentials in candidate-controlled processes or reusable workflow callers. +A second validation after checkout rejects a changed PR identity or NVIDIA ownership before preparation. +Candidate PR runs cannot publish release qualification. The native-runtime producer binds the open PR, candidate commit, base commit, trusted workflow commit, and first workflow attempt. It runs the trusted plan from `main` and passes no GitHub, model-provider, API, or messaging credentials to candidate code. Configure `NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL=enabled` before dispatch. The ARM64 GPU case also requires `NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL`; the workflow provides no fallback runner. @@ -141,7 +145,7 @@ RUN_URL="$(jq -r '.[0].url' <<<"$MATCHES")" gh run watch "$RUN_ID" --repo NVIDIA/NemoClaw --exit-status RUN_JSON="$(gh api "repos/NVIDIA/NemoClaw/actions/runs/${RUN_ID}")" jq -e --arg sha "$WORKFLOW_SHA" ' - .run_attempt == 1 and + .run_attempt >= 1 and .head_sha == $sha and .status == "completed" and .conclusion == "success" @@ -151,10 +155,12 @@ test "$(jq -r .state <<<"$CURRENT_PR")" = open test "$(jq -r .head.sha <<<"$CURRENT_PR")" = "$HEAD_SHA" test "$(jq -r .base.sha <<<"$CURRENT_PR")" = "$BASE_SHA" test "$(jq -r .head.repo.full_name <<<"$CURRENT_PR")" = "$HEAD_REPOSITORY" +test "$(jq -r .head.repo.owner.login <<<"$CURRENT_PR")" = "$HEAD_OWNER" +test "$(jq -r .head.repo.owner.type <<<"$CURRENT_PR")" = "$HEAD_OWNER_TYPE" ``` Return the PR number, head repository, head SHA, base SHA, workflow SHA, correlation ID, workflow URL, and result. -A changed head repository, head SHA, or base SHA invalidates the evidence and requires a new run. +A changed head repository, repository owner, head SHA, or base SHA invalidates the evidence and requires a new run. ## Select the Main Mode diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 4f037e4e2ab..b4c287e189c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -10,12 +10,12 @@ on: workflow_dispatch: inputs: targets: - description: "Optional comma-separated typed target IDs. On trusted main runs, empty jobs and targets select all supported registry E2E tests. PR revision runs use the trusted controller target matrix." + description: "Optional comma-separated typed target IDs. Empty jobs and targets select the default suite. NVIDIA-owned PR revisions may select any supported target." required: false default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix." + description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson dispatch and DGX Spark require their opt-in flags. NVIDIA-owned PR revisions may select any supported E2E job; external PR revisions retain the bounded controller matrix." required: false default: "" type: string @@ -74,7 +74,7 @@ on: default: "" type: string review_reason: - description: Optional review reason containing 10 to 500 printable characters. + description: Deprecated optional audit note retained for dispatcher compatibility; not used for authorization. required: false default: "" type: string @@ -102,7 +102,7 @@ permissions: concurrency: group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && format('full-{0}', github.run_id)) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }} - cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }} + cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} @@ -206,41 +206,6 @@ jobs: catalogue_brave_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_brave_nvidia_inference_matrix }} runner_routing: ${{ steps.runner_routing.outputs.runner_routing }} steps: - - id: controller_matrix - name: Build trusted controller target matrix - if: ${{ inputs.checkout_sha != '' }} - env: - JOBS: ${{ inputs.jobs }} - TARGETS: ${{ inputs.targets }} - shell: bash - run: | - set -euo pipefail - test_matrix='[]' - case "${JOBS}:${TARGETS}" in - :) - matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]' - test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/vllm-docker-storage.test.ts","project":"integration"}]' - ;; - inference-routing: | managed-image-protected-runtime: | native-runtime-qualification-producer: | :jetson-nvmap-gpu) - matrix='[]' - ;; - :ubuntu-repo-cloud-langchain-deepagents-code) - matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"}]' - ;; - :ubuntu-repo-docker-post-reboot-recovery) - matrix='[{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' - ;; - :ubuntu-repo-cloud-langchain-deepagents-code,ubuntu-repo-docker-post-reboot-recovery) - matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' - ;; - *) - echo "::error::PR E2E target is not approved by the trusted controller" >&2 - exit 1 - ;; - esac - printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" - printf 'test_matrix=%s\n' "${test_matrix}" >> "${GITHUB_OUTPUT}" - - id: runner_routing name: Build trusted larger-runner routing env: @@ -262,24 +227,18 @@ jobs: runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger}')" printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}" - - name: Authenticate manual PR dispatch + - id: candidate_authorization + name: Authenticate manual PR dispatch if: ${{ inputs.checkout_sha != '' }} env: - ACTOR: ${{ github.actor }} - ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }} BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} GITHUB_TOKEN: ${{ github.token }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} - - INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable }} + INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }} JOBS: ${{ inputs.jobs }} PR_NUMBER: ${{ inputs.pr_number }} - REVIEW_REASON: ${{ inputs.review_reason }} - RUN_ATTEMPT: ${{ github.run_attempt }} - TARGETS: ${{ inputs.targets }} - TRIGGERING_ACTOR: ${{ github.triggering_actor }} WORKFLOW_EVENT: ${{ github.event_name }} WORKFLOW_REF: ${{ github.ref }} WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -288,115 +247,78 @@ jobs: run: | set -euo pipefail - read_collaborator_permission() { - local maintainer="$1" - local attempt curl_exit failure http_status permission_file - permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")" - - for attempt in 1 2 3; do - : >"$permission_file" - if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \ - --output "$permission_file" --write-out "%{http_code}" \ - --header "Authorization: Bearer ${GITHUB_TOKEN}" \ - --header "Accept: application/vnd.github+json" \ - --header "X-GitHub-Api-Version: 2022-11-28" \ - "https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \ - 2>/dev/null)"; then - if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then - if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then - if (( attempt > 1 )); then - echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2 - fi - cat "$permission_file" - rm -f "$permission_file" - return 0 - fi - echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2 - rm -f "$permission_file" - return 1 - fi - if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then - failure="HTTP ${http_status}" - case "$http_status" in - 408 | 429 | 5??) ;; - *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;; - esac - else - echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2 - rm -f "$permission_file" - return 1 - fi - else - curl_exit=$? - case "$curl_exit" in - 5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;; - *) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;; - esac - fi - - if (( attempt == 3 )); then - echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2 - rm -f "$permission_file" - return 1 - fi - echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2 - sleep "$attempt" - done - } - - require_maintainer() { - local maintainer="$1" - [[ "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ && "$maintainer" != -* && "$maintainer" != *- ]] || { - echo "::error::Manual PR E2E actor is invalid" >&2 - exit 1 - } - local permission_json - permission_json="$(read_collaborator_permission "$maintainer")" - if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then - echo "::error::Manual PR E2E permission response did not match the actor" >&2 - exit 1 - fi - case "$(jq -r '.role_name // ""' <<< "$permission_json")" in - maintain | admin) ;; - *) echo "::error::Manual PR E2E requires a repository maintainer or administrator" >&2; exit 1 ;; - esac - } - [[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || { echo "::error::Manual PR E2E must be dispatched from main" >&2 exit 1 } - [[ "$RUN_ATTEMPT" == "1" ]] || { echo "::error::Manual PR E2E cannot be rerun" >&2; exit 1; } - case "${JOBS}:${TARGETS}:${INCLUDE_LAUNCHABLE}:${ALLOW_JETSON_DISPATCH}" in - ::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | native-runtime-qualification-producer::false:false | :jetson-nvmap-gpu:false:true) ;; - *) - echo "::error::Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag" >&2 - exit 1 - ;; - esac [[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer" >&2; exit 1; } [[ "$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::checkout_repository must be an owner/repository name" >&2; exit 1; } [[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA" >&2; exit 1; } [[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::base_sha must be a lowercase 40-character SHA" >&2; exit 1; } - [[ "$REVIEW_REASON" =~ ^[[:print:]]+$ ]] && - (( ${#REVIEW_REASON} >= 10 && ${#REVIEW_REASON} <= 500 )) || - { echo "::error::review_reason must contain 10 to 500 printable characters" >&2; exit 1; } [[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; } - require_maintainer "$ACTOR" - if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then - require_maintainer "$TRIGGERING_ACTOR" - fi - pull_json="$(curl --fail --silent --show-error --proto '=https' \ --header "Authorization: Bearer ${GITHUB_TOKEN}" \ --header "Accept: application/vnd.github+json" \ --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; } + [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request must target NVIDIA/NemoClaw" >&2; exit 1; } + [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request must target main" >&2; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; } + nvidia_owned=false + if [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && + "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]]; then + nvidia_owned=true + fi + if [[ "$INCLUDE_LAUNCHABLE" == "true" || ",${JOBS}," == *",staging-brev-launchable,"* ]]; then + [[ "$nvidia_owned" == "true" ]] || { + echo "::error::Launchable PR E2E requires an NVIDIA-owned source repository" >&2 + exit 1 + } + [[ "$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw" ]] || { + echo "::error::Launchable PR E2E requires a branch in NVIDIA/NemoClaw" >&2 + exit 1 + } + fi + printf 'nvidia_owned=%s\n' "$nvidia_owned" >> "$GITHUB_OUTPUT" + + - id: controller_matrix + name: Build trusted controller target matrix + if: ${{ inputs.checkout_sha != '' && steps.candidate_authorization.outputs.nvidia_owned != 'true' }} + env: + JOBS: ${{ inputs.jobs }} + TARGETS: ${{ inputs.targets }} + shell: bash + run: | + set -euo pipefail + test_matrix='[]' + case "${JOBS}:${TARGETS}" in + :) + matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]' + test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/vllm-docker-storage.test.ts","project":"integration"}]' + ;; + inference-routing: | managed-image-protected-runtime: | native-runtime-qualification-producer: | :jetson-nvmap-gpu) + matrix='[]' + ;; + :ubuntu-repo-cloud-langchain-deepagents-code) + matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"}]' + ;; + :ubuntu-repo-docker-post-reboot-recovery) + matrix='[{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' + ;; + :ubuntu-repo-cloud-langchain-deepagents-code,ubuntu-repo-docker-post-reboot-recovery) + matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]' + ;; + *) + echo "::error::PR E2E target is not approved by the trusted controller" >&2 + exit 1 + ;; + esac + printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}" + printf 'test_matrix=%s\n' "${test_matrix}" >> "${GITHUB_OUTPUT}" - name: Authorize release qualification waiver if: ${{ github.event_name == 'workflow_dispatch' && (inputs.release_qualification_waived_jobs != '' || inputs.release_qualification_waiver_reason != '') }} @@ -599,7 +521,7 @@ jobs: path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json - name: Authorize Launchable E2E maintainer dispatch - if: ${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} + if: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} env: ACTOR: ${{ github.actor }} GITHUB_TOKEN: ${{ github.token }} @@ -706,6 +628,7 @@ jobs: CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} CHECKOUT_SHA: ${{ inputs.checkout_sha }} GITHUB_TOKEN: ${{ github.token }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} PR_NUMBER: ${{ inputs.pr_number }} shell: bash run: | @@ -717,9 +640,18 @@ jobs: --header "X-GitHub-Api-Version: 2022-11-28" \ "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")" [[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; } + [[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]] || { echo "::error::pull request base repository changed before execution" >&2; exit 1; } + [[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]] || { echo "::error::pull request base branch changed before execution" >&2; exit 1; } [[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before execution" >&2; exit 1; } [[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before execution" >&2; exit 1; } [[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; } + if [[ "$NVIDIA_OWNED" == "true" ]]; then + [[ "$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA" && + "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization" ]] || { + echo "::error::PR source repository ownership changed before execution" >&2 + exit 1 + } + fi - id: e2e_credentials name: Authorize E2E credentials @@ -729,6 +661,7 @@ jobs: CHECKOUT_SHA: ${{ inputs.checkout_sha }} EVENT_NAME: ${{ github.event_name }} EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} REF: ${{ github.ref }} WORKFLOW_REPOSITORY: ${{ github.repository }} WORKFLOW_SHA: ${{ github.workflow_sha }} @@ -737,7 +670,7 @@ jobs: set -euo pipefail credentials_allowed=false if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" && - "$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY" && + "$NVIDIA_OWNED" == "true" && "$EVENT_NAME" == "workflow_dispatch" && "$REF" == "refs/heads/main" && "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && @@ -941,6 +874,8 @@ jobs: BEFORE_SHA: ${{ github.event.before }} CANDIDATE_SHA: ${{ github.sha }} RELEASE_QUALIFICATION_WAIVED_JOBS: ${{ inputs.release_qualification_waived_jobs }} + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: ${{ (inputs.checkout_sha == '' || steps.e2e_credentials.outputs.allowed == 'true') && 'true' || 'false' }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} run: | set -euo pipefail if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then @@ -970,7 +905,7 @@ jobs: export CHANGED_FILES fi npx tsx tools/e2e/workflow-plan.mts --ci-output - if [ -n "${CHECKOUT_SHA}" ]; then + if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]; then matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")" expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")" actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")" @@ -2685,7 +2620,7 @@ jobs: staging-brev-launchable: name: Exact staging Brev Launchable needs: generate-matrix - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} + if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }} runs-on: ubuntu-latest timeout-minutes: 180 permissions: @@ -2711,10 +2646,10 @@ jobs: - id: workspace name: Prepare the trusted lane env: - BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_API_KEY || '' }} + BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.BREV_API_KEY || '' }} BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32 BREV_CLI_VERSION: 0.6.334 - BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_ORG_ID || '' }} + BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.BREV_ORG_ID || '' }} run: | set -euo pipefail work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")" @@ -2729,8 +2664,8 @@ jobs: - name: Build, deploy, verify, test, and clean up env: BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }} - GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} - NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} WORK_DIR: ${{ steps.workspace.outputs.work_dir }} run: tools/e2e/brev-launchable-e2e.sh @@ -2773,9 +2708,9 @@ jobs: name: Authenticate to Docker Hub uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f with: - auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }} - username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }} - token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_TOKEN || '' }} + auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && '1' || '0' }} + username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} - name: Configure live E2E trace directory env: @@ -2855,7 +2790,7 @@ jobs: - name: Run live E2E tests env: E2E_TARGET_ID: ${{ matrix.id }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} TARGET_ID: ${{ matrix.id }} run: | set -euo pipefail @@ -3049,8 +2984,8 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-nvidia-api: name: ${{ matrix.display_name }} @@ -3088,9 +3023,9 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} + NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} catalogue-nvidia-inference: name: ${{ matrix.display_name }} @@ -3128,9 +3063,9 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} catalogue-github-read: name: ${{ matrix.display_name }} @@ -3168,8 +3103,8 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} catalogue-brave-nvidia-inference: name: ${{ matrix.display_name }} @@ -3208,10 +3143,10 @@ jobs: artifact_layout: ${{ matrix.artifact_layout }} trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }} secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }} - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + DOCKERHUB_USERNAME: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + DOCKERHUB_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} + BRAVE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.BRAVE_API_KEY || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} openshell-gateway-auth-contract: needs: generate-matrix @@ -4824,7 +4759,7 @@ jobs: - name: Run all-agent GPU, local inference, rollback, and cleanup qualification env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} shell: bash run: | set -euo pipefail @@ -4942,7 +4877,7 @@ jobs: - name: Run Hermes live Vitest test env: - NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts @@ -5338,7 +5273,7 @@ jobs: - name: Run cloud-onboard live Vitest test env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" @@ -5436,17 +5371,17 @@ jobs: # the legacy fake-token defaults, optional _REAL secret overrides, # provider placeholder redaction checks, and WhatsApp QR-only parity. env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - TELEGRAM_BOT_TOKEN_REAL: ${{ secrets.TELEGRAM_BOT_TOKEN_REAL }} - DISCORD_BOT_TOKEN_REAL: ${{ secrets.DISCORD_BOT_TOKEN_REAL }} - SLACK_BOT_TOKEN_REAL: ${{ secrets.SLACK_BOT_TOKEN_REAL }} - SLACK_APP_TOKEN_REAL: ${{ secrets.SLACK_APP_TOKEN_REAL }} - TELEGRAM_ALLOWED_IDS: ${{ secrets.TELEGRAM_ALLOWED_IDS }} - TELEGRAM_AUTHORIZED_CHAT_IDS: ${{ secrets.TELEGRAM_AUTHORIZED_CHAT_IDS }} - TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} - TELEGRAM_CHAT_ID_E2E: ${{ secrets.TELEGRAM_CHAT_ID_E2E }} - DISCORD_CHANNEL_ID_E2E: ${{ secrets.DISCORD_CHANNEL_ID_E2E }} - SLACK_CHANNEL_ID_E2E: ${{ secrets.SLACK_CHANNEL_ID_E2E }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + TELEGRAM_BOT_TOKEN_REAL: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.TELEGRAM_BOT_TOKEN_REAL || '' }} + DISCORD_BOT_TOKEN_REAL: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DISCORD_BOT_TOKEN_REAL || '' }} + SLACK_BOT_TOKEN_REAL: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.SLACK_BOT_TOKEN_REAL || '' }} + SLACK_APP_TOKEN_REAL: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.SLACK_APP_TOKEN_REAL || '' }} + TELEGRAM_ALLOWED_IDS: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.TELEGRAM_ALLOWED_IDS || '' }} + TELEGRAM_AUTHORIZED_CHAT_IDS: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.TELEGRAM_AUTHORIZED_CHAT_IDS || '' }} + TELEGRAM_CHAT_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.TELEGRAM_CHAT_ID || '' }} + TELEGRAM_CHAT_ID_E2E: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.TELEGRAM_CHAT_ID_E2E || '' }} + DISCORD_CHANNEL_ID_E2E: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DISCORD_CHANNEL_ID_E2E || '' }} + SLACK_CHANNEL_ID_E2E: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.SLACK_CHANNEL_ID_E2E || '' }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/messaging-providers.test.ts diff --git a/test/e2e/README.md b/test/e2e/README.md index ed0f54e6116..35cea3ad08b 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -258,11 +258,11 @@ The planner partitions that set into GitHub Actions matrices, one for each execu The execution profile owns the credentials available to its target step: - `standard` displays `no provider credential` and receives no NVIDIA API credential. -- `nvidia-api` displays `NVIDIA API key` and receives `NVIDIA_API_KEY` on trusted `main` runs. -- `nvidia-inference` displays `NVIDIA inference API key` and receives `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs. +- `nvidia-api` displays `NVIDIA API key` and receives `NVIDIA_API_KEY` on trusted `main` runs and authenticated NVIDIA-owned PR runs. +- `nvidia-inference` displays `NVIDIA inference API key` and receives `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs and authenticated NVIDIA-owned PR runs. - `github-read` displays `GitHub read token` and receives the job-scoped `GITHUB_TOKEN` only for the target step when `trusted_main` is `true`. - The reusable workflow enforces this boundary; PR revision callers set `trusted_main` to `false`, so their target steps receive no `GITHUB_TOKEN`. -- `brave-nvidia-inference` displays `Brave and NVIDIA inference API keys` and receives `BRAVE_API_KEY` and `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs. + The reusable workflow enforces this boundary; an authenticated NVIDIA-owned PR caller sets `trusted_main` to `true`, while an external PR caller sets it to `false` and receives no `GITHUB_TOKEN`. +- `brave-nvidia-inference` displays `Brave and NVIDIA inference API keys` and receives `BRAVE_API_KEY` and `NVIDIA_INFERENCE_API_KEY` on trusted `main` runs and authenticated NVIDIA-owned PR runs. GitHub Actions renders each catalogue execution as ` / `. All catalogue profiles call `.github/workflows/e2e-standard-profile.yaml`. @@ -1097,6 +1097,8 @@ teardown discards that filesystem. These credentials remain valid until they expire or an administrator revokes them in their issuing services. If cleanup fails, remove the recorded Brev workspace. Rotate or revoke each credential to remove later access. +For an NVIDIA-owned PR revision, the job builds and runs the exact candidate commit with this same credential boundary. +The PR branch must be in `NVIDIA/NemoClaw` because the image producer does not accept a sibling-repository candidate. The `NEMOCLAW_STAGING_LAUNCHABLE_ID` repository Actions variable selects the standing Launchable. Keep its value equal to the Launchable ID in the default @@ -1112,15 +1114,21 @@ The `totalRunnerMinutes` field contains the cumulative runner time for those sum A later successful attempt sets `action` to `passed-after-retry` and `flaky` to `true`. The observer ignores manual PR runs and a run superseded by a newer `main` push. -For a PR revision run, a repository maintainer or administrator leaves `jobs` and `targets` empty. The run selects: +GitHub's workflow-dispatch permission is the actor authorization for a PR revision run. +The workflow does not add a second `maintain` or `admin` role gate. +Before checkout, it verifies the open PR, exact target repository and `main` branch, current source repository and commit, base commit, and trusted workflow commit from the GitHub PR API. + +When the API reports that the PR source repository owner is the `NVIDIA` organization, empty `jobs` and `targets` select: - every default-selected free-standing workflow E2E except `Exact staging Brev Launchable`; -- every catalogue target in the `standard` profile; +- every catalogue target across all credential profiles; - every shared credential-free test; and -- these controller-selected registry targets: `ubuntu-policy-custom-missing-presets-negative`, `ubuntu-repo-cloud-langchain-deepagents-code`, `ubuntu-repo-cloud-openclaw`, and `ubuntu-repo-docker-post-reboot-recovery`. +- every default registry target. -The PR selection does not forward an NVIDIA API key, `BRAVE_API_KEY`, or `GITHUB_TOKEN` to the candidate checkout. +An NVIDIA-owned PR may also select any supported E2E job or target. +For an external PR, the bounded controller matrix remains in effect and the workflow does not forward repository credentials to candidate-controlled processes or reusable workflow callers. The run skips `jetson-nvmap-gpu` unless `allow_jetson_dispatch` is `true`. +Jetson and Launchable dispatch additionally require the PR branch to be in `NVIDIA/NemoClaw`; their operator and image-producer backends do not accept a sibling-repository candidate. It skips `llama-cpp-dgx-spark-plan` and `llama-cpp-dgx-spark-qualification` unless their runner-queue flag is `true`. The trusted workflow definition remains on `main` and binds the latest PR commit to the current PR base SHA. @@ -1139,9 +1147,10 @@ The risk plan selects the `openshell-gateway-upgrade` catalogue target and the The catalogue target covers the installer-driven OpenShell gateway upgrade handoff. The typed target covers the LangChain Deep Agents Code sandbox recreation path. -A trusted manual `main` run with empty selectors exposes these values to candidate-controlled job processes: +An NVIDIA-owned PR run with empty selectors exposes these values to candidate-controlled job processes: - Long-lived API keys from repository secrets: `NVIDIA_INFERENCE_API_KEY`, `NVIDIA_API_KEY`, and `BRAVE_API_KEY`. +- Docker Hub credentials from `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN`, available to candidate processes through the job's temporary Docker configuration until cleanup. - Long-lived messaging credentials from repository secrets: `TELEGRAM_BOT_TOKEN_REAL`, `DISCORD_BOT_TOKEN_REAL`, `SLACK_BOT_TOKEN_REAL`, and `SLACK_APP_TOKEN_REAL`. - The job-scoped `GITHUB_TOKEN`, exposed only to the target step in the `token-rotation` and `openshell-gateway-upgrade` catalogue executions. It has `contents: read` access. @@ -1149,7 +1158,7 @@ A trusted manual `main` run with empty selectors exposes these values to candida GitHub Actions invalidates it after the reusable workflow job. - Messaging account and channel identifiers from repository secrets: `TELEGRAM_ALLOWED_IDS`, `TELEGRAM_AUTHORIZED_CHAT_IDS`, `TELEGRAM_CHAT_ID`, `TELEGRAM_CHAT_ID_E2E`, `DISCORD_CHANNEL_ID_E2E`, and `SLACK_CHANNEL_ID_E2E`. -The workflow does not rotate or revoke these API keys or messaging credentials. To remove later access, rotate or revoke every listed credential in the external service that issued it. The workflow cannot erase identifiers copied by candidate code. Review the complete candidate diff before dispatch. +The workflow does not rotate or revoke these API keys, Docker Hub credentials, or messaging credentials. To remove later access, rotate or revoke every listed credential in the external service that issued it. The workflow cannot erase identifiers copied by candidate code. Review the complete candidate diff before dispatch. Live targets can create external resources. After a failure, inspect the workflow artifacts and remove resources that target cleanup did not remove. @@ -1167,8 +1176,7 @@ Verify that the old value is invalid. After you accept this credential boundary, dispatch `native-runtime-qualification-producer` from trusted `main` for a same-repository open PR. Use the first workflow attempt. The executing workflow commit and `workflow_sha` input must equal the exact PR-recorded base commit. -The actor must have repository `maintain` or `admin` permission. -If `github.triggering_actor` differs from the actor, it must also have one of those permissions. +The dispatcher must have GitHub permission to run the workflow; the E2E workflow adds no second actor-role check. The trusted workflow binds the candidate commit, base commit, workflow commit, repository, PR, run, attempt, and 24-case plan. The unprivileged installer and live-test processes run with `env -i` under a temporary account. @@ -1202,9 +1210,8 @@ For a manual PR run, provide these inputs: - The PR source repository. - The lowercase 40-character PR base SHA. - The exact SHA of the trusted workflow commit on `main`. -- A review reason containing 10 to 500 printable characters. -For the default PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. +For the default NVIDIA-owned PR revision selection, leave `jobs` and `targets` empty and keep `include_staging_brev_launchable=false`. Keep `allow_jetson_dispatch=false` and `allow_dgx_spark_runner_queue=false` for the default PR revision selection. If `allow_dgx_spark_runner_queue=true`, GitHub can pause the qualification job for the `approve-dgx-spark-image-qualification` environment. An authorized environment reviewer must approve it before qualification starts. @@ -1216,8 +1223,10 @@ To select native runtime qualification evidence production, set `jobs=native-run Leave `targets` empty and keep `include_staging_brev_launchable=false`. For this producer run, the executing workflow SHA, `workflow_sha` input, and PR base SHA must match. Confirm that the PR comes from `NVIDIA/NemoClaw`, the required ephemeral runner variables are configured, and the workflow has not been rerun. -A trusted `main` workflow pre-checkout step requires current `maintain` or `admin` permission. The workflow validates the exact open PR and selected mode before candidate code runs. -A second validation after checkout rejects a changed candidate commit, base commit, or PR source repository before preparation. +A trusted `main` workflow pre-checkout step validates the exact open PR and records whether its source repository has API-confirmed `NVIDIA` organization ownership. +That ownership authorizes the full ordinary plan and credential profiles; external sources retain the bounded controller plan. +A second validation after checkout rejects a changed candidate commit, base commit, PR source repository, or NVIDIA ownership before preparation. +Candidate runs cannot publish release qualification. The Actions run is advisory for the pull request and is not a required merge context. Treat it as passing evidence only when the `E2E` workflow concludes with `success` for the recorded PR number, PR source repository, candidate commit SHA, base commit SHA, and executing workflow SHA. @@ -1251,10 +1260,10 @@ configuration, or the unified E2E workflow. Compatibility schema fields may classify that guidance as required, but rendered advisor guidance remains non-authoritative. Model advice is additive and cannot downgrade the deterministic floor. PR Review Advisor recommendations remain advisory. -A maintainer decides whether to dispatch this trusted selection for the current PR -revision. The manual PR selection includes the credential-free -`inference-routing` catalogue target. It does not dispatch secret-backed targets such as -`network-policy` for PR revisions. The Advisor comment labels that boundary. +A repository-authorized user decides whether to dispatch this trusted selection for the current PR revision. +For API-confirmed NVIDIA-owned sources, the selection may include secret-backed targets such as `network-policy`. +External PR revisions retain the credential-free controller boundary. +The Advisor comment labels the requested coverage, but does not restrict an NVIDIA-owned PR to that recommendation. No PR E2E controller dispatches the risk plan. The `full-e2e` target enforces a separate hard acceptance contract for the diff --git a/test/e2e/RETRY_INVENTORY.md b/test/e2e/RETRY_INVENTORY.md index 64c05e7c1b8..9f3b9a47884 100644 --- a/test/e2e/RETRY_INVENTORY.md +++ b/test/e2e/RETRY_INVENTORY.md @@ -17,7 +17,7 @@ Exhaustion remains failed. | `hosted-runner-recovery` | Confirmed GitHub-hosted runner loss; `tools/e2e/hosted-runner-recovery.mts`, `tools/e2e/hosted-runner-loss*.mts` | Authenticated runner-allocation or internal-runner evidence that remains identical across 2 consecutive reads | 2 immediate evidence reads and at most 1 recovery request; no delay | GitHub reruns a workflow attempt | GitHub Actions | Dedicated runner-loss classifications | Source and recovery run links plus authenticated job evidence | External owner; governed by #7146, not this policy | | `pr-rerun-reconciliation` | PR E2E dispatch reconciliation; `tools/e2e/pr-e2e-dispatch-reconciliation.mts`, `tools/e2e/pr-e2e-retry-receipt.mts` | Trusted dispatch receipt state | Contract-defined single reconciliation | Reconciles workflow and commit identity before action | GitHub Actions | Receipt-specific terminal states | Signed workflow identity and receipt | External scope; governed by #7206 | | `github-publication-read` | GitHub API reads; `tools/e2e/base-image-publication.mts` | Fetch error, 408, rate limit, or 5xx | 3 attempts; Retry-After/rate-limit reset or linear delay capped at 10s | Read-only | GitHub API | Returned parsed selection on success; thrown terminal HTTP/fetch error on failure or exhaustion | Caller artifact records the returned publication selection; terminal errors identify exhausted fetch or HTTP status without response content | Eligible bounded read; existing implementation retained | -| `trusted-controller-collaborator-permission-read` | Collaborator-permission reads for manual PR dispatch, release waiver, and Launchable publication; `.github/workflows/e2e.yaml` | Curl exit 5, 6, 7, 16, 18, 28, 35, 52, 55, 56, 92, 95, or 96; HTTP 408, 429, or 5xx | 3 attempts; linear 1s then 2s | Read-only GitHub API request | GitHub API | Transient API read versus terminal authentication, authorization, actor, or response failure | Operation name, attempt number, and sanitized failure class or HTTP status; no response body, header, or token | Eligible bounded read; HTTP 401, 403, 404, and 422, malformed responses, actor failures, and insufficient roles remain terminal; no cached permission or workflow rerun | +| `trusted-controller-collaborator-permission-read` | Collaborator-permission reads for release waiver and direct-main Launchable E2E; `.github/workflows/e2e.yaml` | Curl exit 5, 6, 7, 16, 18, 28, 35, 52, 55, 56, 92, 95, or 96; HTTP 408, 429, or 5xx | 3 attempts; linear 1s then 2s | Read-only GitHub API request | GitHub API | Transient API read versus terminal authentication, authorization, actor, or response failure | Operation name, attempt number, and sanitized failure class or HTTP status; no response body, header, or token | Eligible bounded read; HTTP 401, 403, 404, and 422, malformed responses, actor failures, and insufficient roles remain terminal; no cached permission or workflow rerun | | `inference-switch-ts` | Verified inference route update; `test/e2e/fixtures/inference-switch-retry.ts` | Timeout, reset, DNS/connectivity/connect error, request transport error, or exact 502/503/504 status; authentication, authorization, policy, malformed-input, and invalid-request signals take precedence | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Shared `RetryEvidence` classifications | Every attempt classification and aggregate outcome; command artifacts remain separate and redacted | Uses `runBoundedRetry`; deterministic verification mismatches stop; no `--no-verify` exhaustion bypass | | `inference-switch-shell` | Verified shell inference route update; `test/e2e/lib/inference-switch-retry.sh` | Same bounded transient and terminal-precedence signatures as the TypeScript helper | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Exit status remains failed on exhaustion | Existing command output and retry progress | Bounded compatibility helper; no `--no-verify` exhaustion bypass | | `provider-install-standard` | Provider validation during Brave, cron, device-auth, Hermes-switch, network-policy, and restricted onboarding | `isTransientProviderValidationFailure` allowlist only | 1 local or 3 CI attempts; linear 10s backoff | Repeats the same desired onboarding state; restricted paths destroy the prior sandbox before retry | Inference provider | Transient allowlist versus terminal install failure | Per-attempt command artifacts; restricted paths add a terminal skip artifact | Existing bounded paths; no deterministic install retry | diff --git a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts index 4d3d48d0b72..533cc110919 100644 --- a/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts +++ b/test/e2e/support/dockerhub-auth-workflow-boundary.test.ts @@ -330,7 +330,7 @@ describe("shared Docker Hub authentication workflow boundary (#6961)", () => { const errors = validateMutation((workflow) => { const auth = namedStep(workflow.jobs.live, AUTH_STEP_NAME)!; const ungatedPredicate = - "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')"; + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == ''"; auth.with = { "auth-required": `\${{ ${ungatedPredicate} && '1' || '0' }}`, username: `\${{ ${ungatedPredicate} && secrets.DOCKERHUB_USERNAME || '' }}`, diff --git a/test/e2e/support/e2e-collaborator-permission-retry.test.ts b/test/e2e/support/e2e-collaborator-permission-retry.test.ts index 635d785e2ae..c0123f0e3dd 100644 --- a/test/e2e/support/e2e-collaborator-permission-retry.test.ts +++ b/test/e2e/support/e2e-collaborator-permission-retry.test.ts @@ -25,20 +25,15 @@ type PermissionScenario = | "transport-exhaustion"; const AUTHORIZATION_STEPS: AuthorizationStep[] = [ - { - deniedMessage: "Manual PR E2E requires a repository maintainer or administrator", - mismatchMessage: "Manual PR E2E permission response did not match the actor", - name: "Authenticate manual PR dispatch", - }, { deniedMessage: "Release qualification waiver requires a repository administrator", mismatchMessage: "Release qualification waiver permission response did not match the actor", name: "Authorize release qualification waiver", }, { - deniedMessage: "Launchable image publication requires a repository maintainer or administrator", - mismatchMessage: "Launchable image publication permission response did not match the actor", - name: "Authorize Launchable image publication", + deniedMessage: "Launchable E2E requires a repository maintainer or administrator", + mismatchMessage: "Launchable E2E permission response did not match the actor", + name: "Authorize Launchable E2E maintainer dispatch", }, ]; @@ -138,12 +133,12 @@ printf '%s\n' "$1" >>"$SLEEP_LOG" ALLOW_JETSON_DISPATCH: "false", BASE_SHA: "b".repeat(40), CHECKOUT_REPOSITORY: "contributor/NemoClaw", - CHECKOUT_SHA: stepName === "Authenticate manual PR dispatch" ? "a".repeat(40) : "", + CHECKOUT_SHA: "", CURL_LOG: curlLog, EXPECTED_WORKFLOW_SHA: workflowSha, GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "private-test-token", - INCLUDE_LAUNCHABLE: stepName === "Authenticate manual PR dispatch" ? "false" : "true", + INCLUDE_LAUNCHABLE: "true", JOBS: "", PATH: `${fixture}:${process.env.PATH ?? ""}`, PERMISSION_ATTEMPT_FILE: attemptFile, diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 78c65a2f3c5..9e7e898a189 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -278,6 +278,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; expect.arrayContaining([ "Manual PR credential authorization must expose only the authorization result", 'Manual PR credential authorization must retain "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', + 'Manual PR credential authorization must retain "$NVIDIA_OWNED" == "true"', 'Manual PR credential authorization must retain "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', ]), ); @@ -287,14 +288,25 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "matching repository and requested SHAs", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: true, workflowShaMatches: true, expectedAllowed: true, }, { - caseName: "a checkout repository outside NVIDIA/NemoClaw", + caseName: "an NVIDIA-owned sibling repository", + checkoutRepository: "NVIDIA/NemoClaw-E2E", + nvidiaOwned: true, + workflowRepository: "NVIDIA/NemoClaw", + checkoutShaMatches: true, + workflowShaMatches: true, + expectedAllowed: true, + }, + { + caseName: "a checkout repository outside NVIDIA", checkoutRepository: "contributor/NemoClaw", + nvidiaOwned: false, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: true, workflowShaMatches: true, @@ -303,6 +315,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "a workflow repository outside NVIDIA/NemoClaw", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "contributor/NemoClaw", checkoutShaMatches: true, workflowShaMatches: true, @@ -311,6 +324,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "checkout_sha differs from the checked-out commit", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: false, workflowShaMatches: true, @@ -319,6 +333,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; { caseName: "a requested workflow SHA that differs from the running workflow", checkoutRepository: "NVIDIA/NemoClaw", + nvidiaOwned: true, workflowRepository: "NVIDIA/NemoClaw", checkoutShaMatches: true, workflowShaMatches: false, @@ -328,6 +343,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "sets E2E credential access to $expectedAllowed for $caseName (#9047)", ({ checkoutRepository, + nvidiaOwned, workflowRepository, checkoutShaMatches, workflowShaMatches, @@ -360,6 +376,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; EVENT_NAME: "workflow_dispatch", EXPECTED_WORKFLOW_SHA: expectedWorkflowSha, GITHUB_OUTPUT: output, + NVIDIA_OWNED: nvidiaOwned ? "true" : "false", REF: "refs/heads/main", WORKFLOW_REPOSITORY: workflowRepository, WORKFLOW_SHA: workflowSha, @@ -402,46 +419,28 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); - it("limits manual PR runs to controller-approved selectors or Jetson dispatch", () => { + it("requires API-confirmed NVIDIA organization ownership for full PR E2E", () => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", )!; authentication.run = authentication.run!.replace( - "Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag", - "Manual PR E2E accepts arbitrary selectors", - ); - - expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR authentication must retain Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag", + `"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization"`, + `"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "User"`, ); - }); - - it("uses the same controller selectors as the PR Review Advisor", () => { - const workflow = readE2eOperationsWorkflow(); - const authentication = workflow.jobs["generate-matrix"].steps!.find( - (step) => step.name === "Authenticate manual PR dispatch", - )!; - authentication.run = authentication.run!.replace("inference-routing::false:false | ", ""); expect(validateE2eOperationsWorkflow(workflow)).toContain( - "Manual PR authentication must retain ::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | native-runtime-qualification-producer::false:false | :jetson-nvmap-gpu:false:true) ;;", + `Manual PR authentication must retain "$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization"`, ); }); it.each([ - ["maintain", "", "", "false", 0, ""], - ["maintain", "inference-routing", "", "false", 0, ""], - ["maintain", "managed-image-protected-runtime", "", "false", 0, ""], - ["maintain", "native-runtime-qualification-producer", "", "false", 0, ""], - ["maintain", "", "jetson-nvmap-gpu", "true", 0, ""], - ["maintain", "", "jetson-nvmap-gpu", "false", 1, "accepts only empty selectors"], - ["maintain", "network-policy", "", "false", 1, "accepts only empty selectors"], - ["maintain", "gpu-e2e", "", "false", 1, "accepts only empty selectors"], - ["write", "", "", "false", 1, "requires a repository maintainer or administrator"], + ["NVIDIA organization", "NVIDIA", "Organization", true], + ["external organization", "contributor", "Organization", false], + ["lookalike user", "NVIDIA", "User", false], ])( - "requires a maintainer role and bounded selector before manual PR E2E for %s with jobs %s and targets %s", - (role, jobs, targets, allowJetsonDispatch, expectedStatus, expectedStderr) => { + "records NVIDIA ownership for a %s without a duplicate actor-role gate", + (_caseName, ownerLogin, ownerType, expectedNvidiaOwned) => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", @@ -449,20 +448,18 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; const headSha = "a".repeat(40); const baseSha = "b".repeat(40); const workflowSha = "c".repeat(40); + const checkoutRepository = `${ownerLogin}/NemoClaw`; const prefix = [ "curl() {", - ' local url="${@: -1}" output_file="" previous="" argument body', - ' for argument in "$@"; do', - ' if [[ "$previous" == "--output" ]]; then output_file="$argument"; fi', - ' previous="$argument"', - " done", - ' case "$url" in', - ` *collaborators*) body='{"user":{"login":"maintainer"},"role_name":"${role}"}'; if [[ -n "$output_file" ]]; then printf '%s' "$body" >"$output_file"; printf '200'; else printf '%s' "$body"; fi ;;`, - ` *pulls/42) printf '%s' '{"state":"open","head":{"repo":{"full_name":"contributor/NemoClaw"},"sha":"${headSha}"},"base":{"sha":"${baseSha}"}}' ;;`, + ' case "${@: -1}" in', + ` *pulls/42) printf '%s' '{"state":"open","head":{"repo":{"full_name":"${checkoutRepository}","owner":{"login":"${ownerLogin}","type":"${ownerType}"}},"sha":"${headSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${baseSha}"}}' ;;`, " *) return 1 ;;", " esac", "}", ].join("\n"); + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-pr-owner-")); + const output = join(directory, "output"); + writeFileSync(output, ""); const result = spawnSync( "bash", ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", `${prefix}\n${authentication.run}`], @@ -470,21 +467,112 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; encoding: "utf8", env: { ...process.env, - ACTOR: "maintainer", - ALLOW_JETSON_DISPATCH: allowJetsonDispatch, BASE_SHA: baseSha, - CHECKOUT_REPOSITORY: "contributor/NemoClaw", + CHECKOUT_REPOSITORY: checkoutRepository, CHECKOUT_SHA: headSha, EXPECTED_WORKFLOW_SHA: workflowSha, GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "token", + GITHUB_OUTPUT: output, INCLUDE_LAUNCHABLE: "false", + JOBS: "", + PR_NUMBER: "42", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: "refs/heads/main", + WORKFLOW_SHA: workflowSha, + }, + }, + ); + + try { + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(output, "utf8")).toBe( + `nvidia_owned=${expectedNvidiaOwned ? "true" : "false"}\n`, + ); + expect(authentication.run).not.toContain("collaborators/"); + expect(authentication.run).not.toContain("role_name"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }, + ); + + it.each([ + ["NVIDIA inclusion flag", "NVIDIA/NemoClaw", "NVIDIA", "Organization", "true", "", 0, ""], + [ + "NVIDIA job selector", + "NVIDIA/NemoClaw", + "NVIDIA", + "Organization", + "false", + "staging-brev-launchable", + 0, + "", + ], + [ + "external source", + "contributor/NemoClaw", + "contributor", + "Organization", + "true", + "", + 1, + "Launchable PR E2E requires an NVIDIA-owned source repository", + ], + [ + "NVIDIA sibling repository", + "NVIDIA/NemoClaw-fork", + "NVIDIA", + "Organization", + "false", + "staging-brev-launchable", + 1, + "Launchable PR E2E requires a branch in NVIDIA/NemoClaw", + ], + ])( + "authorizes Launchable PR E2E for %s", + ( + _caseName, + checkoutRepository, + ownerLogin, + ownerType, + includeLaunchable, + jobs, + expectedStatus, + expectedError, + ) => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const headSha = "a".repeat(40); + const baseSha = "b".repeat(40); + const workflowSha = "c".repeat(40); + const prefix = [ + "curl() {", + ' case "${@: -1}" in', + ` *pulls/42) printf '%s' '{"state":"open","head":{"repo":{"full_name":"${checkoutRepository}","owner":{"login":"${ownerLogin}","type":"${ownerType}"}},"sha":"${headSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${baseSha}"}}' ;;`, + " *) return 1 ;;", + " esac", + "}", + ].join("\n"); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-c", `${prefix}\n${authentication.run}`], + { + encoding: "utf8", + env: { + ...process.env, + BASE_SHA: baseSha, + CHECKOUT_REPOSITORY: checkoutRepository, + CHECKOUT_SHA: headSha, + EXPECTED_WORKFLOW_SHA: workflowSha, + GITHUB_OUTPUT: "/dev/null", + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + INCLUDE_LAUNCHABLE: includeLaunchable, JOBS: jobs, PR_NUMBER: "42", - REVIEW_REASON: "Reviewed latest PR commit", - RUN_ATTEMPT: "1", - TARGETS: targets, - TRIGGERING_ACTOR: "maintainer", WORKFLOW_EVENT: "workflow_dispatch", WORKFLOW_REF: "refs/heads/main", WORKFLOW_SHA: workflowSha, @@ -493,7 +581,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); expect(result.status, result.stderr).toBe(expectedStatus); - expect(result.stderr).toContain(expectedStderr); + if (expectedError) expect(result.stderr).toContain(`::error::${expectedError}`); }, ); @@ -576,6 +664,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: "", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NVIDIA_OWNED: "false", TARGETS: "", }, }, @@ -659,6 +749,8 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: jobSelector, + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NVIDIA_OWNED: "false", TARGETS: targetSelector, }, }, diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index e72ed4faad9..20faacc374d 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -124,6 +124,8 @@ function executeGenerateMatrixWithPlannerOutput( GITHUB_STEP_SUMMARY: path.join(directory, "summary.md"), INFERENCE_MODE: "mock", JOBS: options.jobs ?? "cloud-onboard", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NVIDIA_OWNED: "false", PATH: `${binDirectory}:${process.env.PATH ?? ""}`, TARGETS: options.targets ?? "", }, diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 4389b510b64..95a00676959 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -112,6 +112,29 @@ describe("e2e workflow boundary", () => { () => expect(validateE2eWorkflowBoundary()).toEqual([]), ); + it.each([ + ["live", "Run live E2E tests", "NVIDIA_INFERENCE_API_KEY"], + ["cloud-onboard", "Run cloud-onboard live Vitest test", "NVIDIA_INFERENCE_API_KEY"], + ["messaging-providers", "Run messaging providers live Vitest test", "SLACK_BOT_TOKEN_REAL"], + ])("rejects unguarded candidate credentials in %s", (jobName, stepName, secret) => { + const workflow = readWorkflow() as { + jobs: Record< + string, + { steps?: Array<{ env?: Record; name?: string }> } + >; + }; + const step = workflow.jobs[jobName]!.steps!.find((candidate) => candidate.name === stepName)!; + step.env![secret] = `\${{ secrets.${secret} }}`; + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + expect.stringContaining( + `${secret} behind trusted-main candidate authorization`, + ), + ]), + ); + }); + it("rejects a Launchable environment gate, authorization drift, and credential boundary drift", () => { const workflow = readWorkflow() as { jobs: Record< @@ -361,7 +384,7 @@ describe("e2e workflow boundary", () => { expect(validateE2eWorkflow(workflow)).toEqual( expect.arrayContaining([ "workflow concurrency must isolate each full dispatch with github.run_id", - "workflow concurrency must not cancel an active Jetson dispatch", + "workflow concurrency must not cancel an active Jetson or Launchable dispatch", "staging-brev-launchable concurrency must queue all pending Launchable E2E runs without cancellation", ]), ); @@ -538,7 +561,7 @@ describe("e2e workflow boundary", () => { expect.arrayContaining([ "generate-matrix job must expose trusted controller matrix output", "trusted controller matrix must pin typed target runner to ubuntu-latest", - "trusted controller matrix step must run before PR checkout", + "external controller matrix must run before PR checkout", ]), ); }); diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index d01c7c82e71..67d3f18e9f8 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -219,7 +219,7 @@ describe("Hermes GPU boundary", () => { expect(errors).toEqual( expect.arrayContaining([ expect.stringContaining( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch", ), ]), ); @@ -237,7 +237,7 @@ describe("Hermes GPU boundary", () => { expect(errors).toEqual( expect.arrayContaining([ expect.stringContaining( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch", ), ]), ); diff --git a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts index 9c29e02cdbb..148e1ad4b11 100644 --- a/test/e2e/support/managed-image-protected-runtime-workflow.test.ts +++ b/test/e2e/support/managed-image-protected-runtime-workflow.test.ts @@ -143,6 +143,23 @@ describe("protected managed-image runtime workflow", () => { ); }); + it("rejects an unguarded qualification credential", () => { + const value = workflow(); + const qualification = namedStep( + value, + "Run all-agent GPU, local inference, rollback, and cleanup qualification", + ); + qualification.env = { NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}" }; + + expect(validateManagedImageProtectedRuntimeWorkflow(value)).toEqual( + expect.arrayContaining([ + expect.stringContaining( + "managed-image-protected-runtime qualification env must bind NVIDIA_API_KEY", + ), + ]), + ); + }); + it("prevents the credentialed qualification step from executing .candidate-runtime files", () => { const value = workflow(); const qualification = namedStep( diff --git a/test/e2e/support/standard-profile-workflow-boundary.test.ts b/test/e2e/support/standard-profile-workflow-boundary.test.ts index 51240513b75..a2bb836f5c2 100644 --- a/test/e2e/support/standard-profile-workflow-boundary.test.ts +++ b/test/e2e/support/standard-profile-workflow-boundary.test.ts @@ -54,6 +54,18 @@ describe("standard E2E execution profile", () => { ); }); + it("rejects an unguarded existing catalogue caller secret", () => { + const workflow = readWorkflow() as { + jobs: Record }>; + }; + workflow.jobs["catalogue-nvidia-api"]!.secrets.NVIDIA_API_KEY = + "${{ secrets.NVIDIA_API_KEY }}"; + + expect(validateStandardProfileWorkflowBoundary(workflow)).toContain( + "catalogue-nvidia-api must receive only its profile secrets", + ); + }); + it("rejects catalogue callers that bypass E2E credential authorization (#9047)", () => { const workflow = readWorkflow() as { jobs: Record }>; diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 9a0a143a086..c5afd7bfc72 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -65,20 +65,6 @@ function expectedCiOutput(plan: ReturnType): string ].join("\n"); } -function prCandidatePlan( - plan: ReturnType, -): ReturnType { - return { - ...plan, - catalogueMatrices: Object.fromEntries( - Object.entries(plan.catalogueMatrices).map(([profile, rows]) => [ - profile, - rows.filter((row) => isPrCandidateCatalogueTarget(catalogueTarget(row.id))), - ]), - ) as ReturnType["catalogueMatrices"], - }; -} - describe("E2E workflow plan", () => { it("defaults to every release-required target and tagged credential-free test", () => { const plan = buildE2eWorkflowPlan(); @@ -451,16 +437,11 @@ describe("E2E workflow plan", () => { ).toThrow("invalid or duplicate display name"); }); - it("omits credentialed catalogue profiles when checkout_sha is set", () => { + it("includes every catalogue profile for an authorized NVIDIA-owned candidate", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-pr-")); const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); const plan = buildE2eWorkflowPlan(); - plan.catalogueMatrices["nvidia-api"] = []; - plan.catalogueMatrices["nvidia-inference"] = []; - plan.catalogueMatrices["github-read"] = []; - plan.catalogueMatrices["brave-nvidia-inference"] = []; - try { writeE2eWorkflowPlanCiOutput( {}, @@ -468,6 +449,7 @@ describe("E2E workflow plan", () => { GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, ); @@ -479,7 +461,37 @@ describe("E2E workflow plan", () => { } }); - it("allows manual PR dispatch only for standard-profile targets", () => { + it("omits credentialed catalogue profiles for an untrusted candidate", () => { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-fork-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + try { + writeE2eWorkflowPlanCiOutput( + {}, + { + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }, + ); + + const lines = readFileSync(output, "utf8").split("\n"); + for (const profile of [ + "nvidia_api", + "nvidia_inference", + "github_read", + "brave_nvidia_inference", + ]) { + expect(lines).toContain(`catalogue_${profile}_matrix=[]`); + } + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("classifies only standard-profile targets as credential-free PR candidates", () => { expect( Object.fromEntries( E2E_TARGET_CATALOGUE.map((target) => [ @@ -678,6 +690,7 @@ describe("E2E workflow plan", () => { GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: "launchable-smoke", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, @@ -685,9 +698,8 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - const expectedPlan = prCandidatePlan(plan); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(expectedPlan)); - expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(expectedPlan)); + expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); } finally { rmSync(directory, { force: true, recursive: true }); } @@ -709,6 +721,7 @@ describe("E2E workflow plan", () => { GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", JOBS: [activeJobs, ...retiredControllerSelectorIds()].join(","), + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "true", TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, @@ -716,9 +729,8 @@ describe("E2E workflow plan", () => { }); expect(result.status, result.stderr).toBe(0); - const expectedPlan = prCandidatePlan(plan); - expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(expectedPlan)); - expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(expectedPlan)); + expect(readFileSync(output, "utf8")).toBe(expectedCiOutput(plan)); + expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); } finally { rmSync(directory, { force: true, recursive: true }); } diff --git a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts index 619499ab985..a3dcc6ed2d4 100644 --- a/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts +++ b/tools/e2e/managed-image-protected-runtime-workflow-boundary.mts @@ -19,6 +19,8 @@ const ACTIVATION_PATH = "ci/protected-managed-image-runtime-activation-v1.json"; const LIVE_TEST_PATH = "test/e2e/live/managed-image-protected-runtime.test.ts"; const REGISTRY_IMAGE = "docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373"; +const GUARDED_NVIDIA_API_KEY = + "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }}"; // Keep lane-specific trust assertions explicit: the multiarch lane executes // candidate code directly, while this GPU lane keeps secrets in trusted code @@ -298,7 +300,7 @@ export function validateManagedImageProtectedRuntimeWorkflow(workflow: WorkflowR "Run all-agent GPU, local inference, rollback, and cleanup qualification", ); requireValues(errors, `${JOB_ID} qualification env`, record(qualification?.env), { - NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}", + NVIDIA_API_KEY: GUARDED_NVIDIA_API_KEY, }); const secretBearingSteps = workflowSteps.filter( (step) => record(step.env).NVIDIA_API_KEY !== undefined, diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index 452d8523b4d..59e26a09c77 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -10,7 +10,7 @@ export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = - "c559e6cd5bf076bed8d359bbca397d4e31fbf3c11123389425917b865544940d"; + "067df18297c3b5e5175dc11de071a0f3c261aa894db6141b8b789d67f5e9c0d1"; export const MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256 = "62cf2ee01ac7192f41fc7b2b071de729da8bacec1e4f693da1ec6f0b1f4723c0"; diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index f89dbf28db3..d7f52d40937 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -8,7 +8,7 @@ import { isDeepStrictEqual } from "node:util"; import ts from "typescript"; import YAML from "yaml"; -import { PR_E2E_MANUAL_CONTROLLER_JOB_IDS, RISK_RULES } from "../advisors/risk-plan.mts"; +import { RISK_RULES } from "../advisors/risk-plan.mts"; import { validateStandardProfileWorkflowBoundary } from "./standard-profile-workflow-boundary.mts"; import { catalogueTarget, E2E_TARGET_CATALOGUE } from "./target-catalogue.mts"; @@ -262,9 +262,9 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } if ( workflow.concurrency?.["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}" ) { - errors.push("Manual PR E2E concurrency must not cancel an active Jetson dispatch"); + errors.push("Manual PR E2E concurrency must not cancel an active Jetson or Launchable dispatch"); } const matrixJob = workflow.jobs["generate-matrix"] ?? {}; @@ -293,24 +293,22 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } const authentication = authenticationIndex >= 0 ? steps[authenticationIndex] : {}; - if (authentication.if !== "${{ inputs.checkout_sha != '' }}") { + if ( + authentication.id !== "candidate_authorization" || + authentication.if !== "${{ inputs.checkout_sha != '' }}" + ) { errors.push("Manual PR authentication must be activated only by checkout_sha"); } const authEnvironment = { - ACTOR: "${{ github.actor }}", - ALLOW_JETSON_DISPATCH: "${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}", BASE_SHA: "${{ inputs.base_sha }}", CHECKOUT_REPOSITORY: "${{ inputs.checkout_repository }}", CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", GITHUB_TOKEN: "${{ github.token }}", - INCLUDE_LAUNCHABLE: "${{ inputs.include_staging_brev_launchable }}", + INCLUDE_LAUNCHABLE: + "${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}", JOBS: "${{ inputs.jobs }}", PR_NUMBER: "${{ inputs.pr_number }}", - REVIEW_REASON: "${{ inputs.review_reason }}", - RUN_ATTEMPT: "${{ github.run_attempt }}", - TARGETS: "${{ inputs.targets }}", - TRIGGERING_ACTOR: "${{ github.triggering_actor }}", WORKFLOW_EVENT: "${{ github.event_name }}", WORKFLOW_REF: "${{ github.ref }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", @@ -320,37 +318,31 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push(`Manual PR authentication must bind ${name}`); } const authSource = String(authentication.run ?? ""); - const acceptedJobCases = [ - "::false:false", - ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS.map((jobId) => `${jobId}::false:false`), - "native-runtime-qualification-producer::false:false", - ":jetson-nvmap-gpu:false:true", - ].join(" | "); - const acceptedNames = [ - ...PR_E2E_MANUAL_CONTROLLER_JOB_IDS, - "native-runtime-qualification-producer", - "jetson-nvmap-gpu with its dispatch flag", - ]; - const acceptedJobNames = `${acceptedNames.slice(0, -1).join(", ")}, or ${acceptedNames.at(-1)}`; for (const fragment of [ '"$WORKFLOW_EVENT" == "workflow_dispatch"', '"$WORKFLOW_REF" == "refs/heads/main"', - '"$RUN_ATTEMPT" == "1"', '"$PR_NUMBER" =~ ^[1-9][0-9]*$', '"$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$', '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', '"$BASE_SHA" =~ ^[a-f0-9]{40}$', - '"$REVIEW_REASON" =~ ^[[:print:]]+$', - "${#REVIEW_REASON} >= 10", - "${#REVIEW_REASON} <= 500", '"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA"', - "Manual PR E2E requires a repository maintainer or administrator", - `${acceptedJobCases}) ;;`, - `Manual PR E2E accepts only empty selectors, ${acceptedJobNames}`, "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", + `[[ "$(jq -r '.base.repo.full_name // ""' <<< "$pull_json")" == "NVIDIA/NemoClaw" ]]`, + `[[ "$(jq -r '.base.ref // ""' <<< "$pull_json")" == "main" ]]`, `[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]]`, `[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]]`, `[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]]`, + '"$INCLUDE_LAUNCHABLE" == "true"', + '",${JOBS}," == *",staging-brev-launchable,"*', + '"$nvidia_owned" == "true"', + "Launchable PR E2E requires an NVIDIA-owned source repository", + '"$CHECKOUT_REPOSITORY" == "NVIDIA/NemoClaw"', + "Launchable PR E2E requires a branch in NVIDIA/NemoClaw", + `"$(jq -r '.head.repo.owner.login // ""' <<< "$pull_json")" == "NVIDIA"`, + `"$(jq -r '.head.repo.owner.type // ""' <<< "$pull_json")" == "Organization"`, + "nvidia_owned=false", + "nvidia_owned=true", + `printf 'nvidia_owned=%s\\n' "$nvidia_owned" >> "$GITHUB_OUTPUT"`, ]) { if (!authSource.includes(fragment)) errors.push(`Manual PR authentication must retain ${fragment}`); @@ -381,13 +373,20 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push("Manual PR checkout validation must skip qualification producer dispatches"); } const validationSource = String(validation.run ?? ""); + if (validation.env?.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}") { + errors.push("Manual PR checkout validation must bind authenticated NVIDIA ownership"); + } for (const fragment of [ '"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', "https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}", "pull request must still be open", + "pull request base repository changed before execution", + "pull request base branch changed before execution", "checkout_repository changed before execution", "checkout_sha changed before execution", "base_sha changed before execution", + '"$NVIDIA_OWNED" == "true"', + "PR source repository ownership changed before execution", ]) { if (!validationSource.includes(fragment)) { errors.push(`Manual PR checkout validation must retain ${fragment}`); @@ -410,6 +409,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EVENT_NAME: "${{ github.event_name }}", EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", + NVIDIA_OWNED: "${{ steps.candidate_authorization.outputs.nvidia_owned }}", REF: "${{ github.ref }}", WORKFLOW_REPOSITORY: "${{ github.repository }}", WORKFLOW_SHA: "${{ github.workflow_sha }}", @@ -422,7 +422,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const authorizationSource = String(credentialAuthorization.run ?? ""); for (const fragment of [ '"$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw"', - '"$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY"', + '"$NVIDIA_OWNED" == "true"', '"$EVENT_NAME" == "workflow_dispatch"', '"$REF" == "refs/heads/main"', '"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', diff --git a/tools/e2e/standard-profile-workflow-boundary.mts b/tools/e2e/standard-profile-workflow-boundary.mts index 22a5b5d0bb4..98f593fd26f 100644 --- a/tools/e2e/standard-profile-workflow-boundary.mts +++ b/tools/e2e/standard-profile-workflow-boundary.mts @@ -26,6 +26,10 @@ const DEFAULT_PROFILE_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-standa const PROFILE_WORKFLOW = "./.github/workflows/e2e-standard-profile.yaml"; const CHECKOUT = "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1"; const EXECUTION_PLAN_SHELL = "/bin/bash --noprofile --norc -e -o pipefail {0}"; +const TRUSTED_CALLER_CREDENTIAL_PREDICATE = + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true')"; +const guardedCallerSecret = (name: string): string => + `\${{ ${TRUSTED_CALLER_CREDENTIAL_PREDICATE} && secrets.${name} || '' }}`; const SKILL_AGENT_UPLOAD_PATH = `${[ "e2e-artifacts/live/skill-agent/evidence-manifest.json", "e2e-artifacts/live/skill-agent/*/artifact-summary.json", @@ -184,7 +188,7 @@ function validateProfileCallers(errors: string[], workflow: WorkflowRecord): voi const callerSecrets = record(job.secrets); if ( Object.keys(callerSecrets).sort().join(",") !== [...contract.secrets].sort().join(",") || - contract.secrets.some((name) => callerSecrets[name] !== `\${{ secrets.${name} }}`) + contract.secrets.some((name) => callerSecrets[name] !== guardedCallerSecret(name)) ) { errors.push(`${contract.job} must receive only its profile secrets`); } diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index f894bfd4400..ee6b514006d 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -188,12 +188,15 @@ const HOST_DEPENDENCY_ACTION_USES = HOST_DEPENDENCY_ACTION_PROVENANCE.reference; const DOCKER_HUB_CLEANUP_KEYS = ["if", "name", "run", "shell"]; // The general E2E workflow runs on push/manual dispatch. Its event set is // intentionally distinct from the reusable image workflow's push/manual boundary. -const TRUSTED_DOCKER_HUB_PREDICATE = - "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == ''"; +const TRUSTED_E2E_CREDENTIAL_PREDICATE = + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true')"; +const guardedE2eSecret = (secret: string): string => + `\${{ ${TRUSTED_E2E_CREDENTIAL_PREDICATE} && secrets.${secret} || '' }}`; +const TRUSTED_DOCKER_HUB_PREDICATE = TRUSTED_E2E_CREDENTIAL_PREDICATE; const GUARDED_DOCKER_HUB_AUTH_REQUIRED = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && '1' || '0' }}`; const GUARDED_DOCKER_HUB_USERNAME = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_USERNAME || '' }}`; const GUARDED_DOCKER_HUB_TOKEN = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_TOKEN || '' }}`; -const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; +const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; const RUNNER_ROUTING_OUTPUT = "${{ steps.runner_routing.outputs.runner_routing }}"; const RUNNER_ROUTING_STEP_NAME = "Build trusted larger-runner routing"; const RUNNER_ROUTING_SCRIPT = [ @@ -1586,7 +1589,7 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== GUARDED_HERMES_E2E_INFERENCE_KEY) { errors.push( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch without a PR checkout and the inference mode condition", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch and the inference mode condition", ); } requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); @@ -1594,6 +1597,46 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { requireRunDoesNotContain(errors, runVitest, "${{ inputs."); } +function validateCandidateCredentialSteps(errors: string[], jobs: WorkflowRecord): void { + const credentialSteps = [ + { + jobName: "cloud-onboard", + stepName: "Run cloud-onboard live Vitest test", + secrets: ["NVIDIA_INFERENCE_API_KEY"], + }, + { + jobName: "messaging-providers", + stepName: "Run messaging providers live Vitest test", + secrets: [ + "NVIDIA_INFERENCE_API_KEY", + "TELEGRAM_BOT_TOKEN_REAL", + "DISCORD_BOT_TOKEN_REAL", + "SLACK_BOT_TOKEN_REAL", + "SLACK_APP_TOKEN_REAL", + "TELEGRAM_ALLOWED_IDS", + "TELEGRAM_AUTHORIZED_CHAT_IDS", + "TELEGRAM_CHAT_ID", + "TELEGRAM_CHAT_ID_E2E", + "DISCORD_CHANNEL_ID_E2E", + "SLACK_CHANNEL_ID_E2E", + ], + }, + ] as const; + + for (const { jobName, stepName, secrets } of credentialSteps) { + const job = asRecord(jobs[jobName]); + const step = requireJobStep(errors, jobName, asSteps(job.steps), stepName); + const environment = asRecord(step?.env); + for (const secret of secrets) { + if (environment[secret] !== guardedE2eSecret(secret)) { + errors.push( + `${jobName} ${stepName} must guard ${secret} behind trusted-main candidate authorization`, + ); + } + } + } +} + function validateHermesTimeoutHeadroom(errors: string[], jobs: WorkflowRecord): void { for (const { innerTest, @@ -1778,9 +1821,9 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): } if ( concurrency["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}" ) { - errors.push("workflow concurrency must not cancel an active Jetson dispatch"); + errors.push("workflow concurrency must not cancel an active Jetson or Launchable dispatch"); } } @@ -1793,14 +1836,16 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord errors.push("staging-brev-launchable must not use a GitHub environment"); } const trustedRun = "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main'"; + const candidateAuthorization = + "(inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true')"; if ( !stringValue(job.if).includes(trustedRun) || - stringValue(job.if).includes("checkout_sha == ''") + !stringValue(job.if).includes(candidateAuthorization) ) { - errors.push("staging-brev-launchable must allow only trusted-main dispatches"); + errors.push("staging-brev-launchable must require trusted-main candidate authorization"); } const expectedSelector = - "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; + "${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; if (job.if !== expectedSelector) { errors.push("staging-brev-launchable must retain trusted manual selection"); } @@ -1813,7 +1858,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord "Authorize Launchable E2E maintainer dispatch", ); const expectedAuthorizationSelector = - "${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; + "${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}"; if (authorization?.if !== expectedAuthorizationSelector) { errors.push("Launchable E2E maintainer authorization must cover exact and full dispatches"); } @@ -1883,7 +1928,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord [runEnv, "GH_TOKEN", "NEMOCLAW_IMAGE_DISPATCH_TOKEN"], [runEnv, "NVIDIA_INFERENCE_API_KEY", "NVIDIA_INFERENCE_API_KEY"], ] as const) { - const expected = `\${{ ${trustedRun} && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.${secret} || '' }}`; + const expected = `\${{ ${trustedRun} && github.event_name == 'workflow_dispatch' && ${candidateAuthorization} && secrets.${secret} || '' }}`; if (env[key] !== expected) { errors.push(`staging-brev-launchable ${key} must use the trusted-run secret guard`); } @@ -2192,11 +2237,13 @@ function validateTrustedE2eDispatchReceipt( } const authentication = namedStep(generateSteps, "Authenticate manual PR dispatch"); + const controllerMatrix = namedStep(generateSteps, "Build trusted controller target matrix"); const waiverAuthorization = namedStep(generateSteps, "Authorize release qualification waiver"); const candidateCheckout = generateSteps.find((step) => stringValue(step.uses).startsWith("actions/checkout@"), ); const authenticationIndex = authentication ? generateSteps.indexOf(authentication) : -1; + const controllerMatrixIndex = controllerMatrix ? generateSteps.indexOf(controllerMatrix) : -1; const waiverAuthorizationIndex = waiverAuthorization ? generateSteps.indexOf(waiverAuthorization) : -1; @@ -2204,9 +2251,9 @@ function validateTrustedE2eDispatchReceipt( const uploadIndex = dispatchUpload ? generateSteps.indexOf(dispatchUpload) : -1; const checkoutIndex = candidateCheckout ? generateSteps.indexOf(candidateCheckout) : -1; const trustedPrefix = [ - "Build trusted controller target matrix", "Build trusted larger-runner routing", "Authenticate manual PR dispatch", + "Build trusted controller target matrix", "Authorize release qualification waiver", "Record trusted E2E dispatch receipt", "Upload trusted E2E dispatch receipt", @@ -2217,7 +2264,8 @@ function validateTrustedE2eDispatchReceipt( trustedPrefix, ) || authenticationIndex < 0 || - waiverAuthorizationIndex !== authenticationIndex + 1 || + controllerMatrixIndex !== authenticationIndex + 1 || + waiverAuthorizationIndex !== controllerMatrixIndex + 1 || receiptIndex !== waiverAuthorizationIndex + 1 || uploadIndex !== receiptIndex + 1 || checkoutIndex <= uploadIndex @@ -2340,8 +2388,11 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (controllerMatrix?.id !== "controller_matrix") { errors.push("trusted controller matrix step must use id controller_matrix"); } - if (controllerMatrix?.if !== "${{ inputs.checkout_sha != '' }}") { - errors.push("trusted controller matrix step must run only for controller dispatches"); + if ( + controllerMatrix?.if !== + "${{ inputs.checkout_sha != '' && steps.candidate_authorization.outputs.nvidia_owned != 'true' }}" + ) { + errors.push("trusted controller matrix step must run only for authenticated external dispatches"); } if (controllerMatrix?.shell !== "bash") { errors.push("trusted controller matrix step must use bash"); @@ -2429,12 +2480,22 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { stringValue(step.uses).startsWith("actions/checkout@"), ); if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); + const candidateAuthorization = generateSteps.find( + (step) => stringValue(step.id) === "candidate_authorization", + ); + if ( + controllerMatrix && + candidateAuthorization && + generateSteps.indexOf(controllerMatrix) <= generateSteps.indexOf(candidateAuthorization) + ) { + errors.push("external controller matrix must run after PR ownership authentication"); + } if ( controllerMatrix && generateCheckout && generateSteps.indexOf(controllerMatrix) >= generateSteps.indexOf(generateCheckout) ) { - errors.push("trusted controller matrix step must run before PR checkout"); + errors.push("external controller matrix must run before PR checkout"); } requireFullShaAction(errors, generateCheckout, "generate-matrix checkout"); if (asRecord(generateCheckout?.with)["persist-credentials"] !== false) { @@ -2464,11 +2525,26 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { ) { errors.push("matrix generation step must pass release qualification waived jobs through env"); } + if ( + generateEnv.NEMOCLAW_E2E_CREDENTIALS_ALLOWED !== + "${{ (inputs.checkout_sha == '' || steps.e2e_credentials.outputs.allowed == 'true') && 'true' || 'false' }}" + ) { + errors.push("matrix generation step must bind NVIDIA-owned candidate authorization"); + } + if ( + generateEnv.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}" + ) { + errors.push("matrix generation step must bind the authenticated PR repository owner"); + } validateInferenceModeGeneration(errors, generate, generateEnv); requireRunContains(errors, generate, "npx tsx tools/e2e/workflow-plan.mts"); requireRunContains(errors, generate, "--ci-output"); requireRunContains(errors, generate, "git diff --name-only --diff-filter=ACMRD"); - requireRunContains(errors, generate, 'if [ -n "${CHECKOUT_SHA}" ]'); + requireRunContains( + errors, + generate, + 'if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]', + ); requireRunContains(errors, generate, "GITHUB_OUTPUT"); requireRunContains(errors, generate, "expected_controller_matrix="); requireRunContains(errors, generate, "actual_controller_matrix="); @@ -2651,8 +2727,10 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (runVitestEnv.TARGET_ID !== "${{ matrix.id }}") { errors.push("live E2E step must pass matrix.id through TARGET_ID env"); } - if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push("live E2E step must receive NVIDIA_INFERENCE_API_KEY from secrets"); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== guardedE2eSecret("NVIDIA_INFERENCE_API_KEY")) { + errors.push( + "live E2E step must guard NVIDIA_INFERENCE_API_KEY behind trusted-main candidate authorization", + ); } requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); requireRunContains(errors, runVitest, "test/e2e/live/registry-targets.test.ts"); @@ -2825,6 +2903,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateStagingBrevLaunchableJob(errors, jobs); validateCatalogueOwnedJobs(errors, jobs); validateHermesE2EJob(errors, jobs); + validateCandidateCredentialSteps(errors, jobs); validateHermesTimeoutHeadroom(errors, jobs); const reportToPr = asRecord(jobs["report-to-pr"]); diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index b7e05b8f313..e002f4941cb 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -684,8 +684,10 @@ export function writeE2eWorkflowPlanCiOutput( controllerMap.retiredSelectorSelected && !hasPlannerSelectors ? emptyE2eWorkflowPlan() : buildE2eWorkflowPlan(plannerSelectors, { changedFiles }); + const candidateRevision = COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? ""); + const credentialsAllowed = environment.NEMOCLAW_E2E_CREDENTIALS_ALLOWED === "true"; const plan = validateE2eWorkflowPlan( - COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? "") + candidateRevision && !credentialsAllowed ? withoutCredentialedCatalogueProfiles(planned) : planned, ); From 8ccda6cd3b892f859a5c9512fe19841f21a76c5a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 21:18:54 -0700 Subject: [PATCH 02/16] fix(e2e): plan from trusted workflow source Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 141 ++++++++++-------- .../e2e-operations-workflow-boundary.test.ts | 4 +- test/e2e/support/e2e-workflow.test.ts | 36 ++++- tools/e2e/operations-workflow-boundary.mts | 8 +- tools/e2e/workflow-boundary.mts | 70 +++++++-- 5 files changed, 177 insertions(+), 82 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1315c3e1841..38064c6f4d7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -73,11 +73,6 @@ on: required: false default: "" type: string - review_reason: - description: Deprecated optional audit note retained for dispatcher compatibility; not used for authorization. - required: false - default: "" - type: string base_sha: description: Optional lowercase PR base SHA for manual exact-revision E2E. required: false @@ -612,7 +607,82 @@ jobs: require_maintainer "$TRIGGERING_ACTOR" fi - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check out trusted E2E planner + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.workflow_sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Set up Node for trusted E2E planning + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6.3.0 + with: + node-version: 22 + + - name: Install trusted E2E planner dependencies + run: npm ci --ignore-scripts --no-audit --no-fund + + - id: matrix + name: Generate E2E target matrix + env: + CHECKOUT_SHA: ${{ inputs.checkout_sha }} + CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }} + CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }} + INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} + JOBS: ${{ inputs.jobs }} + TARGETS: ${{ inputs.targets }} + EVENT_NAME: ${{ github.event_name }} + BEFORE_SHA: ${{ github.event.before }} + CANDIDATE_SHA: ${{ github.sha }} + RELEASE_QUALIFICATION_WAIVED_JOBS: ${{ inputs.release_qualification_waived_jobs }} + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: ${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }} + NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} + run: | + set -euo pipefail + if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then + { + echo 'matrix=[]' + echo 'test_matrix=[]' + echo 'catalogue_standard_matrix=[]' + echo 'catalogue_nvidia_api_matrix=[]' + echo 'catalogue_nvidia_inference_matrix=[]' + echo 'catalogue_github_read_matrix=[]' + echo 'catalogue_brave_nvidia_inference_matrix=[]' + echo 'selected_jobs=["native-runtime-qualification-producer"]' + echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]' + echo 'hermes_selected=false' + echo 'explicit_only_jobs=native-runtime-qualification-producer' + echo 'release_qualification_waived_jobs=[]' + echo 'release_required_jobs=[]' + } >>"$GITHUB_OUTPUT" + exit 0 + fi + if [ "${EVENT_NAME:-}" = "push" ]; then + [[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || { + echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2 + exit 1 + } + CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")" + export CHANGED_FILES + fi + npx tsx tools/e2e/workflow-plan.mts --ci-output + if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]; then + matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")" + expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")" + actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")" + test_matrix="$(sed -n 's/^test_matrix=//p' "${GITHUB_OUTPUT}")" + expected_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${CONTROLLER_TEST_MATRIX}")" + actual_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${test_matrix}")" + if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ] || + [ "${actual_controller_test_matrix}" != "${expected_controller_test_matrix}" ]; then + : > "${GITHUB_OUTPUT}" + echo "::error::E2E planner matrix does not match controller-selected targets or shared tests" >&2 + exit 1 + fi + fi + + - name: Check out E2E candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }} with: repository: ${{ inputs.checkout_repository || github.repository }} @@ -860,65 +930,6 @@ jobs: echo "- Payload digest: \`${PAYLOAD_SHA256}\`" } >>"$GITHUB_STEP_SUMMARY" - - id: matrix - name: Generate E2E target matrix - env: - CHECKOUT_SHA: ${{ inputs.checkout_sha }} - CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }} - CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }} - INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }} - JOBS: ${{ inputs.jobs }} - TARGETS: ${{ inputs.targets }} - EVENT_NAME: ${{ github.event_name }} - BEFORE_SHA: ${{ github.event.before }} - CANDIDATE_SHA: ${{ github.sha }} - RELEASE_QUALIFICATION_WAIVED_JOBS: ${{ inputs.release_qualification_waived_jobs }} - NEMOCLAW_E2E_CREDENTIALS_ALLOWED: ${{ (inputs.checkout_sha == '' || steps.e2e_credentials.outputs.allowed == 'true') && 'true' || 'false' }} - NVIDIA_OWNED: ${{ steps.candidate_authorization.outputs.nvidia_owned }} - run: | - set -euo pipefail - if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then - { - echo 'matrix=[]' - echo 'test_matrix=[]' - echo 'catalogue_standard_matrix=[]' - echo 'catalogue_nvidia_api_matrix=[]' - echo 'catalogue_nvidia_inference_matrix=[]' - echo 'catalogue_github_read_matrix=[]' - echo 'catalogue_brave_nvidia_inference_matrix=[]' - echo 'selected_jobs=["native-runtime-qualification-producer"]' - echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]' - echo 'hermes_selected=false' - echo 'explicit_only_jobs=native-runtime-qualification-producer' - echo 'release_qualification_waived_jobs=[]' - echo 'release_required_jobs=[]' - } >>"$GITHUB_OUTPUT" - exit 0 - fi - if [ "${EVENT_NAME:-}" = "push" ]; then - [[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || { - echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2 - exit 1 - } - CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")" - export CHANGED_FILES - fi - npx tsx tools/e2e/workflow-plan.mts --ci-output - if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]; then - matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")" - expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")" - actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")" - test_matrix="$(sed -n 's/^test_matrix=//p' "${GITHUB_OUTPUT}")" - expected_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${CONTROLLER_TEST_MATRIX}")" - actual_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${test_matrix}")" - if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ] || - [ "${actual_controller_test_matrix}" != "${expected_controller_test_matrix}" ]; then - : > "${GITHUB_OUTPUT}" - echo "::error::E2E planner matrix does not match controller-selected targets or shared tests" >&2 - exit 1 - fi - fi - native-runtime-qualification-producer-plan: needs: generate-matrix if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }} diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index ec3f4358f3b..a27e8f84fa0 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -246,7 +246,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; it("validates manual PR dispatch inputs and the checked-out commit", () => { const workflow = readE2eOperationsWorkflow(); - delete workflow.on?.workflow_dispatch?.inputs?.review_reason; + delete workflow.on?.workflow_dispatch?.inputs?.checkout_repository; const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch", )!; @@ -258,7 +258,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; expect(validateE2eOperationsWorkflow(workflow)).toEqual( expect.arrayContaining([ - "workflow_dispatch review_reason must be an optional string with an empty default", + "workflow_dispatch checkout_repository must be an optional string with an empty default", 'Manual PR authentication must retain "$WORKFLOW_EVENT" == "workflow_dispatch"', 'Manual PR authentication must retain "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$', 'Manual PR checkout validation must retain "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA"', diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 3e103cc5bbb..0bba2ddafd4 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -507,6 +507,40 @@ describe("e2e workflow boundary", () => { ); }); + it("runs the trusted planner before checking out candidate code", () => { + const workflow = readWorkflow() as { + jobs: Record< + string, + { + steps: Array<{ + name?: string; + uses?: string; + with?: Record; + }>; + } + >; + }; + const steps = workflow.jobs["generate-matrix"]!.steps; + const plannerCheckout = steps.find((step) => step.name === "Check out trusted E2E planner")!; + plannerCheckout.with!.ref = "${{ inputs.checkout_sha || github.sha }}"; + const [generate] = steps.splice( + steps.findIndex((step) => step.name === "Generate E2E target matrix"), + 1, + ); + steps.splice( + steps.findIndex((step) => step.name === "Check out E2E candidate") + 1, + 0, + generate!, + ); + + expect(validateE2eWorkflow(workflow)).toEqual( + expect.arrayContaining([ + "trusted E2E planner checkout must use the workflow commit without credentials", + "trusted E2E planning must finish before candidate checkout and execution", + ]), + ); + }); + it("includes deleted owning paths in main-push selection", () => { const workflow = readWorkflow() as { jobs: Record }>; @@ -590,7 +624,7 @@ describe("e2e workflow boundary", () => { ); trusted!.run = trusted!.run!.replace('"runner":"ubuntu-latest"', '"runner":"self-hosted"'); generateMatrix.steps.splice( - generateMatrix.steps.findIndex((step) => step.uses?.startsWith("actions/checkout@")) + 1, + generateMatrix.steps.findIndex((step) => step.name === "Check out E2E candidate") + 1, 0, trusted!, ); diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index b78fbcb23bf..10a95730627 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -230,7 +230,6 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow "pr_number", "checkout_sha", "checkout_repository", - "review_reason", "base_sha", "workflow_sha", "correlation_id", @@ -272,7 +271,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const authenticationIndex = steps.findIndex( (step) => step.name === "Authenticate manual PR dispatch", ); - const checkoutIndex = steps.findIndex((step) => step.uses?.startsWith("actions/checkout@")); + const checkoutIndex = steps.findIndex((step) => step.name === "Check out E2E candidate"); const validationIndex = steps.findIndex((step) => step.name === "Validate manual PR checkout"); const credentialAuthorizationIndex = steps.findIndex( (step) => step.name === "Authorize E2E credentials", @@ -445,6 +444,10 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.name === "Checkout trusted Hermes GPU runtime fixture" && step.with?.repository === "NVIDIA/NemoClaw" && step.with?.ref === "${{ github.workflow_sha }}"; + const trustedE2ePlannerCheckout = + jobName === "generate-matrix" && + step.name === "Check out trusted E2E planner" && + step.with?.ref === "${{ github.workflow_sha }}"; const trustedReportHelperCheckout = jobName === "report-to-pr" && step.name === "Check out the trusted E2E reporting helper" && @@ -530,6 +533,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow step.with?.ref === "${{ github.workflow_sha }}"); const trustedCheckout = trustedHermesFixtureCheckout || + trustedE2ePlannerCheckout || trustedReportHelperCheckout || trustedReleaseQualificationCheckout || trustedRelevantE2eCheckout || diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index ee6b514006d..3a110cb6d9c 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1886,9 +1886,7 @@ function validateStagingBrevLaunchableJob(errors: string[], jobs: WorkflowRecord ]) { requireRunContains(errors, authorization, required); } - const generateCheckout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), - ); + const generateCheckout = namedStep(generateSteps, "Check out E2E candidate"); if ( authorization && generateCheckout && @@ -2082,9 +2080,7 @@ function validateReleaseQualificationWaiverAuthorization( ]) { requireRunContains(errors, authorization, required); } - const checkout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), - ); + const checkout = namedStep(generateSteps, "Check out E2E candidate"); if ( authorization && checkout && @@ -2239,9 +2235,7 @@ function validateTrustedE2eDispatchReceipt( const authentication = namedStep(generateSteps, "Authenticate manual PR dispatch"); const controllerMatrix = namedStep(generateSteps, "Build trusted controller target matrix"); const waiverAuthorization = namedStep(generateSteps, "Authorize release qualification waiver"); - const candidateCheckout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), - ); + const candidateCheckout = namedStep(generateSteps, "Check out E2E candidate"); const authenticationIndex = authentication ? generateSteps.indexOf(authentication) : -1; const controllerMatrixIndex = controllerMatrix ? generateSteps.indexOf(controllerMatrix) : -1; const waiverAuthorizationIndex = waiverAuthorization @@ -2476,9 +2470,22 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { controllerMatrix, `printf 'matrix=%s\\n' "\${matrix}" >> "\${GITHUB_OUTPUT}"`, ); - const generateCheckout = generateSteps.find((step) => - stringValue(step.uses).startsWith("actions/checkout@"), + const trustedPlannerCheckout = requireStep( + errors, + generateSteps, + "Check out trusted E2E planner", ); + const trustedPlannerSetup = requireStep( + errors, + generateSteps, + "Set up Node for trusted E2E planning", + ); + const trustedPlannerInstall = requireStep( + errors, + generateSteps, + "Install trusted E2E planner dependencies", + ); + const generateCheckout = requireStep(errors, generateSteps, "Check out E2E candidate"); if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); const candidateAuthorization = generateSteps.find( (step) => stringValue(step.id) === "candidate_authorization", @@ -2503,6 +2510,45 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { } validateLargerRunnerRouting(errors, jobs, generateMatrix, generateSteps, generateCheckout); const generate = requireStep(errors, generateSteps, "Generate E2E target matrix"); + requireFullShaAction(errors, trustedPlannerCheckout, "trusted E2E planner checkout"); + if ( + !isDeepStrictEqual(asRecord(trustedPlannerCheckout?.with), { + ref: "${{ github.workflow_sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }) + ) { + errors.push("trusted E2E planner checkout must use the workflow commit without credentials"); + } + requireFullShaAction(errors, trustedPlannerSetup, "trusted E2E planner Node setup"); + if ( + !isDeepStrictEqual(asRecord(trustedPlannerSetup?.with), { + "node-version": 22, + }) + ) { + errors.push("trusted E2E planner must use Node 22"); + } + if (trustedPlannerInstall?.run !== "npm ci --ignore-scripts --no-audit --no-fund") { + errors.push("trusted E2E planner dependencies must install without lifecycle scripts"); + } + const trustedPlannerIndex = trustedPlannerCheckout + ? generateSteps.indexOf(trustedPlannerCheckout) + : -1; + const trustedSetupIndex = trustedPlannerSetup ? generateSteps.indexOf(trustedPlannerSetup) : -1; + const trustedInstallIndex = trustedPlannerInstall + ? generateSteps.indexOf(trustedPlannerInstall) + : -1; + const generateIndex = generate ? generateSteps.indexOf(generate) : -1; + const candidateCheckoutIndex = generateCheckout ? generateSteps.indexOf(generateCheckout) : -1; + if ( + trustedPlannerIndex < 0 || + trustedSetupIndex <= trustedPlannerIndex || + trustedInstallIndex <= trustedSetupIndex || + generateIndex <= trustedInstallIndex || + candidateCheckoutIndex <= generateIndex + ) { + errors.push("trusted E2E planning must finish before candidate checkout and execution"); + } const generateEnv = asRecord(generate?.env); if (generateEnv.CHECKOUT_SHA !== "${{ inputs.checkout_sha }}") { errors.push("matrix generation step must bind controller checkout through CHECKOUT_SHA env"); @@ -2527,7 +2573,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { } if ( generateEnv.NEMOCLAW_E2E_CREDENTIALS_ALLOWED !== - "${{ (inputs.checkout_sha == '' || steps.e2e_credentials.outputs.allowed == 'true') && 'true' || 'false' }}" + "${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }}" ) { errors.push("matrix generation step must bind NVIDIA-owned candidate authorization"); } From a8e934bda91a1f1f0786216094d80329cc43f480 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 21:21:23 -0700 Subject: [PATCH 03/16] fix(e2e): keep trusted planner dependencies local Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 2 +- tools/e2e/workflow-boundary.mts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 38064c6f4d7..6311e7942f0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -665,7 +665,7 @@ jobs: CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")" export CHANGED_FILES fi - npx tsx tools/e2e/workflow-plan.mts --ci-output + npx --no-install tsx tools/e2e/workflow-plan.mts --ci-output if [ -n "${CHECKOUT_SHA}" ] && [ "${NVIDIA_OWNED}" != "true" ]; then matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")" expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")" diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 3a110cb6d9c..9d2a530bc7d 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -2583,7 +2583,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { errors.push("matrix generation step must bind the authenticated PR repository owner"); } validateInferenceModeGeneration(errors, generate, generateEnv); - requireRunContains(errors, generate, "npx tsx tools/e2e/workflow-plan.mts"); + requireRunContains(errors, generate, "npx --no-install tsx tools/e2e/workflow-plan.mts"); requireRunContains(errors, generate, "--ci-output"); requireRunContains(errors, generate, "git diff --name-only --diff-filter=ACMRD"); requireRunContains( From 0444124a3f3c034ff5720faaa27db5a48599391d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 21:22:41 -0700 Subject: [PATCH 04/16] docs(e2e): name the latest PR commit Signed-off-by: Carlos Villela --- .agents/skills/nemoclaw-maintainer-e2e/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md index ed6bd8a430f..86587277b15 100644 --- a/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md +++ b/.agents/skills/nemoclaw-maintainer-e2e/SKILL.md @@ -21,7 +21,7 @@ Do not substitute local `npm run test:live-e2e` unless the maintainer explicitly Use this mode when a repository-authorized dispatcher requests E2E for a pull request. GitHub's permission to dispatch the workflow is the actor authorization; do not add a second repository-role check. -The run executes an E2E selection against the current PR head commit while the workflow definition remains on `main`. +The run executes an E2E selection against the latest PR commit while the workflow definition remains on `main`. It is advisory and does not create a required PR check. An empty-selector NVIDIA-owned PR run exposes these values to candidate-controlled job processes: From 7312b2f95738361d4e1c7cf5e23bc67eeaf7c56c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 21:50:42 -0700 Subject: [PATCH 05/16] test(e2e): satisfy growth checks Signed-off-by: Carlos Villela --- .../e2e-operations-workflow-boundary.test.ts | 8 ++++---- test/e2e/support/workflow-plan.test.ts | 17 ++++++++--------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 0b2437ef731..94d9883aff3 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -515,7 +515,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "true", "", 1, - "Launchable PR E2E requires an NVIDIA-owned source repository", + "::error::Launchable PR E2E requires an NVIDIA-owned source repository\n", ], [ "NVIDIA sibling repository", @@ -525,7 +525,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; "false", "staging-brev-launchable", 1, - "Launchable PR E2E requires a branch in NVIDIA/NemoClaw", + "::error::Launchable PR E2E requires a branch in NVIDIA/NemoClaw\n", ], ])( "authorizes Launchable PR E2E for %s", @@ -537,7 +537,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; includeLaunchable, jobs, expectedStatus, - expectedError, + expectedStderr, ) => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( @@ -579,7 +579,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); expect(result.status, result.stderr).toBe(expectedStatus); - if (expectedError) expect(result.stderr).toContain(`::error::${expectedError}`); + expect(result.stderr).toBe(expectedStderr); }, ); diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 7c41ba94249..7ed97b57ad4 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -463,15 +463,14 @@ describe("E2E workflow plan", () => { }, ); - const lines = readFileSync(output, "utf8").split("\n"); - for (const profile of [ - "nvidia_api", - "nvidia_inference", - "github_read", - "brave_nvidia_inference", - ]) { - expect(lines).toContain(`catalogue_${profile}_matrix=[]`); - } + expect(readFileSync(output, "utf8").split("\n")).toEqual( + expect.arrayContaining([ + "catalogue_nvidia_api_matrix=[]", + "catalogue_nvidia_inference_matrix=[]", + "catalogue_github_read_matrix=[]", + "catalogue_brave_nvidia_inference_matrix=[]", + ]), + ); } finally { rmSync(directory, { force: true, recursive: true }); } From 2aeb7250812d4915a51fbdd684660e1a061e27e7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 21:56:08 -0700 Subject: [PATCH 06/16] refactor(e2e): isolate trusted planner validation Signed-off-by: Carlos Villela --- tools/e2e/workflow-boundary.mts | 139 +++++++++++++++++--------------- 1 file changed, 74 insertions(+), 65 deletions(-) diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 500e6ced055..bf991b9dd80 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -2118,6 +2118,79 @@ function validateTrustedE2eDispatchReceipt( } } +function validateTrustedE2ePlannerBoundary( + errors: string[], + generateSteps: WorkflowRecord[], + generate: WorkflowRecord | undefined, + candidateCheckout: WorkflowRecord | undefined, +): void { + const trustedPlannerCheckout = requireStep( + errors, + generateSteps, + "Check out trusted E2E planner", + ); + const trustedPlannerSetup = requireStep( + errors, + generateSteps, + "Set up Node for trusted E2E planning", + ); + const trustedPlannerInstall = requireStep( + errors, + generateSteps, + "Install trusted E2E planner dependencies", + ); + requireFullShaAction(errors, trustedPlannerCheckout, "trusted E2E planner checkout"); + if ( + !isDeepStrictEqual(asRecord(trustedPlannerCheckout?.with), { + ref: "${{ github.workflow_sha }}", + "fetch-depth": 0, + "persist-credentials": false, + }) + ) { + errors.push("trusted E2E planner checkout must use the workflow commit without credentials"); + } + requireFullShaAction(errors, trustedPlannerSetup, "trusted E2E planner Node setup"); + if ( + !isDeepStrictEqual(asRecord(trustedPlannerSetup?.with), { + "node-version": 22, + }) + ) { + errors.push("trusted E2E planner must use Node 22"); + } + if (trustedPlannerInstall?.run !== "npm ci --ignore-scripts --no-audit --no-fund") { + errors.push("trusted E2E planner dependencies must install without lifecycle scripts"); + } + const trustedPlannerIndex = trustedPlannerCheckout + ? generateSteps.indexOf(trustedPlannerCheckout) + : -1; + const trustedSetupIndex = trustedPlannerSetup ? generateSteps.indexOf(trustedPlannerSetup) : -1; + const trustedInstallIndex = trustedPlannerInstall + ? generateSteps.indexOf(trustedPlannerInstall) + : -1; + const generateIndex = generate ? generateSteps.indexOf(generate) : -1; + const candidateCheckoutIndex = candidateCheckout ? generateSteps.indexOf(candidateCheckout) : -1; + if ( + trustedPlannerIndex < 0 || + trustedSetupIndex <= trustedPlannerIndex || + trustedInstallIndex <= trustedSetupIndex || + generateIndex <= trustedInstallIndex || + candidateCheckoutIndex <= generateIndex + ) { + errors.push("trusted E2E planning must finish before candidate checkout and execution"); + } + + const generateEnv = asRecord(generate?.env); + if ( + generateEnv.NEMOCLAW_E2E_CREDENTIALS_ALLOWED !== + "${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }}" + ) { + errors.push("matrix generation step must bind NVIDIA-owned candidate authorization"); + } + if (generateEnv.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}") { + errors.push("matrix generation step must bind the authenticated PR repository owner"); + } +} + export function validateE2eWorkflow(workflowValue: unknown): string[] { const workflow = asRecord(workflowValue); const errors: string[] = []; @@ -2311,21 +2384,6 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { controllerMatrix, `printf 'matrix=%s\\n' "\${matrix}" >> "\${GITHUB_OUTPUT}"`, ); - const trustedPlannerCheckout = requireStep( - errors, - generateSteps, - "Check out trusted E2E planner", - ); - const trustedPlannerSetup = requireStep( - errors, - generateSteps, - "Set up Node for trusted E2E planning", - ); - const trustedPlannerInstall = requireStep( - errors, - generateSteps, - "Install trusted E2E planner dependencies", - ); const generateCheckout = requireStep(errors, generateSteps, "Check out E2E candidate"); if (!generateCheckout) errors.push("generate-matrix job missing checkout step"); const candidateAuthorization = generateSteps.find( @@ -2351,45 +2409,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { } validateLargerRunnerRouting(errors, jobs, generateMatrix, generateSteps, generateCheckout); const generate = requireStep(errors, generateSteps, "Generate E2E target matrix"); - requireFullShaAction(errors, trustedPlannerCheckout, "trusted E2E planner checkout"); - if ( - !isDeepStrictEqual(asRecord(trustedPlannerCheckout?.with), { - ref: "${{ github.workflow_sha }}", - "fetch-depth": 0, - "persist-credentials": false, - }) - ) { - errors.push("trusted E2E planner checkout must use the workflow commit without credentials"); - } - requireFullShaAction(errors, trustedPlannerSetup, "trusted E2E planner Node setup"); - if ( - !isDeepStrictEqual(asRecord(trustedPlannerSetup?.with), { - "node-version": 22, - }) - ) { - errors.push("trusted E2E planner must use Node 22"); - } - if (trustedPlannerInstall?.run !== "npm ci --ignore-scripts --no-audit --no-fund") { - errors.push("trusted E2E planner dependencies must install without lifecycle scripts"); - } - const trustedPlannerIndex = trustedPlannerCheckout - ? generateSteps.indexOf(trustedPlannerCheckout) - : -1; - const trustedSetupIndex = trustedPlannerSetup ? generateSteps.indexOf(trustedPlannerSetup) : -1; - const trustedInstallIndex = trustedPlannerInstall - ? generateSteps.indexOf(trustedPlannerInstall) - : -1; - const generateIndex = generate ? generateSteps.indexOf(generate) : -1; - const candidateCheckoutIndex = generateCheckout ? generateSteps.indexOf(generateCheckout) : -1; - if ( - trustedPlannerIndex < 0 || - trustedSetupIndex <= trustedPlannerIndex || - trustedInstallIndex <= trustedSetupIndex || - generateIndex <= trustedInstallIndex || - candidateCheckoutIndex <= generateIndex - ) { - errors.push("trusted E2E planning must finish before candidate checkout and execution"); - } + validateTrustedE2ePlannerBoundary(errors, generateSteps, generate, generateCheckout); const generateEnv = asRecord(generate?.env); if (generateEnv.CHECKOUT_SHA !== "${{ inputs.checkout_sha }}") { errors.push("matrix generation step must bind controller checkout through CHECKOUT_SHA env"); @@ -2406,17 +2426,6 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { if (generateEnv.TARGETS !== "${{ inputs.targets }}") { errors.push("matrix generation step must pass targets through TARGETS env"); } - if ( - generateEnv.NEMOCLAW_E2E_CREDENTIALS_ALLOWED !== - "${{ (inputs.checkout_sha == '' || steps.candidate_authorization.outputs.nvidia_owned == 'true') && 'true' || 'false' }}" - ) { - errors.push("matrix generation step must bind NVIDIA-owned candidate authorization"); - } - if ( - generateEnv.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}" - ) { - errors.push("matrix generation step must bind the authenticated PR repository owner"); - } validateInferenceModeGeneration(errors, generate, generateEnv); requireRunContains(errors, generate, "npx --no-install tsx tools/e2e/workflow-plan.mts"); requireRunContains(errors, generate, "--ci-output"); From 868d8837a1a53cf8bb6bfe0e77c0601b24a5675c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:18:00 -0700 Subject: [PATCH 07/16] test(e2e): preserve Launchable concurrency Signed-off-by: Carlos Villela --- .../e2e/support/jetson-workflow-boundary.test.ts | 16 ++++++++++++++++ tools/e2e/workflow-boundary.mts | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 5092621e086..fd9f4d1bcd1 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it } from "vitest"; import { + validateE2eWorkflow, validateE2eWorkflowBoundary, validateJetsonDispatchBoundary, } from "../../../tools/e2e/workflow-boundary.mts"; @@ -76,6 +77,21 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { ); }); + it.each([ + [ + "Jetson", + "${{ inputs.checkout_sha != '' && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}", + ], + ["Launchable", "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}"], + ])("rejects concurrency that cancels active %s dispatches", (_dispatch, cancellation) => { + const workflow = readWorkflow(); + workflow.concurrency!["cancel-in-progress"] = cancellation; + + expect(validateE2eWorkflow(workflow)).toContain( + "workflow concurrency must not cancel an active Jetson or Launchable dispatch", + ); + }); + it("rejects candidate execution or credential-bearing controller steps (#8142)", () => { const errors = validateWorkflowMutation((workflow) => { const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index bf991b9dd80..be084ee0562 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1777,9 +1777,9 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): } if ( concurrency["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}" ) { - errors.push("workflow concurrency must not cancel an active Jetson dispatch"); + errors.push("workflow concurrency must not cancel an active Jetson or Launchable dispatch"); } } From 4efaf0685af00b3e45c8cc92b08c5f517329f3f2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:20:48 -0700 Subject: [PATCH 08/16] fix(e2e): align authorized workflow contracts Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 2 +- test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts | 2 +- .../native-runtime-qualification-producer-workflow.test.ts | 2 +- tools/e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index d31d4fe2b77..a1d1b6155a8 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -4622,7 +4622,7 @@ jobs: - name: Run all-agent GPU, local inference, rollback, and cleanup qualification env: - NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + NVIDIA_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.NVIDIA_API_KEY || '' }} shell: bash run: | set -euo pipefail diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index 20faacc374d..188cbceb8b0 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -97,7 +97,7 @@ function executeGenerateMatrixWithPlannerOutput( fakeNpx, [ "#!/usr/bin/env bash", - "expected=(tsx tools/e2e/workflow-plan.mts --ci-output)", + "expected=(--no-install tsx tools/e2e/workflow-plan.mts --ci-output)", 'actual=("$@")', '[[ "${#actual[@]}" -eq "${#expected[@]}" ]] || exit 97', 'for index in "${!expected[@]}"; do [[ "${actual[$index]}" == "${expected[$index]}" ]] || exit 97; done', diff --git a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts index 572cb2744e1..16049dce242 100644 --- a/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts +++ b/test/e2e/support/native-runtime-qualification-producer-workflow.test.ts @@ -43,7 +43,7 @@ function expectRequiredPodmanPackages(run: string): void { describe("native runtime qualification producer workflow", () => { it("keeps candidate execution out of the authenticated controller", () => { const generate = job("generate-matrix"); - const checkout = generate.steps?.find((entry) => entry.uses?.startsWith("actions/checkout@")); + const checkout = step(generate, "Check out E2E candidate"); expect(checkout?.if).toContain("inputs.jobs != 'native-runtime-qualification-producer'"); expect(step(generate, "Validate manual PR checkout").if).toContain( diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index 59e26a09c77..452d8523b4d 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -10,7 +10,7 @@ export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = - "067df18297c3b5e5175dc11de071a0f3c261aa894db6141b8b789d67f5e9c0d1"; + "c559e6cd5bf076bed8d359bbca397d4e31fbf3c11123389425917b865544940d"; export const MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256 = "62cf2ee01ac7192f41fc7b2b071de729da8bacec1e4f693da1ec6f0b1f4723c0"; From b90b90f2277e96af2dedb9be51abd2dc0c4ebaa0 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:35:44 -0700 Subject: [PATCH 09/16] test(e2e): align authorization fixtures Signed-off-by: Carlos Villela --- .../e2e-collaborator-permission-retry.test.ts | 15 ++------------- test/e2e/support/jetson-workflow-boundary.test.ts | 3 ++- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/test/e2e/support/e2e-collaborator-permission-retry.test.ts b/test/e2e/support/e2e-collaborator-permission-retry.test.ts index 2b019bbd0cb..5090a340d66 100644 --- a/test/e2e/support/e2e-collaborator-permission-retry.test.ts +++ b/test/e2e/support/e2e-collaborator-permission-retry.test.ts @@ -25,11 +25,6 @@ type PermissionScenario = | "transport-exhaustion"; const AUTHORIZATION_STEPS: AuthorizationStep[] = [ - { - deniedMessage: "Manual PR E2E requires a repository maintainer or administrator", - mismatchMessage: "Manual PR E2E permission response did not match the actor", - name: "Authenticate manual PR dispatch", - }, { deniedMessage: "Launchable E2E requires a repository maintainer or administrator", mismatchMessage: "Launchable E2E permission response did not match the actor", @@ -104,12 +99,6 @@ if [[ "$url" == *"/collaborators/"*"/permission" ]]; then exit "$curl_exit" fi -if [[ "$url" == *"/pulls/42" ]]; then - printf '%s\n' "pull" >>"$CURL_LOG" - printf '%s' '{"state":"open","head":{"repo":{"full_name":"contributor/NemoClaw"},"sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"base":{"sha":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}}' - exit 0 -fi - exit 2 `, ); @@ -133,12 +122,12 @@ printf '%s\n' "$1" >>"$SLEEP_LOG" ALLOW_JETSON_DISPATCH: "false", BASE_SHA: "b".repeat(40), CHECKOUT_REPOSITORY: "contributor/NemoClaw", - CHECKOUT_SHA: stepName === "Authenticate manual PR dispatch" ? "a".repeat(40) : "", + CHECKOUT_SHA: "", CURL_LOG: curlLog, EXPECTED_WORKFLOW_SHA: workflowSha, GITHUB_REPOSITORY: "NVIDIA/NemoClaw", GITHUB_TOKEN: "private-test-token", - INCLUDE_LAUNCHABLE: stepName === "Authenticate manual PR dispatch" ? "false" : "true", + INCLUDE_LAUNCHABLE: "true", JOBS: "", PATH: `${fixture}:${process.env.PATH ?? ""}`, PERMISSION_ATTEMPT_FILE: attemptFile, diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index fd9f4d1bcd1..725ae08a56c 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -85,7 +85,8 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { ["Launchable", "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}"], ])("rejects concurrency that cancels active %s dispatches", (_dispatch, cancellation) => { const workflow = readWorkflow(); - workflow.concurrency!["cancel-in-progress"] = cancellation; + const concurrency = workflow.concurrency as Record; + concurrency["cancel-in-progress"] = cancellation; expect(validateE2eWorkflow(workflow)).toContain( "workflow concurrency must not cancel an active Jetson or Launchable dispatch", From 2cd4543a65bfa59291b99e7727783e200f108eb2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:42:49 -0700 Subject: [PATCH 10/16] fix(e2e): authorize remaining PR credentials Signed-off-by: Carlos Villela --- .github/workflows/e2e.yaml | 8 ++++---- tools/e2e/mcp-dev-workflow-boundary-digests.mts | 2 +- tools/e2e/workflow-boundary.mts | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a1d1b6155a8..bc80863f88c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -2571,9 +2571,9 @@ jobs: name: Authenticate to Docker Hub uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f with: - auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }} - username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }} - token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_TOKEN || '' }} + auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && '1' || '0' }} + username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_USERNAME || '' }} + token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && secrets.DOCKERHUB_TOKEN || '' }} - name: Configure live E2E trace directory env: @@ -4740,7 +4740,7 @@ jobs: - name: Run Hermes live Vitest test env: - NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} + NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts diff --git a/tools/e2e/mcp-dev-workflow-boundary-digests.mts b/tools/e2e/mcp-dev-workflow-boundary-digests.mts index 452d8523b4d..59e26a09c77 100644 --- a/tools/e2e/mcp-dev-workflow-boundary-digests.mts +++ b/tools/e2e/mcp-dev-workflow-boundary-digests.mts @@ -10,7 +10,7 @@ export const MCP_DEV_JOB_EXECUTION_CONTEXT_SHA256 = export const MCP_DEV_TRUSTED_NODE_SETUP_CONTENT_SHA256 = "504821ad93c57971d0281ef1130ed6008fadd331bd56acb1a6b5e6a3358f3e49"; export const MCP_DEV_TRUSTED_PREFIX_CONTENT_SHA256 = - "c559e6cd5bf076bed8d359bbca397d4e31fbf3c11123389425917b865544940d"; + "067df18297c3b5e5175dc11de071a0f3c261aa894db6141b8b789d67f5e9c0d1"; export const MCP_DEV_POST_INSTALL_TRANSITION_CONTENT_SHA256 = "62cf2ee01ac7192f41fc7b2b071de729da8bacec1e4f693da1ec6f0b1f4723c0"; diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index be084ee0562..4d8432347ba 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -189,11 +189,11 @@ const DOCKER_HUB_CLEANUP_KEYS = ["if", "name", "run", "shell"]; // The general E2E workflow runs on push/manual dispatch. Its event set is // intentionally distinct from the reusable image workflow's push/manual boundary. const TRUSTED_DOCKER_HUB_PREDICATE = - "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == ''"; + "github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true')"; const GUARDED_DOCKER_HUB_AUTH_REQUIRED = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && '1' || '0' }}`; const GUARDED_DOCKER_HUB_USERNAME = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_USERNAME || '' }}`; const GUARDED_DOCKER_HUB_TOKEN = `\${{ ${TRUSTED_DOCKER_HUB_PREDICATE} && secrets.DOCKERHUB_TOKEN || '' }}`; -const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; +const GUARDED_HERMES_E2E_INFERENCE_KEY = `\${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}`; const RUNNER_ROUTING_OUTPUT = "${{ steps.runner_routing.outputs.runner_routing }}"; const RUNNER_ROUTING_STEP_NAME = "Build trusted larger-runner routing"; const RUNNER_ROUTING_SCRIPT = [ @@ -1586,7 +1586,7 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== GUARDED_HERMES_E2E_INFERENCE_KEY) { errors.push( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a trusted main-branch dispatch without a PR checkout and the inference mode condition", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch and the inference mode condition", ); } requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); From 72fdc79a322fe50547f5ac8d571f2ad06bf9fc48 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:45:48 -0700 Subject: [PATCH 11/16] test(e2e): clarify authorized credential cases Signed-off-by: Carlos Villela --- test/e2e/support/hermes-workflow-boundary.test.ts | 4 ++-- tools/e2e/workflow-boundary.mts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index 67d3f18e9f8..0a312e49187 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -219,7 +219,7 @@ describe("Hermes GPU boundary", () => { expect(errors).toEqual( expect.arrayContaining([ expect.stringContaining( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a direct main dispatch or an authorized NVIDIA-owned PR dispatch", ), ]), ); @@ -237,7 +237,7 @@ describe("Hermes GPU boundary", () => { expect(errors).toEqual( expect.arrayContaining([ expect.stringContaining( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a direct main dispatch or an authorized NVIDIA-owned PR dispatch", ), ]), ); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 4d8432347ba..13205a837d6 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1586,7 +1586,7 @@ function validateHermesE2EJob(errors: string[], jobs: WorkflowRecord): void { const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== GUARDED_HERMES_E2E_INFERENCE_KEY) { errors.push( - "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind an authorized trusted-main dispatch and the inference mode condition", + "hermes-e2e run step must guard NVIDIA_INFERENCE_API_KEY behind a direct main dispatch or an authorized NVIDIA-owned PR dispatch, plus the inference mode condition", ); } requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); From 73d760e8a60614511c4b43bb6060e10c8c5731ce Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:46:49 -0700 Subject: [PATCH 12/16] test(e2e): name unauthorized credential case Signed-off-by: Carlos Villela --- test/e2e/support/hermes-workflow-boundary.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/support/hermes-workflow-boundary.test.ts b/test/e2e/support/hermes-workflow-boundary.test.ts index 0a312e49187..b487b4683e1 100644 --- a/test/e2e/support/hermes-workflow-boundary.test.ts +++ b/test/e2e/support/hermes-workflow-boundary.test.ts @@ -225,7 +225,7 @@ describe("Hermes GPU boundary", () => { ); }); - it("rejects live secret exposure to a PR checkout", () => { + it("rejects live secret exposure to a PR checkout without authorization", () => { const errors = wfErrors((workflow) => { const run = step(workflow.jobs["hermes-e2e"], "Run Hermes live Vitest test"); run.env = { From dc8665967124a702a3ba2f90ae28659a69224250 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:52:37 -0700 Subject: [PATCH 13/16] fix(e2e): restrict external default plan --- test/e2e/support/workflow-plan.test.ts | 36 ++++++++++++++++++++++++-- tools/e2e/workflow-plan.mts | 16 +++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 7ed97b57ad4..04578f17071 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -447,7 +447,7 @@ describe("E2E workflow plan", () => { } }); - it("omits credentialed catalogue profiles for an untrusted candidate", () => { + it("limits an unauthorized candidate without selectors to credential-free matrices", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-fork-")); const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); @@ -463,12 +463,44 @@ describe("E2E workflow plan", () => { }, ); - expect(readFileSync(output, "utf8").split("\n")).toEqual( + const outputLines = readFileSync(output, "utf8").split("\n"); + expect(outputLines).toEqual( expect.arrayContaining([ "catalogue_nvidia_api_matrix=[]", "catalogue_nvidia_inference_matrix=[]", "catalogue_github_read_matrix=[]", "catalogue_brave_nvidia_inference_matrix=[]", + "selected_jobs=[]", + "hermes_selected=false", + ]), + ); + expect(outputLines).not.toContain("catalogue_standard_matrix=[]"); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + + it("retains controller-approved jobs for an unauthorized candidate", () => { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-fork-jobs-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + try { + writeE2eWorkflowPlanCiOutput( + { jobs: "cloud-onboard,hermes-e2e" }, + { + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + NEMOCLAW_E2E_CREDENTIALS_ALLOWED: "false", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }, + ); + + const outputLines = readFileSync(output, "utf8").split("\n"); + expect(outputLines).toEqual( + expect.arrayContaining([ + 'selected_jobs=["cloud-onboard","hermes-e2e"]', + "hermes_selected=true", ]), ); } finally { diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index 54f7ccf0639..ff0147db959 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -609,7 +609,10 @@ function expectedHermesSelection( return (selected.length === 0 && !retiredSelectorSelected) || selected.includes(HERMES_JOB_ID); } -function withoutCredentialedCatalogueProfiles(plan: E2eWorkflowPlan): E2eWorkflowPlan { +function restrictUnauthorizedCandidatePlan( + plan: E2eWorkflowPlan, + hasPlannerSelectors: boolean, +): E2eWorkflowPlan { const eligibleRows = (rows: E2eCatalogueMatrixRow[]) => rows.filter((row) => isPrCandidateCatalogueTarget(catalogueTarget(row.id))); return { @@ -620,6 +623,8 @@ function withoutCredentialedCatalogueProfiles(plan: E2eWorkflowPlan): E2eWorkflo eligibleRows(plan.catalogueMatrices[profile]), ]), ) as Record, + selectedJobs: hasPlannerSelectors ? plan.selectedJobs : [], + hermesSelected: hasPlannerSelectors && plan.hermesSelected, }; } @@ -667,13 +672,16 @@ export function writeE2eWorkflowPlanCiOutput( const credentialsAllowed = environment.NEMOCLAW_E2E_CREDENTIALS_ALLOWED === "true"; const plan = validateE2eWorkflowPlan( candidateRevision && !credentialsAllowed - ? withoutCredentialedCatalogueProfiles(planned) + ? restrictUnauthorizedCandidatePlan(planned, hasPlannerSelectors) : planned, ); + const expectedHermes = + candidateRevision && !credentialsAllowed && !hasPlannerSelectors + ? false + : expectedHermesSelection(plannerSelectors, controllerMap.retiredSelectorSelected); if ( !changedFiles && - plan.hermesSelected !== - expectedHermesSelection(plannerSelectors, controllerMap.retiredSelectorSelected) + plan.hermesSelected !== expectedHermes ) { throw new Error("E2E planner changed the trusted Hermes selection"); } From b3e3d9cefe10b5ceec5b9dbccf4015e827619820 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 22:54:28 -0700 Subject: [PATCH 14/16] test(e2e): use approved external selector --- test/e2e/support/workflow-plan.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 04578f17071..cf09f337196 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -486,7 +486,7 @@ describe("E2E workflow plan", () => { const summary = path.join(directory, "summary.md"); try { writeE2eWorkflowPlanCiOutput( - { jobs: "cloud-onboard,hermes-e2e" }, + { jobs: "managed-image-protected-runtime" }, { GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, @@ -499,8 +499,8 @@ describe("E2E workflow plan", () => { const outputLines = readFileSync(output, "utf8").split("\n"); expect(outputLines).toEqual( expect.arrayContaining([ - 'selected_jobs=["cloud-onboard","hermes-e2e"]', - "hermes_selected=true", + 'selected_jobs=["managed-image-protected-runtime"]', + "hermes_selected=false", ]), ); } finally { From f7c0ee0c9ebadf8554a11a52ebf25a35ec4addc8 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 23:07:53 -0700 Subject: [PATCH 15/16] fix(e2e): enforce PR dispatch boundaries --- .github/workflows/e2e.yaml | 5 +- .../e2e-operations-workflow-boundary.test.ts | 113 ++++++++++++++++++ .../support/jetson-workflow-boundary.test.ts | 12 +- tools/e2e/operations-workflow-boundary.mts | 21 ++-- tools/e2e/workflow-boundary.mts | 3 +- 5 files changed, 136 insertions(+), 18 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index bc80863f88c..6f903cd341c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -87,7 +87,7 @@ permissions: concurrency: group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && format('full-{0}', github.run_id)) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }} - cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }} + cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !inputs.include_staging_brev_launchable }} env: NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }} @@ -212,7 +212,7 @@ jobs: - id: candidate_authorization name: Authenticate manual PR dispatch - if: ${{ inputs.checkout_sha != '' }} + if: ${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }} env: BASE_SHA: ${{ inputs.base_sha }} CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }} @@ -466,6 +466,7 @@ jobs: - name: Check out trusted E2E planner uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + repository: ${{ github.repository }} ref: ${{ github.workflow_sha }} fetch-depth: 0 persist-credentials: false diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index 94d9883aff3..c731703eb2b 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -264,6 +264,35 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); + it("runs authentication for any candidate identity input and permits an empty group", () => { + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const validationError = + "Manual PR authentication must run when any candidate identity input is present"; + + expect(validateE2eOperationsWorkflow(workflow)).not.toContain(validationError); + expect(authentication.if).toBe( + "${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }}", + ); + + authentication.if = "${{ inputs.checkout_sha != '' }}"; + expect(validateE2eOperationsWorkflow(workflow)).toContain(validationError); + }); + + it("pins the trusted planner checkout to the workflow repository", () => { + const workflow = readE2eOperationsWorkflow(); + const checkout = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Check out trusted E2E planner", + )!; + checkout.with!.repository = "untrusted/repository"; + + expect(validateE2eOperationsWorkflow(workflow)).toContain( + "generate-matrix checkout must use the selected PR source repository", + ); + }); + it("rejects changes that bypass E2E credential authorization (#9047)", () => { const workflow = readE2eOperationsWorkflow(); delete workflow.jobs["generate-matrix"].outputs!.e2e_credentials_allowed; @@ -495,6 +524,90 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; }, ); + it.each([ + [ + "an invalid source repository name", + "invalid-repository", + "a", + "b", + "c", + "::error::checkout_repository must be an owner/repository name\n", + ], + [ + "a PR commit mismatch", + "NVIDIA/NemoClaw", + "d", + "b", + "c", + "::error::checkout_sha must match the latest PR commit SHA\n", + ], + [ + "a PR base commit mismatch", + "NVIDIA/NemoClaw", + "a", + "d", + "c", + "::error::base_sha must match the PR base SHA\n", + ], + [ + "a trusted workflow commit mismatch", + "NVIDIA/NemoClaw", + "a", + "b", + "d", + "::error::workflow_sha must match the trusted main workflow SHA\n", + ], + ] as const)( + "rejects manual PR authentication for %s", + ( + _caseName, + requestedRepository, + requestedHeadCharacter, + requestedBaseCharacter, + expectedWorkflowCharacter, + expectedStderr, + ) => { + const apiHeadSha = "a".repeat(40); + const apiBaseSha = "b".repeat(40); + const workflowSha = "c".repeat(40); + const workflow = readE2eOperationsWorkflow(); + const authentication = workflow.jobs["generate-matrix"].steps!.find( + (step) => step.name === "Authenticate manual PR dispatch", + )!; + const prefix = [ + "curl() {", + ` printf '%s' '{"state":"open","head":{"repo":{"full_name":"NVIDIA/NemoClaw","owner":{"login":"NVIDIA","type":"Organization"}},"sha":"${apiHeadSha}"},"base":{"repo":{"full_name":"NVIDIA/NemoClaw"},"ref":"main","sha":"${apiBaseSha}"}}'`, + "}", + ].join("\n"); + const result = spawnSync( + "bash", + ["--noprofile", "--norc", "-e", "-o", "pipefail", "-c", `${prefix}\n${authentication.run}`], + { + encoding: "utf8", + env: { + ...process.env, + BASE_SHA: requestedBaseCharacter.repeat(40), + CHECKOUT_REPOSITORY: requestedRepository, + CHECKOUT_SHA: requestedHeadCharacter.repeat(40), + EXPECTED_WORKFLOW_SHA: expectedWorkflowCharacter.repeat(40), + GITHUB_OUTPUT: "/dev/null", + GITHUB_REPOSITORY: "NVIDIA/NemoClaw", + GITHUB_TOKEN: "token", + INCLUDE_LAUNCHABLE: "false", + JOBS: "", + PR_NUMBER: "42", + WORKFLOW_EVENT: "workflow_dispatch", + WORKFLOW_REF: "refs/heads/main", + WORKFLOW_SHA: workflowSha, + }, + }, + ); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(expectedStderr); + }, + ); + it.each([ ["NVIDIA inclusion flag", "NVIDIA/NemoClaw", "NVIDIA", "Organization", "true", "", 0, ""], [ diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 725ae08a56c..3f5e656a0c0 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -78,19 +78,17 @@ describe("Jetson nvmap GPU E2E workflow boundary", () => { }); it.each([ - [ - "Jetson", - "${{ inputs.checkout_sha != '' && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}", - ], + ["Jetson", "${{ inputs.checkout_sha != '' && !inputs.include_staging_brev_launchable }}"], ["Launchable", "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}"], ])("rejects concurrency that cancels active %s dispatches", (_dispatch, cancellation) => { const workflow = readWorkflow(); + const validationError = + "workflow concurrency must not cancel an active Jetson or Launchable dispatch"; + expect(validateE2eWorkflow(workflow)).not.toContain(validationError); const concurrency = workflow.concurrency as Record; concurrency["cancel-in-progress"] = cancellation; - expect(validateE2eWorkflow(workflow)).toContain( - "workflow concurrency must not cancel an active Jetson or Launchable dispatch", - ); + expect(validateE2eWorkflow(workflow)).toContain(validationError); }); it("rejects candidate execution or credential-bearing controller steps (#8142)", () => { diff --git a/tools/e2e/operations-workflow-boundary.mts b/tools/e2e/operations-workflow-boundary.mts index b5144844d86..d4ee5dcb91f 100644 --- a/tools/e2e/operations-workflow-boundary.mts +++ b/tools/e2e/operations-workflow-boundary.mts @@ -261,9 +261,11 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow } if ( workflow.concurrency?.["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !inputs.include_staging_brev_launchable }}" ) { - errors.push("Manual PR E2E concurrency must not cancel an active Jetson or Launchable dispatch"); + errors.push( + "Manual PR E2E concurrency must not cancel an active Jetson or Launchable dispatch", + ); } const matrixJob = workflow.jobs["generate-matrix"] ?? {}; @@ -294,9 +296,10 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const authentication = authenticationIndex >= 0 ? steps[authenticationIndex] : {}; if ( authentication.id !== "candidate_authorization" || - authentication.if !== "${{ inputs.checkout_sha != '' }}" + authentication.if !== + "${{ inputs.pr_number != '' || inputs.checkout_sha != '' || inputs.checkout_repository != '' || inputs.base_sha != '' || inputs.workflow_sha != '' }}" ) { - errors.push("Manual PR authentication must be activated only by checkout_sha"); + errors.push("Manual PR authentication must run when any candidate identity input is present"); } const authEnvironment = { BASE_SHA: "${{ inputs.base_sha }}", @@ -304,8 +307,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow CHECKOUT_SHA: "${{ inputs.checkout_sha }}", EXPECTED_WORKFLOW_SHA: "${{ inputs.workflow_sha }}", GITHUB_TOKEN: "${{ github.token }}", - INCLUDE_LAUNCHABLE: - "${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}", + INCLUDE_LAUNCHABLE: "${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}", JOBS: "${{ inputs.jobs }}", PR_NUMBER: "${{ inputs.pr_number }}", WORKFLOW_EVENT: "${{ github.event_name }}", @@ -372,7 +374,9 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow errors.push("Manual PR checkout validation must skip qualification producer dispatches"); } const validationSource = String(validation.run ?? ""); - if (validation.env?.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}") { + if ( + validation.env?.NVIDIA_OWNED !== "${{ steps.candidate_authorization.outputs.nvidia_owned }}" + ) { errors.push("Manual PR checkout validation must bind authenticated NVIDIA ownership"); } for (const fragment of [ @@ -447,6 +451,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow const trustedE2ePlannerCheckout = jobName === "generate-matrix" && step.name === "Check out trusted E2E planner" && + step.with?.repository === "${{ github.repository }}" && step.with?.ref === "${{ github.workflow_sha }}"; const trustedReportHelperCheckout = jobName === "report-to-pr" && @@ -557,7 +562,7 @@ function validateManualPrDispatch(errors: string[], workflow: OperationsWorkflow !trustedCheckout && step.with?.repository !== "${{ inputs.checkout_repository || github.repository }}" ) { - errors.push(`${jobName} checkout must use the selected PR head repository`); + errors.push(`${jobName} checkout must use the selected PR source repository`); } } } diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 13205a837d6..729e8aaaef8 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1777,7 +1777,7 @@ function validateFullE2eConcurrency(errors: string[], workflow: WorkflowRecord): } if ( concurrency["cancel-in-progress"] !== - "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && inputs.jobs != 'staging-brev-launchable' && !inputs.include_staging_brev_launchable }}" + "${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch && !contains(format(',{0},', inputs.jobs), ',staging-brev-launchable,') && !inputs.include_staging_brev_launchable }}" ) { errors.push("workflow concurrency must not cancel an active Jetson or Launchable dispatch"); } @@ -2142,6 +2142,7 @@ function validateTrustedE2ePlannerBoundary( requireFullShaAction(errors, trustedPlannerCheckout, "trusted E2E planner checkout"); if ( !isDeepStrictEqual(asRecord(trustedPlannerCheckout?.with), { + repository: "${{ github.repository }}", ref: "${{ github.workflow_sha }}", "fetch-depth": 0, "persist-credentials": false, From 16915966deac8a496a6e3c28ee55a0e07fd02ac1 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Mon, 17 Aug 2026 23:10:13 -0700 Subject: [PATCH 16/16] test(e2e): name direct main authentication case --- test/e2e/support/e2e-operations-workflow-boundary.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/support/e2e-operations-workflow-boundary.test.ts b/test/e2e/support/e2e-operations-workflow-boundary.test.ts index c731703eb2b..b85d4d034f4 100644 --- a/test/e2e/support/e2e-operations-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-operations-workflow-boundary.test.ts @@ -264,7 +264,7 @@ const interpolatedNeeds = \${{ toJSON ( needs ) }}; ); }); - it("runs authentication for any candidate identity input and permits an empty group", () => { + it("runs authentication when any candidate identity input is present and skips it when all are empty", () => { const workflow = readE2eOperationsWorkflow(); const authentication = workflow.jobs["generate-matrix"].steps!.find( (step) => step.name === "Authenticate manual PR dispatch",