Skip to content

fix(e2e): add Brev readiness diagnostics #9226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cv merged 6 commits into from
Aug 16, 2026
+425 −162
Merged

fix(e2e): add Brev readiness diagnostics #9226

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
037d8cc
fix(e2e): add Brev readiness diagnostics
jyaunches Aug 15, 2026
ced1b99
test(e2e): cover readiness diagnostic markers
jyaunches Aug 15, 2026
8d270ff
fix(e2e): bound Brev failure diagnostics
jyaunches Aug 15, 2026
600b715
fix(e2e): clarify SSH alias diagnostics
jyaunches Aug 15, 2026
6091e70
test(e2e): separate SSH alias failure states
jyaunches Aug 15, 2026
0039238
fix(e2e): mark incomplete readiness diagnostics
jyaunches Aug 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 123 additions & 65 deletions test/brev-launchable-e2e.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ function executable(file: string, source: string): void {
function fixture(
options: {
bootImage?: string;
brevContainerStatus?: number;
brevHostStatus?: number;
deleteFails?: boolean;
e2eFails?: boolean;
imageRepositorySha?: string;
Expand All @@ -38,8 +40,11 @@ function fixture(
repoSha?: string;
runtimeOverrides?: boolean;
schemaVersion?: number;
hostAliasConfigured?: boolean;
plainAliasConfigured?: boolean;
sshDefaultStatus?: number;
sshHostError?: string;
sshHostProbeStatus?: number;
sshReadyAfter?: number;
sourceRepository?: string;
sourcePath?: string;
Expand All @@ -52,6 +57,7 @@ function fixture(
const workDir = path.join(root, "evidence");
const state = path.join(root, "workspace.json");
const calls = path.join(root, "calls.log");
const diagnosticPhase = path.join(root, "diagnostic-phase");
const refreshAttempts = path.join(root, "refresh-attempts");
const sshAttempts = path.join(root, "ssh-attempts");
const timeoutBlock = path.join(root, "timeout-block");
Expand Down Expand Up @@ -133,6 +139,14 @@ case "$1" in
jq -cn --arg name "$INSTANCE_NAME" --arg shell "$shell" --arg build "$build" \
'{workspaces:[{id:"ws-1",name:$name,status:"RUNNING",shell_status:$shell,build_status:$build}]}' > "$FAKE_STATE" ;;
exec)
if [ "\${3:-}" = true ]; then
if [ "\${4:-}" = --host ]; then
[ "$FAKE_BREV_HOST_STATUS" -eq 0 ] || printf '%s\n' "$FAKE_BREV_HOST_ERROR" >&2
exit "$FAKE_BREV_HOST_STATUS"
fi
[ "$FAKE_BREV_CONTAINER_STATUS" -eq 0 ] || printf '%s\n' "$FAKE_BREV_CONTAINER_ERROR" >&2
exit "$FAKE_BREV_CONTAINER_STATUS"
fi
case "$3" in
*NEMOCLAW_BOOT_IMAGE*)
printf 'NEMOCLAW_BOOT_IMAGE=%s\\n' "$FAKE_BOOT_IMAGE"
Expand Down Expand Up @@ -174,6 +188,20 @@ esac
path.join(bin, "ssh"),
`#!/usr/bin/env bash
set -euo pipefail
if [ "\${1:-}" = -G ]; then
touch "$FAKE_DIAGNOSTIC_PHASE"
alias="\${2:-}"
configured=0
if [ "$alias" = "$INSTANCE_NAME" ]; then
configured="$FAKE_PLAIN_ALIAS_CONFIGURED"
elif [ "$alias" = "$INSTANCE_NAME-host" ]; then
configured="$FAKE_HOST_ALIAS_CONFIGURED"
fi
if [ "$configured" = 1 ]; then hostname=203.0.113.20; else hostname="$alias"; fi
printf 'hostname %s\n' "$hostname"
printf 'user hidden-user\nidentityfile /hidden/private-key\nproxycommand none\n'
exit 0
fi
if [ "\${*: -1}" = true ]; then
required=(-T "-o BatchMode=yes" "-o ConnectTimeout=10" "-o ConnectionAttempts=1" "-o NumberOfPasswordPrompts=0" "-o RequestTTY=no" "-o LogLevel=ERROR")
for argument in "\${required[@]}"; do
Expand All @@ -189,6 +217,13 @@ if [ "\${*: -1}" = true ]; then
exit 0
fi
[ "$target" = "$INSTANCE_NAME-host" ]
if [ -f "$FAKE_DIAGNOSTIC_PHASE" ]; then
if [ "$FAKE_SSH_HOST_PROBE_STATUS" -ne 0 ]; then
printf '%s\n' "$FAKE_SSH_HOST_ERROR" >&2
exit "$FAKE_SSH_HOST_PROBE_STATUS"
fi
exit 0
fi
attempts=0
[ ! -f "$FAKE_SSH_ATTEMPTS" ] || attempts="$(cat "$FAKE_SSH_ATTEMPTS")"
attempts=$((attempts + 1))
Expand Down Expand Up @@ -225,15 +260,24 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n'
CANDIDATE_SHA: candidateSha,
CORRELATION_ID: "11111111-1111-4111-8111-111111111111",
FAKE_BOOT_IMAGE: options.bootImage ?? "projects/brevdevprod/global/images/nemoclaw-test-image",
FAKE_BREV_CONTAINER_ERROR:
"Brev container safe detail; credential=container-secret; endpoint=container.hidden.internal",
FAKE_BREV_CONTAINER_STATUS: String(options.brevContainerStatus ?? 31),
FAKE_BREV_HOST_ERROR:
"Brev host safe detail; token=host-token; endpoint=host-exec.hidden.internal",
FAKE_BREV_HOST_STATUS: String(options.brevHostStatus ?? 32),
FAKE_CALLS: calls,
FAKE_DELETE_FAILS: options.deleteFails ? "1" : "0",
FAKE_DIAGNOSTIC_PHASE: diagnosticPhase,
FAKE_E2E_FAILS: options.e2eFails ? "1" : "0",
FAKE_IMAGE_REPOSITORY_SHA: options.imageRepositorySha ?? "b".repeat(40),
FAKE_HOST_ALIAS_CONFIGURED: options.hostAliasConfigured === false ? "0" : "1",
FAKE_MISSING_PROVISION_RECEIPT: options.missingProvisionReceipt ? "1" : "0",
FAKE_OMIT_RECEIPT_FIELD: options.omitReceiptField ?? "",
FAKE_PROVISION_IMAGE_REPOSITORY_SHA:
options.provisionImageRepositorySha ?? options.imageRepositorySha ?? "b".repeat(40),
FAKE_PROVISION_SHA: options.provisionSha ?? candidateSha,
FAKE_PLAIN_ALIAS_CONFIGURED: options.plainAliasConfigured === false ? "0" : "1",
FAKE_READY: options.ready === false ? "0" : "1",
FAKE_RECEIPT_SHA: options.receiptSha ?? candidateSha,
FAKE_REPO_CLEAN: options.repoClean === false ? "false" : "true",
Expand All @@ -252,6 +296,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n'
FAKE_SSH_HOST_ERROR:
options.sshHostError ??
"ssh: Could not resolve hostname host.hidden.internal: host SSH safe detail; password=ssh-secret; identityfile=/hidden/private-key",
FAKE_SSH_HOST_PROBE_STATUS: String(options.sshHostProbeStatus ?? 34),
FAKE_SSH_READY_AFTER: String(options.sshReadyAfter ?? 1),
FAKE_SOURCE_REPOSITORY: options.sourceRepository ?? "NVIDIA/NemoClaw",
FAKE_SOURCE_PATH: options.sourcePath ?? "/opt/nemoclaw-image/NemoClaw",
Expand All @@ -268,7 +313,7 @@ printf 'NEMOCLAW_FULL_E2E_PASSED\\n'
RUNNER_TEMP: root,
WORK_DIR: workDir,
};
return { calls, env, sshAttempts, state, workDir };
return { calls, env, refreshAttempts, sshAttempts, state, workDir };
}

function run(env: NodeJS.ProcessEnv) {
Expand All @@ -281,7 +326,9 @@ function emittedOutput(result: ReturnType<typeof run>, workDir: string): string

describe("focused staging Brev Launchable lane", () => {
it("binds the producer run, verifies the clean booted SHA, runs E2E, and deletes (#6943)", () => {
const { calls, env, sshAttempts, state, workDir } = fixture({ sshReadyAfter: 6 });
const { calls, env, sshAttempts, state, workDir } = fixture({
sshReadyAfter: 6,
});
const result = run(env);
expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0);
const commands = fs.readFileSync(calls, "utf8");
Expand Down Expand Up @@ -315,28 +362,33 @@ describe("focused staging Brev Launchable lane", () => {
.find((line) => line.startsWith("ssh host readiness attempt 1: "));
expect(readinessCall).toBeDefined();
const readinessArgs = readinessCall?.split(": ").at(1)?.split(" ") ?? [];
expect(readinessArgs).toEqual(
expect.arrayContaining([
"-T",
"BatchMode=yes",
"ConnectTimeout=10",
"ConnectionAttempts=1",
"NumberOfPasswordPrompts=0",
"RequestTTY=no",
"LogLevel=ERROR",
]),
);
expect(readinessArgs.slice(-2)).toEqual(["nclaw-e2e-test-1-host", "true"]);
expect(readinessArgs).toEqual([
"-T",
"-o",
"BatchMode=yes",
"-o",
"ConnectTimeout=10",
"-o",
"ConnectionAttempts=1",
"-o",
"NumberOfPasswordPrompts=0",
"-o",
"RequestTTY=no",
"-o",
"LogLevel=ERROR",
"nclaw-e2e-test-1-host",
"true",
]);
expect(fs.readFileSync(sshAttempts, "utf8").trim()).toBe("6");
expect(commands).toContain("ssh preinstalled full-e2e.test.ts");
expect(commands).not.toContain("nvapi-test-value");
expect(commands).not.toMatch(/rsync|install\.sh|npm (?:ci|install)|git clone/u);
expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).not.toMatch(
/last failure|readiness diagnostics|Readiness classification/u,
);
expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).toContain(
"Waiting up to 900 seconds for host SSH access",
);
expect(fs.readFileSync(path.join(workDir, "lane.log"), "utf8")).not.toMatch(
/last failure|Readiness classification/u,
);
expect(fs.existsSync(state)).toBe(false);
expect(fs.readdirSync(workDir).sort()).toEqual([
"cleanup.json",
Expand Down Expand Up @@ -431,61 +483,77 @@ describe("focused staging Brev Launchable lane", () => {
});
});

it("reports only final refresh and direct host SSH failures with bounded diagnostic redaction", () => {
it("reports only the final sanitized refresh and direct host SSH failures", () => {
const { calls, env, state, workDir } = fixture({
refreshError: `refresh safe detail; api_key=brev-test-secret; endpoint=https://refresh.hidden.internal/path; ${"r".repeat(6_000)}`,
plainAliasConfigured: false,
refreshError: `refresh final safe detail\npassword=hunter2\n${"x".repeat(5_000)}`,
refreshStatus: 35,
sshHostError: `ssh: Could not resolve hostname host.hidden.internal: host SSH safe detail; password=ssh-secret; identityfile=/hidden/private-key; Authorization: Bearer short-token; ${"s".repeat(6_000)}`,
sshHostError:
"hidden-user@example.internal: Permission denied (publickey); host SSH final safe detail; password=ssh-secret; identityfile=/hidden/private-key\nAuthorization: Bearer short-token",
sshReadyAfter: Number.MAX_SAFE_INTEGER,
});
const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "5" });
const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "2" });
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("host SSH readiness timed out");
const commands = fs.readFileSync(calls, "utf8");
expect(commands).toContain("timeout 5s ssh -G nclaw-e2e-test-1");
expect(commands).toContain("timeout 5s ssh -G nclaw-e2e-test-1-host");
expect(commands).toContain("timeout 15s brev exec nclaw-e2e-test-1 true");
expect(commands).toContain("timeout 15s brev exec nclaw-e2e-test-1 true --host");
expect(commands).toMatch(
/timeout [1-5]s ssh -T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR nclaw-e2e-test-1 true/u,
/timeout 15s ssh -T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR nclaw-e2e-test-1 true/u,
);
expect(commands).toMatch(
/timeout [1-5]s ssh -T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR nclaw-e2e-test-1-host true/u,
/timeout 15s ssh -T -o BatchMode=yes -o ConnectTimeout=10 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 -o RequestTTY=no -o LogLevel=ERROR nclaw-e2e-test-1-host true/u,
);
expect(commands).not.toMatch(/ssh -G|brev exec .* true --host/u);
expect(commands).not.toMatch(/NEMOCLAW_BOOT_IMAGE|full-e2e\.test\.ts/u);

const output = emittedOutput(result, workDir);
expect(output).toContain(
"Readiness Brev refresh last failure: status 35; error: refresh safe detail",
"Readiness Brev refresh last failure: status 35; error: refresh final safe detail",
);
expect(output).toContain("Readiness direct host SSH last failure: status 34; error:");
expect(output).toContain("host SSH safe detail");
expect(output).toContain(
"Readiness initial default Brev container probe: status 33; error: default SSH safe detail",
);
expect(output).toContain("host SSH final safe detail");
expect(output).toContain("kex_exchange_identification");
expect(output).toContain(
"Readiness classification: initial default Brev container probe failed; direct host SSH did not succeed before deadline",
);
expect(output).toContain("Readiness SSH alias nclaw-e2e-test-1: missing");
expect(output).toContain("Readiness SSH alias nclaw-e2e-test-1-host: configured");
expect(output).toContain("Readiness probe brev exec container: failure; status 31;");
expect(output).toContain("Readiness probe brev exec host: failure; status 32;");
expect(output).toContain("Readiness probe direct SSH container: failure; status 33;");
expect(output).toContain("Readiness probe direct SSH host: failure; status 34;");
expect(output).toContain("Readiness classification: Brev refresh/configuration failure");
expect(output).not.toContain("stale refresh detail");
expect(output).not.toContain("stale host SSH detail");
const errorLines = fs
const diagnosticErrorLines = fs
.readFileSync(path.join(workDir, "lane.log"), "utf8")
.split("\n")
.filter((line) => line.includes("; error:"));
expect(errorLines).not.toHaveLength(0);
for (const line of errorLines) {
expect(Buffer.byteLength(line.split("; error: ", 2)[1] ?? "")).toBeLessThanOrEqual(512);
expect(diagnosticErrorLines).not.toHaveLength(0);
for (const line of diagnosticErrorLines) {
const error = line.split("; error: ", 2)[1]?.replace(/\)$/u, "") ?? "";
expect(Buffer.byteLength(error)).toBeLessThanOrEqual(512);
}
for (const sensitiveValue of [
for (const secretOrConfiguration of [
"brev-test-secret",
"container-secret",
"default-secret",
"host-token",
"ssh-secret",
"short-token",
"hunter2",
"hidden-user",
"github-test-token",
"nvapi-test-value",
"/hidden/private-key",
"host.hidden.internal",
"host-exec.hidden.internal",
"container.hidden.internal",
"refresh.hidden.internal",
"203.0.113.20",
"identityfile /hidden/private-key",
"user hidden-user",
]) {
expect(output).not.toContain(sensitiveValue);
expect(output).not.toContain(secretOrConfiguration);
}
expect(fs.existsSync(state)).toBe(false);
expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({
Expand All @@ -494,28 +562,22 @@ describe("focused staging Brev Launchable lane", () => {
});

it.each([
[
0,
"initial default Brev container probe succeeded; direct host SSH did not succeed before deadline",
],
[
33,
"initial default Brev container probe failed; direct host SSH did not succeed before deadline",
],
[
125,
"initial default Brev container probe failed; direct host SSH did not succeed before deadline",
],
])("classifies initial default container status %i as %s", (sshDefaultStatus, classification) => {
["default container reachable but host unreachable", { brevContainerStatus: 0 }],
["Brev host execution works but direct host SSH fails", { brevHostStatus: 0 }],
["neither target reachable", {}],
])("classifies %s after the shared readiness deadline", (classification, probeOptions) => {
const { calls, env, state, workDir } = fixture({
sshDefaultStatus,
...probeOptions,
sshReadyAfter: Number.MAX_SAFE_INTEGER,
});
const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "5" });
const result = run({ ...env, BREV_HOST_SSH_TIMEOUT_SECONDS: "1" });
expect(result.status).not.toBe(0);
expect(emittedOutput(result, workDir)).toContain(`Readiness classification: ${classification}`);
const commands = fs.readFileSync(calls, "utf8");
expect(commands).toContain("ssh default diagnostic probe");
expect(commands).toContain("timeout 15s brev exec nclaw-e2e-test-1 true");
expect(commands).toContain("timeout 15s brev exec nclaw-e2e-test-1 true --host");
expect(commands).toMatch(/timeout 15s ssh -T .* nclaw-e2e-test-1 true/u);
expect(commands).toMatch(/timeout 15s ssh -T .* nclaw-e2e-test-1-host true/u);
expect(commands).not.toMatch(/NEMOCLAW_BOOT_IMAGE|full-e2e\.test\.ts/u);
expect(fs.existsSync(state)).toBe(false);
expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({
Expand Down Expand Up @@ -554,15 +616,11 @@ describe("focused staging Brev Launchable lane", () => {
const elapsedMs = performance.now() - startedAt;
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("host SSH readiness timed out");
expect(elapsedMs).toBeLessThan(10_000);
expect(emittedOutput(result, workDir)).toContain(
"Readiness classification: default Brev container and direct host SSH were not probed before deadline",
);
expect(elapsedMs).toBeLessThan(80_000);
const commands = fs.readFileSync(calls, "utf8");
expect(commands).toContain("timeout 1s brev refresh");
expect(commands).not.toMatch(
/ssh default diagnostic probe|ssh host readiness attempt|NEMOCLAW_BOOT_IMAGE|full-e2e\.test\.ts/u,
);
expect(commands).not.toMatch(/NEMOCLAW_BOOT_IMAGE|full-e2e\.test\.ts/u);
expect(commands).toContain("timeout 15s brev exec nclaw-e2e-test-1 true");
expect(fs.existsSync(state)).toBe(false);
expect(JSON.parse(fs.readFileSync(path.join(workDir, "cleanup.json"), "utf8"))).toMatchObject({
status: "ABSENT",
Expand All @@ -587,22 +645,22 @@ describe("focused staging Brev Launchable lane", () => {
});
}, 90_000);

it("caps the five-second poll sleep by the shared readiness deadline", () => {
it("caps the poll sleep by the shared readiness deadline", () => {
const { calls, env, state, workDir } = fixture({
sshReadyAfter: Number.MAX_SAFE_INTEGER,
});
const result = run({
...env,
BREV_HOST_SSH_TIMEOUT_SECONDS: "5",
BREV_HOST_SSH_TIMEOUT_SECONDS: "2",
POLL_SECONDS: "9",
});
expect(result.status).not.toBe(0);
const commands = fs.readFileSync(calls, "utf8");
const readinessCommands = commands.slice(
commands.indexOf("timeout 5s brev refresh"),
commands.indexOf("timeout 2s brev refresh"),
commands.indexOf("timeout 60s brev delete"),
);
expect(readinessCommands).toMatch(/sleep [1-5]/u);
expect(readinessCommands).toMatch(/sleep [12]/u);
expect(readinessCommands).not.toContain("sleep 9");
expect(commands).not.toMatch(/NEMOCLAW_BOOT_IMAGE|full-e2e\.test\.ts/u);
expect(fs.existsSync(state)).toBe(false);
Expand Down
Loading
Loading