fix(relay): async provisioning (VM + LKE) so a dropped connection is recoverable

.claude/integrations/slack/relay/relay.js

@@ -6,7 +6,7 @@ import https from "node:https";
 import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { execFile } from "node:child_process";
+import { execFile, spawn } from "node:child_process";
 import { promisify } from "node:util";
 
 const { App } = pkg;
@@ -353,29 +353,84 @@ const SAFE_ID = (v) => {
   v = String(v || "");
   return /^[A-Za-z0-9._-]+$/.test(v) && v !== "." && v !== ".." && !v.includes("..");
 };
+const readIf = (p) => { try { return fs.readFileSync(p, "utf8").trim(); } catch { return null; } };
+
+// Every per-run artifact a build can leave behind. A reclaim wipes ALL of them
+// so a replacement build never serves the previous run's creds/cluster/logs.
+const RUN_ARTIFACTS = ["status", "summary.env", "ip", "exec_token", "exec_cert.pem", "cluster_id", "kubeconfig.yaml", "provision.log", "pods.txt", "events.txt", "describe.txt", "expires_epoch"];
+// Atomic single-flight claim for a provisioning run dir, guarded by an exclusive
+// mkdir lock (mkdir is atomic) so two concurrent kickoffs for the same run_id
+// can't both claim/reclaim. Inside the lock we re-read state: a run is busy if a
+// build is in flight (started, no terminal status, younger than capSec);
+// otherwise it's a fresh dir or a terminal/stale run, which we reclaim by wiping
+// every prior artifact and recording the initiating actor. claimRun is fully
+// synchronous, so the lock is always released within the same tick.
+// Returns { busy:true } or { ok:true }.
+function claimRun(rd, by, capSec) {
+  fs.mkdirSync(rd, { recursive: true });
+  try { fs.mkdirSync(`${rd}/.lock`); } catch (e) { if (e.code === "EEXIST") return { busy: true }; throw e; }
+  try {
+    const now = Math.floor(Date.now() / 1000);
+    const started = Number(readIf(`${rd}/started`)) || 0;
+    if (fs.existsSync(`${rd}/started`) && !readIf(`${rd}/status`) && now - started < capSec) return { busy: true };
+    for (const f of RUN_ARTIFACTS) { try { fs.rmSync(`${rd}/${f}`, { force: true }); } catch {} }
+    fs.writeFileSync(`${rd}/started`, String(now));
+    fs.writeFileSync(`${rd}/actor`, String(by));
+    return { ok: true };
+  } finally {
+    try { fs.rmSync(`${rd}/.lock`, { recursive: true, force: true }); } catch {}
+  }
+}
+// Bind result reads to the initiating actor: a shared RELAY_KEY + a guessable
+// run_id must not let another roster user read someone else's creds/kubeconfig.
+// A run with no recorded owner is treated as unauthorized, never world-readable.
+const ownerOk = (rd, by) => { const o = readIf(`${rd}/actor`); return o != null && o === String(by); };
 
 async function brokerLinode(action, m, by) {
   if (!process.env.LINODE_TOKEN) return { status: 503, body: "linode_not_configured" };
   if (action === "provision") {
     const { role, run_id, ttl_hours, pmm_qa_ref } = m;
     if (!SAFE_ID(role)) return { status: 400, body: "bad_role" };
     if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" };
+    const rd = `${RUNNER_DIR}/runs/${run_id}`;
+    // Same async model as provision-lke: a VM build can brush past the ~5-min
+    // connection cut, and the exec creds only come back in the final response.
+    // Kick off detached, return the run_id now, poll /linode/provision-result.
+    if (claimRun(rd, by, 900).busy) {
+      return { status: 409, json: true, body: JSON.stringify({ run_id, status: "provisioning", hint: "already running — poll /linode/provision-result" }) };
+    }
     const args = [`${RUNNER_DIR}/up.sh`, String(role), String(run_id)];
     if (ttl_hours != null && Number.isFinite(Number(ttl_hours))) args.push("-var", `ttl_hours=${Number(ttl_hours)}`);
-    const env = { ...process.env, PMM_QA_REF: pmm_qa_ref ? String(pmm_qa_ref) : "main", CLAUDE_CODE_SESSION_ID: `relay:${by}` };
-    console.log(`linode/provision ${role} ${run_id} (ttl=${ttl_hours ?? 24}) by ${by}`);
-    try {
-      await execFileP("bash", args, { env, timeout: 480000, maxBuffer: 10 * 1024 * 1024 });
-      const rd = `${RUNNER_DIR}/runs/${run_id}`;
-      const ip = fs.readFileSync(`${rd}/ip`, "utf8").trim();
-      const exec_token = fs.readFileSync(`${rd}/exec_token`, "utf8").trim();
-      const exec_cert_pem = fs.readFileSync(`${rd}/exec_cert.pem`, "utf8"); // public cert, run.sh pins it
-      return { status: 200, json: true, body: JSON.stringify({ run_id, ip, exec_token, exec_cert_pem }) };
-    } catch (e) {
-      const tail = String(e.stderr || e.stdout || e.message || "").slice(-1500);
-      console.error(`linode/provision failed: ${e.message}\n${tail}`);
-      return { status: 502, json: true, body: JSON.stringify({ error: "provision_failed", detail: tail }) };
+    const env = { ...process.env, PMM_QA_REF: pmm_qa_ref ? String(pmm_qa_ref) : "main", CLAUDE_CODE_SESSION_ID: `relay:${by}`, RUN_DIR: rd };
+    // Detached wrapper: cap at 12 min, tee to provision.log, write a terminal
+    // status file. `ready` requires all three creds so a partial write never reads ready.
+    const wrapper =
+      'timeout 720 bash "$@" >>"$RUN_DIR/provision.log" 2>&1; ec=$?; ' +
+      'if [ "$ec" -eq 0 ] && [ -s "$RUN_DIR/ip" ] && [ -s "$RUN_DIR/exec_token" ] && [ -s "$RUN_DIR/exec_cert.pem" ]; then echo ready >"$RUN_DIR/status"; ' +
+      'else echo "failed:$ec" >"$RUN_DIR/status"; fi';
+    const child = spawn("bash", ["-c", wrapper, "_", ...args], { env, detached: true, stdio: "ignore" });
+    child.unref();
+    console.log(`linode/provision ${role} ${run_id} started (ttl=${ttl_hours ?? 24}) by ${by}`);
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", poll: "/linode/provision-result" }) };
+  }
+  if (action === "provision-result") {
+    const { run_id } = m;
+    if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" };
+    const rd = `${RUNNER_DIR}/runs/${run_id}`;
+    if (!fs.existsSync(rd)) return { status: 404, body: "unknown_run" };
+    if (!ownerOk(rd, by)) return { status: 403, body: "not_your_run" };
+    const status = readIf(`${rd}/status`);
+    if (status === "ready") {
+      const ip = readIf(`${rd}/ip`);
+      const exec_token = readIf(`${rd}/exec_token`);
+      const exec_cert_pem = fs.existsSync(`${rd}/exec_cert.pem`) ? fs.readFileSync(`${rd}/exec_cert.pem`, "utf8") : null;
+      return { status: 200, json: true, body: JSON.stringify({ run_id, status: "ready", ip, exec_token, exec_cert_pem }) };
     }
+    if (status && status.startsWith("failed")) {
+      const tail = (readIf(`${rd}/provision.log`) || "").slice(-1500);
+      return { status: 502, json: true, body: JSON.stringify({ run_id, status, error: "provision_failed", detail: tail }) };
+    }
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning" }) };
   }
   if (action === "destroy") {
     const { run_id } = m;
@@ -400,7 +455,14 @@ async function brokerLinode(action, m, by) {
     const ttlH = ttl_hours != null && Number.isFinite(Number(ttl_hours)) && Number(ttl_hours) > 0 ? Number(ttl_hours) : LKE_DEFAULT_TTL_H;
     const expiresEpoch = Math.floor(Date.now() / 1000) + Math.round(ttlH * 3600);
     const runDir = `${LKE_RUNS_DIR}/${run_id}`;
-    fs.mkdirSync(runDir, { recursive: true });
+    // An LKE HA build takes 10–20 min, but the kubeconfig only comes back in the
+    // final response and a silent long-held connection gets cut by intermediaries
+    // at ~5 min. So provisioning is ASYNC: kick off a detached build, return the
+    // run_id immediately, and let the caller poll /linode/lke-result. A dropped
+    // connection is then fully recoverable — all state lives in runDir on this box.
+    if (claimRun(runDir, by, 3300).busy) {
+      return { status: 409, json: true, body: JSON.stringify({ run_id, status: "provisioning", hint: "already running — poll /linode/lke-result" }) };
+    }
     // Optional passthrough config — light validation, then handed to the script as env vars.
     const cfg = {};
     const pass = (k, envk, re) => { const v = m[k]; if (v != null && (!re || re.test(String(v)))) cfg[envk] = String(v); };
@@ -419,25 +481,44 @@ async function brokerLinode(action, m, by) {
       if (m[key]) { const p = `${runDir}/${fname}`; fs.writeFileSync(p, Buffer.from(String(m[key]), "base64")); cfg[envk] = p; }
     }
     const env = { ...process.env, LINODE_CLI_TOKEN: process.env.LINODE_TOKEN, RUN_ID: String(run_id), RUN_DIR: runDir, TTL_HOURS: String(ttlH), EXPIRES_EPOCH: String(expiresEpoch), CLAUDE_CODE_SESSION_ID: `relay:${by}`, ...cfg };
-    console.log(`linode/provision-lke ${run_id} (ttl=${ttlH}h, expires=${expiresEpoch}) by ${by}`);
-    try {
-      await execFileP("bash", [`${HA_DIR}/create-lke-pmm-ha.sh`], { env, timeout: 1_500_000, maxBuffer: 20 * 1024 * 1024 });
+    fs.writeFileSync(`${runDir}/expires_epoch`, String(expiresEpoch));
+    // Detached wrapper: cap the build at 50 min, tee to provision.log, and write a
+    // terminal `status` file (ready | failed:<code>) the poller reads. `ready`
+    // requires BOTH result artifacts so a partial run never reads ready. unref()
+    // so the build outlives both this request and a relay restart.
+    const wrapper =
+      'timeout 3000 bash "$0" >>"$RUN_DIR/provision.log" 2>&1; ec=$?; ' +
+      'if [ "$ec" -eq 0 ] && [ -s "$RUN_DIR/summary.env" ] && [ -s "$RUN_DIR/kubeconfig.yaml" ]; then echo ready >"$RUN_DIR/status"; ' +
+      'else echo "failed:$ec" >"$RUN_DIR/status"; fi';
+    const child = spawn("bash", ["-c", wrapper, `${HA_DIR}/create-lke-pmm-ha.sh`], { env, detached: true, stdio: "ignore" });
+    child.unref();
+    console.log(`linode/provision-lke ${run_id} started (ttl=${ttlH}h, expires=${expiresEpoch}) by ${by}`);
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", expires_epoch: expiresEpoch, ttl_hours: ttlH, poll: "/linode/lke-result" }) };
+  }
+  if (action === "lke-result") {
+    const { run_id } = m;
+    if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" };
+    const runDir = `${LKE_RUNS_DIR}/${run_id}`;
+    if (!fs.existsSync(runDir)) return { status: 404, body: "unknown_run" };
+    if (!ownerOk(runDir, by)) return { status: 403, body: "not_your_run" };
+    const status = readIf(`${runDir}/status`);
+    const cluster_id = readIf(`${runDir}/cluster_id`);
+    const expiresEpoch = Number(readIf(`${runDir}/expires_epoch`)) || null;
+    if (status === "ready" && fs.existsSync(`${runDir}/summary.env`)) {
       const summary = {};
       for (const line of fs.readFileSync(`${runDir}/summary.env`, "utf8").split("\n")) { const i = line.indexOf("="); if (i > 0) summary[line.slice(0, i)] = line.slice(i + 1).trim(); }
       const kubeconfig_b64 = fs.readFileSync(`${runDir}/kubeconfig.yaml`).toString("base64");
-      const cluster_id = fs.readFileSync(`${runDir}/cluster_id`, "utf8").trim();
-      return { status: 200, json: true, body: JSON.stringify({ run_id, cluster_id, expires_epoch: expiresEpoch, ttl_hours: ttlH, external_ip: summary.external_ip, url: summary.url, kubeconfig_b64, passwords: summary }) };
-    } catch (e) {
-      const tail = String(e.stderr || e.stdout || e.message || "").slice(-2000);
-      console.error(`linode/provision-lke failed: ${e.message}\n${tail}`);
-      // Best-effort: if the cluster was created before the failure, delete it now so
-      // a broken run doesn't wait for the reaper's TTL to stop billing.
-      try {
-        const cid = fs.readFileSync(`${runDir}/cluster_id`, "utf8").trim();
-        if (cid) { console.error(`linode/provision-lke: tearing down partial cluster ${cid}`); await execFileP("bash", [`${HA_DIR}/destroy-lke.sh`, cid], { env, timeout: 240000 }).catch(() => {}); }
-      } catch {}
-      return { status: 502, json: true, body: JSON.stringify({ error: "provision_lke_failed", detail: tail }) };
+      const pods = readIf(`${runDir}/pods.txt`); // HA pod snapshot the create script captured (kubectl runs on the relay, not the caller)
+      return { status: 200, json: true, body: JSON.stringify({ run_id, status: "ready", cluster_id, expires_epoch: expiresEpoch, external_ip: summary.external_ip, url: summary.url, kubeconfig_b64, passwords: summary, pods }) };
+    }
+    if (status && status.startsWith("failed")) {
+      const tail = (readIf(`${runDir}/provision.log`) || "").slice(-2000);
+      const pods = readIf(`${runDir}/pods.txt`); // diagnostics the create script captures on exit
+      const describe = (readIf(`${runDir}/describe.txt`) || "").slice(-6000);
+      return { status: 502, json: true, body: JSON.stringify({ run_id, status, cluster_id, error: "provision_lke_failed", detail: tail, pods, describe }) };
     }
+    // still building — surface a coarse phase so the caller can log progress
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", phase: cluster_id ? "installing" : "creating-cluster", cluster_id, expires_epoch: expiresEpoch }) };
   }
   if (action === "destroy-lke") {
     const { run_id, cluster_id } = m;

.claude/scripts/pmm-ui-login.js

@@ -5,8 +5,11 @@
 // Usage:
 //   PMM_URL='https://<linode-ip>'          node pmm-ui-login.js PMM-14576
 //   ADMIN_PASSWORD='...'                   (optional, defaults to 'pmm3admin!')
-//   PMM_CERT_PATH='runs/<run_id>/pmm_cert.pem'  (required -- pins PMM's own
-//     cert instead of trusting any cert; see pmm-linode-docker-provisioning skill)
+//   PMM_CERT_PATH='runs/<run_id>/pmm_cert.pem'  (pins PMM's own cert instead of
+//     trusting any cert; see linode-docker-provisioning skill) — REQUIRED unless
+//   PMM_UI_INSECURE=1  (HA/LKE only: PMM's cert is self-signed behind the egress
+//     MITM so pinning can't match; disables TLS verification, keeps the
+//     origin-redirect guard)
 //
 // Writes a reusable Playwright storage state to
 // .claude/scripts/.sessions/<SESSION_ID>.json — pass that file to a
@@ -55,24 +58,33 @@ async function main() {
 
   const authToken = Buffer.from(`admin:${adminPassword}`).toString("base64");
 
-  // This script only ever targets PMM, so there's no legitimate case for an
-  // insecure fallback -- require the cert pin rather than silently trusting
-  // any certificate on a connection that's about to carry the admin
-  // password. PMM's cert can't be known before the box exists, but once
-  // PMM is up, its cert is fetched over the already-pinned exec-server (see
-  // pmm-linode-docker-provisioning's readyz step) and passed here via PMM_CERT_PATH.
+  // Single-server (docker) path: PMM's cert is fetched over the already-pinned
+  // exec-server and pinned here (SPKI), so the admin password never rides an
+  // unverified connection. HA/LKE has no exec-server and PMM's cert is
+  // self-signed behind the egress MITM, so pinning can't match — PMM_UI_INSECURE=1
+  // opts out of the pin there. It's explicit and loud, and the origin-redirect
+  // refusal below stays as the compensating control against leaking the password
+  // to the wrong origin.
+  const insecure = process.env.PMM_UI_INSECURE === "1";
   const certPath = process.env.PMM_CERT_PATH;
-  if (!certPath) {
+  let launchArgs = [];
+  if (insecure) {
     console.error(
-      "PMM_CERT_PATH is required -- fetch PMM's cert after readyz (see pmm-linode-docker-provisioning skill) and pass it here.",
+      "WARNING: PMM_UI_INSECURE=1 — TLS verification disabled (no SPKI pin). Use only for HA/LKE (self-signed cert behind the egress MITM).",
     );
-    process.exit(1);
-  }
-  if (!fs.existsSync(certPath)) {
-    console.error(`PMM_CERT_PATH set but not found: ${certPath}`);
-    process.exit(1);
+  } else {
+    if (!certPath) {
+      console.error(
+        "PMM_CERT_PATH is required -- fetch PMM's cert after readyz (see linode-docker-provisioning skill), or set PMM_UI_INSECURE=1 for HA/LKE.",
+      );
+      process.exit(1);
+    }
+    if (!fs.existsSync(certPath)) {
+      console.error(`PMM_CERT_PATH set but not found: ${certPath}`);
+      process.exit(1);
+    }
+    launchArgs = [`--ignore-certificate-errors-spki-list=${spkiPinFromCertFile(certPath)}`];
   }
-  const launchArgs = [`--ignore-certificate-errors-spki-list=${spkiPinFromCertFile(certPath)}`];
 
   // Explicit executablePath: the pre-installed Chromium revision at
   // /opt/pw-browsers can drift from what a freshly `npm install`-ed
@@ -87,7 +99,7 @@ async function main() {
     proxy: proxyOpts.proxy,
   });
   const context = await browser.newContext({
-    ignoreHTTPSErrors: false,
+    ignoreHTTPSErrors: insecure,
     viewport: { width, height },
   });
   const page = await context.newPage();

.claude/scripts/pw-screenshot.js

@@ -5,6 +5,15 @@
 // Usage:
 //   node pw-screenshot.js <url> <output.png> [sessionId]
 //
+// Env knobs:
+//   PMM_UI_INSECURE=1   disable TLS verification (HA/LKE: self-signed cert
+//                       behind the egress MITM — pairs with pmm-ui-login.js's
+//                       same flag)
+//   PW_SCROLL=1         scroll the page top-to-bottom first to force Grafana's
+//                       virtualized panels to render before a fullPage shot
+//   PW_CLICK_TEXT='...' click an element by partial text before capturing
+//   PW_SETTLE_MS, PW_WAIT_SELECTOR, PMM_UI_WIDTH/HEIGHT   as before
+//
 // If <sessionId> is given and .claude/scripts/.sessions/<sessionId>.json
 // exists (written by pmm-ui-login.js), it is reused as the browser context's
 // storage state so PMM pages stay logged in.
@@ -33,21 +42,25 @@ async function main() {
   const width = Number(process.env.PMM_UI_WIDTH || 1920);
   const height = Number(process.env.PMM_UI_HEIGHT || 1080);
 
+  // PMM_UI_INSECURE=1 disables TLS verification (no pin) — for HA/LKE, where
+  // PMM's cert is self-signed and the egress gateway MITMs outbound TLS, so
+  // SPKI-pinning can't match. Strict pinning stays the default everywhere else.
+  const insecure = process.env.PMM_UI_INSECURE === "1";
   // PMM_CERT_PATH (see pmm-ui-login.js) pins PMM's own cert instead of
   // trusting any cert -- optional since this script also screenshots
   // non-PMM pages (e.g. a GitHub Actions run) with a real CA already, which
   // strict verification (the default below) already handles fine.
   const certPath = process.env.PMM_CERT_PATH;
   const launchArgs = [];
-  if (certPath) {
+  if (certPath && !insecure) {
     if (!fs.existsSync(certPath)) {
       console.error(`PMM_CERT_PATH set but not found: ${certPath}`);
       process.exit(1);
     }
     launchArgs.push(`--ignore-certificate-errors-spki-list=${spkiPinFromCertFile(certPath)}`);
   }
 
-  const contextOpts = { ignoreHTTPSErrors: false, viewport: { width, height } };
+  const contextOpts = { ignoreHTTPSErrors: insecure, viewport: { width, height } };
   if (sessionId) {
     if (!/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(sessionId)) {
       console.error(`invalid sessionId '${sessionId}' (letters, digits, '_', '-' only)`);
@@ -116,6 +129,31 @@ async function main() {
     await page.waitForTimeout(settleMs);
   }
 
+  // PW_CLICK_TEXT: click an element by (partial) text before capturing — e.g. to
+  // open a collapsed row or expand a section on the dashboard.
+  const clickText = process.env.PW_CLICK_TEXT;
+  if (clickText) {
+    try {
+      await page.getByText(clickText, { exact: false }).first().click({ timeout: 8000 });
+      await page.waitForTimeout(2500);
+    } catch (e) {
+      console.error("click failed:", clickText, e.message);
+    }
+  }
+
+  // PW_SCROLL=1: Grafana virtualizes panels, so a fullPage shot of a tall HA
+  // dashboard misses panels that never scrolled into view. Scroll through to
+  // force lazy render, then return to the top before capturing.
+  if (process.env.PW_SCROLL === "1") {
+    const scrollH = await page.evaluate(() => document.body.scrollHeight);
+    for (let y = 0; y < scrollH; y += 700) {
+      await page.evaluate((yy) => window.scrollTo(0, yy), y);
+      await page.waitForTimeout(1000);
+    }
+    await page.evaluate(() => window.scrollTo(0, 0));
+    await page.waitForTimeout(1500);
+  }
+
   fs.mkdirSync(path.dirname(path.resolve(outputPath)), { recursive: true });
   await page.screenshot({ path: outputPath, fullPage: true });