fix(relay): async provisioning (VM + LKE) so a dropped connection is recoverable

.claude/integrations/slack/relay/relay.js

@@ -6,7 +6,7 @@ import https from "node:https";
 import crypto from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
-import { execFile } from "node:child_process";
+import { execFile, spawn } from "node:child_process";
 import { promisify } from "node:util";
 
 const { App } = pkg;
@@ -353,29 +353,78 @@ const SAFE_ID = (v) => {
   v = String(v || "");
   return /^[A-Za-z0-9._-]+$/.test(v) && v !== "." && v !== ".." && !v.includes("..");
 };
+const readIf = (p) => { try { return fs.readFileSync(p, "utf8").trim(); } catch { return null; } };
+
+// Atomic single-flight claim for a provisioning run dir. `started` doubles as an
+// O_EXCL lock: a concurrent kickoff for the same run_id fails the exclusive
+// create and is rejected as busy, unless the existing run is terminal (status
+// written) or stale (older than capSec, i.e. its build cap has elapsed), in
+// which case it is reclaimed. Also records the initiating actor so result reads
+// can be bound to it. Returns { busy:true } or { ok:true }.
+function claimRun(rd, by, capSec) {
+  fs.mkdirSync(rd, { recursive: true });
+  const now = Math.floor(Date.now() / 1000);
+  try {
+    fs.writeFileSync(`${rd}/started`, String(now), { flag: "wx" });
+  } catch (e) {
+    if (e.code !== "EEXIST") throw e;
+    const started = Number(readIf(`${rd}/started`)) || 0;
+    if (!readIf(`${rd}/status`) && now - started < capSec) return { busy: true };
+    fs.writeFileSync(`${rd}/started`, String(now)); // reclaim a terminal/stale run
+  }
+  for (const f of ["status", "summary.env"]) { try { fs.rmSync(`${rd}/${f}`, { force: true }); } catch {} }
+  fs.writeFileSync(`${rd}/actor`, String(by));
+  return { ok: true };
+}
+// Bind result reads to the initiating actor: a shared RELAY_KEY + a guessable
+// run_id must not let another roster user read someone else's creds/kubeconfig.
+const ownerOk = (rd, by) => { const o = readIf(`${rd}/actor`); return !o || o === String(by); };
 
 async function brokerLinode(action, m, by) {
   if (!process.env.LINODE_TOKEN) return { status: 503, body: "linode_not_configured" };
   if (action === "provision") {
     const { role, run_id, ttl_hours, pmm_qa_ref } = m;
     if (!SAFE_ID(role)) return { status: 400, body: "bad_role" };
     if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" };
+    const rd = `${RUNNER_DIR}/runs/${run_id}`;
+    // Same async model as provision-lke: a VM build can brush past the ~5-min
+    // connection cut, and the exec creds only come back in the final response.
+    // Kick off detached, return the run_id now, poll /linode/provision-result.
+    if (claimRun(rd, by, 900).busy) {
+      return { status: 409, json: true, body: JSON.stringify({ run_id, status: "provisioning", hint: "already running — poll /linode/provision-result" }) };
+    }
     const args = [`${RUNNER_DIR}/up.sh`, String(role), String(run_id)];
     if (ttl_hours != null && Number.isFinite(Number(ttl_hours))) args.push("-var", `ttl_hours=${Number(ttl_hours)}`);
-    const env = { ...process.env, PMM_QA_REF: pmm_qa_ref ? String(pmm_qa_ref) : "main", CLAUDE_CODE_SESSION_ID: `relay:${by}` };
-    console.log(`linode/provision ${role} ${run_id} (ttl=${ttl_hours ?? 24}) by ${by}`);
-    try {
-      await execFileP("bash", args, { env, timeout: 480000, maxBuffer: 10 * 1024 * 1024 });
-      const rd = `${RUNNER_DIR}/runs/${run_id}`;
-      const ip = fs.readFileSync(`${rd}/ip`, "utf8").trim();
-      const exec_token = fs.readFileSync(`${rd}/exec_token`, "utf8").trim();
-      const exec_cert_pem = fs.readFileSync(`${rd}/exec_cert.pem`, "utf8"); // public cert, run.sh pins it
-      return { status: 200, json: true, body: JSON.stringify({ run_id, ip, exec_token, exec_cert_pem }) };
-    } catch (e) {
-      const tail = String(e.stderr || e.stdout || e.message || "").slice(-1500);
-      console.error(`linode/provision failed: ${e.message}\n${tail}`);
-      return { status: 502, json: true, body: JSON.stringify({ error: "provision_failed", detail: tail }) };
+    const env = { ...process.env, PMM_QA_REF: pmm_qa_ref ? String(pmm_qa_ref) : "main", CLAUDE_CODE_SESSION_ID: `relay:${by}`, RUN_DIR: rd };
+    // Detached wrapper: cap at 12 min, tee to provision.log, write a terminal
+    // status file. `ready` requires all three creds so a partial write never reads ready.
+    const wrapper =
+      'timeout 720 bash "$@" >>"$RUN_DIR/provision.log" 2>&1; ec=$?; ' +
+      'if [ "$ec" -eq 0 ] && [ -s "$RUN_DIR/ip" ] && [ -s "$RUN_DIR/exec_token" ] && [ -s "$RUN_DIR/exec_cert.pem" ]; then echo ready >"$RUN_DIR/status"; ' +
+      'else echo "failed:$ec" >"$RUN_DIR/status"; fi';
+    const child = spawn("bash", ["-c", wrapper, "_", ...args], { env, detached: true, stdio: "ignore" });
+    child.unref();
+    console.log(`linode/provision ${role} ${run_id} started (ttl=${ttl_hours ?? 24}) by ${by}`);
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", poll: "/linode/provision-result" }) };
+  }
+  if (action === "provision-result") {
+    const { run_id } = m;
+    if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" };
+    const rd = `${RUNNER_DIR}/runs/${run_id}`;
+    if (!fs.existsSync(rd)) return { status: 404, body: "unknown_run" };
+    if (!ownerOk(rd, by)) return { status: 403, body: "not_your_run" };
+    const status = readIf(`${rd}/status`);
+    if (status === "ready") {
+      const ip = readIf(`${rd}/ip`);
+      const exec_token = readIf(`${rd}/exec_token`);
+      const exec_cert_pem = fs.existsSync(`${rd}/exec_cert.pem`) ? fs.readFileSync(`${rd}/exec_cert.pem`, "utf8") : null;
+      return { status: 200, json: true, body: JSON.stringify({ run_id, status: "ready", ip, exec_token, exec_cert_pem }) };
     }
+    if (status && status.startsWith("failed")) {
+      const tail = (readIf(`${rd}/provision.log`) || "").slice(-1500);
+      return { status: 502, json: true, body: JSON.stringify({ run_id, status, error: "provision_failed", detail: tail }) };
+    }
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning" }) };
   }
   if (action === "destroy") {
     const { run_id } = m;
@@ -400,7 +449,14 @@ async function brokerLinode(action, m, by) {
     const ttlH = ttl_hours != null && Number.isFinite(Number(ttl_hours)) && Number(ttl_hours) > 0 ? Number(ttl_hours) : LKE_DEFAULT_TTL_H;
     const expiresEpoch = Math.floor(Date.now() / 1000) + Math.round(ttlH * 3600);
     const runDir = `${LKE_RUNS_DIR}/${run_id}`;
-    fs.mkdirSync(runDir, { recursive: true });
+    // An LKE HA build takes 10–20 min, but the kubeconfig only comes back in the
+    // final response and a silent long-held connection gets cut by intermediaries
+    // at ~5 min. So provisioning is ASYNC: kick off a detached build, return the
+    // run_id immediately, and let the caller poll /linode/lke-result. A dropped
+    // connection is then fully recoverable — all state lives in runDir on this box.
+    if (claimRun(runDir, by, 3300).busy) {
+      return { status: 409, json: true, body: JSON.stringify({ run_id, status: "provisioning", hint: "already running — poll /linode/lke-result" }) };
+    }
     // Optional passthrough config — light validation, then handed to the script as env vars.
     const cfg = {};
     const pass = (k, envk, re) => { const v = m[k]; if (v != null && (!re || re.test(String(v)))) cfg[envk] = String(v); };
@@ -419,25 +475,44 @@ async function brokerLinode(action, m, by) {
       if (m[key]) { const p = `${runDir}/${fname}`; fs.writeFileSync(p, Buffer.from(String(m[key]), "base64")); cfg[envk] = p; }
     }
     const env = { ...process.env, LINODE_CLI_TOKEN: process.env.LINODE_TOKEN, RUN_ID: String(run_id), RUN_DIR: runDir, TTL_HOURS: String(ttlH), EXPIRES_EPOCH: String(expiresEpoch), CLAUDE_CODE_SESSION_ID: `relay:${by}`, ...cfg };
-    console.log(`linode/provision-lke ${run_id} (ttl=${ttlH}h, expires=${expiresEpoch}) by ${by}`);
-    try {
-      await execFileP("bash", [`${HA_DIR}/create-lke-pmm-ha.sh`], { env, timeout: 1_500_000, maxBuffer: 20 * 1024 * 1024 });
+    fs.writeFileSync(`${runDir}/expires_epoch`, String(expiresEpoch));
+    // Detached wrapper: cap the build at 50 min, tee to provision.log, and write a
+    // terminal `status` file (ready | failed:<code>) the poller reads. `ready`
+    // requires BOTH result artifacts so a partial run never reads ready. unref()
+    // so the build outlives both this request and a relay restart.
+    const wrapper =
+      'timeout 3000 bash "$0" >>"$RUN_DIR/provision.log" 2>&1; ec=$?; ' +
+      'if [ "$ec" -eq 0 ] && [ -s "$RUN_DIR/summary.env" ] && [ -s "$RUN_DIR/kubeconfig.yaml" ]; then echo ready >"$RUN_DIR/status"; ' +
+      'else echo "failed:$ec" >"$RUN_DIR/status"; fi';
+    const child = spawn("bash", ["-c", wrapper, `${HA_DIR}/create-lke-pmm-ha.sh`], { env, detached: true, stdio: "ignore" });
+    child.unref();
+    console.log(`linode/provision-lke ${run_id} started (ttl=${ttlH}h, expires=${expiresEpoch}) by ${by}`);
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", expires_epoch: expiresEpoch, ttl_hours: ttlH, poll: "/linode/lke-result" }) };
+  }
+  if (action === "lke-result") {
+    const { run_id } = m;
+    if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" };
+    const runDir = `${LKE_RUNS_DIR}/${run_id}`;
+    if (!fs.existsSync(runDir)) return { status: 404, body: "unknown_run" };
+    if (!ownerOk(runDir, by)) return { status: 403, body: "not_your_run" };
+    const status = readIf(`${runDir}/status`);
+    const cluster_id = readIf(`${runDir}/cluster_id`);
+    const expiresEpoch = Number(readIf(`${runDir}/expires_epoch`)) || null;
+    if (status === "ready" && fs.existsSync(`${runDir}/summary.env`)) {
       const summary = {};
       for (const line of fs.readFileSync(`${runDir}/summary.env`, "utf8").split("\n")) { const i = line.indexOf("="); if (i > 0) summary[line.slice(0, i)] = line.slice(i + 1).trim(); }
       const kubeconfig_b64 = fs.readFileSync(`${runDir}/kubeconfig.yaml`).toString("base64");
-      const cluster_id = fs.readFileSync(`${runDir}/cluster_id`, "utf8").trim();
-      return { status: 200, json: true, body: JSON.stringify({ run_id, cluster_id, expires_epoch: expiresEpoch, ttl_hours: ttlH, external_ip: summary.external_ip, url: summary.url, kubeconfig_b64, passwords: summary }) };
-    } catch (e) {
-      const tail = String(e.stderr || e.stdout || e.message || "").slice(-2000);
-      console.error(`linode/provision-lke failed: ${e.message}\n${tail}`);
-      // Best-effort: if the cluster was created before the failure, delete it now so
-      // a broken run doesn't wait for the reaper's TTL to stop billing.
-      try {
-        const cid = fs.readFileSync(`${runDir}/cluster_id`, "utf8").trim();
-        if (cid) { console.error(`linode/provision-lke: tearing down partial cluster ${cid}`); await execFileP("bash", [`${HA_DIR}/destroy-lke.sh`, cid], { env, timeout: 240000 }).catch(() => {}); }
-      } catch {}
-      return { status: 502, json: true, body: JSON.stringify({ error: "provision_lke_failed", detail: tail }) };
+      const pods = readIf(`${runDir}/pods.txt`); // HA pod snapshot the create script captured (kubectl runs on the relay, not the caller)
+      return { status: 200, json: true, body: JSON.stringify({ run_id, status: "ready", cluster_id, expires_epoch: expiresEpoch, external_ip: summary.external_ip, url: summary.url, kubeconfig_b64, passwords: summary, pods }) };
+    }
+    if (status && status.startsWith("failed")) {
+      const tail = (readIf(`${runDir}/provision.log`) || "").slice(-2000);
+      const pods = readIf(`${runDir}/pods.txt`); // diagnostics the create script captures on exit
+      const describe = (readIf(`${runDir}/describe.txt`) || "").slice(-6000);
+      return { status: 502, json: true, body: JSON.stringify({ run_id, status, cluster_id, error: "provision_lke_failed", detail: tail, pods, describe }) };
     }
+    // still building — surface a coarse phase so the caller can log progress
+    return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", phase: cluster_id ? "installing" : "creating-cluster", cluster_id, expires_epoch: expiresEpoch }) };
   }
   if (action === "destroy-lke") {
     const { run_id, cluster_id } = m;

.claude/skills/linode-docker-provisioning/SKILL.md

@@ -55,25 +55,47 @@ mkdir -p "$RUN_DIR"
 ACTOR="$(gh api user --jq .login 2>/dev/null)"
 
 # ttl_hours + pmm_qa_ref are optional; add keep-alive handling below.
-curl -sS -m 600 --fail-with-body -X POST "$RELAY/linode/provision" \
+# 1) Kick off the build — returns immediately with {run_id, status:"provisioning"}.
+curl -sS -m 60 --fail-with-body -X POST "$RELAY/linode/provision" \
   -H "X-Relay-Secret: $RELAY_KEY" -H "X-Actor: $ACTOR" -H "Content-Type: application/json" \
-  -d "$(jq -n --arg r "$ROLE" --arg id "$RUN_ID" '{role:$r, run_id:$id}')" >"$RUN_DIR/provision.json"
-
-# Unpack what the session-side helpers (run.sh/sync.sh/extend.sh) need locally,
-# and tag the run so the SessionEnd hook can tear down exactly our own VMs.
-jq -r .ip            "$RUN_DIR/provision.json" >"$RUN_DIR/ip"
-jq -r .exec_token    "$RUN_DIR/provision.json" >"$RUN_DIR/exec_token"; chmod 600 "$RUN_DIR/exec_token"
-jq -r .exec_cert_pem "$RUN_DIR/provision.json" >"$RUN_DIR/exec_cert.pem"
-printf '%s' "$RELAY"                      >"$RUN_DIR/relay"        # marks this run relay-brokered (holds the relay URL for the SessionEnd hook)
+  -d "$(jq -n --arg r "$ROLE" --arg id "$RUN_ID" '{role:$r, run_id:$id}')" >"$RUN_DIR/provision-start.json"
+
+# Mark the run relay-brokered NOW so the SessionEnd hook can tear it down even if we lose the poll.
+printf '%s' "$RELAY"                      >"$RUN_DIR/relay"        # relay URL for the SessionEnd hook
 printf '%s' "${CLAUDE_CODE_SESSION_ID:-}" >"$RUN_DIR/session_id"   # scopes the SessionEnd hook
+
+# 2) Poll for the result — a dropped connection is recoverable (state is on the relay).
+deadline=$(( $(date +%s) + 900 ))
+while :; do
+  code=$(curl -sS -m 60 -o "$RUN_DIR/provision.json" -w '%{http_code}' -X POST "$RELAY/linode/provision-result" \
+    -H "X-Relay-Secret: $RELAY_KEY" -H "X-Actor: $ACTOR" -H "Content-Type: application/json" \
+    -d "$(jq -n --arg id "$RUN_ID" '{run_id:$id}')")
+  case "$code" in
+    200) echo "VM ready"; break;;
+    202) echo "provisioning…";;
+    502) echo "provisioning FAILED:"; jq -r '.detail // .' "$RUN_DIR/provision.json"; break;;
+    *)   echo "unexpected $code:"; cat "$RUN_DIR/provision.json";;
+  esac
+  [ "$(date +%s)" -lt "$deadline" ] || { echo "timed out"; break; }
+  sleep 15
+done
+
+# 3) Unpack what the session-side helpers (run.sh/sync.sh/extend.sh) need locally.
+if jq -e .exec_token "$RUN_DIR/provision.json" >/dev/null 2>&1; then
+  jq -r .ip            "$RUN_DIR/provision.json" >"$RUN_DIR/ip"
+  jq -r .exec_token    "$RUN_DIR/provision.json" >"$RUN_DIR/exec_token"; chmod 600 "$RUN_DIR/exec_token"
+  jq -r .exec_cert_pem "$RUN_DIR/provision.json" >"$RUN_DIR/exec_cert.pem"
+else
+  echo "no exec creds — tear the run down (Cleanup) before retrying"
+fi
 ```
 
 `role` is `test-runner` or `investigator` (free text, just for the tag). The relay:
 - Creates a Linode VM (default `g6-standard-6`, Ubuntu 24.04) with a firewall open only on 443, tagged `pmm-qa-ephemeral`.
 - Waits for the exec-server to answer, then for cloud-init to finish installing Docker + Ansible and scheduling its own self-destruct timer (default 24h — see Cleanup below).
 - `git clone`s `percona/pmm-qa` onto the box at `/root/pmm-qa` — `main` by default, or pass `"pmm_qa_ref":"<branch>"` in the POST body (must already be pushed; see "Never code on the Linode VM" above).
 
-Works from the **default** proxied-HTTPS environment — no special network policy needed. Takes 2-4 minutes; the relay call blocks until the box is fully ready. After this, `run.sh`/`sync.sh`/`extend.sh` are addressed exactly as before by `<run_id>` — they use the local `exec_token` + `exec_cert.pem`, never the account token. **Teardown is the exception:** it goes through the relay's `/linode/destroy` (see Cleanup), not a local `down.sh`, since destroying the VM needs the account token that no longer lives in this environment.
+Works from the **default** proxied-HTTPS environment — no special network policy needed. Provisioning is **async**: the first call returns a `run_id`, then you poll `/linode/provision-result` until `200` (ready — creds in the body) or `502` (failed). The build runs on the relay and all state lives in its run dir, so a dropped connection is recoverable by re-polling the same `run_id` (usually 2-4 min). After this, `run.sh`/`sync.sh`/`extend.sh` are addressed exactly as before by `<run_id>` — they use the local `exec_token` + `exec_cert.pem`, never the account token. **Teardown is the exception:** it goes through the relay's `/linode/destroy` (see Cleanup), not a local `down.sh`, since destroying the VM needs the account token that no longer lives in this environment.
 
 **Keep-alive:** for an explicit "leave it running" request, add `"ttl_hours":<N>` to the POST body **and** `touch "$RUN_DIR/keep-alive"` — the marker tells the SessionEnd hook to leave this VM up (its on-box self-destruct timer still reaps it after `ttl_hours`).