diff --git a/.claude/agents/investigator.md b/.claude/agents/investigator.md index 7e573ff19..eda27687b 100644 --- a/.claude/agents/investigator.md +++ b/.claude/agents/investigator.md @@ -36,7 +36,7 @@ Once you have what you're investigating and a ref, use marker `## Failures fixed ## Workflow -1. **Dedup (mandatory — stop if this is already tracked)** — start with: fetch open `percona/pmm-qa` PRs and read bodies (title alone isn't enough) with repo-scoped REST (`gh pr list --json` is GraphQL and 403s here — see `repos`): `gh api "repos/percona/pmm-qa/pulls?state=open&per_page=50" --jq '.[] | {number, title, body}'`; if any identifier from the failure list already appears under the `## Failures fixed (investigator)` marker → **stop immediately**, reply with that PR URL. For a question or suspected bug more likely to be a product problem: also check whether an existing Jira ticket already describes the same thing (still sitting in `New` or another non-delivered status) before investigating fresh — if one exists, link it instead of duplicating the work. Either way, if nothing turns up, continue. +1. **Dedup (mandatory — stop if this is already tracked)** — start with: fetch open `percona/pmm-qa` PRs and read bodies (title alone isn't enough) with repo-scoped REST (`gh pr list --json` is GraphQL and 403s here — see `repos`): `gh api "repos/percona/pmm-qa/pulls?state=open&per_page=50" --jq '.[] | {number, title, body}'`; if any identifier from the failure list already appears under the `## Failures fixed (investigator)` marker → **stop immediately**, reply with that PR URL. For a question or suspected bug more likely to be a product problem: also check whether an existing Jira ticket already describes the same thing (still sitting in `New` or another non-delivered status) before investigating fresh — search via the **relay** (`jira` skill → `search` action, JQL, PMM-scoped), **not** the Atlassian MCP (its search needs interactive auth that isn't there in a Routine/headless run). If one exists, link it instead of duplicating the work. Either way, if nothing turns up, continue. 2. **Reproduce** — Follow `linode-docker-provisioning` to bring up a throwaway Linode VM at the given ref, and run the exact command(s) or tests that failed, or walk through the described scenario step by step if this is a question from someone. Watch what actually happens — a question only skips reproduction when reading the code makes the answer completely unambiguous and needs zero investigation; memory alone never counts. 3. **Classify from what you observed** — one decision tree: - **Didn't reproduce at all** → likely an infra flake (known failure) or not enough detail to confirm (a report). Say so and stop. If this was a bug relayed secondhand, ask for more specific repro steps (exact version, exact clicks/commands) rather than guessing further. diff --git a/.claude/integrations/slack/relay/relay.js b/.claude/integrations/slack/relay/relay.js index a1946878c..8e6fb964e 100644 --- a/.claude/integrations/slack/relay/relay.js +++ b/.claude/integrations/slack/relay/relay.js @@ -6,7 +6,7 @@ import https from "node:https"; import crypto from "node:crypto"; import fs from "node:fs"; import path from "node:path"; -import { execFile } from "node:child_process"; +import { execFile, spawn } from "node:child_process"; import { promisify } from "node:util"; const { App } = pkg; @@ -353,6 +353,55 @@ const SAFE_ID = (v) => { v = String(v || ""); return /^[A-Za-z0-9._-]+$/.test(v) && v !== "." && v !== ".." && !v.includes(".."); }; +const readIf = (p) => { try { return fs.readFileSync(p, "utf8").trim(); } catch { return null; } }; +// Index of a top-level JQL `ORDER BY` (outside any quoted string), or -1. Used to keep +// the caller's WHERE clause and any sort clause separate when forcing `project = PMM`. +const findOrderBy = (s) => { + let q = null; + for (let i = 0; i < s.length; i++) { + const ch = s[i]; + if (q) { + if (ch === "\\") { i++; continue; } + if (ch === q) q = null; + } else if (ch === '"' || ch === "'") { + q = ch; + } else if ((ch === "o" || ch === "O") && (i === 0 || /\s/.test(s[i - 1])) && /^order\s+by\b/i.test(s.slice(i))) { + return i; + } + } + return -1; +}; + +// Every per-run artifact a build can leave behind. A reclaim wipes ALL of them +// so a replacement build never serves the previous run's creds/cluster/logs. +const RUN_ARTIFACTS = ["status", "summary.env", "ip", "exec_token", "exec_cert.pem", "cluster_id", "kubeconfig.yaml", "provision.log", "pods.txt", "events.txt", "describe.txt", "expires_epoch"]; +// Atomic single-flight claim for a provisioning run dir, guarded by an exclusive +// mkdir lock (mkdir is atomic) so two concurrent kickoffs for the same run_id +// can't both claim/reclaim. Inside the lock we re-read state: a run is busy if a +// build is in flight (started, no terminal status, younger than capSec); +// otherwise it's a fresh dir or a terminal/stale run, which we reclaim by wiping +// every prior artifact and recording the initiating actor. claimRun is fully +// synchronous, so the lock is always released within the same tick. +// Returns { busy:true } or { ok:true }. +function claimRun(rd, by, capSec) { + fs.mkdirSync(rd, { recursive: true }); + try { fs.mkdirSync(`${rd}/.lock`); } catch (e) { if (e.code === "EEXIST") return { busy: true }; throw e; } + try { + const now = Math.floor(Date.now() / 1000); + const started = Number(readIf(`${rd}/started`)) || 0; + if (fs.existsSync(`${rd}/started`) && !readIf(`${rd}/status`) && now - started < capSec) return { busy: true }; + for (const f of RUN_ARTIFACTS) { try { fs.rmSync(`${rd}/${f}`, { force: true }); } catch {} } + fs.writeFileSync(`${rd}/started`, String(now)); + fs.writeFileSync(`${rd}/actor`, String(by)); + return { ok: true }; + } finally { + try { fs.rmSync(`${rd}/.lock`, { recursive: true, force: true }); } catch {} + } +} +// Bind result reads to the initiating actor: a shared RELAY_KEY + a guessable +// run_id must not let another roster user read someone else's creds/kubeconfig. +// A run with no recorded owner is treated as unauthorized, never world-readable. +const ownerOk = (rd, by) => { const o = readIf(`${rd}/actor`); return o != null && o === String(by); }; async function brokerLinode(action, m, by) { if (!process.env.LINODE_TOKEN) return { status: 503, body: "linode_not_configured" }; @@ -360,22 +409,45 @@ async function brokerLinode(action, m, by) { const { role, run_id, ttl_hours, pmm_qa_ref } = m; if (!SAFE_ID(role)) return { status: 400, body: "bad_role" }; if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" }; + const rd = `${RUNNER_DIR}/runs/${run_id}`; + // Same async model as provision-lke: a VM build can brush past the ~5-min + // connection cut, and the exec creds only come back in the final response. + // Kick off detached, return the run_id now, poll /linode/provision-result. + if (claimRun(rd, by, 900).busy) { + return { status: 409, json: true, body: JSON.stringify({ run_id, status: "provisioning", hint: "already running — poll /linode/provision-result" }) }; + } const args = [`${RUNNER_DIR}/up.sh`, String(role), String(run_id)]; if (ttl_hours != null && Number.isFinite(Number(ttl_hours))) args.push("-var", `ttl_hours=${Number(ttl_hours)}`); - const env = { ...process.env, PMM_QA_REF: pmm_qa_ref ? String(pmm_qa_ref) : "main", CLAUDE_CODE_SESSION_ID: `relay:${by}` }; - console.log(`linode/provision ${role} ${run_id} (ttl=${ttl_hours ?? 24}) by ${by}`); - try { - await execFileP("bash", args, { env, timeout: 480000, maxBuffer: 10 * 1024 * 1024 }); - const rd = `${RUNNER_DIR}/runs/${run_id}`; - const ip = fs.readFileSync(`${rd}/ip`, "utf8").trim(); - const exec_token = fs.readFileSync(`${rd}/exec_token`, "utf8").trim(); - const exec_cert_pem = fs.readFileSync(`${rd}/exec_cert.pem`, "utf8"); // public cert, run.sh pins it - return { status: 200, json: true, body: JSON.stringify({ run_id, ip, exec_token, exec_cert_pem }) }; - } catch (e) { - const tail = String(e.stderr || e.stdout || e.message || "").slice(-1500); - console.error(`linode/provision failed: ${e.message}\n${tail}`); - return { status: 502, json: true, body: JSON.stringify({ error: "provision_failed", detail: tail }) }; + const env = { ...process.env, PMM_QA_REF: pmm_qa_ref ? String(pmm_qa_ref) : "main", CLAUDE_CODE_SESSION_ID: `relay:${by}`, RUN_DIR: rd }; + // Detached wrapper: cap at 12 min, tee to provision.log, write a terminal + // status file. `ready` requires all three creds so a partial write never reads ready. + const wrapper = + 'timeout 720 bash "$@" >>"$RUN_DIR/provision.log" 2>&1; ec=$?; ' + + 'if [ "$ec" -eq 0 ] && [ -s "$RUN_DIR/ip" ] && [ -s "$RUN_DIR/exec_token" ] && [ -s "$RUN_DIR/exec_cert.pem" ]; then echo ready >"$RUN_DIR/status"; ' + + 'else echo "failed:$ec" >"$RUN_DIR/status"; fi'; + const child = spawn("bash", ["-c", wrapper, "_", ...args], { env, detached: true, stdio: "ignore" }); + child.unref(); + console.log(`linode/provision ${role} ${run_id} started (ttl=${ttl_hours ?? 24}) by ${by}`); + return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", poll: "/linode/provision-result" }) }; + } + if (action === "provision-result") { + const { run_id } = m; + if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" }; + const rd = `${RUNNER_DIR}/runs/${run_id}`; + if (!fs.existsSync(rd)) return { status: 404, body: "unknown_run" }; + if (!ownerOk(rd, by)) return { status: 403, body: "not_your_run" }; + const status = readIf(`${rd}/status`); + if (status === "ready") { + const ip = readIf(`${rd}/ip`); + const exec_token = readIf(`${rd}/exec_token`); + const exec_cert_pem = fs.existsSync(`${rd}/exec_cert.pem`) ? fs.readFileSync(`${rd}/exec_cert.pem`, "utf8") : null; + return { status: 200, json: true, body: JSON.stringify({ run_id, status: "ready", ip, exec_token, exec_cert_pem }) }; + } + if (status && status.startsWith("failed")) { + const tail = (readIf(`${rd}/provision.log`) || "").slice(-1500); + return { status: 502, json: true, body: JSON.stringify({ run_id, status, error: "provision_failed", detail: tail }) }; } + return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning" }) }; } if (action === "destroy") { const { run_id } = m; @@ -400,7 +472,14 @@ async function brokerLinode(action, m, by) { const ttlH = ttl_hours != null && Number.isFinite(Number(ttl_hours)) && Number(ttl_hours) > 0 ? Number(ttl_hours) : LKE_DEFAULT_TTL_H; const expiresEpoch = Math.floor(Date.now() / 1000) + Math.round(ttlH * 3600); const runDir = `${LKE_RUNS_DIR}/${run_id}`; - fs.mkdirSync(runDir, { recursive: true }); + // An LKE HA build takes 10–20 min, but the kubeconfig only comes back in the + // final response and a silent long-held connection gets cut by intermediaries + // at ~5 min. So provisioning is ASYNC: kick off a detached build, return the + // run_id immediately, and let the caller poll /linode/lke-result. A dropped + // connection is then fully recoverable — all state lives in runDir on this box. + if (claimRun(runDir, by, 3300).busy) { + return { status: 409, json: true, body: JSON.stringify({ run_id, status: "provisioning", hint: "already running — poll /linode/lke-result" }) }; + } // Optional passthrough config — light validation, then handed to the script as env vars. const cfg = {}; const pass = (k, envk, re) => { const v = m[k]; if (v != null && (!re || re.test(String(v)))) cfg[envk] = String(v); }; @@ -419,25 +498,44 @@ async function brokerLinode(action, m, by) { if (m[key]) { const p = `${runDir}/${fname}`; fs.writeFileSync(p, Buffer.from(String(m[key]), "base64")); cfg[envk] = p; } } const env = { ...process.env, LINODE_CLI_TOKEN: process.env.LINODE_TOKEN, RUN_ID: String(run_id), RUN_DIR: runDir, TTL_HOURS: String(ttlH), EXPIRES_EPOCH: String(expiresEpoch), CLAUDE_CODE_SESSION_ID: `relay:${by}`, ...cfg }; - console.log(`linode/provision-lke ${run_id} (ttl=${ttlH}h, expires=${expiresEpoch}) by ${by}`); - try { - await execFileP("bash", [`${HA_DIR}/create-lke-pmm-ha.sh`], { env, timeout: 1_500_000, maxBuffer: 20 * 1024 * 1024 }); + fs.writeFileSync(`${runDir}/expires_epoch`, String(expiresEpoch)); + // Detached wrapper: cap the build at 50 min, tee to provision.log, and write a + // terminal `status` file (ready | failed:) the poller reads. `ready` + // requires BOTH result artifacts so a partial run never reads ready. unref() + // so the build outlives both this request and a relay restart. + const wrapper = + 'timeout 3000 bash "$0" >>"$RUN_DIR/provision.log" 2>&1; ec=$?; ' + + 'if [ "$ec" -eq 0 ] && [ -s "$RUN_DIR/summary.env" ] && [ -s "$RUN_DIR/kubeconfig.yaml" ]; then echo ready >"$RUN_DIR/status"; ' + + 'else echo "failed:$ec" >"$RUN_DIR/status"; fi'; + const child = spawn("bash", ["-c", wrapper, `${HA_DIR}/create-lke-pmm-ha.sh`], { env, detached: true, stdio: "ignore" }); + child.unref(); + console.log(`linode/provision-lke ${run_id} started (ttl=${ttlH}h, expires=${expiresEpoch}) by ${by}`); + return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", expires_epoch: expiresEpoch, ttl_hours: ttlH, poll: "/linode/lke-result" }) }; + } + if (action === "lke-result") { + const { run_id } = m; + if (!SAFE_ID(run_id)) return { status: 400, body: "bad_run_id" }; + const runDir = `${LKE_RUNS_DIR}/${run_id}`; + if (!fs.existsSync(runDir)) return { status: 404, body: "unknown_run" }; + if (!ownerOk(runDir, by)) return { status: 403, body: "not_your_run" }; + const status = readIf(`${runDir}/status`); + const cluster_id = readIf(`${runDir}/cluster_id`); + const expiresEpoch = Number(readIf(`${runDir}/expires_epoch`)) || null; + if (status === "ready" && fs.existsSync(`${runDir}/summary.env`)) { const summary = {}; for (const line of fs.readFileSync(`${runDir}/summary.env`, "utf8").split("\n")) { const i = line.indexOf("="); if (i > 0) summary[line.slice(0, i)] = line.slice(i + 1).trim(); } const kubeconfig_b64 = fs.readFileSync(`${runDir}/kubeconfig.yaml`).toString("base64"); - const cluster_id = fs.readFileSync(`${runDir}/cluster_id`, "utf8").trim(); - return { status: 200, json: true, body: JSON.stringify({ run_id, cluster_id, expires_epoch: expiresEpoch, ttl_hours: ttlH, external_ip: summary.external_ip, url: summary.url, kubeconfig_b64, passwords: summary }) }; - } catch (e) { - const tail = String(e.stderr || e.stdout || e.message || "").slice(-2000); - console.error(`linode/provision-lke failed: ${e.message}\n${tail}`); - // Best-effort: if the cluster was created before the failure, delete it now so - // a broken run doesn't wait for the reaper's TTL to stop billing. - try { - const cid = fs.readFileSync(`${runDir}/cluster_id`, "utf8").trim(); - if (cid) { console.error(`linode/provision-lke: tearing down partial cluster ${cid}`); await execFileP("bash", [`${HA_DIR}/destroy-lke.sh`, cid], { env, timeout: 240000 }).catch(() => {}); } - } catch {} - return { status: 502, json: true, body: JSON.stringify({ error: "provision_lke_failed", detail: tail }) }; + const pods = readIf(`${runDir}/pods.txt`); // HA pod snapshot the create script captured (kubectl runs on the relay, not the caller) + return { status: 200, json: true, body: JSON.stringify({ run_id, status: "ready", cluster_id, expires_epoch: expiresEpoch, external_ip: summary.external_ip, url: summary.url, kubeconfig_b64, passwords: summary, pods }) }; + } + if (status && status.startsWith("failed")) { + const tail = (readIf(`${runDir}/provision.log`) || "").slice(-2000); + const pods = readIf(`${runDir}/pods.txt`); // diagnostics the create script captures on exit + const describe = (readIf(`${runDir}/describe.txt`) || "").slice(-6000); + return { status: 502, json: true, body: JSON.stringify({ run_id, status, cluster_id, error: "provision_lke_failed", detail: tail, pods, describe }) }; } + // still building — surface a coarse phase so the caller can log progress + return { status: 202, json: true, body: JSON.stringify({ run_id, status: "provisioning", phase: cluster_id ? "installing" : "creating-cluster", cluster_id, expires_epoch: expiresEpoch }) }; } if (action === "destroy-lke") { const { run_id, cluster_id } = m; @@ -521,13 +619,14 @@ setTimeout(reapLke, 60_000).unref(); // first sweep shortly after boot async function brokerJira(action, m, by) { if (!JIRA_EMAIL || !JIRA_API_TOKEN) return { status: 503, body: "jira_not_configured" }; const issue = m.issue; - // Every action except `create` operates on an existing PMM issue. - if (action !== "create" && !/^PMM-\d+$/.test(issue || "")) return { status: 400, body: "issue_must_be_a_PMM_key" }; + // Every action except create/search operates on an existing PMM issue. + if (action !== "create" && action !== "search" && !/^PMM-\d+$/.test(issue || "")) return { status: 400, body: "issue_must_be_a_PMM_key" }; const base = "https://perconadev.atlassian.net/rest/api/2"; const auth = "Basic " + Buffer.from(`${JIRA_EMAIL}:${JIRA_API_TOKEN}`).toString("base64"); + const JIRA_TIMEOUT_MS = 30_000; // bound every Jira call so a hung upstream can't wedge the handler const jira = (path, init = {}) => - fetch(`${base}${path}`, { ...init, headers: { Authorization: auth, "Content-Type": "application/json", Accept: "application/json", ...(init.headers || {}) } }); - console.log(`jira/${action} ${issue || m.summary || ""} by ${by}`); + fetch(`${base}${path}`, { ...init, signal: AbortSignal.timeout(JIRA_TIMEOUT_MS), headers: { Authorization: auth, "Content-Type": "application/json", Accept: "application/json", ...(init.headers || {}) } }); + console.log(`jira/${action} ${issue || m.summary || m.jql || ""} by ${by}`); let r; try { if (action === "create") { @@ -543,6 +642,24 @@ async function brokerJira(action, m, by) { if (!fields.summary || !fields.issuetype?.name) return { status: 400, body: "summary_and_issuetype_required" }; if (fields.issuetype.name === "Bug" && fields.customfield_10059 === undefined) fields.customfield_10059 = [{ value: "Yes" }]; r = await jira(`/issue`, { method: "POST", body: JSON.stringify({ fields }) }); + } else if (action === "search") { + // JQL search, FORCED to the PMM project — lets callers (e.g. Investigator + // dedup) find existing tickets through the relay's service account instead + // of an interactively-authenticated MCP. Read-only, PMM-only. Uses the + // enhanced /search/jql endpoint (classic /search is sunset on Jira Cloud). + const raw = String(m.jql || "").trim(); + // Find ORDER BY only OUTSIDE quoted strings, so a value like `summary ~ "order by"` + // stays in the WHERE clause instead of being split as a sort clause. JQL quotes with + // ' or " and escapes with a backslash. + const oi = findOrderBy(raw); + const where = oi >= 0 ? raw.slice(0, oi).trim() : raw; + const order = oi >= 0 ? raw.slice(oi) : ""; + const jql = `project = PMM${where ? ` AND (${where})` : ""}${order ? ` ${order}` : ""}`; + const maxResults = Math.min(Math.max(Math.floor(Number(m.maxResults) || 20), 1), 100); + const fields = (Array.isArray(m.fields) ? m.fields + : String(m.fields || "summary,status,issuetype,updated").split(",")) + .map((s) => String(s).trim()).filter(Boolean); + r = await jira(`/search/jql`, { method: "POST", body: JSON.stringify({ jql, maxResults, fields }) }); } else if (action === "read") { const fields = m.fieldsCsv || "summary,description,status,customfield_10083,customfield_10492,comment"; r = await jira(`/issue/${issue}?fields=${encodeURIComponent(fields)}`); @@ -558,7 +675,7 @@ async function brokerJira(action, m, by) { } else if (action === "attach") { const fd = new FormData(); fd.append("file", new Blob([Buffer.from(String(m.content_b64 || ""), "base64")]), String(m.filename || "evidence.png")); - r = await fetch(`${base}/issue/${issue}/attachments`, { method: "POST", headers: { Authorization: auth, "X-Atlassian-Token": "no-check" }, body: fd }); + r = await fetch(`${base}/issue/${issue}/attachments`, { method: "POST", signal: AbortSignal.timeout(JIRA_TIMEOUT_MS), headers: { Authorization: auth, "X-Atlassian-Token": "no-check" }, body: fd }); } else { return { status: 400, body: "unknown_action" }; } diff --git a/.claude/scripts/pmm-ui-login.js b/.claude/scripts/pmm-ui-login.js index 59070ec37..b70e4c7de 100644 --- a/.claude/scripts/pmm-ui-login.js +++ b/.claude/scripts/pmm-ui-login.js @@ -5,8 +5,11 @@ // Usage: // PMM_URL='https://' node pmm-ui-login.js PMM-14576 // ADMIN_PASSWORD='...' (optional, defaults to 'pmm3admin!') -// PMM_CERT_PATH='runs//pmm_cert.pem' (required -- pins PMM's own -// cert instead of trusting any cert; see pmm-linode-docker-provisioning skill) +// PMM_CERT_PATH='runs//pmm_cert.pem' (pins PMM's own cert instead of +// trusting any cert; see linode-docker-provisioning skill) — REQUIRED unless +// PMM_UI_INSECURE=1 (HA/LKE only: PMM's cert is self-signed behind the egress +// MITM so pinning can't match; disables TLS verification, keeps the +// origin-redirect guard) // // Writes a reusable Playwright storage state to // .claude/scripts/.sessions/.json — pass that file to a @@ -55,24 +58,33 @@ async function main() { const authToken = Buffer.from(`admin:${adminPassword}`).toString("base64"); - // This script only ever targets PMM, so there's no legitimate case for an - // insecure fallback -- require the cert pin rather than silently trusting - // any certificate on a connection that's about to carry the admin - // password. PMM's cert can't be known before the box exists, but once - // PMM is up, its cert is fetched over the already-pinned exec-server (see - // pmm-linode-docker-provisioning's readyz step) and passed here via PMM_CERT_PATH. + // Single-server (docker) path: PMM's cert is fetched over the already-pinned + // exec-server and pinned here (SPKI), so the admin password never rides an + // unverified connection. HA/LKE has no exec-server and PMM's cert is + // self-signed behind the egress MITM, so pinning can't match — PMM_UI_INSECURE=1 + // opts out of the pin there. It's explicit and loud, and the origin-redirect + // refusal below stays as the compensating control against leaking the password + // to the wrong origin. + const insecure = process.env.PMM_UI_INSECURE === "1"; const certPath = process.env.PMM_CERT_PATH; - if (!certPath) { + let launchArgs = []; + if (insecure) { console.error( - "PMM_CERT_PATH is required -- fetch PMM's cert after readyz (see pmm-linode-docker-provisioning skill) and pass it here.", + "WARNING: PMM_UI_INSECURE=1 — TLS verification disabled (no SPKI pin). Use only for HA/LKE (self-signed cert behind the egress MITM).", ); - process.exit(1); - } - if (!fs.existsSync(certPath)) { - console.error(`PMM_CERT_PATH set but not found: ${certPath}`); - process.exit(1); + } else { + if (!certPath) { + console.error( + "PMM_CERT_PATH is required -- fetch PMM's cert after readyz (see linode-docker-provisioning skill), or set PMM_UI_INSECURE=1 for HA/LKE.", + ); + process.exit(1); + } + if (!fs.existsSync(certPath)) { + console.error(`PMM_CERT_PATH set but not found: ${certPath}`); + process.exit(1); + } + launchArgs = [`--ignore-certificate-errors-spki-list=${spkiPinFromCertFile(certPath)}`]; } - const launchArgs = [`--ignore-certificate-errors-spki-list=${spkiPinFromCertFile(certPath)}`]; // Explicit executablePath: the pre-installed Chromium revision at // /opt/pw-browsers can drift from what a freshly `npm install`-ed @@ -87,7 +99,7 @@ async function main() { proxy: proxyOpts.proxy, }); const context = await browser.newContext({ - ignoreHTTPSErrors: false, + ignoreHTTPSErrors: insecure, viewport: { width, height }, }); const page = await context.newPage(); diff --git a/.claude/scripts/pw-screenshot.js b/.claude/scripts/pw-screenshot.js index bc7621b95..7c0946a14 100644 --- a/.claude/scripts/pw-screenshot.js +++ b/.claude/scripts/pw-screenshot.js @@ -5,6 +5,15 @@ // Usage: // node pw-screenshot.js [sessionId] // +// Env knobs: +// PMM_UI_INSECURE=1 disable TLS verification (HA/LKE: self-signed cert +// behind the egress MITM — pairs with pmm-ui-login.js's +// same flag) +// PW_SCROLL=1 scroll the page top-to-bottom first to force Grafana's +// virtualized panels to render before a fullPage shot +// PW_CLICK_TEXT='...' click an element by partial text before capturing +// PW_SETTLE_MS, PW_WAIT_SELECTOR, PMM_UI_WIDTH/HEIGHT as before +// // If is given and .claude/scripts/.sessions/.json // exists (written by pmm-ui-login.js), it is reused as the browser context's // storage state so PMM pages stay logged in. @@ -33,13 +42,17 @@ async function main() { const width = Number(process.env.PMM_UI_WIDTH || 1920); const height = Number(process.env.PMM_UI_HEIGHT || 1080); + // PMM_UI_INSECURE=1 disables TLS verification (no pin) — for HA/LKE, where + // PMM's cert is self-signed and the egress gateway MITMs outbound TLS, so + // SPKI-pinning can't match. Strict pinning stays the default everywhere else. + const insecure = process.env.PMM_UI_INSECURE === "1"; // PMM_CERT_PATH (see pmm-ui-login.js) pins PMM's own cert instead of // trusting any cert -- optional since this script also screenshots // non-PMM pages (e.g. a GitHub Actions run) with a real CA already, which // strict verification (the default below) already handles fine. const certPath = process.env.PMM_CERT_PATH; const launchArgs = []; - if (certPath) { + if (certPath && !insecure) { if (!fs.existsSync(certPath)) { console.error(`PMM_CERT_PATH set but not found: ${certPath}`); process.exit(1); @@ -47,7 +60,7 @@ async function main() { launchArgs.push(`--ignore-certificate-errors-spki-list=${spkiPinFromCertFile(certPath)}`); } - const contextOpts = { ignoreHTTPSErrors: false, viewport: { width, height } }; + const contextOpts = { ignoreHTTPSErrors: insecure, viewport: { width, height } }; if (sessionId) { if (!/^[A-Za-z0-9][A-Za-z0-9_-]*$/.test(sessionId)) { console.error(`invalid sessionId '${sessionId}' (letters, digits, '_', '-' only)`); @@ -116,6 +129,40 @@ async function main() { await page.waitForTimeout(settleMs); } + // PW_CLICK_TEXT: click an element by (partial) text before capturing — e.g. to + // open a collapsed row or expand a section on the dashboard. + const clickText = process.env.PW_CLICK_TEXT; + if (clickText) { + try { + await page.getByText(clickText, { exact: false }).first().click({ timeout: 8000 }); + await page.waitForTimeout(2500); + } catch (e) { + console.error("click failed:", clickText, e.message); + } + } + + // PW_SCROLL=1: Grafana virtualizes panels, so a fullPage shot of a tall HA + // dashboard misses panels that never scrolled into view. Scroll through to + // force lazy render, then return to the top before capturing. + if (process.env.PW_SCROLL === "1") { + // Re-read scrollHeight each step: lazy panels expand the page as they render, + // so a height captured once up front would stop the loop short and miss lower + // panels. Continue until the bottom is reached and the height has settled. + // Cap the iterations so a page that keeps growing can't hang the shot. + let y = 0; + let prevH = -1; + for (let i = 0; i < 60; i++) { + const scrollH = await page.evaluate(() => document.body.scrollHeight); + if (y >= scrollH && scrollH === prevH) break; + prevH = scrollH; + await page.evaluate((yy) => window.scrollTo(0, yy), y); + await page.waitForTimeout(1000); + y += 700; + } + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(1500); + } + fs.mkdirSync(path.dirname(path.resolve(outputPath)), { recursive: true }); await page.screenshot({ path: outputPath, fullPage: true }); diff --git a/.claude/skills/jira/SKILL.md b/.claude/skills/jira/SKILL.md index 44040ec86..afa45dc59 100644 --- a/.claude/skills/jira/SKILL.md +++ b/.claude/skills/jira/SKILL.md @@ -23,7 +23,10 @@ the relay roster-checks. The relay: Do **not** call the Atlassian MCP connector tools (kept documented below only for reference), and do **not** hit `perconadev.atlassian.net` directly — the -token isn't in this environment. Connector approval is also still broken for +token isn't in this environment. This includes **searching for existing +tickets**: use the relay `search` action (JQL), not the Atlassian Rovo search — +the connector needs interactive auth that isn't there in a Routine/headless run, +so it fails closed. The relay `search` is the supported dedup path. Connector approval is also still broken for Routine grants ([claude-code#61015](https://github.com/anthropics/claude-code/issues/61015)), so the connector path stays disabled; the relay path has no approval gate. @@ -112,6 +115,12 @@ J create "$(jq -n --arg s "..." '{issuetype:"Bug", summary:$s, fields:{customfie J read "$(jq -n --arg i PMM-15188 '{issue:$i}')" J read "$(jq -n --arg i PMM-15188 '{issue:$i,fieldsCsv:"summary,status"}')" +# search — JQL to find existing tickets (e.g. dedup before create). The project +# is FORCED to PMM, so write only the rest of the clause. Read-only. ORDER BY ok; +# maxResults<=100 (default 20); fields optional. Use THIS, never the Atlassian MCP. +J search "$(jq -n --arg q 'text ~ "cannot add MySQL 8.4" AND statusCategory != Done ORDER BY updated DESC' \ + '{jql:$q, maxResults:20, fields:"summary,status,issuetype,updated"}')" + # comment — visibility is FORCED to Developers by the relay; you cannot post public J comment "$(jq -n --arg i PMM-15188 --arg b "h2. QA results"$'\n'"..." '{issue:$i,body:$b}')" @@ -127,8 +136,9 @@ J attach "$(jq -n --arg i PMM-15188 --arg f fb-checks.png \ --arg c "$(base64 -w0 fb-checks.png)" '{issue:$i,filename:$f,content_b64:$c}')" ``` -Available actions: `create`, `read`, `comment`, `field`, `transitions`, +Available actions: `create`, `read`, `search`, `comment`, `field`, `transitions`, `transition`, `attach` — the full set the old direct-REST path had **plus -`create`** (project forced to `PMM`), minus delete (the relay refuses that by -construction). The **mandatory Developers-only visibility rule** is enforced by +`create`** (project forced to `PMM`) **and `search`** (JQL, also PMM-scoped, so +dedup goes through the relay instead of the Atlassian MCP), minus delete (the +relay refuses that by construction). The **mandatory Developers-only visibility rule** is enforced by the relay itself, so it holds even if a caller forgets it. diff --git a/.claude/skills/linode-docker-provisioning/SKILL.md b/.claude/skills/linode-docker-provisioning/SKILL.md index aef4f1b1b..9ee41ba8a 100644 --- a/.claude/skills/linode-docker-provisioning/SKILL.md +++ b/.claude/skills/linode-docker-provisioning/SKILL.md @@ -49,31 +49,55 @@ possession gate; `X-Actor` is the identity. ```bash RELAY=https://139-162-176-43.ip.linodeusercontent.com # fixed prod relay (reserved IP) RUN_ID= # e.g. PMM-15196 (see "Pick a run_id") -ROLE= # test-runner or investigator (free text, tag only) +ROLE= # test-runner or investigator (safe id: [A-Za-z0-9._-], tag only) RUN_DIR="terraform/linode-runner/runs/$RUN_ID" mkdir -p "$RUN_DIR" ACTOR="$(gh api user --jq .login 2>/dev/null)" # ttl_hours + pmm_qa_ref are optional; add keep-alive handling below. -curl -sS -m 600 --fail-with-body -X POST "$RELAY/linode/provision" \ +# 1) Kick off the build — returns immediately with {run_id, status:"provisioning"}. +curl -sS -m 60 --fail-with-body -X POST "$RELAY/linode/provision" \ -H "X-Relay-Secret: $RELAY_KEY" -H "X-Actor: $ACTOR" -H "Content-Type: application/json" \ - -d "$(jq -n --arg r "$ROLE" --arg id "$RUN_ID" '{role:$r, run_id:$id}')" >"$RUN_DIR/provision.json" - -# Unpack what the session-side helpers (run.sh/sync.sh/extend.sh) need locally, -# and tag the run so the SessionEnd hook can tear down exactly our own VMs. -jq -r .ip "$RUN_DIR/provision.json" >"$RUN_DIR/ip" -jq -r .exec_token "$RUN_DIR/provision.json" >"$RUN_DIR/exec_token"; chmod 600 "$RUN_DIR/exec_token" -jq -r .exec_cert_pem "$RUN_DIR/provision.json" >"$RUN_DIR/exec_cert.pem" -printf '%s' "$RELAY" >"$RUN_DIR/relay" # marks this run relay-brokered (holds the relay URL for the SessionEnd hook) + -d "$(jq -n --arg r "$ROLE" --arg id "$RUN_ID" '{role:$r, run_id:$id}')" >"$RUN_DIR/provision-start.json" + +# Mark the run relay-brokered NOW so the SessionEnd hook can tear it down even if we lose the poll. +printf '%s' "$RELAY" >"$RUN_DIR/relay" # relay URL for the SessionEnd hook printf '%s' "${CLAUDE_CODE_SESSION_ID:-}" >"$RUN_DIR/session_id" # scopes the SessionEnd hook + +# 2) Poll for the result — a dropped connection is recoverable (state is on the relay). +deadline=$(( $(date +%s) + 900 )) +while :; do + code=$(curl -sS -m 60 -o "$RUN_DIR/provision.json" -w '%{http_code}' -X POST "$RELAY/linode/provision-result" \ + -H "X-Relay-Secret: $RELAY_KEY" -H "X-Actor: $ACTOR" -H "Content-Type: application/json" \ + -d "$(jq -n --arg id "$RUN_ID" '{run_id:$id}')") + case "$code" in + 200) echo "VM ready"; break;; + 202) echo "provisioning…";; + 502) echo "provisioning FAILED:"; jq -r '.detail // .' "$RUN_DIR/provision.json"; break;; + *) echo "unexpected $code:"; cat "$RUN_DIR/provision.json";; + esac + [ "$(date +%s)" -lt "$deadline" ] || { echo "timed out"; break; } + sleep 15 +done + +# 3) Unpack what the session-side helpers (run.sh/sync.sh/extend.sh) need locally. +if jq -e .exec_token "$RUN_DIR/provision.json" >/dev/null 2>&1; then + jq -r .ip "$RUN_DIR/provision.json" >"$RUN_DIR/ip" + jq -r .exec_token "$RUN_DIR/provision.json" >"$RUN_DIR/exec_token"; chmod 600 "$RUN_DIR/exec_token" + jq -r .exec_cert_pem "$RUN_DIR/provision.json" >"$RUN_DIR/exec_cert.pem" +else + echo "no exec creds — tear the run down (Cleanup) before retrying" +fi ``` -`role` is `test-runner` or `investigator` (free text, just for the tag). The relay: +`role` is `test-runner` or `investigator` — a tag only, but it must be a safe +identifier (`[A-Za-z0-9._-]`, no spaces or `..`); the relay rejects anything else +with `400 bad_role`. The relay: - Creates a Linode VM (default `g6-standard-6`, Ubuntu 24.04) with a firewall open only on 443, tagged `pmm-qa-ephemeral`. - Waits for the exec-server to answer, then for cloud-init to finish installing Docker + Ansible and scheduling its own self-destruct timer (default 24h — see Cleanup below). - `git clone`s `percona/pmm-qa` onto the box at `/root/pmm-qa` — `main` by default, or pass `"pmm_qa_ref":""` in the POST body (must already be pushed; see "Never code on the Linode VM" above). -Works from the **default** proxied-HTTPS environment — no special network policy needed. Takes 2-4 minutes; the relay call blocks until the box is fully ready. After this, `run.sh`/`sync.sh`/`extend.sh` are addressed exactly as before by `` — they use the local `exec_token` + `exec_cert.pem`, never the account token. **Teardown is the exception:** it goes through the relay's `/linode/destroy` (see Cleanup), not a local `down.sh`, since destroying the VM needs the account token that no longer lives in this environment. +Works from the **default** proxied-HTTPS environment — no special network policy needed. Provisioning is **async**: the first call returns a `run_id`, then you poll `/linode/provision-result` until `200` (ready — creds in the body) or `502` (failed). The build runs on the relay and all state lives in its run dir, so a dropped connection is recoverable by re-polling the same `run_id` (usually 2-4 min). After this, `run.sh`/`sync.sh`/`extend.sh` are addressed exactly as before by `` — they use the local `exec_token` + `exec_cert.pem`, never the account token. **Teardown is the exception:** it goes through the relay's `/linode/destroy` (see Cleanup), not a local `down.sh`, since destroying the VM needs the account token that no longer lives in this environment. **Keep-alive:** for an explicit "leave it running" request, add `"ttl_hours":` to the POST body **and** `touch "$RUN_DIR/keep-alive"` — the marker tells the SessionEnd hook to leave this VM up (its on-box self-destruct timer still reaps it after `ttl_hours`). diff --git a/.claude/skills/linode-ha-provisioning/SKILL.md b/.claude/skills/linode-ha-provisioning/SKILL.md index 6b41191d9..e773413f8 100644 --- a/.claude/skills/linode-ha-provisioning/SKILL.md +++ b/.claude/skills/linode-ha-provisioning/SKILL.md @@ -29,22 +29,51 @@ ACTOR="$(gh api user --jq .login 2>/dev/null)" # ttl_hours optional (default 24). Overridable: node_count/node_type/region/ # k8s_version/namespace, and for FB — pmm_chart/deps_chart, pmm_set/deps_set, # chart_version, or pmm_values_b64/deps_values_b64 (a values.yaml, base64). See "Custom charts". -curl -sS -m 1800 --fail-with-body -X POST "$RELAY/linode/provision-lke" \ +# 1) Kick off the build — returns immediately with {run_id, status:"provisioning"}. +# The cluster builds server-side on the relay; this call does NOT hold open. +curl -sS -m 60 --fail-with-body -X POST "$RELAY/linode/provision-lke" \ -H "X-Relay-Secret: $RELAY_KEY" -H "X-Actor: $ACTOR" -H "Content-Type: application/json" \ - -d "$(jq -n --arg id "$RUN_ID" '{run_id:$id}')" >"$RUN_DIR/provision.json" + -d "$(jq -n --arg id "$RUN_ID" '{run_id:$id}')" >"$RUN_DIR/provision-start.json" -# Unpack: kubeconfig for kubectl, cluster_id marker for teardown, session tag. -jq -r .kubeconfig_b64 "$RUN_DIR/provision.json" | base64 -d >"$RUN_DIR/kubeconfig.yaml"; chmod 600 "$RUN_DIR/kubeconfig.yaml" -jq -r .cluster_id "$RUN_DIR/provision.json" >"$RUN_DIR/lke" # marks this run LKE-brokered (holds cluster_id) +# Mark the run LKE-brokered NOW so teardown/reaper work even if we lose the poll. printf '%s' "$RELAY" >"$RUN_DIR/relay" # relay URL for the SessionEnd hook printf '%s' "${CLAUDE_CODE_SESSION_ID:-}" >"$RUN_DIR/session_id" # scopes the SessionEnd hook -export KUBECONFIG="$PWD/$RUN_DIR/kubeconfig.yaml" -jq -r '"URL: \(.url)\nadmin password: \(.passwords.pmm_admin_password)"' "$RUN_DIR/provision.json" +: >"$RUN_DIR/lke" # marks this run LKE-brokered; destroy-lke keys by run_id + +# 2) Poll for the result — a dropped connection is recoverable (state is on the relay). +deadline=$(( $(date +%s) + 2400 )) +while :; do + code=$(curl -sS -m 60 -o "$RUN_DIR/provision.json" -w '%{http_code}' -X POST "$RELAY/linode/lke-result" \ + -H "X-Relay-Secret: $RELAY_KEY" -H "X-Actor: $ACTOR" -H "Content-Type: application/json" \ + -d "$(jq -n --arg id "$RUN_ID" '{run_id:$id}')") + case "$code" in + 200) echo "cluster ready"; break;; + 202) echo "provisioning… ($(jq -r '.phase // "?"' "$RUN_DIR/provision.json"))";; + 502) echo "provisioning FAILED:"; jq -r '.detail // .' "$RUN_DIR/provision.json"; break;; + *) echo "unexpected $code:"; cat "$RUN_DIR/provision.json";; + esac + [ "$(date +%s)" -lt "$deadline" ] || { echo "timed out waiting for cluster"; break; } + sleep 30 +done + +# 3) On success, unpack kubeconfig + cluster_id marker. +if jq -e .kubeconfig_b64 "$RUN_DIR/provision.json" >/dev/null 2>&1; then + jq -r .kubeconfig_b64 "$RUN_DIR/provision.json" | base64 -d >"$RUN_DIR/kubeconfig.yaml"; chmod 600 "$RUN_DIR/kubeconfig.yaml" + jq -r .cluster_id "$RUN_DIR/provision.json" >"$RUN_DIR/lke" # holds cluster_id (teardown also works by run_id) + export KUBECONFIG="$PWD/$RUN_DIR/kubeconfig.yaml" + jq -r '"URL: \(.url)\nadmin password: \(.passwords.pmm_admin_password)"' "$RUN_DIR/provision.json" +else + echo "no kubeconfig — tear the run down (step: Teardown) before retrying" +fi ``` -The call blocks while the cluster + operators + PMM + HAProxy + LoadBalancer come -up (often 10–20 min; the relay allows up to 25) and returns only once the cluster -is ready. `kubectl`/`helm` then work locally against `$KUBECONFIG`. Defaults (all +Provisioning is **async**: the first call returns a `run_id` at once, then you poll +`/linode/lke-result` until `200` (ready — kubeconfig in the body), `502` (failed — +`detail` has the log tail), or your deadline. This survives a dropped connection: +the build runs on the relay (capped at 35 min) and all state lives in its run dir, +so re-polling the same `run_id` always returns the current state. The cluster + +operators + PMM + HAProxy + LoadBalancer usually take 10–20 min. `kubectl`/`helm` +then work locally against `$KUBECONFIG`. Defaults (all overridable in the POST body): `region=us-east`, `node_type=g6-standard-4`, `node_count=3` (Raft quorum, tolerates one node down); `k8s_version` defaults to the latest LKE offers (versions roll — a retired pin 400s). @@ -74,7 +103,7 @@ Standing up the cluster isn't the test. Exercise what the change actually touche - Leader status: PMM's HA API / `pmm_ha_*` metrics; confirm exactly one leader. - **Leader failover** for leader-only work (backups, scheduler, checks, telemetry, cleaner, versionCache): delete the leader pod, confirm a new leader is elected and the singleton work resumes there once — not zero times, not on every replica. - Shared state: confirm data written on one replica is visible via another (it lives in the shared PG/ClickHouse/VM, not local `/srv`). -- UI via the LoadBalancer IP (`ui-evidence`), using the admin password from the provision response (`.passwords.pmm_admin_password` in `provision.json`). +- UI evidence (`ui-evidence` → "HA / LKE variant"): reach PMM by the hostname `.url` from `provision.json` (**not** the raw LB IP — the egress proxy refuses raw-IP HTTPS), log in with `PMM_UI_INSECURE=1` (self-signed cert) and `.passwords.pmm_admin_password`, and pass `PW_SCROLL=1` for the tall HA dashboards. Use the existing `pmm-ui-login.js` + `pw-screenshot.js` helpers — don't write a bespoke capture script. ## Teardown — mandatory, every path diff --git a/.claude/skills/linode-ha-provisioning/scripts/create-lke-pmm-ha.sh b/.claude/skills/linode-ha-provisioning/scripts/create-lke-pmm-ha.sh index b6e9de311..2915e03ec 100755 --- a/.claude/skills/linode-ha-provisioning/scripts/create-lke-pmm-ha.sh +++ b/.claude/skills/linode-ha-provisioning/scripts/create-lke-pmm-ha.sh @@ -15,7 +15,7 @@ RUN_ID="${RUN_ID:-$(date +%Y%m%d-%H%M%S)}" CLUSTER_LABEL="${CLUSTER_LABEL:-pmm-ha-${RUN_ID}}" REGION="${REGION:-us-east}" K8S_VERSION="${K8S_VERSION:-}" # resolved to the latest LKE offers below if unset (versions roll) -NODE_TYPE="${NODE_TYPE:-g6-standard-4}" +NODE_TYPE="${NODE_TYPE:-g6-standard-6}" # 16GB/6vCPU — 8GB/4vCPU (standard-4) starves the 3-replica HA stack (pods stay Pending) NODE_COUNT="${NODE_COUNT:-3}" # >=3 keeps a Raft quorum with one node down NAMESPACE="${NAMESPACE:-pmm}" @@ -105,6 +105,20 @@ until linode-cli lke kubeconfig-view "$CLUSTER_ID" --json \ done export KUBECONFIG="$KUBECONFIG_FILE" log "KUBECONFIG: $KUBECONFIG" +# Always capture pod state on exit (success OR failure) so a stuck bring-up is +# debuggable from the run dir without the cluster still being alive. +_diag() { + kubectl get pods -n "$NAMESPACE" -o wide >"$RUN_DIR/pods.txt" 2>&1 || true + kubectl get events -n "$NAMESPACE" --sort-by=.metadata.creationTimestamp >"$RUN_DIR/events.txt" 2>&1 || true + kubectl describe pods -n "$NAMESPACE" >"$RUN_DIR/describe.txt" 2>&1 || true +} +trap _diag EXIT +# Linode reports the pool "ready" before the nodes register with the k8s API +# server, so `kubectl wait --all` would hit an empty list and fail immediately +# ("no matching resources found"). Wait for the nodes to appear first, then wait +# for Ready. +log "Waiting for $NODE_COUNT node(s) to register with the API server..." +until [ "$(kubectl get nodes --no-headers 2>/dev/null | grep -c .)" -ge "$NODE_COUNT" ]; do sleep 10; done kubectl wait --for=condition=Ready nodes --all --timeout=300s kubectl get nodes @@ -125,6 +139,9 @@ helm install pmm-operators "$DEPS_CHART" --namespace "$NAMESPACE" "${deps_args[@ for op in victoria-metrics-operator altinity-clickhouse-operator pg-operator; do log "Waiting for operator: $op" + # Same empty-match race: the pod may not exist the instant helm returns, and + # `kubectl wait -l` errors on zero matches. Wait for it to appear, then Ready. + until kubectl get pod -l "app.kubernetes.io/name=$op" -n "$NAMESPACE" --no-headers 2>/dev/null | grep -q .; do sleep 5; done kubectl wait --for=condition=ready pod \ -l "app.kubernetes.io/name=$op" -n "$NAMESPACE" --timeout=300s done @@ -155,7 +172,26 @@ pmm_args=(); [ -n "$CHART_VERSION" ] && pmm_args+=(--version "$CHART_VERSION") log "Installing PMM HA from $PMM_CHART" helm install pmm-ha "$PMM_CHART" --namespace "$NAMESPACE" "${pmm_args[@]}" -log "Waiting for HAProxy front end (up to 15m)..." +# HAProxy fronts the PMM server, and its readiness gate depends on the backend +# PMM pods coming up first — so wait for the PMM server StatefulSet to be Ready +# before waiting on HAProxy, which both sequences the bring-up and makes a stuck +# backend show up as a backend timeout (clearer than an opaque HAProxy timeout). +# Wait for the PMM server StatefulSet to FULLY roll out before HAProxy. `rollout +# status` blocks until readyReplicas == spec.replicas, so it covers the complete +# expected replica set — not a one-time pod snapshot that can miss replicas +# created later — and fails (set -e) on timeout, so a degraded cluster is never +# published as ready. The EXIT trap has already captured pods/describe. +log "Waiting for the PMM server StatefulSet to appear (up to 5m)..." +pmm_sts="" +pmm_appear_deadline=$(( $(date +%s) + 300 )) +until pmm_sts="$(kubectl get statefulset -n "$NAMESPACE" -o name | grep -E '/pmm-ha$|/pmm-ha-server$' | head -1)"; [ -n "$pmm_sts" ]; do + [ "$(date +%s)" -lt "$pmm_appear_deadline" ] || { echo "ERROR: PMM server StatefulSet never appeared within 5m" >&2; exit 1; } + sleep 10 +done +log "Waiting for $pmm_sts to roll out (all replicas Ready, up to 20m)..." +kubectl rollout status "$pmm_sts" -n "$NAMESPACE" --timeout=20m + +log "Waiting for HAProxy front end (up to 20m)..." haproxy_deadline=$(( $(date +%s) + 900 )) until kubectl get pods -n "$NAMESPACE" -o name | grep -q pmm-ha-haproxy; do [ "$(date +%s)" -lt "$haproxy_deadline" ] || { echo "ERROR: pmm-ha-haproxy pods never appeared within 15m" >&2; exit 1; } @@ -166,7 +202,7 @@ done # shellcheck disable=SC2046 kubectl wait --for=condition=ready \ $(kubectl get pods -n "$NAMESPACE" -o name | grep pmm-ha-haproxy) \ - -n "$NAMESPACE" --timeout=15m + -n "$NAMESPACE" --timeout=20m kubectl get pods -n "$NAMESPACE" # --- external access --------------------------------------------------------- @@ -180,13 +216,34 @@ until [ -n "$EXTERNAL_IP" ]; do [ -n "$EXTERNAL_IP" ] || sleep 15 done +# --- PMM actually serving? (not just pods Ready) ----------------------------- +# Pods Ready doesn't guarantee the HTTP front end is reachable end-to-end (LB +# wiring, HAProxy backends). This box (the relay) reaches the public LB IP +# directly — no egress proxy here — so confirm PMM answers over the LoadBalancer +# before publishing the run as ready. Fail the build (set -e) if it never does, +# so the relay never reports `ready` for a cluster whose UI can't be opened. +# Require an exact 200 — `curl -f` treats 3xx as success, so a redirect could +# otherwise pass this gate without PMM actually serving. Check %{http_code}. +log "Verifying PMM returns 200 from https://$EXTERNAL_IP/v1/readyz (up to 10m)..." +pmm_up_deadline=$(( $(date +%s) + 600 )) +until [ "$(curl -k -sS -m 10 -o /dev/null -w '%{http_code}' "https://$EXTERNAL_IP/v1/readyz" 2>/dev/null)" = "200" ]; do + [ "$(date +%s)" -lt "$pmm_up_deadline" ] || { echo "ERROR: PMM did not return 200 from /v1/readyz on the LB within 10m" >&2; exit 1; } + sleep 10 +done +log "PMM is serving (/v1/readyz 200)." + # --- persist run artifacts --------------------------------------------------- { echo "cluster_label=$CLUSTER_LABEL" echo "cluster_id=$CLUSTER_ID" echo "expires_epoch=$EXPIRES_EPOCH" echo "external_ip=$EXTERNAL_IP" - echo "url=https://$EXTERNAL_IP" + # The QA session reaches PMM through the agent egress proxy, which refuses raw-IP + # HTTPS but allows Linode's per-IP rDNS hostname. Hand back the hostname URL so + # the UI is actually openable (curl/Playwright still need -k / ignoreHTTPSErrors + # for PMM's self-signed cert). + echo "external_host=$(echo "$EXTERNAL_IP" | tr '.' '-').ip.linodeusercontent.com" + echo "url=https://$(echo "$EXTERNAL_IP" | tr '.' '-').ip.linodeusercontent.com" echo "pmm_admin_password=$PMM_PW" echo "postgres_password=$PG_PW" echo "grafana_password=$GF_PW" diff --git a/.claude/skills/ui-evidence/SKILL.md b/.claude/skills/ui-evidence/SKILL.md index 8177d3174..76b74485e 100644 --- a/.claude/skills/ui-evidence/SKILL.md +++ b/.claude/skills/ui-evidence/SKILL.md @@ -13,6 +13,8 @@ This environment ships Chromium pre-installed with Playwright already pointed at All three accept an optional `PMM_CERT_PATH` env var — set it to the cert `linode-docker-provisioning` step 2 fetched (`terraform/linode-runner/runs//pmm_cert.pem`) whenever the URL is PMM's own, so the browser pins that exact cert (via Chromium's `--ignore-certificate-errors-spki-list`) instead of falling back to `ignoreHTTPSErrors`. Omit it for non-PMM URLs (e.g. a GitHub Actions run), which already have a real CA. +On the **HA / LKE** path there is no exec-server to fetch a pinnable cert and PMM's cert is self-signed behind the egress MITM, so pinning can't match — pass **`PMM_UI_INSECURE=1`** to `pmm-ui-login.js` / `pw-screenshot.js` instead of `PMM_CERT_PATH` (see the HA variant below). Don't write a bespoke HA screenshot script — the same two helpers cover it. + ## Log into PMM UI and screenshot ```bash @@ -35,6 +37,27 @@ PMM_CERT_PATH="$PMM_CERT_PATH" node .claude/scripts/pw-screenshot.js \ Session name `PMM-14576` above — reuse the same ticket key for follow-up screenshots (or a recording) so the login isn't repeated. +## HA / LKE variant (self-signed cert, tall dashboards) + +Same two helpers, three differences: `PMM_UI_INSECURE=1` instead of a cert pin; reach PMM by the **hostname** `url` the relay's `lke-result` returned (never the raw LB IP — the egress proxy refuses raw-IP HTTPS); and pass `PW_SCROLL=1` so Grafana's virtualized HA panels render before the fullPage shot. + +```bash +# from the linode-ha-provisioning run: $RUN_DIR/provision.json holds url + passwords +PMM_URL="$(jq -r .url "$RUN_DIR/provision.json")" +ADMIN_PASSWORD="$(jq -r .passwords.pmm_admin_password "$RUN_DIR/provision.json")" + +PMM_URL="$PMM_URL" ADMIN_PASSWORD="$ADMIN_PASSWORD" PMM_UI_INSECURE=1 \ + node .claude/scripts/pmm-ui-login.js PMM-13860 + +PMM_UI_INSECURE=1 PW_SCROLL=1 PW_SETTLE_MS=15000 \ + node .claude/scripts/pw-screenshot.js \ + "$PMM_URL/graph/d/pmm-ha-health-overview" \ + "/tmp/PMM-13860-ha-overview.png" \ + PMM-13860 # reuse the session for each dashboard +``` + +`PW_CLICK_TEXT='...'` clicks an element by partial text first (e.g. to expand a collapsed row). Login once, then one `pw-screenshot.js` per dashboard. + ## Record a short clip instead of a screenshot For a flow that's clearer as motion than a still (e.g. an alert firing, a dashboard panel updating): diff --git a/docs/agents/AUTOMATIONS.md b/docs/agents/AUTOMATIONS.md index 76c824c11..939f5f389 100644 --- a/docs/agents/AUTOMATIONS.md +++ b/docs/agents/AUTOMATIONS.md @@ -306,32 +306,37 @@ One dispatch, one gate: consolidating the old `/announce`, `/jira-act`, `/provis - [x] GitHub connector activated for the org - [x] `gh --version`, `terraform version`, `json-diff --version`, `ffmpeg -version` succeed after a fresh SessionStart hook run - [x] Connector permission prompts — understood, not repo-fixable: in web sessions the prompt is **enforced by the claude.ai host layer** — no `permissions.allow` spelling and no PreToolUse allow-hook can suppress it (all tested live 2026-08-06). Routine runs are governed by the Routine's own connector list instead (once #61015 is fixed). Consequently settings.json allowlists only `mcp__github`, the one MCP entry that verifiably works (project-provisioned server). Useful facts: settings/hook edits hot-reload mid-session; connector server names vary across sessions (`Atlassian_Rovo` vs `Atlassian-Rovo`); an agent cannot see whether a prompt fired — verification needs a human watching. -- [ ] **Provisioning is blocked in Routine runs by the auto-mode classifier — needs a human-applied settings fix.** Hit live on 2026-08-10: Investigator fired on a red `E2E tests Matrix` run, got through dedup, then `terraform/linode-runner/up.sh investigator ` was denied ("Blocked by classifier"), so it could not reproduce and closed with no verdict. Mechanics, all observed in that run: - - The allowlist is not what adjudicated it. `permissions.allow` holds bare tool names, and a bare `"Bash"` entry does **not** count as a shell pre-authorization — the command still went to the classifier, which judged `terraform apply` (billable infra, firewall to `0.0.0.0/0`, irreversible) as needing a human. Ordinary `git`/`python3`/`grep` passed untouched, so this is content-based, not a blanket Bash denial. `defaultMode: acceptEdits` is irrelevant here: it auto-accepts *file edits*, not Bash. - - The rules currently in #1143 cannot work: `Bash(*linode-runner/up.sh *)` leads with `*`, but Bash rules are **prefix** matches, not globs. They have also never been active in a Routine — those runs read `/root/.claude/settings.json`, planted from `main`, which has no such entries. - - A path-prefix rule is fragile regardless: the shell cwd is the repo root but is not guaranteed stable (observed resetting to `/home/user` mid-session), and the README's own `PMM_QA_REF= up.sh …` form does not *start* with the script path. Hence the `autoMode.allow` entry below, which is path- and prefix-agnostic, is the primary lever; the `Bash(...)` rules are the cheap documented fast path. - - **An agent cannot apply this itself.** Editing `.claude/settings.json` is refused even with the user explicitly asking for it in-session — a hard boundary (self-escalation), not a soft prompt, and correctly so: an agent able to grant itself provisioning rights defeats the permission system. Do not expect a future agent run to fix this; a human edits the file. +- [ ] **Provisioning is blocked in Routine runs by the auto-mode classifier — needs a human-applied `autoMode` config.** Hit live 2026-08-10 and again 2026-08-14 ([run 31765312223](https://github.com/percona/pmm-qa/actions/runs/31765312223)): Investigator got through dedup, then every attempt to reach the provisioning relay was denied with `Blocked by classifier`. A Routine cannot show a permission prompt, so the run just continued and closed with no verdict. Mechanics — the earlier notes here were partly wrong; **corrected against the official docs 2026-08-14** ([auto-mode-config](https://code.claude.com/docs/en/auto-mode-config), [permissions](https://code.claude.com/docs/en/permissions), [server-managed-settings](https://code.claude.com/docs/en/server-managed-settings)): + - **The lever is `autoMode`, not `permissions.allow`.** Auto mode routes every tool call through a classifier that blocks anything aimed *outside the trusted environment*. The denial is **destination-based**: a plain `GET /health` to the relay host is denied while `curl https://api.github.com/rate_limit` (a repo remote) returns 200. In auto mode, **broad `permissions.allow` entries — a bare `"Bash"`, `Bash(*)` — are suspended** (the classifier evaluates the command regardless); only *narrow* rules (`Bash(npm test)`) carry over and resolve before it. So no `permissions.allow` spelling fixes this — `autoMode.environment` (trusting the relay's provider domain) is the primary lever, and `autoMode.allow` clears the "creating billable/irreversible infra" soft block. + - **The classifier reads `autoMode` only from user scope (`~/.claude/settings.json`) and [managed settings](https://code.claude.com/docs/en/server-managed-settings) — NOT from project `.claude/settings.json`.** A block committed to this repo's `.claude/settings.json` is therefore inert *as project settings*; it takes effect **only because the qa-linode setup script copies that file to `/root/.claude/settings.json` (user scope)**. The durable, org-wide home is **server-managed settings** — an org Owner pastes it at `claude.ai/admin-settings/claude-code`; it is the only managed channel that reaches cloud/web sessions and it is highest precedence. The repo→user-scope copy is the self-service fallback that lands via PR today (then touch the setup script to bust its ~7-day snapshot cache). If it goes to managed settings it applies org-wide, so every `environment` line is written to be true for all of Percona, not scoped to QA. + - **Correction to the earlier claim:** `Bash(...)` rules are **globs, not prefix-only**, and a **leading `*` does match** — the docs give `Bash(* install)` matching any command ending in a space followed by `install` ([permissions § Wildcard patterns](https://code.claude.com/docs/en/permissions)). The old note that `Bash(*linode-runner/up.sh *)` "leads with `*`, but Bash rules are prefix matches" was wrong on the semantics. (It was separately true that those entries were absent from the `settings.json` planted from `main` at the time.) + - **Provisioning moved to the relay (#1163):** `up.sh`/`down.sh` run on the relay now, so the session-side action is a `curl` to the relay, not a local `terraform apply`. The five `Bash(*linode-runner/*.sh *)` entries are obsolete — **remove them**; `run.sh`/`sync.sh`/`extend.sh` (still session-side, reaching `exec-.nip.io`) are covered by the `autoMode` block below (the `exec-*.nip.io` trust + the provisioning/driving `allow` entry). Keep the other bare tool-name / `mcp__*` allows — those are permission-layer grants, a separate gate the classifier doesn't touch. + - **An agent cannot apply the `settings.json` change itself** — editing `.claude/settings.json` is a self-escalation boundary (an agent granting itself provisioning rights). A human commits it, or an Owner pastes the `autoMode` block into managed settings. - Replacement for the five broken entries, plus the classifier lever (`"$defaults"` must stay, it inherits the built-in rules): + The `autoMode` block to add (keep every `"$defaults"` — it splices in the built-in rules; omitting it discards them. `soft_deny`/`hard_deny` are deliberately left unset so their built-ins stay intact): ```json - "Bash(terraform/linode-runner/up.sh:*)", - "Bash(terraform/linode-runner/run.sh:*)", - "Bash(terraform/linode-runner/sync.sh:*)", - "Bash(terraform/linode-runner/extend.sh:*)", - "Bash(terraform/linode-runner/down.sh:*)" - ``` - - ```json - "autoMode": { - "allow": [ - "$defaults", - "Provisioning, driving and tearing down this repository's throwaway QA VMs via terraform/linode-runner/up.sh, run.sh, sync.sh, extend.sh and down.sh -- at any path prefix, and including invocations led by environment assignments such as PMM_QA_REF=, ALLOWED_INBOUND_CIDR=, DOCKER_VERSION= or CLIENT_VERSION=. These are single-run, self-destructing Linode instances (on-box systemd TTL timer, default 24h) created by the test-runner and investigator agents and by this repo's scheduled Routines. Standing up one is the expected first step of any reproduction and down.sh destroying one is the mandatory last step; neither is an unusual or unintended side effect here." - ] + { + "autoMode": { + "environment": [ + "$defaults", + "Organization: Percona, an open-source database software company (github.com/percona and github.com/Percona-Lab). Primary use of Claude Code: software engineering across many teams and repositories. Some sessions additionally stand up and tear down throwaway test infrastructure for QA and CI; that is one workflow among many, not the primary use.", + "Source control: the Percona GitHub organizations github.com/percona and github.com/Percona-Lab, and every repository under them — two GitHub orgs but a single trust domain (one company). A repository not attached to a session was simply not selected at creation (cross-org repos can't be attached mid-session), not foreign. Percona has many more repositories than any one session touches.", + "Repository visibility: mixed across Percona — most repositories are public, some are private.", + "Cloud provider(s): Linode (Akamai). Used to create short-lived, self-destructing test instances and throwaway Kubernetes (LKE) clusters for QA and CI, through the Linode API at api.linode.com.", + "Trusted internal domains: *.linodeusercontent.com and *.ip.linodeusercontent.com — the automation relay and throwaway-cluster load balancers are reached at Linode reverse-DNS hostnames of the form .ip.linodeusercontent.com, whose address changes on every rebuild (sometimes daily), so the provider domain is trusted by wildcard rather than any fixed host. exec-*.nip.io — a throwaway instance's own exec service is reached at exec-.nip.io, derived from that instance's IP and not knowable in advance; only the exec-prefixed host is trusted, not arbitrary nip.io names that map to any public IP. perconadev.atlassian.net — Percona's Jira. *.cd.percona.com — Percona's Jenkins CI.", + "Key internal services these sessions talk to: an automation relay that brokers infrastructure and ticketing actions (Linode VM and LKE provisioning/teardown, Jira, chat notifications) at a *.ip.linodeusercontent.com host whose address is not fixed; Percona Jenkins at pmm.cd.percona.com (and *.cd.percona.com); Percona Jira at perconadev.atlassian.net; Launchable at *.launchableinc.com; the Linode API at api.linode.com; the Terraform registry at registry.terraform.io; and Percona's public Docker Hub namespaces docker.io/percona and docker.io/perconalab.", + "Additional context: many of these are cloud (web) sessions that root at a parent directory above several cloned repositories, so the classifier's working-repo default may not resolve to a Percona repo — the Source control entry above states the trust domain explicitly. The test infrastructure these sessions create is single-run and self-destructing (Linode VMs carry an on-box TTL timer; throwaway LKE clusters carry an expiry tag a reaper enforces), so standing one up and destroying it are the ordinary first and last steps of a task, not unusual or irreversible side effects." + ], + "allow": [ + "$defaults", + "Provisioning, driving and tearing down single-run, self-destructing test infrastructure is expected routine work here, not an unusual or unintended side effect. This covers: creating a throwaway Linode VM or a throwaway Linode Kubernetes (LKE) cluster to reproduce a failure or run a test; POSTing provision and teardown requests to the automation relay; running the session-side helper scripts that drive an already-provisioned instance over its exec service at exec-.nip.io; and any of the above led by environment-variable assignments such as PMM_QA_REF=, DOCKER_VERSION= or CLIENT_VERSION=. Narrowing a run's inbound exposure with ALLOWED_INBOUND_CIDR=/32 is likewise routine; opening it to the whole internet (0.0.0.0/0) is a deliberate, visible choice made at the call site, not a widening this entry pre-approves. These instances self-destruct on their own (on-box TTL timer / expiry-tag reaper); standing one up is the normal first step of a reproduction and destroying it is the mandatory last step." + ] + } } ``` - Zero-cost verification after applying: run `terraform/linode-runner/up.sh` with no arguments. It exits on `ROLE="${1:?usage…}"` before `mkdir`, before terraform, before anything billable — a usage line instead of a denial means the rule works and nothing was created. Deliberately **not** recommended: `defaultMode: bypassPermissions` or `disableAutoMode`, either of which fixes this in one line and also switches off the thing that would stop a genuinely bad command in an unattended run. + Verify after applying: `claude auto-mode config` confirms the entries loaded with `"$defaults"` expanded (validated 2026-08-14 in an isolated `CLAUDE_CONFIG_DIR` — the block parses and merges, `soft_deny`/`hard_deny` stay at their defaults); `claude auto-mode critique` gives an AI review of the custom prose. Functional check: `curl -sS https:///health` returns 200 instead of `Blocked by classifier`, or fire Investigator and confirm it reaches provisioning. Deliberately **not** recommended: `defaultMode: bypassPermissions`, `disableAutoMode`, or `autoMode.classifyAllShell` — the first two switch off the thing that would stop a genuinely bad command in an unattended run. - [x] `INVESTIGATOR_ROUTINE_TOKEN` added as a repo secret so `notify-investigator.yml` can actually fire - [x] Live Claude Code Remote Routines updated to match this architecture (Test Doctor renamed to Investigator, FB Validator handled) — see "Updating the live Routines" below for what changed - [x] **Cross-org access to `Percona-Lab/*` — solved: attach the repo at session/Routine creation** (multi-repo, verified live 2026-08-06: `gh api` on both repos and `gh run rerun --failed -R Percona-Lab/pmm-submodules` all work). Single-repo sessions stay blocked by design: mid-session `add_repo` is refused cross-tier, and a PAT env var can never widen scope (the proxy swaps credentials and enforces scope itself). Fallbacks in an unattached session: anonymous git read, or data passed in via the trigger payload. Details for agents live in the `repos` skill.