feat(heartbeat): add local process watchdog telemetry and auto-remediation

Author: MoltGus

doc/DEVELOPING.md

@@ -261,6 +261,25 @@ Expected:
 - `/api/health` returns `{"status":"ok"}`
 - `/api/companies` returns a JSON array
 
+## Local Process Watchdog
+
+Heartbeat recovery includes a local-process watchdog for detached child processes (`claude_local`, `codex_local`, `cursor`, `gemini_local`, `opencode_local`, `pi_local`).
+
+Company snapshot endpoint:
+
+```sh
+curl "http://localhost:3100/api/companies/<company-id>/process-watchdog"
+```
+
+Main env knobs:
+
+- `HEARTBEAT_PROCESS_WATCHDOG_ENABLED=true|false`
+- `HEARTBEAT_PROCESS_WATCHDOG_MAX_LOCAL_PROCESSES=<count>` (default `48`)
+- `HEARTBEAT_PROCESS_WATCHDOG_MAX_MEMORY_USAGE_PERCENT=<1-99>` (default `85`)
+- `HEARTBEAT_PROCESS_WATCHDOG_DETACHED_IDLE_TIMEOUT_MS=<ms>` (default `21600000` / 6h)
+- `HEARTBEAT_PROCESS_WATCHDOG_DETACHED_IDLE_TIMEOUT_UNDER_PRESSURE_MS=<ms>` (default `1200000` / 20m)
+- `HEARTBEAT_PROCESS_WATCHDOG_KILL_GRACE_MS=<ms>` (default `1500`)
+
 ## Reset Local Dev Database
 
 To wipe local dev data and start fresh:

doc/PRODUCT.md

@@ -140,7 +140,7 @@ ZeroInc’s core identity is a **control plane for autonomous AI companies**, ce
    The mental model should not change between local solo use and shared/private or public/cloud deployment.
 
 7. **Safe autonomy**
-   Auto mode is allowed; hidden token burn is not.
+   Auto mode is allowed; hidden token burn is not. Local runtime process pressure must be guarded (watchdog + telemetry + auto-remediation for detached stale runs).
 
 8. **Thin core, rich edges**
    Put optional chat, knowledge, and special surfaces into plugins/extensions rather than bloating the control plane.

doc/SPEC-implementation.md

@@ -105,6 +105,7 @@ A lightweight scheduler/worker in the server process handles:
 
 - heartbeat trigger checks
 - stuck run detection
+- local-process lifecycle watchdog (detached/orphan detection + safe auto-remediation under pressure)
 - budget threshold checks
 
 Separate queue infrastructure is not required for V1.
@@ -587,6 +588,7 @@ Behavior:
 - stream stdout/stderr to run logs
 - mark run status on exit code/timeout
 - cancel sends SIGTERM then SIGKILL after grace
+- persist process metadata (`process_pid`, `process_started_at`) for watchdog telemetry/recovery
 
 ## 11.3 HTTP Adapter
 
@@ -656,6 +658,20 @@ Scheduler must skip invocation when:
 - an existing run is active
 - hard budget limit has been hit
 
+## 11.7 Local Process Watchdog
+
+For local child-process adapters (`claude_local`, `codex_local`, `cursor`, `gemini_local`, `opencode_local`, `pi_local`):
+
+- runtime telemetry must include `run_id -> process_pid -> process_started_at -> last_heartbeat_at`
+- detached runs (`process_detached`) are monitored with idle thresholds
+- when host pressure crosses configured limits (local process count and/or memory usage), stale detached runs are terminated automatically
+- max-duration runs are force-failed and their child PID is explicitly signaled (SIGTERM, then SIGKILL)
+
+Operator API surfaces:
+
+- `GET /api/companies/:companyId/live-runs` includes `processPid`, `processStartedAt`, `lastHeartbeatAt`, `errorCode`
+- `GET /api/companies/:companyId/process-watchdog` returns watchdog snapshot (pressure + run telemetry)
+
 ## 12. Governance and Approval Flows
 
 ## 12.1 Hiring

doc/plans/2026-03-25-zeroinc-companies-study.md

@@ -44,6 +44,13 @@ Arquivo: `server/src/services/heartbeat.ts`
 
 ## 3. Recomendações priorizadas
 
+## 3.0 Status de execução (2026-03-25)
+
+1. P0.1 Outcome Ledger + gates de entregável: implementado.
+2. P0.2 Discovery Engine + estoque por pilar: implementado.
+3. P0.3 Contrato explícito de bloqueio humano + inbox SLA: implementado.
+4. P0.4 Guardião de ciclo de vida de processos locais: implementado (watchdog com limiares de pressão, auto-remediação de detached stale e telemetria por run/pid).
+
 ## P0 (executar agora)
 
 1. Outcome Ledger (entregável real como fonte de verdade)
@@ -124,4 +131,3 @@ Arquivo: `server/src/services/heartbeat.ts`
 ## 6. Conclusão
 
 A direção correta não é aumentar quantidade de agentes; é endurecer o sistema de geração e validação de trabalho real. O ZeroInc já tem bons blocos de governança. O próximo salto é transformar esses blocos em um ciclo contínuo de entrega verificável, com autonomia alta e escalonamento humano explícito.
-

server/src/__tests__/heartbeat-process-recovery.test.ts

@@ -94,6 +94,25 @@ function spawnAliveProcess() {
   });
 }
 
+function isPidAlive(pid: number | null | undefined) {
+  if (typeof pid !== "number" || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function waitForPidExit(pid: number, timeoutMs = 3000) {
+  const start = Date.now();
+  while (Date.now() - start < timeoutMs) {
+    if (!isPidAlive(pid)) return true;
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  return !isPidAlive(pid);
+}
+
 describe("heartbeat orphaned process recovery", () => {
   let db!: ReturnType<typeof createDb>;
   let instance: EmbeddedPostgresInstance | null = null;
@@ -141,13 +160,17 @@ describe("heartbeat orphaned process recovery", () => {
     includeIssue?: boolean;
     runErrorCode?: string | null;
     runError?: string | null;
+    startedAt?: Date;
+    updatedAt?: Date;
   }) {
     const companyId = randomUUID();
     const agentId = randomUUID();
     const runId = randomUUID();
     const wakeupRequestId = randomUUID();
     const issueId = randomUUID();
     const now = new Date();
+    const startedAt = input?.startedAt ?? now;
+    const updatedAt = input?.updatedAt ?? startedAt;
     const issuePrefix = `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`;
 
     await db.insert(companies).values({
@@ -195,8 +218,9 @@ describe("heartbeat orphaned process recovery", () => {
       processLossRetryCount: input?.processLossRetryCount ?? 0,
       errorCode: input?.runErrorCode ?? null,
       error: input?.runError ?? null,
-      startedAt: now,
-      updatedAt: now,
+      startedAt,
+      processStartedAt: startedAt,
+      updatedAt,
     });
 
     if (input?.includeIssue !== false) {
@@ -320,4 +344,61 @@ describe("heartbeat orphaned process recovery", () => {
     expect(run?.errorCode).toBeNull();
     expect(run?.error).toBeNull();
   });
+
+  it("terminates detached local processes when watchdog idle timeout is exceeded", async () => {
+    const child = spawnAliveProcess();
+    childProcesses.add(child);
+    expect(child.pid).toBeTypeOf("number");
+
+    const detachedAt = new Date(Date.now() - 5 * 60 * 1000);
+    const { runId } = await seedRunFixture({
+      processPid: child.pid ?? null,
+      includeIssue: false,
+      runErrorCode: "process_detached",
+      runError: `Lost in-memory process handle, but child pid ${child.pid} is still alive`,
+      startedAt: detachedAt,
+      updatedAt: detachedAt,
+    });
+    const heartbeat = heartbeatService(db);
+
+    const result = await heartbeat.reapOrphanedRuns({
+      watchdog: {
+        detachedIdleTimeoutMs: 1_000,
+        detachedIdleTimeoutUnderPressureMs: 1_000,
+        killGraceMs: 100,
+      },
+    });
+
+    expect(result.reaped).toBe(1);
+    expect(result.runIds).toEqual([runId]);
+
+    const run = await heartbeat.getRun(runId);
+    expect(run?.status).toBe("failed");
+    expect(run?.errorCode).toBe("process_detached_timeout");
+    expect(await waitForPidExit(child.pid ?? -1)).toBe(true);
+  });
+
+  it("kills the child pid when max run duration is exceeded", async () => {
+    const child = spawnAliveProcess();
+    childProcesses.add(child);
+    expect(child.pid).toBeTypeOf("number");
+
+    const startedAt = new Date(Date.now() - 3 * 60 * 60 * 1000);
+    const { runId } = await seedRunFixture({
+      processPid: child.pid ?? null,
+      includeIssue: false,
+      startedAt,
+      updatedAt: startedAt,
+    });
+    const heartbeat = heartbeatService(db);
+
+    const result = await heartbeat.reapOrphanedRuns();
+    expect(result.reaped).toBe(1);
+    expect(result.runIds).toEqual([runId]);
+
+    const run = await heartbeat.getRun(runId);
+    expect(run?.status).toBe("failed");
+    expect(run?.errorCode).toBe("max_duration_exceeded");
+    expect(await waitForPidExit(child.pid ?? -1)).toBe(true);
+  });
 });

server/src/routes/agents.ts

@@ -2150,10 +2150,14 @@ export function agentRoutes(db: Db) {
       triggerDetail: heartbeatRuns.triggerDetail,
       startedAt: heartbeatRuns.startedAt,
       finishedAt: heartbeatRuns.finishedAt,
+      lastHeartbeatAt: heartbeatRuns.updatedAt,
       createdAt: heartbeatRuns.createdAt,
       agentId: heartbeatRuns.agentId,
       agentName: agentsTable.name,
       adapterType: agentsTable.adapterType,
+      processPid: heartbeatRuns.processPid,
+      processStartedAt: heartbeatRuns.processStartedAt,
+      errorCode: heartbeatRuns.errorCode,
       issueId: sql<string | null>`${heartbeatRuns.contextSnapshot} ->> 'issueId'`.as("issueId"),
     };
 
@@ -2192,6 +2196,16 @@ export function agentRoutes(db: Db) {
     res.json(liveRuns);
   });
 
+  router.get("/companies/:companyId/process-watchdog", async (req, res) => {
+    const companyId = req.params.companyId as string;
+    assertCompanyAccess(req, companyId);
+
+    const limitParam = req.query.limit as string | undefined;
+    const limit = limitParam ? Math.max(1, Math.min(500, parseInt(limitParam, 10) || 200)) : 200;
+    const snapshot = await heartbeat.getLocalProcessTelemetry(companyId, { limit });
+    res.json(snapshot);
+  });
+
   router.get("/heartbeat-runs/:runId", async (req, res) => {
     const runId = req.params.runId as string;
     const run = await heartbeat.getRun(runId);
@@ -2317,10 +2331,14 @@ export function agentRoutes(db: Db) {
         triggerDetail: heartbeatRuns.triggerDetail,
         startedAt: heartbeatRuns.startedAt,
         finishedAt: heartbeatRuns.finishedAt,
+        lastHeartbeatAt: heartbeatRuns.updatedAt,
         createdAt: heartbeatRuns.createdAt,
         agentId: heartbeatRuns.agentId,
         agentName: agentsTable.name,
         adapterType: agentsTable.adapterType,
+        processPid: heartbeatRuns.processPid,
+        processStartedAt: heartbeatRuns.processStartedAt,
+        errorCode: heartbeatRuns.errorCode,
       })
       .from(heartbeatRuns)
       .innerJoin(agentsTable, eq(heartbeatRuns.agentId, agentsTable.id))