fix(build-db): make deploys resilient to malformed tenant/job data

.github/workflows/deploy-freshness.yml

@@ -0,0 +1,66 @@
+name: Deploy freshness
+
+# Watchdog for a silently-frozen deploy. The daily refresh chain
+# (Scrape → Build + deploy) can fail after the scrape succeeds — e.g. a
+# malformed tenant row that aborts build-db — in which case the live
+# manifest simply stops advancing while every check stays green. Nothing
+# surfaces the freeze.
+#
+# This job fetches the live manifest and fails when its built_at is older
+# than the threshold. A failed scheduled run shows up as a red check and
+# triggers GitHub's standard failed-workflow notification, so a stalled
+# deploy can no longer go unnoticed.
+#
+# Threshold is 36h — comfortably past the ~24h refresh cadence so a single
+# skipped/late run doesn't false-alarm, but tight enough to catch a real
+# freeze within a day.
+
+on:
+  schedule:
+    # Every 6 hours (00:00, 06:00, 12:00, 18:00 UTC).
+    - cron: "0 */6 * * *"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  freshness:
+    name: Check live manifest freshness
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - name: Fail if the live deploy is stale
+        env:
+          # Must match site/src/lib/site-config.ts SITE_ORIGIN and the
+          # PAGES_BASE mirror in build-deploy.yml — update in lockstep
+          # when the domain moves.
+          MANIFEST_URL: "https://openroles.today/data/manifest.json"
+          MAX_AGE_HOURS: "36"
+        run: |
+          set -euo pipefail
+          if ! body="$(curl -fsSL --max-time 30 "$MANIFEST_URL")"; then
+            echo "::error::could not fetch $MANIFEST_URL — the site may be down or Pages unreachable"
+            exit 1
+          fi
+          built_at="$(printf '%s' "$body" | jq -r '.built_at // empty')"
+          if [ -z "$built_at" ]; then
+            echo "::error::manifest at $MANIFEST_URL has no built_at field"
+            exit 1
+          fi
+          if ! built_epoch="$(date -u -d "$built_at" +%s 2>/dev/null)"; then
+            echo "::error::manifest built_at is not a parseable timestamp: $built_at"
+            exit 1
+          fi
+          now_epoch="$(date -u +%s)"
+          age_hours=$(( (now_epoch - built_epoch) / 3600 ))
+          echo "manifest built_at=$built_at — age ${age_hours}h (threshold ${MAX_AGE_HOURS}h)"
+          if [ "$age_hours" -ge "$MAX_AGE_HOURS" ]; then
+            echo "::error::deploy is stale: live manifest built_at=$built_at is ${age_hours}h old (threshold ${MAX_AGE_HOURS}h). The Scrape → Build + deploy chain is likely failing — check the build-deploy workflow."
+            exit 1
+          fi
+          echo "deploy is fresh (${age_hours}h < ${MAX_AGE_HOURS}h)"

scraper/src/cli.test.ts

@@ -442,6 +442,94 @@ describe("runBuildDbCommand", () => {
     expect(manifest.tenants_live).toBe(1);
   });
 
+  it("skips invalid tenant rows and builds from the valid ones", async () => {
+    const dir = tmpDir();
+    const inputDir = join(dir, "in");
+    const outputDir = join(dir, "out");
+    mkdirSync(inputDir);
+    writeFileSync(join(inputDir, "out.json"), JSON.stringify(emptyOutput()));
+    const tenantsPath = join(dir, "tenants.json");
+    writeFileSync(
+      tenantsPath,
+      JSON.stringify([
+        {
+          ats: "greenhouse",
+          slug: "alpha",
+          status: "live",
+          last_probed_at: "2026-04-26T00:00:00Z",
+        },
+        // Invalid: underscore in slug — the row that used to abort the whole build.
+        {
+          ats: "workable",
+          slug: "foo_bar",
+          status: "live",
+          last_probed_at: "2026-04-26T00:00:00Z",
+        },
+        // Invalid: dotted vendor domain in slug.
+        { ats: "ashby", slug: "kos.ai", status: "live", last_probed_at: "2026-04-26T00:00:00Z" },
+        { ats: "lever", slug: "beta", status: "dead", last_probed_at: "2026-04-26T00:00:00Z" },
+      ]),
+    );
+    const code = await runBuildDbCommand([
+      "--input",
+      inputDir,
+      "--output-dir",
+      outputDir,
+      "--short-sha",
+      "abc1234",
+      "--tenants",
+      tenantsPath,
+    ]);
+    expect(code).toBe(0);
+    expect(existsSync(join(outputDir, "jobs.abc1234.sqlite"))).toBe(true);
+    const manifest = JSON.parse(readFileSync(join(outputDir, "manifest.json"), "utf8"));
+    // Only the two valid rows land; the two malformed slugs are dropped.
+    expect(manifest.tenants_total).toBe(2);
+    expect(manifest.tenants_live).toBe(1);
+  });
+
+  it("skips invalid job rows within a valid scrape output and keeps the rest", async () => {
+    const dir = tmpDir();
+    const inputDir = join(dir, "in");
+    const outputDir = join(dir, "out");
+    mkdirSync(inputDir);
+    const goodJob = {
+      id: "a".repeat(64),
+      ats: "greenhouse",
+      tenant_slug: "acme",
+      source_id: "req-1",
+      title: "Staff Engineer",
+      company: "Acme",
+      level: "staff",
+      level_rank: 5,
+      workplace_type: "remote",
+      is_recruiter_post: false,
+      first_seen_at: "2026-04-26T00:00:00Z",
+      last_seen_at: "2026-04-26T00:00:00Z",
+      url: "https://boards.greenhouse.io/acme/jobs/1",
+    };
+    const badJob = {
+      ...goodJob,
+      id: "b".repeat(64),
+      tenant_slug: "bad_slug",
+      url: "https://boards.greenhouse.io/acme/jobs/2",
+    };
+    const output = { ...(emptyOutput() as Record<string, unknown>), jobs: [goodJob, badJob] };
+    writeFileSync(join(inputDir, "greenhouse.json"), JSON.stringify(output));
+    const code = await runBuildDbCommand([
+      "--input",
+      inputDir,
+      "--output-dir",
+      outputDir,
+      "--short-sha",
+      "abc1234",
+    ]);
+    expect(code).toBe(0);
+    const manifest = JSON.parse(readFileSync(join(outputDir, "manifest.json"), "utf8"));
+    // The malformed job is dropped; the well-formed one survives.
+    expect(manifest.total_rows).toBe(1);
+  });
+
   it("wraps malformed scrape JSON with the file path in the error", async () => {
     const dir = tmpDir();
     const inputDir = join(dir, "in");

scraper/src/cli.ts

@@ -23,6 +23,12 @@ import { z } from "zod";
 import { isSafeFetchHost } from "./ats/common.ts";
 import { fetchWorkdaySite } from "./ats/workday-site-fetch.ts";
 import { buildDb } from "./db/build-db.ts";
+import {
+  formatSkipSummary,
+  partitionScrapeOutput,
+  partitionTenants,
+  type SkippedRow,
+} from "./db/resilient-parse.ts";
 import { emitSlimIndex } from "./db/slim-index.ts";
 import { diskClusterIdxCache } from "./harvest/cc-s3.ts";
 import { SNAPSHOT_ID_RE } from "./harvest/cdx.ts";
@@ -408,25 +414,28 @@ export async function runBuildDbCommand(argv: ReadonlyArray<string>): Promise<nu
   // not crash the daily refresh.
   const entries = (await readdir(args.input)).sort();
   const outputs: ScrapeOutput[] = [];
+  const skippedJobs: SkippedRow[] = [];
   for (const name of entries) {
     if (!name.endsWith(".json")) continue;
     const path = join(args.input, name);
     const raw = await readJsonOrThrow(path, "build-db");
-    const parsed = ScrapeOutputSchema.safeParse(raw);
-    if (!parsed.success) {
-      console.error(
-        `build-db: skipping ${name}: ${parsed.error.issues
-          .map((i) => `${i.path.join(".")} ${i.message}`)
-          .join("; ")}`,
-      );
+    const part = partitionScrapeOutput(raw);
+    if (part.output === null) {
+      console.error(`build-db: skipping ${name}: ${part.envelopeError}`);
       continue;
     }
-    outputs.push(parsed.data);
+    outputs.push(part.output);
+    for (const s of part.skipped) skippedJobs.push(s);
   }
+  const jobSummary = formatSkipSummary("job", skippedJobs);
+  if (jobSummary !== null) console.error(jobSummary);
 
   let tenants: Tenant[] = [];
   if (args.tenants !== undefined) {
-    tenants = z.array(TenantSchema).parse(await readJsonOrThrow(args.tenants, "build-db"));
+    const part = partitionTenants(await readJsonOrThrow(args.tenants, "build-db"));
+    tenants = part.valid;
+    const tenantSummary = formatSkipSummary("tenant", part.skipped);
+    if (tenantSummary !== null) console.error(tenantSummary);
   }
 
   const builtAt = new Date().toISOString();

scraper/src/db/resilient-parse.test.ts

@@ -0,0 +1,176 @@
+import { describe, expect, it } from "bun:test";
+import {
+  formatSkipSummary,
+  MAX_SKIP_SAMPLES,
+  partitionScrapeOutput,
+  partitionTenants,
+  type SkippedRow,
+} from "./resilient-parse.ts";
+
+function validTenant(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    ats: "greenhouse",
+    slug: "acme",
+    status: "live",
+    last_probed_at: "2026-07-01T00:00:00Z",
+    ...overrides,
+  };
+}
+
+const HEX_ID = "a".repeat(64);
+
+function validJob(overrides: Record<string, unknown> = {}): Record<string, unknown> {
+  return {
+    id: HEX_ID,
+    ats: "greenhouse",
+    tenant_slug: "acme",
+    source_id: "req-1",
+    title: "Staff Engineer",
+    company: "Acme",
+    level: "staff",
+    level_rank: 5,
+    workplace_type: "remote",
+    is_recruiter_post: false,
+    first_seen_at: "2026-07-01T00:00:00Z",
+    last_seen_at: "2026-07-01T00:00:00Z",
+    url: "https://boards.greenhouse.io/acme/jobs/1",
+    ...overrides,
+  };
+}
+
+function validEnvelope(jobs: unknown[]): Record<string, unknown> {
+  return {
+    ats: "greenhouse",
+    jobs,
+    tenant_results: [],
+    metrics: {
+      started_at: "2026-07-01T00:00:00Z",
+      finished_at: "2026-07-01T00:00:00Z",
+      duration_ms: 0,
+      requests_made: 0,
+      requests_failed: 0,
+      requests_retried: 0,
+      bytes_received: 0,
+    },
+  };
+}
+
+describe("partitionTenants", () => {
+  it("keeps all rows when every tenant is valid (no skips)", () => {
+    const raw = [validTenant({ slug: "acme" }), validTenant({ slug: "globex", status: "dead" })];
+    const { valid, skipped } = partitionTenants(raw);
+    expect(valid.map((t) => t.slug)).toEqual(["acme", "globex"]);
+    expect(skipped).toHaveLength(0);
+  });
+
+  it("drops only the invalid rows and keeps the valid ones", () => {
+    const raw = [
+      validTenant({ slug: "acme" }),
+      validTenant({ ats: "workable", slug: "foo_bar" }), // underscore — invalid slug
+      validTenant({ slug: "globex" }),
+      validTenant({ ats: "ashby", slug: "kos.ai" }), // dot — invalid slug
+    ];
+    const { valid, skipped } = partitionTenants(raw);
+    expect(valid.map((t) => t.slug)).toEqual(["acme", "globex"]);
+    expect(skipped).toHaveLength(2);
+    expect(skipped[0]).toMatchObject({ ats: "workable", slug: "foo_bar", field: "slug" });
+    expect(skipped[1]).toMatchObject({ ats: "ashby", slug: "kos.ai", field: "slug" });
+    expect(skipped[0]?.reason).toContain("slug");
+  });
+
+  it("labels a row whose ats/slug are themselves missing as '?'", () => {
+    const { valid, skipped } = partitionTenants([{ status: "live" }]);
+    expect(valid).toHaveLength(0);
+    expect(skipped[0]).toMatchObject({ ats: "?", slug: "?" });
+  });
+
+  it("returns a single skip when the input is not an array", () => {
+    const { valid, skipped } = partitionTenants({ not: "an array" });
+    expect(valid).toHaveLength(0);
+    expect(skipped).toHaveLength(1);
+    expect(skipped[0]?.reason).toContain("not a JSON array");
+  });
+});
+
+describe("partitionScrapeOutput", () => {
+  it("keeps every job when all are valid", () => {
+    const raw = validEnvelope([validJob(), validJob({ id: "b".repeat(64) })]);
+    const part = partitionScrapeOutput(raw);
+    expect(part.output?.jobs).toHaveLength(2);
+    expect(part.skipped).toHaveLength(0);
+    expect(part.envelopeError).toBeUndefined();
+  });
+
+  it("drops only invalid jobs and keeps the rest", () => {
+    const raw = validEnvelope([
+      validJob({ id: "a".repeat(64) }),
+      validJob({ id: "b".repeat(64), tenant_slug: "bad_slug" }), // underscore
+      validJob({ id: "c".repeat(64) }),
+    ]);
+    const part = partitionScrapeOutput(raw);
+    expect(part.output?.jobs.map((j) => j.id)).toEqual(["a".repeat(64), "c".repeat(64)]);
+    expect(part.skipped).toHaveLength(1);
+    expect(part.skipped[0]).toMatchObject({ slug: "bad_slug", field: "tenant_slug" });
+  });
+
+  it("falls back to the envelope ats when a bad job omits its own ats", () => {
+    const raw = validEnvelope([{ tenant_slug: "acme" }]); // missing everything else
+    const part = partitionScrapeOutput(raw);
+    expect(part.output?.jobs).toHaveLength(0);
+    expect(part.skipped[0]).toMatchObject({ ats: "greenhouse", slug: "acme" });
+  });
+
+  it("returns envelopeError and null output when the envelope is invalid", () => {
+    const part = partitionScrapeOutput({ ats: "greenhouse" }); // no jobs/metrics/tenant_results
+    expect(part.output).toBeNull();
+    expect(part.skipped).toHaveLength(0);
+    expect(part.envelopeError).toBeTruthy();
+  });
+
+  it("preserves the envelope's tenant_results and metrics", () => {
+    const raw = validEnvelope([validJob()]);
+    const part = partitionScrapeOutput(raw);
+    expect(part.output?.ats).toBe("greenhouse");
+    expect(part.output?.tenant_results).toEqual([]);
+    expect(part.output?.metrics.duration_ms).toBe(0);
+  });
+});
+
+describe("formatSkipSummary", () => {
+  it("returns null when nothing was skipped", () => {
+    expect(formatSkipSummary("tenant", [])).toBeNull();
+  });
+
+  it("names offenders and reports the count (singular)", () => {
+    const rows: SkippedRow[] = [
+      { ats: "workable", slug: "foo_bar", field: "slug", reason: "slug ..." },
+    ];
+    expect(formatSkipSummary("tenant", rows)).toBe(
+      "build-db: skipped 1 invalid tenant row: workable/foo_bar (slug)",
+    );
+  });
+
+  it("uses the plural form and joins multiple offenders", () => {
+    const rows: SkippedRow[] = [
+      { ats: "workable", slug: "foo_bar", field: "slug", reason: "" },
+      { ats: "ashby", slug: "kos.ai", field: "slug", reason: "" },
+    ];
+    expect(formatSkipSummary("tenant", rows)).toBe(
+      "build-db: skipped 2 invalid tenant rows: workable/foo_bar (slug), ashby/kos.ai (slug)",
+    );
+  });
+
+  it("caps the sample list and reports the overflow total", () => {
+    const rows: SkippedRow[] = Array.from({ length: MAX_SKIP_SAMPLES + 5 }, (_, i) => ({
+      ats: "greenhouse",
+      slug: `t${i}`,
+      field: "slug",
+      reason: "",
+    }));
+    const summary = formatSkipSummary("job", rows);
+    expect(summary).toContain(`skipped ${MAX_SKIP_SAMPLES + 5} invalid job rows`);
+    expect(summary).toContain("+5 more");
+    // Only MAX_SKIP_SAMPLES offenders are named before the "+N more".
+    expect(summary?.match(/greenhouse\//g)).toHaveLength(MAX_SKIP_SAMPLES);
+  });
+});