From aa37b6a0119027702753674c9596171cb15260cd Mon Sep 17 00:00:00 2001 From: datascry Date: Mon, 13 Jul 2026 12:00:26 -0700 Subject: [PATCH 1/3] fix(build-db): skip and warn on malformed rows instead of aborting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A single tenant or job row that fails schema validation used to throw at build-db and abort the whole deploy: the SQLite build never finished, the Pages deploy was skipped, and the live manifest froze at the last good build while scrapes kept succeeding — a silent outage. One underscore or dotted domain in a slug was enough to stall the site indefinitely. Parse each row with safeParse: valid rows proceed unchanged, invalid rows are dropped and recorded, and one summary line per kind names the offenders on stderr. Jobs are now validated per-row rather than per-file, so one bad posting no longer discards a tenant's entire corpus. Orphan jobs whose tenant row was skipped are left in place — the slim index reads jobs directly with no tenant join, matching prior behavior. --- scraper/src/cli.test.ts | 88 +++++++++++++ scraper/src/cli.ts | 27 ++-- scraper/src/db/resilient-parse.test.ts | 176 +++++++++++++++++++++++++ scraper/src/db/resilient-parse.ts | 170 ++++++++++++++++++++++++ 4 files changed, 452 insertions(+), 9 deletions(-) create mode 100644 scraper/src/db/resilient-parse.test.ts create mode 100644 scraper/src/db/resilient-parse.ts diff --git a/scraper/src/cli.test.ts b/scraper/src/cli.test.ts index 78c7ad3..26d1fa3 100644 --- a/scraper/src/cli.test.ts +++ b/scraper/src/cli.test.ts @@ -442,6 +442,94 @@ describe("runBuildDbCommand", () => { expect(manifest.tenants_live).toBe(1); }); + it("skips invalid tenant rows and builds from the valid ones", async () => { + const dir = tmpDir(); + const inputDir = join(dir, "in"); + const outputDir = join(dir, "out"); + mkdirSync(inputDir); + writeFileSync(join(inputDir, "out.json"), JSON.stringify(emptyOutput())); + const tenantsPath = join(dir, "tenants.json"); + writeFileSync( + tenantsPath, + JSON.stringify([ + { + ats: "greenhouse", + slug: "alpha", + status: "live", + last_probed_at: "2026-04-26T00:00:00Z", + }, + // Invalid: underscore in slug — the row that used to abort the whole build. + { + ats: "workable", + slug: "foo_bar", + status: "live", + last_probed_at: "2026-04-26T00:00:00Z", + }, + // Invalid: dotted vendor domain in slug. + { ats: "ashby", slug: "kos.ai", status: "live", last_probed_at: "2026-04-26T00:00:00Z" }, + { ats: "lever", slug: "beta", status: "dead", last_probed_at: "2026-04-26T00:00:00Z" }, + ]), + ); + const code = await runBuildDbCommand([ + "--input", + inputDir, + "--output-dir", + outputDir, + "--short-sha", + "abc1234", + "--tenants", + tenantsPath, + ]); + expect(code).toBe(0); + expect(existsSync(join(outputDir, "jobs.abc1234.sqlite"))).toBe(true); + const manifest = JSON.parse(readFileSync(join(outputDir, "manifest.json"), "utf8")); + // Only the two valid rows land; the two malformed slugs are dropped. + expect(manifest.tenants_total).toBe(2); + expect(manifest.tenants_live).toBe(1); + }); + + it("skips invalid job rows within a valid scrape output and keeps the rest", async () => { + const dir = tmpDir(); + const inputDir = join(dir, "in"); + const outputDir = join(dir, "out"); + mkdirSync(inputDir); + const goodJob = { + id: "a".repeat(64), + ats: "greenhouse", + tenant_slug: "acme", + source_id: "req-1", + title: "Staff Engineer", + company: "Acme", + level: "staff", + level_rank: 5, + workplace_type: "remote", + is_recruiter_post: false, + first_seen_at: "2026-04-26T00:00:00Z", + last_seen_at: "2026-04-26T00:00:00Z", + url: "https://boards.greenhouse.io/acme/jobs/1", + }; + const badJob = { + ...goodJob, + id: "b".repeat(64), + tenant_slug: "bad_slug", + url: "https://boards.greenhouse.io/acme/jobs/2", + }; + const output = { ...(emptyOutput() as Record), jobs: [goodJob, badJob] }; + writeFileSync(join(inputDir, "greenhouse.json"), JSON.stringify(output)); + const code = await runBuildDbCommand([ + "--input", + inputDir, + "--output-dir", + outputDir, + "--short-sha", + "abc1234", + ]); + expect(code).toBe(0); + const manifest = JSON.parse(readFileSync(join(outputDir, "manifest.json"), "utf8")); + // The malformed job is dropped; the well-formed one survives. + expect(manifest.total_rows).toBe(1); + }); + it("wraps malformed scrape JSON with the file path in the error", async () => { const dir = tmpDir(); const inputDir = join(dir, "in"); diff --git a/scraper/src/cli.ts b/scraper/src/cli.ts index 0e0635c..70161df 100644 --- a/scraper/src/cli.ts +++ b/scraper/src/cli.ts @@ -23,6 +23,12 @@ import { z } from "zod"; import { isSafeFetchHost } from "./ats/common.ts"; import { fetchWorkdaySite } from "./ats/workday-site-fetch.ts"; import { buildDb } from "./db/build-db.ts"; +import { + formatSkipSummary, + partitionScrapeOutput, + partitionTenants, + type SkippedRow, +} from "./db/resilient-parse.ts"; import { emitSlimIndex } from "./db/slim-index.ts"; import { diskClusterIdxCache } from "./harvest/cc-s3.ts"; import { SNAPSHOT_ID_RE } from "./harvest/cdx.ts"; @@ -408,25 +414,28 @@ export async function runBuildDbCommand(argv: ReadonlyArray): Promise `${i.path.join(".")} ${i.message}`) - .join("; ")}`, - ); + const part = partitionScrapeOutput(raw); + if (part.output === null) { + console.error(`build-db: skipping ${name}: ${part.envelopeError}`); continue; } - outputs.push(parsed.data); + outputs.push(part.output); + for (const s of part.skipped) skippedJobs.push(s); } + const jobSummary = formatSkipSummary("job", skippedJobs); + if (jobSummary !== null) console.error(jobSummary); let tenants: Tenant[] = []; if (args.tenants !== undefined) { - tenants = z.array(TenantSchema).parse(await readJsonOrThrow(args.tenants, "build-db")); + const part = partitionTenants(await readJsonOrThrow(args.tenants, "build-db")); + tenants = part.valid; + const tenantSummary = formatSkipSummary("tenant", part.skipped); + if (tenantSummary !== null) console.error(tenantSummary); } const builtAt = new Date().toISOString(); diff --git a/scraper/src/db/resilient-parse.test.ts b/scraper/src/db/resilient-parse.test.ts new file mode 100644 index 0000000..324e00a --- /dev/null +++ b/scraper/src/db/resilient-parse.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it } from "bun:test"; +import { + formatSkipSummary, + MAX_SKIP_SAMPLES, + partitionScrapeOutput, + partitionTenants, + type SkippedRow, +} from "./resilient-parse.ts"; + +function validTenant(overrides: Record = {}): Record { + return { + ats: "greenhouse", + slug: "acme", + status: "live", + last_probed_at: "2026-07-01T00:00:00Z", + ...overrides, + }; +} + +const HEX_ID = "a".repeat(64); + +function validJob(overrides: Record = {}): Record { + return { + id: HEX_ID, + ats: "greenhouse", + tenant_slug: "acme", + source_id: "req-1", + title: "Staff Engineer", + company: "Acme", + level: "staff", + level_rank: 5, + workplace_type: "remote", + is_recruiter_post: false, + first_seen_at: "2026-07-01T00:00:00Z", + last_seen_at: "2026-07-01T00:00:00Z", + url: "https://boards.greenhouse.io/acme/jobs/1", + ...overrides, + }; +} + +function validEnvelope(jobs: unknown[]): Record { + return { + ats: "greenhouse", + jobs, + tenant_results: [], + metrics: { + started_at: "2026-07-01T00:00:00Z", + finished_at: "2026-07-01T00:00:00Z", + duration_ms: 0, + requests_made: 0, + requests_failed: 0, + requests_retried: 0, + bytes_received: 0, + }, + }; +} + +describe("partitionTenants", () => { + it("keeps all rows when every tenant is valid (no skips)", () => { + const raw = [validTenant({ slug: "acme" }), validTenant({ slug: "globex", status: "dead" })]; + const { valid, skipped } = partitionTenants(raw); + expect(valid.map((t) => t.slug)).toEqual(["acme", "globex"]); + expect(skipped).toHaveLength(0); + }); + + it("drops only the invalid rows and keeps the valid ones", () => { + const raw = [ + validTenant({ slug: "acme" }), + validTenant({ ats: "workable", slug: "foo_bar" }), // underscore — invalid slug + validTenant({ slug: "globex" }), + validTenant({ ats: "ashby", slug: "kos.ai" }), // dot — invalid slug + ]; + const { valid, skipped } = partitionTenants(raw); + expect(valid.map((t) => t.slug)).toEqual(["acme", "globex"]); + expect(skipped).toHaveLength(2); + expect(skipped[0]).toMatchObject({ ats: "workable", slug: "foo_bar", field: "slug" }); + expect(skipped[1]).toMatchObject({ ats: "ashby", slug: "kos.ai", field: "slug" }); + expect(skipped[0]?.reason).toContain("slug"); + }); + + it("labels a row whose ats/slug are themselves missing as '?'", () => { + const { valid, skipped } = partitionTenants([{ status: "live" }]); + expect(valid).toHaveLength(0); + expect(skipped[0]).toMatchObject({ ats: "?", slug: "?" }); + }); + + it("returns a single skip when the input is not an array", () => { + const { valid, skipped } = partitionTenants({ not: "an array" }); + expect(valid).toHaveLength(0); + expect(skipped).toHaveLength(1); + expect(skipped[0]?.reason).toContain("not a JSON array"); + }); +}); + +describe("partitionScrapeOutput", () => { + it("keeps every job when all are valid", () => { + const raw = validEnvelope([validJob(), validJob({ id: "b".repeat(64) })]); + const part = partitionScrapeOutput(raw); + expect(part.output?.jobs).toHaveLength(2); + expect(part.skipped).toHaveLength(0); + expect(part.envelopeError).toBeUndefined(); + }); + + it("drops only invalid jobs and keeps the rest", () => { + const raw = validEnvelope([ + validJob({ id: "a".repeat(64) }), + validJob({ id: "b".repeat(64), tenant_slug: "bad_slug" }), // underscore + validJob({ id: "c".repeat(64) }), + ]); + const part = partitionScrapeOutput(raw); + expect(part.output?.jobs.map((j) => j.id)).toEqual(["a".repeat(64), "c".repeat(64)]); + expect(part.skipped).toHaveLength(1); + expect(part.skipped[0]).toMatchObject({ slug: "bad_slug", field: "tenant_slug" }); + }); + + it("falls back to the envelope ats when a bad job omits its own ats", () => { + const raw = validEnvelope([{ tenant_slug: "acme" }]); // missing everything else + const part = partitionScrapeOutput(raw); + expect(part.output?.jobs).toHaveLength(0); + expect(part.skipped[0]).toMatchObject({ ats: "greenhouse", slug: "acme" }); + }); + + it("returns envelopeError and null output when the envelope is invalid", () => { + const part = partitionScrapeOutput({ ats: "greenhouse" }); // no jobs/metrics/tenant_results + expect(part.output).toBeNull(); + expect(part.skipped).toHaveLength(0); + expect(part.envelopeError).toBeTruthy(); + }); + + it("preserves the envelope's tenant_results and metrics", () => { + const raw = validEnvelope([validJob()]); + const part = partitionScrapeOutput(raw); + expect(part.output?.ats).toBe("greenhouse"); + expect(part.output?.tenant_results).toEqual([]); + expect(part.output?.metrics.duration_ms).toBe(0); + }); +}); + +describe("formatSkipSummary", () => { + it("returns null when nothing was skipped", () => { + expect(formatSkipSummary("tenant", [])).toBeNull(); + }); + + it("names offenders and reports the count (singular)", () => { + const rows: SkippedRow[] = [ + { ats: "workable", slug: "foo_bar", field: "slug", reason: "slug ..." }, + ]; + expect(formatSkipSummary("tenant", rows)).toBe( + "build-db: skipped 1 invalid tenant row: workable/foo_bar (slug)", + ); + }); + + it("uses the plural form and joins multiple offenders", () => { + const rows: SkippedRow[] = [ + { ats: "workable", slug: "foo_bar", field: "slug", reason: "" }, + { ats: "ashby", slug: "kos.ai", field: "slug", reason: "" }, + ]; + expect(formatSkipSummary("tenant", rows)).toBe( + "build-db: skipped 2 invalid tenant rows: workable/foo_bar (slug), ashby/kos.ai (slug)", + ); + }); + + it("caps the sample list and reports the overflow total", () => { + const rows: SkippedRow[] = Array.from({ length: MAX_SKIP_SAMPLES + 5 }, (_, i) => ({ + ats: "greenhouse", + slug: `t${i}`, + field: "slug", + reason: "", + })); + const summary = formatSkipSummary("job", rows); + expect(summary).toContain(`skipped ${MAX_SKIP_SAMPLES + 5} invalid job rows`); + expect(summary).toContain("+5 more"); + // Only MAX_SKIP_SAMPLES offenders are named before the "+N more". + expect(summary?.match(/greenhouse\//g)).toHaveLength(MAX_SKIP_SAMPLES); + }); +}); diff --git a/scraper/src/db/resilient-parse.ts b/scraper/src/db/resilient-parse.ts new file mode 100644 index 0000000..a9fa452 --- /dev/null +++ b/scraper/src/db/resilient-parse.ts @@ -0,0 +1,170 @@ +// Resilient, per-row parsing for build-db inputs. +// +// The daily refresh reads a merged tenant list and a directory of scrape +// outputs, then feeds the valid rows into `buildDb`. A single malformed row +// — an underscore in a tenant slug, a dotted vendor domain, a control char +// in a job title — must never abort the build: the site would freeze at the +// last good deploy while scrapes keep succeeding, and the failure is silent. +// +// These helpers replace the eager `z.array(Schema).parse(...)` (throws on the +// first bad row) with a per-row `safeParse`: valid rows pass through unchanged, +// invalid rows are dropped and recorded so the caller can log one summary line. + +import { + ATSIdSchema, + type Job, + JobSchema, + ScrapeMetricsSchema, + type ScrapeOutput, + type Tenant, + TenantResultSchema, + TenantSchema, +} from "@openroles/shared"; +import { z } from "zod"; + +/** Maximum offending rows named in a single summary line before truncation. */ +export const MAX_SKIP_SAMPLES = 20; + +/** + * A row rejected during resilient parsing, tagged with a best-effort identity + * so the summary log points an operator straight at the offending tenant/job. + */ +export interface SkippedRow { + /** The row's `ats` value, or "?" when it was itself missing/non-string. */ + readonly ats: string; + /** The row's slug, or "?" when it was missing/non-string. */ + readonly slug: string; + /** Compact label of the first field that failed, e.g. "slug" or "title". */ + readonly field: string; + /** Full Zod issue string, for the record — `field message; field message`. */ + readonly reason: string; +} + +/** Valid rows plus the rows that were dropped. */ +export interface Partition { + readonly valid: T[]; + readonly skipped: SkippedRow[]; +} + +/** A scrape-output file split into its valid jobs and the jobs that were dropped. */ +export interface ScrapeOutputPartition { + /** The rebuilt output with only valid jobs, or null when the envelope itself was invalid. */ + readonly output: ScrapeOutput | null; + /** Jobs dropped from an otherwise-valid envelope. */ + readonly skipped: SkippedRow[]; + /** Set only when the envelope (ats / metrics / tenant_results) failed — the whole file is unusable. */ + readonly envelopeError?: string; +} + +/** Best-effort (ats, slug) label from an unvalidated record. */ +function labelOf(raw: unknown, slugKey: "slug" | "tenant_slug"): { ats: string; slug: string } { + const rec = typeof raw === "object" && raw !== null ? (raw as Record) : {}; + const ats = typeof rec["ats"] === "string" && rec["ats"].length > 0 ? rec["ats"] : "?"; + const slugVal = rec[slugKey]; + const slug = typeof slugVal === "string" && slugVal.length > 0 ? slugVal : "?"; + return { ats, slug }; +} + +/** Path of the first issue, e.g. "slug" or "metrics.started_at" — "(root)" when empty. */ +function fieldOf(error: z.ZodError): string { + const first = error.issues[0]; + if (first === undefined) return "(root)"; + return first.path.length > 0 ? first.path.join(".") : "(root)"; +} + +/** Full human-readable reason string joining every issue. */ +function reasonOf(error: z.ZodError): string { + return error.issues + .map((i) => `${i.path.length > 0 ? i.path.join(".") : "(root)"} ${i.message}`) + .join("; "); +} + +/** + * Split a raw tenants array into valid `Tenant`s and dropped rows. + * + * A non-array input (corrupt merge) yields zero valid rows and a single + * skip describing the shape error, rather than throwing. + */ +export function partitionTenants(raw: unknown): Partition { + const valid: Tenant[] = []; + const skipped: SkippedRow[] = []; + if (!Array.isArray(raw)) { + skipped.push({ + ats: "?", + slug: "?", + field: "(root)", + reason: "tenants input is not a JSON array", + }); + return { valid, skipped }; + } + for (const entry of raw) { + const parsed = TenantSchema.safeParse(entry); + if (parsed.success) { + valid.push(parsed.data); + continue; + } + const { ats, slug } = labelOf(entry, "slug"); + skipped.push({ ats, slug, field: fieldOf(parsed.error), reason: reasonOf(parsed.error) }); + } + return { valid, skipped }; +} + +// Envelope with a lenient jobs field: validate ats / metrics / tenant_results +// up front (a broken envelope means the whole file is unusable) but defer each +// job to a per-row safeParse so one bad posting can't drop a tenant's corpus. +const LenientScrapeOutputSchema = z.object({ + ats: ATSIdSchema, + jobs: z.array(z.unknown()), + tenant_results: z.array(TenantResultSchema), + metrics: ScrapeMetricsSchema, +}); + +/** + * Split one scrape-output file into a rebuilt output carrying only valid jobs + * plus the jobs that were dropped. When the envelope itself is invalid the + * whole file is unusable — `output` is null and `envelopeError` explains why. + */ +export function partitionScrapeOutput(raw: unknown): ScrapeOutputPartition { + const envelope = LenientScrapeOutputSchema.safeParse(raw); + if (!envelope.success) { + return { output: null, skipped: [], envelopeError: reasonOf(envelope.error) }; + } + const jobs: Job[] = []; + const skipped: SkippedRow[] = []; + for (const entry of envelope.data.jobs) { + const parsed = JobSchema.safeParse(entry); + if (parsed.success) { + jobs.push(parsed.data); + continue; + } + const label = labelOf(entry, "tenant_slug"); + skipped.push({ + ats: label.ats === "?" ? envelope.data.ats : label.ats, + slug: label.slug, + field: fieldOf(parsed.error), + reason: reasonOf(parsed.error), + }); + } + const output: ScrapeOutput = { + ats: envelope.data.ats, + jobs, + tenant_results: envelope.data.tenant_results, + metrics: envelope.data.metrics, + }; + return { output, skipped }; +} + +/** + * One-line stderr summary for a batch of skipped rows, or null when nothing was + * skipped. Names up to {@link MAX_SKIP_SAMPLES} offenders and always reports the + * true total, e.g. + * `build-db: skipped 3 invalid tenant rows: workable/foo_bar (slug), ashby/kos.ai (slug)`. + */ +export function formatSkipSummary(kind: string, skipped: ReadonlyArray): string | null { + if (skipped.length === 0) return null; + const samples = skipped.slice(0, MAX_SKIP_SAMPLES).map((s) => `${s.ats}/${s.slug} (${s.field})`); + const more = skipped.length - samples.length; + const suffix = more > 0 ? `, +${more} more` : ""; + const plural = skipped.length === 1 ? "" : "s"; + return `build-db: skipped ${skipped.length} invalid ${kind} row${plural}: ${samples.join(", ")}${suffix}`; +} From e0e5b3eaa84f152812806a7992f9a5415da2d07b Mon Sep 17 00:00:00 2001 From: datascry Date: Mon, 13 Jul 2026 12:02:28 -0700 Subject: [PATCH 2/3] test(scraper): guard committed tenant data against schema violations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-db runs only post-merge, so a malformed tenant row passes pull-request CI and first surfaces at deploy — the exact blind spot that froze the live site. Validate every committed data/tenants/*.json entry against TenantSchema in the test suite (which pr.yml runs), failing with the file, row index, slug, and violated rule so the source gets fixed before merge rather than silently dropped on every refresh. --- scraper/tests/tenant-data.test.ts | 74 +++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 scraper/tests/tenant-data.test.ts diff --git a/scraper/tests/tenant-data.test.ts b/scraper/tests/tenant-data.test.ts new file mode 100644 index 0000000..c61c4aa --- /dev/null +++ b/scraper/tests/tenant-data.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { TenantSchema } from "@openroles/shared"; + +// Data-integrity guard. build-db is NOT run in pr.yml — it only runs +// post-merge in build-deploy.yml — so a tenant row that violates +// TenantSchema (an underscore or dotted domain in a slug, a bad status, +// a malformed timestamp) sails through PR CI and only detonates at deploy +// time, freezing the live site at the last good build. +// +// This test loads every committed data/tenants/*.json and validates each +// entry against the same schema build-db uses, so bad tenant data fails +// `bun run test` at PR time. build-db still skips such rows defensively +// (see resilient-parse.ts) — this guard exists so a human fixes the source +// instead of silently shedding tenants on every refresh. +const TENANTS_DIR = join(import.meta.dir, "..", "..", "data", "tenants"); + +interface BadRow { + readonly file: string; + readonly index: number; + readonly slug: string; + readonly rule: string; +} + +function collectBadRows(): BadRow[] { + const bad: BadRow[] = []; + const files = readdirSync(TENANTS_DIR) + .filter((f) => f.endsWith(".json")) + .sort(); + for (const file of files) { + const raw: unknown = JSON.parse(readFileSync(join(TENANTS_DIR, file), "utf8")); + if (!Array.isArray(raw)) { + bad.push({ file, index: -1, slug: "(root)", rule: "file is not a JSON array" }); + continue; + } + raw.forEach((entry, index) => { + const parsed = TenantSchema.safeParse(entry); + if (parsed.success) return; + const rec = (typeof entry === "object" && entry !== null ? entry : {}) as Record< + string, + unknown + >; + const slug = typeof rec["slug"] === "string" ? rec["slug"] : "(missing)"; + const rule = parsed.error.issues + .map((i) => `${i.path.length > 0 ? i.path.join(".") : "(root)"}: ${i.message}`) + .join("; "); + bad.push({ file, index, slug, rule }); + }); + } + return bad; +} + +describe("data/tenants/*.json integrity", () => { + it("finds at least one tenant file to validate", () => { + const files = readdirSync(TENANTS_DIR).filter((f) => f.endsWith(".json")); + expect(files.length).toBeGreaterThan(0); + }); + + it("validates every committed tenant row against TenantSchema", () => { + const bad = collectBadRows(); + const report = bad + .slice(0, 50) + .map((b) => ` ${b.file}[${b.index}] slug=${b.slug} — ${b.rule}`) + .join("\n"); + const more = bad.length > 50 ? `\n …and ${bad.length - 50} more` : ""; + expect( + bad.length, + bad.length === 0 + ? "" + : `${bad.length} invalid tenant row(s) in data/tenants/. Fix or remove them — build-db would drop these at deploy time:\n${report}${more}`, + ).toBe(0); + }); +}); From 269a1bd385d46c1895a1c6e00de4ea9ecc5fd208 Mon Sep 17 00:00:00 2001 From: datascry Date: Mon, 13 Jul 2026 12:03:42 -0700 Subject: [PATCH 3/3] ci: add scheduled deploy-freshness watchdog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refresh chain can fail after the scrape succeeds — build-db aborting on a bad row, a Pages hiccup — leaving the live manifest frozen while every check stays green. Nothing surfaces the freeze. Add a workflow that runs every 6h (and on dispatch), fetches the live manifest, and fails when built_at is older than 36h. A red scheduled run plus GitHub's failed-workflow notification makes a stalled deploy visible within a day; the failure message reports the actual staleness. It needs no new deps — plain curl, jq, and date on the runner. --- .github/workflows/deploy-freshness.yml | 66 ++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/deploy-freshness.yml diff --git a/.github/workflows/deploy-freshness.yml b/.github/workflows/deploy-freshness.yml new file mode 100644 index 0000000..bf0225d --- /dev/null +++ b/.github/workflows/deploy-freshness.yml @@ -0,0 +1,66 @@ +name: Deploy freshness + +# Watchdog for a silently-frozen deploy. The daily refresh chain +# (Scrape → Build + deploy) can fail after the scrape succeeds — e.g. a +# malformed tenant row that aborts build-db — in which case the live +# manifest simply stops advancing while every check stays green. Nothing +# surfaces the freeze. +# +# This job fetches the live manifest and fails when its built_at is older +# than the threshold. A failed scheduled run shows up as a red check and +# triggers GitHub's standard failed-workflow notification, so a stalled +# deploy can no longer go unnoticed. +# +# Threshold is 36h — comfortably past the ~24h refresh cadence so a single +# skipped/late run doesn't false-alarm, but tight enough to catch a real +# freeze within a day. + +on: + schedule: + # Every 6 hours (00:00, 06:00, 12:00, 18:00 UTC). + - cron: "0 */6 * * *" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + freshness: + name: Check live manifest freshness + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Fail if the live deploy is stale + env: + # Must match site/src/lib/site-config.ts SITE_ORIGIN and the + # PAGES_BASE mirror in build-deploy.yml — update in lockstep + # when the domain moves. + MANIFEST_URL: "https://openroles.today/data/manifest.json" + MAX_AGE_HOURS: "36" + run: | + set -euo pipefail + if ! body="$(curl -fsSL --max-time 30 "$MANIFEST_URL")"; then + echo "::error::could not fetch $MANIFEST_URL — the site may be down or Pages unreachable" + exit 1 + fi + built_at="$(printf '%s' "$body" | jq -r '.built_at // empty')" + if [ -z "$built_at" ]; then + echo "::error::manifest at $MANIFEST_URL has no built_at field" + exit 1 + fi + if ! built_epoch="$(date -u -d "$built_at" +%s 2>/dev/null)"; then + echo "::error::manifest built_at is not a parseable timestamp: $built_at" + exit 1 + fi + now_epoch="$(date -u +%s)" + age_hours=$(( (now_epoch - built_epoch) / 3600 )) + echo "manifest built_at=$built_at — age ${age_hours}h (threshold ${MAX_AGE_HOURS}h)" + if [ "$age_hours" -ge "$MAX_AGE_HOURS" ]; then + echo "::error::deploy is stale: live manifest built_at=$built_at is ${age_hours}h old (threshold ${MAX_AGE_HOURS}h). The Scrape → Build + deploy chain is likely failing — check the build-deploy workflow." + exit 1 + fi + echo "deploy is fresh (${age_hours}h < ${MAX_AGE_HOURS}h)"