diff --git a/app/api/deep-dive/route.ts b/app/api/deep-dive/route.ts index d60a3f1..c9571a6 100644 --- a/app/api/deep-dive/route.ts +++ b/app/api/deep-dive/route.ts @@ -4,40 +4,7 @@ export const maxDuration = 120; import { buildPromptForTemplate } from "@/lib/deep-dive-templates"; import { parseSseChunk } from "@/lib/utils/sse"; - -type ResearchCitation = { claim: string; source_url: string; source_text?: string }; - -function extractResult(data: unknown): unknown { - if (!data || typeof data !== "object" || Array.isArray(data)) return data ?? null; - const d = data as Record; - if ("result" in d) return d.result; - const { citations: _c, ...rest } = d; - return Object.keys(rest).length > 0 ? rest : null; -} - -function extractCitations(data: unknown): ResearchCitation[] { - if (!data || typeof data !== "object" || Array.isArray(data)) return []; - const raw = (data as Record).citations; - if (!Array.isArray(raw)) return []; - return raw.flatMap((item: unknown): ResearchCitation[] => { - if (!item || typeof item !== "object" || Array.isArray(item)) return []; - const r = item as Record; - if (typeof r.source_url !== "string") return []; - try { - const u = new URL(r.source_url); - if (u.protocol !== "https:" && u.protocol !== "http:") return []; - } catch { - return []; - } - return [ - { - claim: typeof r.claim === "string" ? r.claim : "", - source_url: r.source_url, - source_text: typeof r.source_text === "string" ? r.source_text : undefined - } - ]; - }); -} +import { extractCitations, extractResult } from "@/lib/tabstack/research"; type DeepDiveRequest = { competitorId?: string; diff --git a/lib/tabstack/__tests__/research.test.ts b/lib/tabstack/__tests__/research.test.ts index 5bfa02e..5cefeea 100644 --- a/lib/tabstack/__tests__/research.test.ts +++ b/lib/tabstack/__tests__/research.test.ts @@ -292,6 +292,130 @@ describe("runResearch", () => { await expect(runResearch({ query: "q", mode: "fast" })).rejects.toThrow("Network failure"); }); + + it("extracts citations from metadata.citedPages (actual Tabstack API shape)", async () => { + const { runResearch } = await import("@/lib/tabstack/research"); + researchMock.mockResolvedValue( + makeStream([ + { + event: "complete", + data: { + report: "Browser Use raised $17M [1].", + metadata: { + citedPages: [ + { id: "v1", url: "https://browser-use.com/posts/seed-round", title: "We Raised $17M", claims: [], sourceQueries: [] } + ] + } + } + } + ]) + ); + + const result = await runResearch({ query: "q", mode: "fast" }); + + expect(result.citations).toHaveLength(1); + expect(result.citations[0].source_url).toBe("https://browser-use.com/posts/seed-round"); + expect(result.citations[0].claim).toBe("We Raised $17M"); + }); + + it("prefers citedPages over data.citations when both are present", async () => { + const { runResearch } = await import("@/lib/tabstack/research"); + researchMock.mockResolvedValue( + makeStream([ + { + event: "complete", + data: { + citations: [{ claim: "From citations field", source_url: "https://citations.com" }], + metadata: { + citedPages: [ + { id: "v1", url: "https://citedpages.com", title: "From citedPages", claims: [], sourceQueries: [] } + ] + } + } + } + ]) + ); + + const result = await runResearch({ query: "q", mode: "fast" }); + + expect(result.citations).toHaveLength(1); + expect(result.citations[0].source_url).toBe("https://citedpages.com"); + }); + + it("uses first string from citedPages.claims when populated, falls back to title otherwise", async () => { + const { runResearch } = await import("@/lib/tabstack/research"); + researchMock.mockResolvedValue( + makeStream([ + { + event: "complete", + data: { + metadata: { + citedPages: [ + { id: "v1", url: "https://a.com", title: "Title A", claims: ["Specific claim"], sourceQueries: [] }, + { id: "v2", url: "https://b.com", title: "Title B", claims: [], sourceQueries: [] } + ] + } + } + } + ]) + ); + + const result = await runResearch({ query: "q", mode: "fast" }); + + expect(result.citations).toHaveLength(2); + expect(result.citations[0].claim).toBe("Specific claim"); + expect(result.citations[1].claim).toBe("Title B"); + }); + + it("falls through to data.citations when all citedPages entries have invalid URLs", async () => { + const { runResearch } = await import("@/lib/tabstack/research"); + researchMock.mockResolvedValue( + makeStream([ + { + event: "complete", + data: { + citations: [{ claim: "Valid fallback", source_url: "https://citations.com" }], + metadata: { + citedPages: [ + { id: "v1", url: "javascript:alert(1)", title: "Bad", claims: [] }, + { id: "v2", url: "file:///etc/passwd", title: "Also bad", claims: [] } + ] + } + } + } + ]) + ); + + const result = await runResearch({ query: "q", mode: "fast" }); + + expect(result.citations).toHaveLength(1); + expect(result.citations[0].source_url).toBe("https://citations.com"); + }); + + it("rejects citedPages entries with non-http/https URLs", async () => { + const { runResearch } = await import("@/lib/tabstack/research"); + researchMock.mockResolvedValue( + makeStream([ + { + event: "complete", + data: { + metadata: { + citedPages: [ + { id: "v1", url: "https://safe.com", title: "Safe", claims: [] }, + { id: "v2", url: "javascript:alert(1)", title: "Bad", claims: [] }, + { id: "v3", url: "file:///etc/passwd", title: "Also bad", claims: [] } + ] + } + } + } + ]) + ); + + const result = await runResearch({ query: "q", mode: "fast" }); + + expect(result.citations).toHaveLength(1); + expect(result.citations[0].source_url).toBe("https://safe.com"); + }); }); describe("runResearch with self-context injection", () => { diff --git a/lib/tabstack/research.ts b/lib/tabstack/research.ts index f11ba68..79ac45e 100644 --- a/lib/tabstack/research.ts +++ b/lib/tabstack/research.ts @@ -79,44 +79,60 @@ export type ResearchInput = { fallback?: LoggerCallMetadata["fallback"]; }; -export function extractCitations(data: unknown): ResearchCitation[] { - if (!isPlainObject(data)) { - return []; +function isSafeHttpUrl(url: string): boolean { + try { + const { protocol } = new URL(url); + return protocol === "https:" || protocol === "http:"; + } catch { + return false; } +} - const raw = data["citations"]; - - if (!Array.isArray(raw)) { +export function extractCitations(data: unknown): ResearchCitation[] { + if (!isPlainObject(data)) { return []; } - return raw.flatMap((item: unknown): ResearchCitation[] => { - if (!isPlainObject(item)) { - return []; - } - - if (typeof item["source_url"] !== "string") { - return []; + // Primary: data.metadata.citedPages — the shape the Tabstack API actually returns. + // Each entry has { id, url, title, claims[], sourceQueries[] }. + const meta = data["metadata"]; + if (isPlainObject(meta)) { + const citedPages = meta["citedPages"]; + if (Array.isArray(citedPages) && citedPages.length > 0) { + const normalized = citedPages.flatMap((item: unknown): ResearchCitation[] => { + if (!isPlainObject(item)) return []; + if (typeof item["url"] !== "string" || !isSafeHttpUrl(item["url"])) return []; + const claims = Array.isArray(item["claims"]) ? item["claims"] : []; + const firstClaim = claims.find((c: unknown): c is string => typeof c === "string"); + return [ + { + claim: firstClaim ?? (typeof item["title"] === "string" ? item["title"] : ""), + source_url: item["url"] + } + ]; + }); + if (normalized.length > 0) return normalized; } + } - // Validate source_url is a safe http/https URL to prevent XSS/SSRF at ingestion - try { - const parsed = new URL(item["source_url"]); - if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { - return []; - } - } catch { - return []; - } + // Fallback: data.citations — forward-compat shape for a future normalized API field + // and for test fixtures that predate the citedPages discovery. + const raw = data["citations"]; + if (Array.isArray(raw) && raw.length > 0) { + return raw.flatMap((item: unknown): ResearchCitation[] => { + if (!isPlainObject(item)) return []; + if (typeof item["source_url"] !== "string" || !isSafeHttpUrl(item["source_url"])) return []; + return [ + { + claim: typeof item["claim"] === "string" ? item["claim"] : "", + source_url: item["source_url"], + source_text: typeof item["source_text"] === "string" ? item["source_text"] : undefined + } + ]; + }); + } - return [ - { - claim: typeof item["claim"] === "string" ? item["claim"] : "", - source_url: item["source_url"], - source_text: typeof item["source_text"] === "string" ? item["source_text"] : undefined - } - ]; - }); + return []; } export function extractResult(data: unknown): unknown {