fix(deep-dive): extract citations from metadata.citedPages

app/api/deep-dive/route.ts

@@ -4,40 +4,7 @@ export const maxDuration = 120;
 
 import { buildPromptForTemplate } from "@/lib/deep-dive-templates";
 import { parseSseChunk } from "@/lib/utils/sse";
-
-type ResearchCitation = { claim: string; source_url: string; source_text?: string };
-
-function extractResult(data: unknown): unknown {
-  if (!data || typeof data !== "object" || Array.isArray(data)) return data ?? null;
-  const d = data as Record<string, unknown>;
-  if ("result" in d) return d.result;
-  const { citations: _c, ...rest } = d;
-  return Object.keys(rest).length > 0 ? rest : null;
-}
-
-function extractCitations(data: unknown): ResearchCitation[] {
-  if (!data || typeof data !== "object" || Array.isArray(data)) return [];
-  const raw = (data as Record<string, unknown>).citations;
-  if (!Array.isArray(raw)) return [];
-  return raw.flatMap((item: unknown): ResearchCitation[] => {
-    if (!item || typeof item !== "object" || Array.isArray(item)) return [];
-    const r = item as Record<string, unknown>;
-    if (typeof r.source_url !== "string") return [];
-    try {
-      const u = new URL(r.source_url);
-      if (u.protocol !== "https:" && u.protocol !== "http:") return [];
-    } catch {
-      return [];
-    }
-    return [
-      {
-        claim: typeof r.claim === "string" ? r.claim : "",
-        source_url: r.source_url,
-        source_text: typeof r.source_text === "string" ? r.source_text : undefined
-      }
-    ];
-  });
-}
+import { extractCitations, extractResult } from "@/lib/tabstack/research";
 
 type DeepDiveRequest = {
   competitorId?: string;

lib/tabstack/__tests__/research.test.ts

@@ -292,6 +292,130 @@ describe("runResearch", () => {
 
     await expect(runResearch({ query: "q", mode: "fast" })).rejects.toThrow("Network failure");
   });
+
+  it("extracts citations from metadata.citedPages (actual Tabstack API shape)", async () => {
+    const { runResearch } = await import("@/lib/tabstack/research");
+    researchMock.mockResolvedValue(
+      makeStream([
+        {
+          event: "complete",
+          data: {
+            report: "Browser Use raised $17M [1].",
+            metadata: {
+              citedPages: [
+                { id: "v1", url: "https://browser-use.com/posts/seed-round", title: "We Raised $17M", claims: [], sourceQueries: [] }
+              ]
+            }
+          }
+        }
+      ])
+    );
+
+    const result = await runResearch({ query: "q", mode: "fast" });
+
+    expect(result.citations).toHaveLength(1);
+    expect(result.citations[0].source_url).toBe("https://browser-use.com/posts/seed-round");
+    expect(result.citations[0].claim).toBe("We Raised $17M");
+  });
+
+  it("prefers citedPages over data.citations when both are present", async () => {
+    const { runResearch } = await import("@/lib/tabstack/research");
+    researchMock.mockResolvedValue(
+      makeStream([
+        {
+          event: "complete",
+          data: {
+            citations: [{ claim: "From citations field", source_url: "https://citations.com" }],
+            metadata: {
+              citedPages: [
+                { id: "v1", url: "https://citedpages.com", title: "From citedPages", claims: [], sourceQueries: [] }
+              ]
+            }
+          }
+        }
+      ])
+    );
+
+    const result = await runResearch({ query: "q", mode: "fast" });
+
+    expect(result.citations).toHaveLength(1);
+    expect(result.citations[0].source_url).toBe("https://citedpages.com");
+  });
+
+  it("uses first string from citedPages.claims when populated, falls back to title otherwise", async () => {
+    const { runResearch } = await import("@/lib/tabstack/research");
+    researchMock.mockResolvedValue(
+      makeStream([
+        {
+          event: "complete",
+          data: {
+            metadata: {
+              citedPages: [
+                { id: "v1", url: "https://a.com", title: "Title A", claims: ["Specific claim"], sourceQueries: [] },
+                { id: "v2", url: "https://b.com", title: "Title B", claims: [], sourceQueries: [] }
+              ]
+            }
+          }
+        }
+      ])
+    );
+
+    const result = await runResearch({ query: "q", mode: "fast" });
+
+    expect(result.citations).toHaveLength(2);
+    expect(result.citations[0].claim).toBe("Specific claim");
+    expect(result.citations[1].claim).toBe("Title B");
+  });
+
+  it("falls through to data.citations when all citedPages entries have invalid URLs", async () => {
+    const { runResearch } = await import("@/lib/tabstack/research");
+    researchMock.mockResolvedValue(
+      makeStream([
+        {
+          event: "complete",
+          data: {
+            citations: [{ claim: "Valid fallback", source_url: "https://citations.com" }],
+            metadata: {
+              citedPages: [
+                { id: "v1", url: "javascript:alert(1)", title: "Bad", claims: [] },
+                { id: "v2", url: "file:///etc/passwd", title: "Also bad", claims: [] }
+              ]
+            }
+          }
+        }
+      ])
+    );
+
+    const result = await runResearch({ query: "q", mode: "fast" });
+
+    expect(result.citations).toHaveLength(1);
+    expect(result.citations[0].source_url).toBe("https://citations.com");
+  });
+
+  it("rejects citedPages entries with non-http/https URLs", async () => {
+    const { runResearch } = await import("@/lib/tabstack/research");
+    researchMock.mockResolvedValue(
+      makeStream([
+        {
+          event: "complete",
+          data: {
+            metadata: {
+              citedPages: [
+                { id: "v1", url: "https://safe.com", title: "Safe", claims: [] },
+                { id: "v2", url: "javascript:alert(1)", title: "Bad", claims: [] },
+                { id: "v3", url: "file:///etc/passwd", title: "Also bad", claims: [] }
+              ]
+            }
+          }
+        }
+      ])
+    );
+
+    const result = await runResearch({ query: "q", mode: "fast" });
+
+    expect(result.citations).toHaveLength(1);
+    expect(result.citations[0].source_url).toBe("https://safe.com");
+  });
 });
 
 describe("runResearch with self-context injection", () => {

lib/tabstack/research.ts

@@ -79,44 +79,60 @@ export type ResearchInput = {
   fallback?: LoggerCallMetadata["fallback"];
 };
 
-export function extractCitations(data: unknown): ResearchCitation[] {
-  if (!isPlainObject(data)) {
-    return [];
+function isSafeHttpUrl(url: string): boolean {
+  try {
+    const { protocol } = new URL(url);
+    return protocol === "https:" || protocol === "http:";
+  } catch {
+    return false;
   }
+}
 
-  const raw = data["citations"];
-
-  if (!Array.isArray(raw)) {
+export function extractCitations(data: unknown): ResearchCitation[] {
+  if (!isPlainObject(data)) {
     return [];
   }
 
-  return raw.flatMap((item: unknown): ResearchCitation[] => {
-    if (!isPlainObject(item)) {
-      return [];
-    }
-
-    if (typeof item["source_url"] !== "string") {
-      return [];
+  // Primary: data.metadata.citedPages — the shape the Tabstack API actually returns.
+  // Each entry has { id, url, title, claims[], sourceQueries[] }.
+  const meta = data["metadata"];
+  if (isPlainObject(meta)) {
+    const citedPages = meta["citedPages"];
+    if (Array.isArray(citedPages) && citedPages.length > 0) {
+      const normalized = citedPages.flatMap((item: unknown): ResearchCitation[] => {
+        if (!isPlainObject(item)) return [];
+        if (typeof item["url"] !== "string" || !isSafeHttpUrl(item["url"])) return [];
+        const claims = Array.isArray(item["claims"]) ? item["claims"] : [];
+        const firstClaim = claims.find((c: unknown): c is string => typeof c === "string");
+        return [
+          {
+            claim: firstClaim ?? (typeof item["title"] === "string" ? item["title"] : ""),
+            source_url: item["url"]
+          }
+        ];
+      });
+      if (normalized.length > 0) return normalized;
     }
+  }
 
-    // Validate source_url is a safe http/https URL to prevent XSS/SSRF at ingestion
-    try {
-      const parsed = new URL(item["source_url"]);
-      if (parsed.protocol !== "https:" && parsed.protocol !== "http:") {
-        return [];
-      }
-    } catch {
-      return [];
-    }
+  // Fallback: data.citations — forward-compat shape for a future normalized API field
+  // and for test fixtures that predate the citedPages discovery.
+  const raw = data["citations"];
+  if (Array.isArray(raw) && raw.length > 0) {
+    return raw.flatMap((item: unknown): ResearchCitation[] => {
+      if (!isPlainObject(item)) return [];
+      if (typeof item["source_url"] !== "string" || !isSafeHttpUrl(item["source_url"])) return [];
+      return [
+        {
+          claim: typeof item["claim"] === "string" ? item["claim"] : "",
+          source_url: item["source_url"],
+          source_text: typeof item["source_text"] === "string" ? item["source_text"] : undefined
+        }
+      ];
+    });
+  }
 
-    return [
-      {
-        claim: typeof item["claim"] === "string" ? item["claim"] : "",
-        source_url: item["source_url"],
-        source_text: typeof item["source_text"] === "string" ? item["source_text"] : undefined
-      }
-    ];
-  });
+  return [];
 }
 
 export function extractResult(data: unknown): unknown {