From 08a2aff99c1a945cb0ea26fd8157586056ab87a3 Mon Sep 17 00:00:00 2001 From: devcool20 Date: Fri, 22 May 2026 01:02:24 +0530 Subject: [PATCH] Add unified web search and fetch tool --- README.md | 3 +- api/mcp.ts | 2 +- api/well-known-mcp-config.ts | 7 +- env.example | 3 +- npm.readme.md | 3 +- server.json | 2 +- src/mcp-handler.ts | 7 + src/tools/webSearchFetch.ts | 279 ++++++++++++++++++++++++ tests/fixtures/exaResponses.ts | 61 ++++++ tests/unit/mcp-handler.test.ts | 15 +- tests/unit/tools/validation.test.ts | 4 +- tests/unit/tools/webSearchFetch.test.ts | 251 +++++++++++++++++++++ 12 files changed, 625 insertions(+), 12 deletions(-) create mode 100644 src/tools/webSearchFetch.ts create mode 100644 tests/unit/tools/webSearchFetch.test.ts diff --git a/README.md b/README.md index 028b86ff..6ba3dbe6 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,7 @@ Use the npm package with your API key. [Get your API key](https://dashboard.exa. | Tool | Description | | ---- | ----------- | | `web_search_exa` | Search the web for any topic and get clean, ready-to-use content | +| `web_search_fetch_exa` | Search the web and fetch full content from the top results in one call | | `web_fetch_exa` | Get the full content of a specific webpage from a known URL | **Off by Default:** @@ -307,7 +308,7 @@ Use the npm package with your API key. [Get your API key](https://dashboard.exa. Enable additional tools with the `tools` parameter: ``` -https://mcp.exa.ai/mcp?exaApiKey=YOUR_KEY&tools=web_search_exa,web_search_advanced_exa,web_fetch_exa +https://mcp.exa.ai/mcp?exaApiKey=YOUR_KEY&tools=web_search_exa,web_search_fetch_exa,web_search_advanced_exa,web_fetch_exa ``` ## Agent Skills (Claude Skills) diff --git a/api/mcp.ts b/api/mcp.ts index 69a8f2a2..760648fc 100644 --- a/api/mcp.ts +++ b/api/mcp.ts @@ -316,7 +316,7 @@ async function checkRateLimits(ip: string, debug: boolean): Promise; + +const DEFAULT_FETCH_NUM_RESULTS = 3; +const MAX_FETCH_NUM_RESULTS = 10; +const DEFAULT_MAX_CHARACTERS = 4000; + +const categorySchema = z + .enum(["company", "research paper", "news", "pdf", "github", "personal site", "people", "financial report"]) + .optional() + .describe("Filter results to a specific category"); + +function normalizeNumber(value: number | undefined, defaultValue: number, options: { allowZero?: boolean } = {}): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + return defaultValue; + } + + if (value < 0 || (!options.allowZero && value === 0)) { + return defaultValue; + } + + return Math.floor(value); +} + +function getString(value: Record, key: string): string | undefined { + return typeof value[key] === "string" ? value[key] : undefined; +} + +function formatSearchResults(results: Record[]): string { + const lines = ["# Search Results", ""]; + + results.forEach((result, index) => { + lines.push(`${index + 1}. ${getString(result, "title") || "(no title)"}`); + lines.push(` URL: ${getString(result, "url") || "N/A"}`); + lines.push(` Published: ${getString(result, "publishedDate") || "N/A"}`); + lines.push(` Author: ${getString(result, "author") || "N/A"}`); + + const highlights = Array.isArray(result.highlights) + ? result.highlights.filter((highlight): highlight is string => typeof highlight === "string") + : []; + + if (highlights.length > 0) { + lines.push(" Highlights:"); + highlights.forEach((highlight) => lines.push(` - ${highlight}`)); + } + + lines.push(""); + }); + + return lines.join("\n").trim(); +} + +function formatCrawledContents(results: Record[], selectedUrls: string[]): string { + const lines = ["# Crawled Contents", ""]; + + if (results.length === 0) { + lines.push("No crawled content returned."); + return lines.join("\n").trim(); + } + + results.forEach((result, index) => { + const url = getString(result, "url") || selectedUrls[index] || "N/A"; + + lines.push(`## ${index + 1}. ${getString(result, "title") || "(no title)"}`); + lines.push(`URL: ${url}`); + + const publishedDate = getString(result, "publishedDate"); + if (publishedDate) { + lines.push(`Published: ${publishedDate.split("T")[0]}`); + } + + const author = getString(result, "author"); + if (author) { + lines.push(`Author: ${author}`); + } + + lines.push(""); + lines.push(getString(result, "text") || "No text content returned."); + lines.push(""); + lines.push("---"); + lines.push(""); + }); + + return lines.join("\n").trim(); +} + +function formatCrawlErrors(errors: ExaSearchStatus[]): string { + if (errors.length === 0) { + return ""; + } + + const lines = ["# Crawl Errors", ""]; + errors.forEach((error) => { + const statusCode = error.error?.httpStatusCode ? ` (${error.error.httpStatusCode})` : ""; + lines.push(`- Error crawling ${error.id}: ${error.error?.tag || "unknown error"}${statusCode}`); + }); + + return lines.join("\n").trim(); +} + +export function registerWebSearchFetchTool(server: McpServer, config?: WebSearchFetchConfig): void { + server.tool( + "web_search_fetch_exa", + `Search the web for a topic and immediately crawl the full content of the top results in a single call. + +Best for: High-performance research tasks where you need full page contents from the top results. Reduces round-trip latency. +Returns: A list of search results accompanied by the full markdown content of the top fetched pages.`, + { + query: lenientString().describe("Natural language search query optimized for semantic search."), + numResults: lenientOptionalNumber().describe("Number of search results to return (default: 10)."), + category: categorySchema, + fetchNumResults: lenientOptionalNumber().describe("Number of top search results to immediately fetch content for (default: 3, max: 10)."), + maxCharacters: lenientOptionalPositiveNumber().describe("Maximum characters to extract per fetched page (default: 4000)."), + }, + { + readOnlyHint: true, + destructiveHint: false, + openWorldHint: false, + idempotentHint: true, + }, + async ({ query, numResults, category, fetchNumResults, maxCharacters }) => { + const toolId = "web_search_fetch_exa"; + const requestId = `${toolId}-${Date.now()}-${Math.random().toString(36).substring(2, 7)}`; + const logger = createRequestLogger(requestId, toolId); + + logger.start(query); + + try { + const exa = new Exa(config?.exaApiKey || process.env.EXA_API_KEY || ""); + const normalizedNumResults = normalizeNumber(numResults, API_CONFIG.DEFAULT_NUM_RESULTS); + const requestedFetchNumResults = Math.min( + normalizeNumber(fetchNumResults, DEFAULT_FETCH_NUM_RESULTS, { allowZero: true }), + MAX_FETCH_NUM_RESULTS, + ); + const normalizedMaxCharacters = normalizeNumber(maxCharacters, DEFAULT_MAX_CHARACTERS); + + const searchRequest: ExaSearchRequest = { + query, + type: config?.defaultSearchType || "auto", + numResults: normalizedNumResults, + ...(category && { category: category as SearchCategory }), + contents: { + highlights: true, + }, + }; + + checkpoint("web_search_fetch_search_request_prepared"); + logger.log("Sending search request to Exa API"); + + const searchResponse = await retryWithBackoff(() => + exa.request( + API_CONFIG.ENDPOINTS.SEARCH, + "POST", + searchRequest, + undefined, + integrationHeaders("web-search-fetch-mcp", config), + ), + ); + + checkpoint("web_search_fetch_search_response_received"); + logger.log("Received search response from Exa API"); + + const sanitizedSearch = sanitizeSearchResponse(searchResponse); + const searchResults = Array.isArray(sanitizedSearch.results) ? sanitizedSearch.results : []; + + if (searchResults.length === 0) { + checkpoint("web_search_fetch_complete"); + return { + content: [{ + type: "text" as const, + text: "No search results found for query. Scrape phase bypassed.", + }], + }; + } + + const adjustedFetchNumResults = Math.min(requestedFetchNumResults, searchResults.length); + const selectedUrls = searchResults + .slice(0, adjustedFetchNumResults) + .map((result) => getString(result, "url")) + .filter((url): url is string => Boolean(url)); + + const sections = [formatSearchResults(searchResults)]; + let sanitizedContents: Record = {}; + let crawlErrors: ExaSearchStatus[] = []; + + if (adjustedFetchNumResults === 0) { + sections.push("# Crawled Contents\n\nScrape phase bypassed because fetchNumResults was set to 0."); + } else if (selectedUrls.length === 0) { + sections.push("# Crawled Contents\n\nScrape phase bypassed because no result URLs were available."); + } else { + const crawlRequest = { + ids: selectedUrls, + contents: { + text: { + maxCharacters: normalizedMaxCharacters, + }, + }, + }; + + checkpoint("web_search_fetch_crawl_request_prepared"); + logger.log(`Sending crawl request for ${selectedUrls.length} URL(s) to Exa API`); + + const contentsResponse = await retryWithBackoff(() => + exa.request( + "/contents", + "POST", + crawlRequest, + undefined, + integrationHeaders("web-search-fetch-mcp", config), + ), + ); + + checkpoint("web_search_fetch_crawl_response_received"); + logger.log("Received crawl response from Exa API"); + + const rawStatuses = Array.isArray(contentsResponse?.statuses) ? contentsResponse.statuses : []; + crawlErrors = rawStatuses.filter((status) => status.status === "error"); + sanitizedContents = sanitizeContentsResponse(contentsResponse); + const crawledResults = Array.isArray(sanitizedContents.results) ? sanitizedContents.results : []; + + sections.push(formatCrawledContents(crawledResults, selectedUrls)); + + const formattedErrors = formatCrawlErrors(crawlErrors); + if (formattedErrors) { + sections.push(formattedErrors); + } + } + + const meta: Record = {}; + if (typeof sanitizedSearch.searchTime === "number") { + meta.searchTime = sanitizedSearch.searchTime; + } + if (typeof sanitizedContents.searchTime === "number") { + meta.crawlTime = sanitizedContents.searchTime; + } + if (sanitizedSearch.costDollars) { + meta.searchCostDollars = sanitizedSearch.costDollars; + } + if (sanitizedContents.costDollars) { + meta.crawlCostDollars = sanitizedContents.costDollars; + } + + const result = { + content: [{ + type: "text" as const, + text: sections.join("\n\n---\n\n"), + _meta: meta, + }], + }; + + checkpoint("web_search_fetch_complete"); + logger.complete(); + return result; + } catch (error) { + logger.error(error); + return formatToolError(error, toolId, config?.userProvidedApiKey); + } + }, + ); +} diff --git a/tests/fixtures/exaResponses.ts b/tests/fixtures/exaResponses.ts index 37c2c92e..0538b2eb 100644 --- a/tests/fixtures/exaResponses.ts +++ b/tests/fixtures/exaResponses.ts @@ -29,6 +29,40 @@ export const emptySearchResponse = { results: [], } satisfies WebSearchFixtureResponse; +export const multiSearchResponse = { + requestId: "multi-search-request", + resolvedSearchType: "auto", + results: [ + { + id: "result-1", + title: "Result One", + url: "https://example.com/one", + publishedDate: "2026-04-01T12:00:00.000Z", + author: "Example Author", + highlights: ["First highlight"], + }, + { + id: "result-2", + title: "Result Two", + url: "https://example.com/two", + publishedDate: "2026-04-02T12:00:00.000Z", + author: "Second Author", + highlights: ["Second highlight"], + }, + { + id: "result-3", + title: "Result Three", + url: "https://example.com/three", + highlights: ["Third highlight"], + }, + ], + searchTime: 0.5, + costDollars: { + total: 0.001, + search: { semantic: 0.001 }, + }, +} satisfies WebSearchFixtureResponse; + export const contentsResponse = { requestId: "contents-request", results: [ @@ -64,3 +98,30 @@ export const contentsErrorResponse = { }, ], } satisfies WebContentsFixtureResponse; + +export const searchFetchContentsResponse = { + requestId: "search-fetch-contents-request", + results: [ + { + id: "page-one", + title: "Fetched One", + url: "https://example.com/one", + publishedDate: "2026-04-03T12:00:00.000Z", + author: "Fetched Author", + text: "Full page one text", + }, + ], + statuses: [ + { + id: "https://example.com/two", + status: "error", + source: "contents", + error: { tag: "forbidden", httpStatusCode: 403 }, + }, + ], + searchTime: 0.31, + costDollars: { + total: 0.002, + contents: { text: 0.002 }, + }, +} satisfies WebContentsFixtureResponse; diff --git a/tests/unit/mcp-handler.test.ts b/tests/unit/mcp-handler.test.ts index a3adaf1e..19e4fd8f 100644 --- a/tests/unit/mcp-handler.test.ts +++ b/tests/unit/mcp-handler.test.ts @@ -18,7 +18,11 @@ describe("initializeMcpServer", () => { initializeMcpServer(server); - expect(server.tools.map((tool) => tool.name)).toEqual(["web_search_exa", "web_fetch_exa"]); + expect(server.tools.map((tool) => tool.name)).toEqual([ + "web_search_exa", + "web_search_fetch_exa", + "web_fetch_exa", + ]); expect(server.prompts.map((prompt) => prompt.name)).toEqual(["web_search_help"]); expect(server.resources.map((resource) => resource.name)).toEqual(["tools_list"]); @@ -36,6 +40,7 @@ describe("initializeMcpServer", () => { expect(toolsList).toEqual( expect.arrayContaining([ expect.objectContaining({ id: "web_search_exa", enabled: true }), + expect.objectContaining({ id: "web_search_fetch_exa", enabled: true }), expect.objectContaining({ id: "web_fetch_exa", enabled: true }), expect.objectContaining({ id: "web_search_advanced_exa", enabled: false }), ]), @@ -46,11 +51,15 @@ describe("initializeMcpServer", () => { const server = new FakeMcpServer(); initializeMcpServer(server, { - enabledTools: ["web_search_advanced_exa", "crawling_exa", "deep_search_exa"], + enabledTools: ["web_search_fetch_exa", "web_search_advanced_exa", "crawling_exa", "deep_search_exa"], userProvidedApiKey: false, }); - expect(server.tools.map((tool) => tool.name)).toEqual(["web_search_advanced_exa", "crawling_exa"]); + expect(server.tools.map((tool) => tool.name)).toEqual([ + "web_search_fetch_exa", + "web_search_advanced_exa", + "crawling_exa", + ]); }); it("only registers deep_search_exa when the user provided an API key", () => { diff --git a/tests/unit/tools/validation.test.ts b/tests/unit/tools/validation.test.ts index e2583ace..922d6eff 100644 --- a/tests/unit/tools/validation.test.ts +++ b/tests/unit/tools/validation.test.ts @@ -185,13 +185,15 @@ describe("advertised JSON Schema retains usable type info", () => { it("each registered tool advertises non-empty inputSchema fields", async () => { const { registerWebSearchTool } = await import("../../../src/tools/webSearch.js"); + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); const { registerWebFetchTool } = await import("../../../src/tools/webFetch.js"); const server = new FakeMcpServer(); registerWebSearchTool(server as any); + registerWebSearchFetchTool(server as any); registerWebFetchTool(server as any); - for (const toolName of ["web_search_exa", "web_fetch_exa"]) { + for (const toolName of ["web_search_exa", "web_search_fetch_exa", "web_fetch_exa"]) { const inputSchema = server.getTool(toolName).inputSchema as Record; expect(Object.keys(inputSchema).length, `${toolName} should expose fields`).toBeGreaterThan(0); diff --git a/tests/unit/tools/webSearchFetch.test.ts b/tests/unit/tools/webSearchFetch.test.ts new file mode 100644 index 00000000..410f8510 --- /dev/null +++ b/tests/unit/tools/webSearchFetch.test.ts @@ -0,0 +1,251 @@ +import { ExaError } from "exa-js"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + emptySearchResponse, + multiSearchResponse, + searchFetchContentsResponse, +} from "../../fixtures/exaResponses.js"; +import { FakeMcpServer } from "../../helpers/fakeMcpServer.js"; + +const { ExaMock, exaConstructorMock, requestMock } = vi.hoisted(() => { + const requestMock = vi.fn(); + const exaConstructorMock = vi.fn(); + class ExaMock { + request = requestMock; + + constructor(...args: unknown[]) { + exaConstructorMock(...args); + } + } + + return { + ExaMock, + exaConstructorMock, + requestMock, + }; +}); + +vi.mock("exa-js", async (importOriginal) => ({ + ...(await importOriginal()), + Exa: ExaMock, +})); + +vi.mock("agnost", () => ({ + checkpoint: vi.fn(), +})); + +describe("registerWebSearchFetchTool", () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => undefined); + }); + + it("searches, fetches the top results, and formats content with crawl errors", async () => { + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); + const server = new FakeMcpServer(); + requestMock.mockResolvedValueOnce(multiSearchResponse).mockResolvedValueOnce(searchFetchContentsResponse); + + registerWebSearchFetchTool(server as any, { + exaApiKey: "test-key", + defaultSearchType: "fast", + exaSource: "test-suite", + mcpSessionId: "session-123", + }); + + const result = await server.getTool("web_search_fetch_exa").handler({ + query: "AI hiring trends", + numResults: 3, + category: "news", + fetchNumResults: 2, + maxCharacters: 500, + }); + + expect(exaConstructorMock).toHaveBeenCalledWith("test-key"); + expect(requestMock).toHaveBeenNthCalledWith( + 1, + "/search", + "POST", + { + query: "AI hiring trends", + type: "fast", + numResults: 3, + category: "news", + contents: { + highlights: true, + }, + }, + undefined, + { + "x-exa-integration": "web-search-fetch-mcp:test-suite", + "x-exa-mcp-session-id": "session-123", + }, + ); + expect(requestMock).toHaveBeenNthCalledWith( + 2, + "/contents", + "POST", + { + ids: ["https://example.com/one", "https://example.com/two"], + contents: { + text: { + maxCharacters: 500, + }, + }, + }, + undefined, + { + "x-exa-integration": "web-search-fetch-mcp:test-suite", + "x-exa-mcp-session-id": "session-123", + }, + ); + + expect(result).toMatchObject({ + content: [ + { + type: "text", + _meta: { + searchTime: 0.5, + crawlTime: 0.31, + }, + }, + ], + }); + + const text = (result as any).content[0].text; + expect(text).toContain("# Search Results"); + expect(text).toContain("1. Result One"); + expect(text).toContain("# Crawled Contents"); + expect(text).toContain("## 1. Fetched One"); + expect(text).toContain("Full page one text"); + expect(text).toContain("# Crawl Errors"); + expect(text).toContain("Error crawling https://example.com/two: forbidden (403)"); + expect((result as any).isError).toBeUndefined(); + }); + + it("uses defaults and clamps fetchNumResults to the returned result count", async () => { + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); + const server = new FakeMcpServer(); + requestMock.mockResolvedValueOnce(multiSearchResponse).mockResolvedValueOnce(searchFetchContentsResponse); + + registerWebSearchFetchTool(server as any); + + await server.getTool("web_search_fetch_exa").handler({ + query: "AI hiring trends", + numResults: -1, + fetchNumResults: 99, + }); + + expect(requestMock).toHaveBeenNthCalledWith( + 1, + "/search", + "POST", + expect.objectContaining({ + query: "AI hiring trends", + type: "auto", + numResults: 10, + }), + undefined, + { "x-exa-integration": "web-search-fetch-mcp" }, + ); + expect(requestMock).toHaveBeenNthCalledWith( + 2, + "/contents", + "POST", + { + ids: ["https://example.com/one", "https://example.com/two", "https://example.com/three"], + contents: { + text: { + maxCharacters: 4000, + }, + }, + }, + undefined, + { "x-exa-integration": "web-search-fetch-mcp" }, + ); + }); + + it("returns search results only when fetchNumResults is zero", async () => { + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); + const server = new FakeMcpServer(); + requestMock.mockResolvedValueOnce(multiSearchResponse); + + registerWebSearchFetchTool(server as any); + + const result = await server.getTool("web_search_fetch_exa").handler({ + query: "AI hiring trends", + fetchNumResults: 0, + }); + + expect(requestMock).toHaveBeenCalledTimes(1); + expect((result as any).content[0].text).toContain("# Search Results"); + expect((result as any).content[0].text).toContain( + "Scrape phase bypassed because fetchNumResults was set to 0.", + ); + }); + + it("returns a clear message and skips contents when search has no results", async () => { + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); + const server = new FakeMcpServer(); + requestMock.mockResolvedValue(emptySearchResponse); + + registerWebSearchFetchTool(server as any); + + await expect( + server.getTool("web_search_fetch_exa").handler({ + query: "nothing", + }), + ).resolves.toEqual({ + content: [{ + type: "text", + text: "No search results found for query. Scrape phase bypassed.", + }], + }); + expect(requestMock).toHaveBeenCalledTimes(1); + }); + + it("returns a formatted tool error when a request throws", async () => { + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); + const server = new FakeMcpServer(); + requestMock.mockRejectedValue(new ExaError("Unauthorized", 401, "2026-04-29T12:00:00.000Z")); + + registerWebSearchFetchTool(server as any, { userProvidedApiKey: true }); + + await expect( + server.getTool("web_search_fetch_exa").handler({ + query: "AI hiring trends", + }), + ).resolves.toEqual({ + content: [ + { + type: "text", + text: "web_search_fetch_exa error (401): Unauthorized\nTimestamp: 2026-04-29T12:00:00.000Z", + }, + ], + isError: true, + }); + }); + + it("returns a formatted tool error when the contents request throws", async () => { + const { registerWebSearchFetchTool } = await import("../../../src/tools/webSearchFetch.js"); + const server = new FakeMcpServer(); + requestMock + .mockResolvedValueOnce(multiSearchResponse) + .mockRejectedValueOnce(new ExaError("contents failed", 400, "2026-04-29T12:00:00.000Z")); + + registerWebSearchFetchTool(server as any, { userProvidedApiKey: true }); + + await expect( + server.getTool("web_search_fetch_exa").handler({ + query: "AI hiring trends", + }), + ).resolves.toEqual({ + content: [ + { + type: "text", + text: "web_search_fetch_exa error (400): contents failed\nTimestamp: 2026-04-29T12:00:00.000Z", + }, + ], + isError: true, + }); + }); +});