diff --git a/.agents/skills/graphistry-mcp/SKILL.md b/.agents/skills/graphistry-mcp/SKILL.md new file mode 100644 index 0000000..91479a6 --- /dev/null +++ b/.agents/skills/graphistry-mcp/SKILL.md @@ -0,0 +1,260 @@ +--- +name: graphistry-mcp +description: "Drive a live Graphistry visualization session from any MCP client. Covers connecting to the viz MCP endpoint, personal-key and JWT auth, session ownership, the tool surface, and GFQL sent as JSON over the wire. Use when an external agent must query or recolor the graph a user is currently looking at." +--- + +# Graphistry MCP + +## Scope + +Use this skill to operate a **live Graphistry visualization session** from an MCP client: inspect +its schema, run GFQL queries against it, and create colored collections the user sees update in +their browser. + +## Two servers share the name "graphistry mcp" + +Pick the right one before writing any code. + +| Server | Use it for | +|---|---| +| **Viz MCP** — `https:///mcp` | Reading and mutating the graph a user already has open. Session-scoped. **This skill.** | +| **PyGraphistry MCP** — `github.com/graphistry/graphistry-mcp` | Building and uploading new visualizations from scratch. Hub-scoped, unrelated to this skill. | + +If the task starts from a graph the user is looking at, use the viz MCP. If it starts from a +DataFrame or a file, use the PyGraphistry MCP or the `pygraphistry` skill. + +## Connect + +Streamable HTTP at `POST /mcp`. `tools/call` carries a credential on the `Authorization` header; +`initialize` and `tools/list` do not. + +Two credentials work. Prefer a personal key: it does not expire, so a long-running agent will not +start returning `401` partway through a conversation the way a JWT does. + +| Credential | Header | Expires | +|---|---|---| +| Personal key `key_id:key` | `Authorization: Bearer :` or `PersonalKey :` | no | +| Viewer JWT | `Authorization: Bearer ` | yes, about an hour | + +Create a personal key on the Graphistry account page at `/users/personal/key/`. Both schemes are +accepted for it because most MCP clients only expose a single "Bearer token" field. + +```bash +curl -si https:///mcp \ + -H "Authorization: Bearer $GRAPHISTRY_PERSONAL_KEY" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' \ + -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{ + "protocolVersion":"2024-11-05","capabilities":{}, + "clientInfo":{"name":"my-agent","version":"1.0"}}}' +``` + +`initialize` returns an `Mcp-Session-Id` response header. Send it back as the `mcp-session-id` +header on every subsequent request. This is the MCP transport session and is distinct from the +Graphistry viz `session_id` the tools take. If it expires the server answers `404` — reinitialize +and retry rather than treating the call as failed. + +The server currently reports protocol version `2024-11-05`. + +Client config for MCP-aware tools: + +```json +{ + "mcpServers": { + "graphistry": { + "type": "http", + "url": "https:///mcp", + "headers": { "Authorization": "Bearer ${GRAPHISTRY_PERSONAL_KEY}" } + } + } +} +``` + +## Session model + +Tools other than `list_sessions` require a `session_id` identifying a live viz session. + +- **A client cannot create a session.** A session exists because a user has the graph open in a + browser. Discover one with `list_sessions`. +- **Pass the session you were given.** `list_sessions` returns bare ids with nothing to tell them + apart, so when a user has two graphs open an agent that discovers one instead of using the one it + was handed will silently answer about the wrong graph. A client that knows which graph is on + screen should supply `session_id` rather than discover it. +- **Only the owner may mutate.** The credential's user must be the user whose browser holds that + session, or the collection tools return + `403 Forbidden: only the session owner may modify this session`. Reads may still succeed for a + non-owner who has access to the underlying dataset; without it they return + `403 Forbidden: caller cannot access this session`. `list_sessions` shows only the caller's own + sessions, so a shared service account will not see an end user's session. +- **A closed tab means no session.** With no live owner the call returns + `410 Session owner unavailable`. A `410` can also mean the owner lookup itself failed, so retry + once before concluding the graph is gone. +- **`list_sessions` can return `[]` while your session is alive.** It sees only the sessions owned + by the server process that happened to receive the request. Emptiness is not evidence of no + session — another reason to pass the `session_id` you were given. + +Other statuses a client must tell apart: + +| Status | Meaning | Retry? | +|---|---|---| +| `403 Talk2Graph is not enabled for this account` | credential is valid, the account lacks the entitlement | no — every `tools/call` will fail | +| `503 Entitlement check unavailable` | the access check itself failed | yes | +| `504 Tool call timeout` | the owning process did not answer in time | yes | +| `400 Missing mcp-session-id` | the header was absent | send the header; this is not the 404 case | +| `404 Unknown or expired mcp-session-id` | the transport session lapsed | reinitialize | + +The transport session expires after 5 minutes idle, sliding on each use, so a slow agent should +expect a `404` and reinitialize rather than treat it as a failure. A revoked credential keeps +working for up to 60 seconds because verification is cached. + +## Tool surface + +Read: + +| Tool | Required | Optional | +|---|---|---| +| `list_sessions` | — | — | +| `get_session_info` | `session_id` | — | +| `query_graph` | `session_id`, `gfql_operations` | `output_type` (`shape` default, `nodes`, `edges`, `all`), `format` | +| `list_collections` | `session_id` | — | + +Mutate (these change what the user sees): + +| Tool | Required | Optional | +|---|---|---| +| `create_collection` | `session_id`, `name`, `gfql_operations`, and one of `node_color` / `palette` | the other of `node_color` / `palette`, `description` | +| `update_collection` | `session_id`, `collection_id` | `name`, `node_color`, `palette` | +| `delete_collection` | `session_id`, `collection_id` | — | +| `reorder_collections` | `session_id`, `order` | — | +| `reset_collections` | `session_id` | — | + +`query_graph` returns at most 50 rows when the expression names columns and 20 when it does not, +projected to the id column plus the columns the expression referenced; the reported count is the +true total. A value list read from a truncated result cannot prove that a value is absent. + +`create_collection` keeps at most 10 collections, evicting the oldest, replaces any existing +collection of the same name, and returns a cached result for an identical call repeated within +30 seconds. `list_collections` reports only collections this MCP session created, never ones the +user made in the browser, and it is emptied when the user's socket reconnects. + +`gfql_operations` is a **JSON-encoded string**, not a nested object. `order` lists collection ids +top-to-bottom; the first renders on top. For a solid color pass a hex `node_color`; use `palette` +only when a named palette is explicitly requested. On `update_collection`, omit a field to leave +it unchanged — do not pass `null`. + +## GFQL over the wire + +Sent as a JSON string. The operation types you need are `Node`, `Edge`, `Call`, `Let`, and `Ref`. + +```json +[{"type": "Node", "filter_dict": {"category": "Malware"}}] +``` + +`Node` filters; a predicate object such as `{"type": "GT", "val": 10}` inside `filter_dict` +thresholds. `Edge` traverses and returns its matched relationship subgraph — it does not project a +single endpoint kind. Every row-pipeline step (`rows`, `group_by`, `order_by`, `limit`) is a +`Call`. `Let` with a `Ref` isolates one endpoint kind and is passed as a **top-level object, never +wrapped in an array**. `query_graph` also accepts one read-only Cypher string. + +Full JSON for each form, including the aggregation pipeline and the `Let`/`Ref` projection: +`references/gfql-wire-format.md`. + +## Shapes that fail silently + +These return `success: true` with wrong or empty results rather than erroring. **A zero-row answer +produced by any of them is not evidence of a real zero.** + +- `filter_nodes_by_dict` / `filter_edges_by_dict` match exact stored values only. A predicate + object matches nothing — threshold with a `Node` operation instead. +- "External/public IP" and "internal/private IP" are derived properties, not requirements for + a dedicated status column. Inspect the live schema and values for the actual IP column first. + Unless the dataset documents another policy, treat RFC1918 IPv4 (`10/8`, `172.16/12`, + `192.168/16`), loopback, link-local, unspecified, multicast, and shared CGNAT (`100.64/10`) + as non-external; apply analogous IPv6 ULA/link-local/loopback/unspecified/multicast rules. + Documentation/reserved examples such as `198.51.100.0/24` and `203.0.113.0/24` are not + RFC1918 internal addresses, so do not silently discard them unless the dataset explicitly + defines them as non-external fixtures. Match the exact stored representation; do not apply + IPv4 rules to IPv6, host:port, URL, or CIDR strings without validating them. Use a positive + GFQL pattern where supported, or `IsIn` only from an exhaustive value set. A limited/sample + table cannot prove a complement; if the value set is truncated or GFQL cannot express the + classification, report that limitation instead of claiming a complete collection. +- Put both graph filters before `rows`. After a `rows` on the edge table a filter is dropped and + the aggregation reports unfiltered counts. +- `where_rows` accepts `expr` (a string like `"n > 10"`) and `filter_dict`. In `filter_dict` only + exact stored values match, so a predicate object there matches nothing; use `expr` to compare. +- `get_session_info` returns column names only, never stored values, so it can never establish + that a value is absent. + +## Errors you will actually see + +These fail loudly with a specific message. Read it — it names the cause, so it is worth acting on +rather than rewriting the expression blindly. + +- `rows` defaults to `table: "nodes"`. Grouping by an edge column without `{"table": "edges"}` + fails as a missing column. +- `Edge` requires `direction`, one of `forward`, `reverse`, `undirected`. `both` is invalid. + `create_collection` fills in `undirected` when it is absent, so an expression that validated + there can still fail in `query_graph` — set it explicitly. +- A bare `{"type": "rows"}`, `{"type": "group_by"}`, `{"type": "order_by"}`, or + `{"type": "limit"}` is invalid. Wrap every one as a `Call`. +- Predicate `type` names are case-sensitive: `GT` works, `gt` fails with an opaque error. +- `create_collection` takes JSON GFQL only. Cypher is accepted by `query_graph` alone. +- `create_collection` refuses an expression whose result stops identifying graph nodes — one that + groups rows, projects the id away, renames or drops it, or reads the edge table. Ordering by a + community column is refused separately, because that ranks by community id rather than size. Both + refusals name the repair: select the nodes themselves, usually a `Node` filter, or report the + sizes with `query_graph` instead. Act on it rather than reporting failure. +- If a collection is refused because the service could **not evaluate** the expression, that is a + different case: report the failure. Do not reword the filter and retry — the expression was not + the problem. + +## Workflow + +1. `list_sessions` to find the live session. +2. `get_session_info` for the schema. Choose columns from the list it returns, most useful first. + If a line ends with `(+K more not listed)` the list is truncated and the count in parentheses is + the real total — a column's absence from the list is not evidence it does not exist, so ask + rather than invent a name. `unavailable (schema could not be read)` means the schema was never + read; treat that as a failed inspection, not as a graph without columns. Being listed is + necessary but not sufficient: some columns are computed for rendering and cannot be filtered. +3. **Inspect stored values before filtering on them.** Natural-language nouns from the user are + unobserved candidates, never literals. Aggregate the distinct values of the relevant column + first and compare case-sensitively — `Hashtag` will not match a search for `hashtag`. +4. Validate the expression with `query_graph` before mutating. +5. `create_collection` reusing that exact validated JSON unchanged, and report the match count it + returns. `create_collection` accepts a narrower set of shapes than `query_graph` validates: a + chain that aggregates or leaves the node axis will pass `query_graph` and still be refused here. + When that happens the refusal names the repair — use it. The count is not guaranteed — when + the preflight cannot produce one the collection is still applied and the reply says so. Report "applied, count unavailable" rather than inventing + a number or omitting the outcome. + +Stop and report a GFQL error rather than retrying with an invented operation name. A failed query +is not an inspection result. + +Zero rows never establishes absence when any value in the expression came from the user's wording +rather than from a tool result. Confirm every value was tool-observed before reporting an empty +result. + +## Decision Rules + +- Use this skill when the graph is already open and the task is to query or recolor it. +- Route to `pygraphistry-gfql` for GFQL written as Python AST objects (`n()`, `e_forward()`) or + for `g.gfql()` in a notebook. This skill covers the JSON wire format only. +- Route to `pygraphistry` to build or upload a graph. +- Route to `graphistry-rest-api` for auth, upload, and dataset endpoints. + +## Safety Rules + +- Keep credentials in environment variables. Never hardcode a username, password, personal key, or + JWT. +- Send the credential in the `Authorization` header. Never put one in a URL query parameter. No + documented tool parameter takes a credential, so it stays out of the model's context — never + add one to `arguments` yourself. +- Treat mutation tools as user-visible: only create, update, or delete a collection when the user + asked to change the visualization. + +## References + +- Graphistry Hub docs: https://hub.graphistry.com/docs/ +- MCP specification: https://modelcontextprotocol.io/specification/ +- REST auth docs: https://hub.graphistry.com/docs/api/1/rest/auth/ diff --git a/.agents/skills/graphistry-mcp/references/gfql-wire-format.md b/.agents/skills/graphistry-mcp/references/gfql-wire-format.md new file mode 100644 index 0000000..0187a1f --- /dev/null +++ b/.agents/skills/graphistry-mcp/references/gfql-wire-format.md @@ -0,0 +1,120 @@ +# GFQL wire format + +Full JSON forms for `gfql_operations`. The decision-critical rules live in `SKILL.md`; this file +holds the complete examples. + +`gfql_operations` is always a **JSON-encoded string**, never a nested object. The operation types +you need are `Node`, `Edge`, `Call`, `Let`, and `Ref`. + +## Node + +Filters nodes. Equality against a stored value: + +```json +[{"type": "Node", "filter_dict": {"category": "Malware"}}] +``` + +Threshold with a predicate object. The predicate goes inside `filter_dict`, keyed by column: + +```json +[{"type": "Node", "filter_dict": {"event_count": {"type": "GT", "val": 10}}}] +``` + +Predicate `type` names are **case-sensitive** — `GT` works, `gt` fails with an opaque error — and +the key carrying the operand differs by family: + +| Family | Types | Operand key | +|---|---|---| +| comparison | `GT` `LT` `GE` `LE` `EQ` `NE` | `val` | +| range | `Between` | `lower`, `upper` | +| set | `IsIn` | `options` | +| string | `Contains` `Startswith` `Endswith` `Match` `Fullmatch` | `pat` | +| null | `IsNA` `NotNA` `IsNull` `NotNull` | none | +| shape | `IsNumeric` `IsAlpha` `Duplicated` and similar | none | + +The server normalizes casing and key names only for `filter_dict` on a top-level operation array. +Inside `edge_match`, inside a `Let`, and anywhere in `create_collection`, write the exact form. + +An **empty** `Node` operation (`{"type": "Node"}`) accumulates both endpoints of a preceding +`Edge` rather than filtering. Use it when the request names two node kinds joined by "and"/"or". + +## Edge + +Traverses. Takes `direction`, `hops`, and `edge_match`: + +```json +[{"type": "Edge", "direction": "undirected", "hops": 1, + "edge_match": {"rel_type": "mentioned"}}] +``` + +`direction` must be `forward`, `reverse`, or `undirected`. `both` is invalid and fails. + +`Edge` returns its matched relationship subgraph — it does **not** project a single endpoint kind. +Both endpoints come back. To isolate one kind, use `Let`/`Ref` below. + +## Call + +Every row-pipeline step is a `Call`. A bare `{"type": "rows"}`, `{"type": "group_by"}`, +`{"type": "order_by"}`, or `{"type": "limit"}` is invalid. + +```json +[{"type":"Call","function":"rows","params":{"table":"edges"}}, + {"type":"Call","function":"group_by","params":{"keys":[""],"aggregations":[["n","count"]]}}, + {"type":"Call","function":"order_by","params":{"keys":[["n","desc"]]}}, + {"type":"Call","function":"limit","params":{"value":20}}] +``` + +`rows` defaults to `table: "nodes"`. Set it to the table holding the columns you group by, or the +query fails on a missing column. + +This chain is a `query_graph` answer, not a collection. It aggregates away the node id, so +`create_collection` refuses it — color the nodes with a separate `Node` filter built from the +values it returns. + +`aggregations` is a list of `[alias, function, column]` triples. Only `count` may be the +2-element `[alias, "count"]`; every other function — `count_distinct`, `sum`, `min`, `max`, `avg`, +`mean`, `collect`, `collect_distinct` — requires the column it aggregates, and `"*"` is rejected +for them. Counting rows where the question asked for distinct values returns a plausible table +answering a different question. + +`order_by` `keys` is a list of `[column, direction]` pairs; the direction is mandatory and must be +`asc` or `desc`. + +Put graph filters (`filter_nodes_by_dict`, `filter_edges_by_dict`) before `rows`. After a `rows` +on the edge table the filter is dropped and the aggregation reports unfiltered counts. + +`where_rows` accepts `expr` (a string like `"n > 10"`) and `filter_dict`. In `filter_dict` only +exact stored values match, so a predicate object there matches nothing; use `expr` to compare. + +## Let and Ref + +Isolate one endpoint kind: bind the edge subgraph, then filter that binding through the `Ref`'s +`chain`. Pass as a **top-level object, never wrapped in an array**: + +```json +{"type":"Let","bindings":{ + "connections":{"type":"Edge","direction":"undirected","hops":1,"edge_match":{"":""}}, + "requested_endpoints":{"type":"Ref","ref":"connections","chain":[{"type":"Node","filter_dict":{"":""}}]}}} +``` + +Use this when the request names one endpoint kind modified by a relationship — "accounts connected +by transfers" wants accounts, not the transfers and their far side. + +When the request instead names two kinds joined by "and"/"or", use an `Edge` followed by an empty +`Node` so both sides accumulate. + +## Cypher + +`query_graph` accepts one whole read-only Cypher string in place of the operation list. One +statement per call; no writes. `create_collection` does not — it takes JSON GFQL only. + +## Output + +`output_type` selects what comes back: `shape` (default, counts only), `nodes`, `edges`, or `all`. +Use `nodes` or `edges` to read actual values — `shape` on an aggregation returns the number of +aggregation rows, not the values inside them, which is an easy way to misread a result. + +Rows are capped at 50 when the expression names columns and 20 when it does not, and are projected +to the id column plus the columns the expression referenced. The reported count is the true total, +so a group_by with more distinct values than the cap is truncated: a value list read from it cannot +prove that some value is absent. diff --git a/.agents/skills/graphistry/SKILL.md b/.agents/skills/graphistry/SKILL.md index b6283ab..ee3444e 100644 --- a/.agents/skills/graphistry/SKILL.md +++ b/.agents/skills/graphistry/SKILL.md @@ -6,7 +6,8 @@ description: > "graphistry plot", or any question mixing Python SDK and REST API concerns. Also triggers on "graphistry SDK vs API", "graphistry authentication", "how do I share a graphistry graph", "Graphistry Hub", or any ambiguous graphistry request before the interface is clear. - Routes to pygraphistry for Python SDK tasks, graphistry-rest-api for curl/REST tasks. + Routes to pygraphistry for Python SDK tasks, graphistry-rest-api for curl/REST tasks, + graphistry-mcp for driving a live viz session from an MCP client. Proactively suggest when the user mentions graph visualization or network analysis and has not yet chosen an interface. --- @@ -18,6 +19,7 @@ Use this skill as the shared entrypoint across Graphistry interfaces. ## Route By Interface - Python SDK tasks (`import graphistry`, DataFrame shaping, `.plot()`, `.gfql()` including Cypher/Let/DAG, PyGraphistry notebooks): use `pygraphistry`. - REST API tasks (`curl`, `/api/v2/...`, JWT/Bearer auth, upload endpoints, `graph.html` URL params): use `graphistry-rest-api`. +- MCP client tasks (driving a live viz session from an external agent, `POST /mcp`, GFQL as JSON, collections): use `graphistry-mcp` if available. - JavaScript/TypeScript SDK tasks (`@graphistry/*`, browser/frontend integrations): use `graphistry-js` if available. ## Mixed Requests diff --git a/CHANGELOG.md b/CHANGELOG.md index e61fba7..3d3204d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [Development] +### Added +- **Skills / graphistry-mcp**: New skill for driving a live Graphistry visualization session from an MCP client. Covers connecting over Streamable HTTP, personal-key and JWT auth, the session-ownership model, the nine-tool surface, GFQL as the JSON wire format, and the query shapes that return `success: true` with empty or wrong results. +- **Skills / graphistry**: Route MCP client tasks to `graphistry-mcp`. + --- ## [0.5.2 - 2026-07-26] diff --git a/README.md b/README.md index ad2e3b1..4981333 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Strong frontier models often already know core Graphistry/PyGraphistry patterns - `graphistry`: umbrella router across interfaces (SDK + REST; JS-ready routing path). - `graphistry-rest-api`: REST specialist for auth, upload lifecycle, URL controls, sessions, and sharing safety. +- `graphistry-mcp`: viz MCP specialist for driving a live visualization session from an external MCP client. - `pygraphistry`: Python SDK router. - `pygraphistry-core`: auth, shaping, and first plot workflows. - `pygraphistry-visualization`: bindings/encodings/layout/privacy/share patterns. @@ -27,6 +28,7 @@ npx skills add graphistry/graphistry-skills \ --agent claude-code \ --skill graphistry \ --skill graphistry-rest-api \ + --skill graphistry-mcp \ --skill pygraphistry \ --skill pygraphistry-core \ --skill pygraphistry-gfql \ diff --git a/evals/journeys/graphistry_mcp_guardrails_v1.json b/evals/journeys/graphistry_mcp_guardrails_v1.json new file mode 100644 index 0000000..e4f2583 --- /dev/null +++ b/evals/journeys/graphistry_mcp_guardrails_v1.json @@ -0,0 +1,171 @@ +{ + "id": "graphistry_mcp_guardrails_v1", + "description": "Credential handling and silent-failure guardrails for the Graphistry viz MCP tool surface.", + "eval_intent": "guardrail", + "tags": [ + "graphistry-mcp", + "guardrails", + "quality" + ], + "cases": [ + { + "id": "credential_in_authorization_header", + "prompt": "Show how an MCP client authenticates a tools/call request to the Graphistry viz MCP endpoint. Keep it under 10 non-empty lines.", + "checks": { + "regex": [ + "(?i)authorization", + "(?i)(header|bearer|personalkey)", + "(?i)(environ|env var|getenv|process\\.env|\\$\\{?[A-Z_]{4,})" + ], + "must_not_regex": [ + "[?&](token|key|jwt|api_key|personal_key)=", + "(?i)(password|personal_key|jwt)\\s*[:=]\\s*['\\\"][A-Za-z0-9_\\-\\.]{8,}['\\\"]" + ], + "max_lines": 16 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "Send the credential on the Authorization header, read from an environment variable. Never place it in a URL query parameter and never hardcode it.", + "rubric": [ + "Credential must travel on the Authorization header.", + "Credential must come from an environment variable, never a literal.", + "Any credential in a URL query parameter should score zero." + ], + "required_concepts": [ + "Authorization" + ], + "forbidden_concepts": [ + "query parameter" + ] + } + }, + { + "id": "no_credential_in_tool_arguments", + "prompt": "Should an MCP client put the Graphistry credential inside the tool call's arguments object? Answer in under 5 non-empty lines.", + "checks": { + "regex": [ + "(?i)\\b(no|never|not)\\b", + "(?i)(authorization|header)" + ], + "max_lines": 5 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "No. No documented tool parameter takes a credential, so it stays out of the model's context. It belongs on the Authorization header.", + "rubric": [ + "Must answer no.", + "Should note no tool parameter accepts a credential and that it belongs on the header.", + "Answering yes should score zero." + ], + "required_concepts": [ + "Authorization" + ] + } + }, + { + "id": "zero_rows_not_absence", + "prompt": "You filtered on a value taken from how the user phrased their request, and query_graph returned zero rows. How do you report this? Answer in under 8 non-empty lines.", + "checks": { + "regex": [ + "(?i)(assumption|unverified|untested|unobserved|guess|candidate|(their|your|user'?s?)\\s+(wording|phrasing)|not\\s+(from|observed|checked))", + "(?i)(distinct|value_counts|top-n|top n|inspect|aggregate|observed values|schema)" + ], + "max_lines": 8 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "Zero rows does not establish absence when the value came from the user's wording rather than a tool result. Aggregate the column's distinct values and compare case-sensitively before reporting an empty result.", + "rubric": [ + "Must state zero rows is not evidence of absence for an unobserved value.", + "Must propose confirming the value against observed data before reporting.", + "Reporting a confident 'none exist' should score zero." + ], + "required_concepts": [ + "absence" + ], + "forbidden_concepts": [ + "confirmed none exist" + ] + } + }, + { + "id": "dict_filter_cannot_threshold", + "prompt": "In the Graphistry viz MCP JSON wire format, can a filter_nodes_by_dict entry express a greater-than threshold on a numeric column? Answer in under 6 non-empty lines.", + "checks": { + "regex": [ + "(?i)\\b(no|cannot|can'?t|does not|doesn'?t)\\b", + "(?i)(exact|equality|equals|membership|literal)", + "(?i)(node|predicate|gfql)" + ], + "max_lines": 6 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "No. filter_nodes_by_dict matches exact stored values only, so a predicate object there matches nothing and returns success with zero rows. Use a Node operation to threshold.", + "rubric": [ + "Must answer no and say it matches exact stored values only.", + "Must note the failure is silent rather than an error.", + "Must redirect to a Node operation for thresholds.", + "Scope is the JSON wire format, not the PyGraphistry Python API where filter_nodes_by_dict accepts an ASTPredicate." + ], + "required_concepts": [ + "exact", + "Node" + ] + } + }, + { + "id": "schema_cannot_prove_absence", + "prompt": "Can get_session_info establish that a particular value does not appear in a column? Answer in under 5 non-empty lines.", + "checks": { + "regex": [ + "(?i)\\b(no|cannot|can't)\\b", + "(?i)(column\\s*\\**\\s*names|names\\**\\s*only|never.{0,25}values|not.{0,25}stored)" + ], + "max_lines": 5 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "No. get_session_info returns column names only, never stored values, so it can never establish that a value is absent.", + "rubric": [ + "Must answer no.", + "Must say get_session_info returns column names, not stored values.", + "Claiming it can prove absence should score zero." + ], + "required_concepts": [ + "column names" + ] + } + }, + { + "id": "act_on_collection_refusal", + "prompt": "`create_collection` returned this error: \"That expression returns a table that no longer identifies graph nodes - it groups rows, projects columns away, or reads the edge table - so there is nothing to color. A collection needs an expression that selects the nodes themselves, such as a Node filter.\" What do you do next? Answer in under 8 non-empty lines.", + "checks": { + "regex": [ + "(?i)node[\\s`*]*(filter|operation)|select\\w*[\\s`*]+(the\\s+)?node|return\\w*[\\s`*]+(graph\\s+)?node|node[\\s`*]+(id|identity)|filter_dict", + "(?i)(rebuild|rewrite|replace|instead|new expression|change the|retry|re-run)" + ], + "max_lines": 8 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "Act on the repair the error names: replace the aggregating chain with an expression that selects the nodes themselves, typically a Node filter built from the values the aggregation returned, then create the collection from that.", + "rubric": [ + "Must act on the refusal rather than reporting failure and stopping.", + "Must build an expression that selects nodes, typically a Node filter.", + "Reporting the failure without attempting the named repair should score near zero.", + "This differs from a refusal where the service could not evaluate the expression, which should be reported." + ], + "required_concepts": [ + "Node" + ] + } + } + ] +} diff --git a/evals/journeys/graphistry_mcp_session_model_v1.json b/evals/journeys/graphistry_mcp_session_model_v1.json new file mode 100644 index 0000000..2ae187c --- /dev/null +++ b/evals/journeys/graphistry_mcp_session_model_v1.json @@ -0,0 +1,138 @@ +{ + "id": "graphistry_mcp_session_model_v1", + "description": "Session discovery, ownership, and status triage for an MCP client driving a live Graphistry viz session.", + "eval_intent": "realistic_capability", + "tags": [ + "graphistry-mcp", + "session", + "errors" + ], + "cases": [ + { + "id": "client_cannot_create_session", + "prompt": "An MCP client must operate on a Graphistry visualization session. How does it obtain a session_id? Answer in under 8 non-empty lines.", + "checks": { + "regex": [ + "(?i)list_sessions", + "(?i)((cannot|can'?t|not|no|never)\\s+create|already open|user (has|have|already))" + ], + "must_not_regex": [ + "(?i)sessions?/create" + ], + "max_lines": 8 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "A client cannot create a session. A session exists because a user has the graph open in a browser; discover one with list_sessions, or use the session_id you were handed.", + "rubric": [ + "Must state the client cannot create a session.", + "Must name list_sessions as the discovery path.", + "Inventing a session-creation tool should score zero." + ], + "required_concepts": [ + "list_sessions" + ] + } + }, + { + "id": "empty_list_sessions_not_absence", + "prompt": "list_sessions returns an empty array, but the user says their graph is open in the browser. What do you conclude and what do you do? Answer in under 8 non-empty lines.", + "checks": { + "regex": [ + "(?i)(session[_ ]?id|list_sessions|credential|identity|owner|host|process)" + ], + "max_lines": 8 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "Emptiness is not evidence of no session. list_sessions sees only sessions owned by the server process that received the request, so pass the session_id you were given rather than concluding the graph is gone.", + "rubric": [ + "Must say an empty result is not evidence that no session exists.", + "Should attribute it to list_sessions seeing only its own process's sessions.", + "Concluding the session is gone should score near zero." + ], + "required_concepts": [ + "session_id" + ] + } + }, + { + "id": "status_410_triage", + "prompt": "A tools/call to the Graphistry viz MCP returns 410. What does it mean and do you retry? Answer in under 6 non-empty lines.", + "checks": { + "regex": [ + "410", + "(?i)(owner unavailable|closed|no live owner|tab)", + "(?i)retry" + ], + "max_lines": 6 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "410 Session owner unavailable: no live owner, typically a closed tab. It can also mean the owner lookup failed, so retry once before concluding the graph is gone.", + "rubric": [ + "Must identify 410 as the owner being unavailable / tab closed.", + "Must say to retry once before concluding the session is gone.", + "Confusing it with the 404 expired-transport-session case should reduce score." + ], + "required_concepts": [ + "410", + "retry" + ] + } + }, + { + "id": "which_mcp_server", + "prompt": "A user has a graph open in their browser and asks an agent to recolor it. Which Graphistry MCP server should the agent talk to, and why? Answer in under 6 non-empty lines.", + "checks": { + "regex": [ + "(?i)(viz|visuali[sz]ation)", + "(?i)(/mcp|session)" + ], + "max_lines": 6 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "The viz MCP at https:///mcp, because the task starts from a graph the user already has open and is session-scoped. The PyGraphistry MCP is for building and uploading new visualizations.", + "rubric": [ + "Must select the viz MCP endpoint, not the PyGraphistry MCP project.", + "Should justify it by the graph already being open / session-scoped work.", + "Selecting the PyGraphistry MCP should score zero." + ], + "required_concepts": [ + "viz" + ] + } + }, + { + "id": "filters_before_rows", + "prompt": "A GFQL chain filters the graph and then aggregates over the edge table with rows. What ordering does the chain require, and what goes wrong otherwise? Answer in under 6 non-empty lines.", + "checks": { + "regex": [ + "(?i)(before|first|precede|ahead of)", + "(?i)rows", + "(?i)(filter|dropped|unfiltered)" + ], + "max_lines": 6 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "Both graph filters must come before rows. After a rows on the edge table a filter is dropped and the aggregation silently reports unfiltered counts.", + "rubric": [ + "Must state filters go before rows.", + "Must note the failure is silent: unfiltered counts, not an error.", + "Claiming order does not matter should score zero." + ], + "required_concepts": [ + "rows", + "before" + ] + } + } + ] +} diff --git a/evals/journeys/graphistry_mcp_wire_format_v1.json b/evals/journeys/graphistry_mcp_wire_format_v1.json new file mode 100644 index 0000000..99f6450 --- /dev/null +++ b/evals/journeys/graphistry_mcp_wire_format_v1.json @@ -0,0 +1,122 @@ +{ + "id": "graphistry_mcp_wire_format_v1", + "description": "JSON GFQL wire-format correctness for the Graphistry viz MCP tool surface.", + "eval_intent": "execution_grade", + "tags": [ + "graphistry-mcp", + "gfql", + "wire-format" + ], + "cases": [ + { + "id": "edge_direction_explicit", + "prompt": "Write the JSON GFQL operation for a one-hop forward edge traversal to send to the Graphistry viz MCP query_graph tool. JSON only, under 12 non-empty lines.", + "checks": { + "regex": [ + "\\\\?\"type\\\\?\"\\s*:\\s*\\\\?\"Edge\\\\?\"", + "\\\\?\"direction\\\\?\"\\s*:\\s*\\\\?\"(forward|reverse|undirected)\\\\?\"" + ], + "must_not_regex": [ + "\\\\?\"direction\\\\?\"\\s*:\\s*\\\\?\"both\\\\?\"" + ], + "max_lines": 12 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "{\"type\": \"Edge\", \"direction\": \"forward\", \"hops\": 1}", + "rubric": [ + "Must emit an Edge operation with an explicit direction of forward, reverse, or undirected.", + "Using 'both' as a direction is invalid and should score near zero.", + "Omitting direction entirely is wrong: query_graph does not default it the way create_collection does." + ], + "required_concepts": [ + "Edge", + "direction" + ] + } + }, + { + "id": "rows_wrapped_as_call", + "prompt": "Write the JSON GFQL for a rows operation over the edge table, as it must appear inside a chain sent to the Graphistry viz MCP query_graph tool. JSON only, under 12 non-empty lines.", + "checks": { + "regex": [ + "\"Call\"", + "\"rows\"", + "\"table\"\\s*:\\s*\"edges\"" + ], + "max_lines": 12 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "{\"type\": \"Call\", \"function\": \"rows\", \"params\": {\"table\": \"edges\"}}", + "rubric": [ + "rows must be wrapped as a Call operation; a bare {\"type\": \"rows\"} is invalid.", + "rows defaults to the nodes table, so table must be set to edges explicitly.", + "Answer should be JSON, not Python AST helpers." + ], + "required_concepts": [ + "Call", + "rows", + "edges" + ] + } + }, + { + "id": "predicate_type_case_sensitive", + "prompt": "Write the JSON GFQL node filter selecting nodes whose score is greater than 10, for the Graphistry viz MCP query_graph tool. JSON only, under 12 non-empty lines.", + "checks": { + "regex": [ + "\"GT\"" + ], + "must_not_regex": [ + "\"type\"\\s*:\\s*\"gt\"" + ], + "max_lines": 12 + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "{\"type\": \"Node\", \"filter_dict\": {\"score\": {\"type\": \"GT\", \"val\": 10}}}", + "rubric": [ + "Predicate type names are case-sensitive: GT is correct, gt fails with an opaque error.", + "A threshold must be expressed as a predicate on a Node operation." + ], + "required_concepts": [ + "GT" + ] + } + }, + { + "id": "create_collection_json_only", + "prompt": "Can a Cypher query string be passed to the Graphistry viz MCP create_collection tool? Answer in under 6 non-empty lines.", + "checks": { + "regex": [ + "(?i)\\b(no|cannot|can'?t|does not|doesn'?t)\\b", + "(?i)(json|gfql)" + ], + "max_lines": 6, + "must_not_regex": [ + "(?i)^\\s*yes\\b", + "(?i)create_collection\\s+(accepts|takes|supports|allows)\\s+(a\\s+|either\\s+)?cypher" + ] + }, + "oracle": { + "enabled": true, + "min_score": 0.8, + "reference_answer": "No. create_collection takes JSON GFQL only; Cypher is accepted by query_graph alone.", + "rubric": [ + "Must say create_collection does not accept Cypher.", + "Must note that query_graph is the tool that does accept Cypher.", + "Claiming create_collection accepts Cypher should score zero." + ], + "required_concepts": [ + "create_collection", + "query_graph", + "JSON" + ] + } + } + ] +} diff --git a/evals/skills_profiles.json b/evals/skills_profiles.json index ab81e23..145b3b8 100644 --- a/evals/skills_profiles.json +++ b/evals/skills_profiles.json @@ -8,5 +8,9 @@ "pygraphistry-ai", "pygraphistry-connectors", "graphistry-rest-api" + ], + "graphistry_mcp": [ + "graphistry", + "graphistry-mcp" ] }