chore(entity-caching): benchmark suite and dev tooling (6/6)

.gitignore

@@ -29,3 +29,11 @@ go.work.sum
 
 # Claude Code
 .claude/worktrees/
+/.playwright-mcp
+
+# PR review workspace — local-only, never commit
+/TODO.md
+/DONE.md
+/FEEDBACK.md
+/COMMENTS.md
+/docs/superpowers/

Makefile

@@ -187,6 +187,51 @@ docker-build-minikube: docker-build-local
 
 	rm -f mk-*.tar
 
+demo-compose:
+	cd demo && $(MAKE) compose-cache
+
+DEMO_STARTUP_ATTEMPTS ?= 60
+DEMO_STARTUP_SLEEP ?= 0.5
+
+demo:
+	@echo "Composing subgraph schemas..."
+	cd demo && $(MAKE) compose-cache
+	@echo "Starting subgraphs and router..."
+	@echo "Playground will be at http://localhost:3002/"
+	@set -e; \
+	cd demo && go run cmd/all/main.go & \
+	pid=$$!; \
+	trap 'kill $$pid 2>/dev/null || true' EXIT INT TERM HUP; \
+	for p in 4012 4013 4014; do \
+		for i in $$(seq 1 $(DEMO_STARTUP_ATTEMPTS)); do \
+			nc -z 127.0.0.1 $$p && break; \
+			sleep $(DEMO_STARTUP_SLEEP); \
+		done; \
+		nc -z 127.0.0.1 $$p || { echo "subgraph $$p did not start" >&2; exit 1; }; \
+	done; \
+	cd router && go run cmd/router/main.go --config ../demo/router-cache.yaml
+
+benchmark-cache-demo:
+	pnpm dlx tsx benchmark/scripts/run_suite.ts --all \
+		$(if $(VUS),--vus $(VUS),) \
+		$(if $(DURATION),--duration $(DURATION),) \
+		$(if $(RAMP_UP),--ramp-up $(RAMP_UP),) \
+		$(if $(RAMP_DOWN),--ramp-down $(RAMP_DOWN),)
+
+benchmark-cache-demo-scenario:
+	@if [ -z "$(SCENARIO)" ]; then \
+		echo "Usage: make benchmark-cache-demo-scenario SCENARIO=<scenario_name>"; \
+		exit 1; \
+	fi
+	pnpm dlx tsx benchmark/scripts/run_suite.ts --scenario $(SCENARIO) \
+		$(if $(VUS),--vus $(VUS),) \
+		$(if $(DURATION),--duration $(DURATION),) \
+		$(if $(RAMP_UP),--ramp-up $(RAMP_UP),) \
+		$(if $(RAMP_DOWN),--ramp-down $(RAMP_DOWN),)
+
+benchmark-cache-demo-validate:
+	pnpm dlx tsx benchmark/scripts/run_suite.ts validate
+
 run-subgraphs-local:
 	cd demo && go run cmd/all/main.go
 

benchmark/.gitignore

@@ -0,0 +1,5 @@
+.run/
+.tmp-metrics/
+.tmp-pprof/
+results/*
+!results/.gitkeep

benchmark/README.md

@@ -0,0 +1,114 @@
+# Cache Demo Benchmark Suite
+
+Local benchmark harness for the cache demo on `localhost:3002`, using:
+
+- the existing `cache-demo` subgraphs
+- a dedicated Redis Docker container for L2 cache storage on `localhost:6399`
+- k6 for request load
+- router Prometheus and pprof for runtime capture
+
+## Prerequisites
+
+- `go` with the repo’s expected toolchain
+- `pnpm`
+- `k6`
+- `docker`
+- free local ports:
+  - `3002`
+  - `4012`
+  - `4013`
+  - `4014`
+  - `6060`
+  - `8088`
+  - `6399`
+
+## Key Files
+
+- `benchmark/router-cache.redis.yaml`: Redis-backed router config for the benchmark
+- `benchmark/scenarios/cache-demo.json`: scenario manifest
+- `benchmark/queries/*.graphql`: canonical benchmark operations
+- `benchmark/fixtures/*.response.json`: uncached router response fixtures
+- `benchmark/k6/cache_demo.js`: k6 runner with exact response assertion
+
+## Scenarios
+
+- `article_simple`
+- `articles_by_ids_batch`
+- `listing_composite_key`
+- `venue_nested_key`
+- `user_profile_header_sensitive`
+- `catalogs_partial_load`
+- `request_scoped_viewer_articles`
+- `viewer_articles_deep_nested`
+
+## Auth Profiles
+
+The demo uses fake bearer tokens:
+
+- `alice` -> `Bearer token-alice`
+- `bob` -> `Bearer token-bob`
+- `charlie` -> `Bearer token-charlie`
+
+Auth-sensitive scenarios must not fall back to anonymous requests.
+
+## Commands
+
+Validate the manifest and checked-in fixtures:
+
+```bash
+make benchmark-cache-demo-validate
+```
+
+Run the full suite with default load settings:
+
+```bash
+make benchmark-cache-demo
+```
+
+Run one scenario with default load settings:
+
+```bash
+make benchmark-cache-demo-scenario SCENARIO=article_simple
+```
+
+Run one scenario with direct control over k6 stages:
+
+```bash
+pnpm dlx tsx benchmark/scripts/run_suite.ts \
+  --scenario article_simple \
+  --vus 10 \
+  --duration 30s \
+  --ramp-up 5s \
+  --ramp-down 5s
+```
+
+## Output Layout
+
+Result bundles are written under:
+
+```text
+benchmark/results/<timestamp>/<scenario>/<mode>/
+```
+
+Each mode directory contains:
+
+- `summary.json`
+- `k6-summary.json`
+- `metrics-before.prom`
+- `metrics-after.prom`
+- `metrics-delta.json`
+- `redis-info-before.txt`
+- `redis-info-after.txt`
+- `redis-docker-stats-before.json`
+- `redis-docker-stats-after.json`
+- `equivalence.json`
+- `pprof/router_cpu.pb.gz`
+- `pprof/router_heap.pb.gz`
+
+`summary.json` also records the warmup request count, k6 stage config, parsed k6 summary, and selected Redis INFO and Docker stats values so downstream interpretation does not need to scrape raw artifacts first.
+
+## Notes
+
+- The harness owns the stack. If the required ports are already in use, startup fails rather than benchmarking against a dirty environment.
+- L2 cache storage is intentionally externalized to Redis so router memory measurements do not include the full L2 object footprint.
+- The suite always performs deterministic uncached equivalence checks before load generation and serial warmup requests before each mode run.

benchmark/fixtures/article_simple.response.json

@@ -0,0 +1 @@
+{"data":{"article":{"id":"1","title":"Introduction to GraphQL Caching","authorName":"Alice"}}}

benchmark/fixtures/articles_by_ids_batch.response.json

@@ -0,0 +1 @@
+{"data":{"articlesByIds":[{"id":"1","title":"Introduction to GraphQL Caching","authorName":"Alice"},{"id":"2","title":"Advanced Federation Patterns","authorName":"Bob"},{"id":"3","title":"Cache Invalidation Strategies","authorName":"Charlie"}]}}

benchmark/fixtures/catalogs_partial_load.response.json

@@ -0,0 +1 @@
+{"data":{"catalogs":[{"id":"c1","name":"Electronics","description":"Consumer electronics, gadgets, and accessories.","lastUpdated":"2025-03-15T08:00:00Z","itemCount":342},{"id":"c2","name":"Books","description":"Fiction, non-fiction, technical books, and audiobooks.","lastUpdated":"2025-03-20T12:00:00Z","itemCount":1205},{"id":"c3","name":"Clothing","description":"Men's, women's, and children's apparel.","lastUpdated":"2025-03-25T16:00:00Z","itemCount":567}]}}

benchmark/fixtures/listing_composite_key.response.json

@@ -0,0 +1 @@
+{"data":{"listing":{"sellerId":"s1","sku":"WIDGET-01","title":"Premium Widget","price":29.99,"currency":"USD","inStock":true}}}

benchmark/fixtures/request_scoped_viewer_articles.response.json

@@ -0,0 +1 @@
+{"data":{"currentViewer":{"id":"v1","name":"Alice","email":"alice@example.com"},"articles":[{"id":"1","title":"Introduction to GraphQL Caching","currentViewer":{"id":"v1","name":"Alice"},"viewCount":12453,"rating":4.7},{"id":"2","title":"Advanced Federation Patterns","currentViewer":{"id":"v1","name":"Alice"},"viewCount":8921,"rating":4.3},{"id":"3","title":"Cache Invalidation Strategies","currentViewer":{"id":"v1","name":"Alice"},"viewCount":15678,"rating":4.9},{"id":"4","title":"Performance Tuning with Entity Caching","currentViewer":{"id":"v1","name":"Alice"},"viewCount":6234,"rating":4.1}]}}

benchmark/fixtures/user_profile_header_sensitive.alice.response.json

@@ -0,0 +1 @@
+{"data":{"userProfile":{"id":"u1","username":"alice","role":"admin","email":"alice@example.com"}}}

benchmark/fixtures/user_profile_header_sensitive.bob.response.json

@@ -0,0 +1 @@
+{"data":{"userProfile":{"id":"u2","username":"bob","role":"editor","email":"bob@example.com"}}}

benchmark/fixtures/venue_nested_key.response.json

@@ -0,0 +1 @@
+{"data":{"venue":{"address":{"id":"v1"},"name":"Grand Conference Hall","capacity":500,"city":"Berlin"}}}

benchmark/k6/cache_demo.js

@@ -0,0 +1,123 @@
+import http from "k6/http";
+import { check, fail } from "k6";
+import { Counter, Rate } from "k6/metrics";
+
+const payload = JSON.parse(__ENV.BENCHMARK_PAYLOAD || "{}");
+
+if (!payload.url || !payload.query) {
+  fail("BENCHMARK_PAYLOAD must include url and query");
+}
+
+const mismatchRate = new Rate("response_mismatch_rate");
+const graphqlErrorRate = new Rate("graphql_error_rate");
+const mismatchCount = new Counter("response_mismatch_count");
+
+function normalizeJson(value) {
+  if (Array.isArray(value)) {
+    return value.map((item) => normalizeJson(item));
+  }
+
+  if (value !== null && typeof value === "object") {
+    const out = {};
+    for (const key of Object.keys(value).sort()) {
+      out[key] = normalizeJson(value[key]);
+    }
+    return out;
+  }
+
+  return value;
+}
+
+function normalizeResponseForComparison(value) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    return normalizeJson(value);
+  }
+
+  const response = JSON.parse(JSON.stringify(value));
+  const extensions = response.extensions;
+
+  if (extensions && typeof extensions === "object" && !Array.isArray(extensions)) {
+    delete extensions.trace;
+
+    if (Object.keys(extensions).length === 0) {
+      delete response.extensions;
+    }
+  }
+
+  return normalizeJson(response);
+}
+
+const expectedBody = JSON.stringify(
+  normalizeResponseForComparison(payload.expectedBody),
+);
+
+export const options = payload.options || {
+  stages: [
+    { duration: "30s", target: 20 },
+    { duration: "2m", target: 20 },
+    { duration: "10s", target: 0 },
+  ],
+};
+
+export default function () {
+  const mergedHeaders = Object.assign(
+    { "content-type": "application/json" },
+    payload.headers || {},
+  );
+
+  const response = http.post(
+    payload.url,
+    JSON.stringify({
+      operationName: payload.operationName,
+      query: payload.query,
+      variables: payload.variables || {},
+    }),
+    {
+      headers: mergedHeaders,
+    },
+  );
+
+  const ok = check(response, {
+    "http status is 200": (res) => res.status === 200,
+  });
+
+  if (!ok) {
+    mismatchRate.add(true);
+    mismatchCount.add(1);
+    graphqlErrorRate.add(false);
+    return;
+  }
+
+  let body;
+  try {
+    body = response.json();
+  } catch (_err) {
+    // Parse failure is both a response mismatch and a graphql-error-equivalent
+    // (the server returned a non-JSON body or truncated JSON under load).
+    mismatchRate.add(true);
+    mismatchCount.add(1);
+    graphqlErrorRate.add(true);
+    return;
+  }
+
+  const hasGraphqlErrors = Array.isArray(body?.errors) && body.errors.length > 0;
+  graphqlErrorRate.add(hasGraphqlErrors);
+
+  const sameBody =
+    JSON.stringify(normalizeResponseForComparison(body)) === expectedBody;
+
+  const assertionOk = check(
+    { body, sameBody, hasGraphqlErrors },
+    {
+      "graphql errors absent": (data) => !data.hasGraphqlErrors,
+      "response matches expected fixture": (data) => data.sameBody,
+    },
+  );
+
+  if (!assertionOk || !sameBody) {
+    mismatchRate.add(true);
+    mismatchCount.add(1);
+  } else {
+    mismatchRate.add(false);
+  }
+}

benchmark/queries/article_simple.graphql

@@ -0,0 +1,7 @@
+query ArticleSimple($id: ID!) {
+  article(id: $id) {
+    id
+    title
+    authorName
+  }
+}

benchmark/queries/articles_by_ids_batch.graphql

@@ -0,0 +1,7 @@
+query ArticlesByIdsBatch($ids: [ID!]!) {
+  articlesByIds(ids: $ids) {
+    id
+    title
+    authorName
+  }
+}

benchmark/queries/catalogs_partial_load.graphql

@@ -0,0 +1,9 @@
+query CatalogsPartialLoad {
+  catalogs {
+    id
+    name
+    description
+    lastUpdated
+    itemCount
+  }
+}

benchmark/queries/listing_composite_key.graphql

@@ -0,0 +1,10 @@
+query ListingCompositeKey($key: ListingKey!) {
+  listing(key: $key) {
+    sellerId
+    sku
+    title
+    price
+    currency
+    inStock
+  }
+}

benchmark/queries/request_scoped_viewer_articles.graphql

@@ -0,0 +1,17 @@
+query RequestScopedViewerArticles {
+  currentViewer {
+    id
+    name
+    email
+  }
+  articles {
+    id
+    title
+    currentViewer {
+      id
+      name
+    }
+    viewCount
+    rating
+  }
+}

benchmark/queries/user_profile_header_sensitive.graphql

@@ -0,0 +1,8 @@
+query UserProfileHeaderSensitive($id: ID!) {
+  userProfile(id: $id) {
+    id
+    username
+    role
+    email
+  }
+}

benchmark/queries/venue_nested_key.graphql

@@ -0,0 +1,10 @@
+query VenueNestedKey($location: VenueLocationKey!) {
+  venue(location: $location) {
+    address {
+      id
+    }
+    name
+    capacity
+    city
+  }
+}