feat(seo): enumerate CMS routes in sitemap.xml and llms.txt

CHANGELOG.md

@@ -34,6 +34,7 @@ latest tag; security fixes land on the latest release (see `SECURITY.md`).
 
 ### Added
 
+- `sitemap.xml` and `/llms.txt` now read from Sanity when it's configured, instead of only ever listing the home page and `/ai`. Both enumerate the same CMS content through one shared helper (`lib/seo/routes.ts`), gated on `isConfigured('sanity')` — a fresh clone with no CMS env set gets the static routes only, and a CMS that's configured but unreachable (bad project ID, deleted dataset) degrades the same way instead of 500ing the sitemap. `lib/seo/site.ts` now exports the normalized `BASE_URL` that `app/robots.ts`, `app/sitemap.ts`, and `/llms.txt` all resolve absolute URLs from, so the three can't drift out of sync on the fallback chain. (#319)
 - `post-checkout` lefthook hook that clears `.next/types` + `.next/dev/types` on branch switches (git flag `1` only — file checkouts untouched, rebases skipped), killing the ghost `tsc` errors caused by the previous branch's generated route types. Types regenerate on the next dev/build.
 - Minimal Playwright E2E harness (`e2e/home.e2e.ts`) — smoke spec covering page render, zero console errors, and zero critical/serious a11y violations (via `@axe-core/playwright`). Run with `bun run test:e2e`. Specs use `.e2e.ts` extension so `bun test` ignores them. (#e2e)
 - Instant-navigation regression test (`e2e/instant-navigation.e2e.ts`) using `@next/playwright`'s `instant()` helper — wraps the 404 page's "Go Home" link click and asserts the home shell paints without waiting on the network, so a refactor that de-opts instant navigation (a `cookies()` read leaking into a shared layout, a moved Suspense boundary) fails CI instead of going unnoticed. (#339)

app/llms.txt/route.ts

@@ -1,3 +1,4 @@
+import { getCmsRoutes } from '@/lib/seo/routes'
 import { formatList, SITE } from '@/lib/seo/site'
 
 /**
@@ -6,8 +7,11 @@ import { formatList, SITE } from '@/lib/seo/site'
  *
  * This is the cheapest possible AEO (answer-engine optimization) win: one
  * static endpoint that states the entity in the format crawlers already
- * expect. The body is generated from `lib/seo/site.ts` rather than
- * hand-written so it can never drift from the JSON-LD graph or on-page copy.
+ * expect. The entity section is generated from `lib/seo/site.ts` rather
+ * than hand-written so it can never drift from the JSON-LD graph or
+ * on-page copy; the content list is generated from the same
+ * `getCmsRoutes()` that feeds `app/sitemap.ts`, so both surfaces describe
+ * the same set of pages.
  *
  * No `export const dynamic = 'force-static'` here — this project runs
  * Next's Cache Components (`cacheComponents: true`), which forbids the
@@ -36,8 +40,26 @@ function buildAbout(): string {
   return clauses.join(' ')
 }
 
+function buildContent(
+  cmsRoutes: Awaited<ReturnType<typeof getCmsRoutes>>
+): string {
+  if (cmsRoutes.length === 0) return ''
+
+  const links = cmsRoutes
+    .map((route) => `- [${route.label}](${SITE.url}${route.path})`)
+    .join('\n')
+
+  return `
+
+## Content
+
+${links}`
+}
+
 async function buildBody(): Promise<string> {
   'use cache'
+  const cmsRoutes = await getCmsRoutes()
+
   return `# ${SITE.name}
 
 > ${SITE.description}
@@ -49,7 +71,7 @@ ${buildAbout()}
 ## Key pages
 
 - [Home](${SITE.url}/): ${SITE.description}
-- [Machine view](${SITE.url}/ai): Plain-HTML index of every page, for agents.
+- [Machine view](${SITE.url}/ai): Plain-HTML index of every page, for agents.${buildContent(cmsRoutes)}
 `
 }
 

app/robots.ts

@@ -1,6 +1,6 @@
 import type { MetadataRoute } from 'next'
 
-import { APP_BASE_URL } from '@/lib/env'
+import { BASE_URL } from '@/lib/seo/site'
 
 const DISALLOW = ['/api/draft-mode/']
 
@@ -43,6 +43,6 @@ export default function robots(): MetadataRoute.Robots {
         disallow: DISALLOW,
       },
     ],
-    sitemap: `${APP_BASE_URL}/sitemap.xml`,
+    sitemap: `${BASE_URL}/sitemap.xml`,
   }
 }

app/sitemap.ts

@@ -1,25 +1,35 @@
 import type { MetadataRoute } from 'next'
 
-import { APP_BASE_URL } from '@/lib/env'
+import { getCmsRoutes, STATIC_ROUTES } from '@/lib/seo/routes'
+import { BASE_URL } from '@/lib/seo/site'
 
+/**
+ * Static routes are listed in `lib/seo/routes.ts` (`STATIC_ROUTES`) —
+ * shared with `/llms.txt` so the two surfaces can't drift. New static
+ * routes must be added there and to `PAGES` in `app/(site)/ai/page.tsx`;
+ * the machine view (`/ai`) has no link from the design, so crawlers only
+ * discover it here.
+ *
+ * CMS-driven routes (`getCmsRoutes`) are appended when the Sanity
+ * integration is configured; a fresh clone with no CMS env set gets the
+ * static routes only.
+ */
 export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  const baseRoutes: MetadataRoute.Sitemap = [
-    {
-      url: APP_BASE_URL,
-      lastModified: new Date(),
-      changeFrequency: 'daily',
-      priority: 1,
-    },
-    // The machine view (`app/(site)/ai/page.tsx`) has no link from the design, so
-    // crawlers only discover it here. New routes must be added both here
-    // and to `PAGES` in `app/(site)/ai/page.tsx`.
-    {
-      url: `${APP_BASE_URL}/ai`,
-      lastModified: new Date(),
-      changeFrequency: 'monthly',
-      priority: 0.5,
-    },
-  ]
+  const cmsRoutes = await getCmsRoutes()
 
-  return baseRoutes
+  const staticEntries: MetadataRoute.Sitemap = STATIC_ROUTES.map((route) => ({
+    url: `${BASE_URL}${route.path}`,
+    lastModified: new Date(),
+    changeFrequency: route.changeFrequency,
+    priority: route.priority,
+  }))
+
+  const cmsEntries: MetadataRoute.Sitemap = cmsRoutes.map((route) => ({
+    url: `${BASE_URL}${route.path}`,
+    lastModified: route.lastModified,
+    changeFrequency: 'weekly',
+    priority: 0.6,
+  }))
+
+  return [...staticEntries, ...cmsEntries]
 }

lib/seo/routes.ts

@@ -0,0 +1,139 @@
+import type { MetadataRoute } from 'next'
+import { defineQuery } from 'next-sanity'
+import { z } from 'zod'
+
+import { isConfigured } from '@/integrations/registry'
+import { sanityFetch } from '@/integrations/sanity/live'
+import { urlForReference } from '@/integrations/sanity/utils/link'
+
+/**
+ * Route enumeration shared by `app/sitemap.ts` and `app/llms.txt/route.ts` —
+ * the sitemap and the machine-readable content list must never disagree
+ * about which URLs exist, so both read from here instead of keeping their
+ * own copies.
+ */
+
+export interface StaticRoute {
+  path: string
+  changeFrequency: NonNullable<MetadataRoute.Sitemap[number]['changeFrequency']>
+  priority: number
+}
+
+export interface ContentRoute {
+  path: string
+  label: string
+  lastModified: Date
+}
+
+/**
+ * Routes with no CMS backing. `/ai` has no link from the design, so
+ * `app/sitemap.ts` is the only place crawlers discover it — see
+ * `app/(site)/ai/page.tsx`, which links the same route from its own
+ * hardcoded `PAGES` list for the human/agent-facing machine view.
+ *
+ * The `(examples)` route group (`/sanity`) is a Sanity wiring tutorial for
+ * developers, not real site content — deliberately excluded here so
+ * search/AI indexes don't cite a demo page as part of the site.
+ */
+export const STATIC_ROUTES: readonly StaticRoute[] = [
+  { path: '/', changeFrequency: 'daily', priority: 1 },
+  { path: '/ai', changeFrequency: 'monthly', priority: 0.5 },
+]
+
+const staticPaths = new Set(STATIC_ROUTES.map((route) => route.path))
+
+/**
+ * Every document type with a `slug` — kept permissive (`nullable()` fields)
+ * because this validates a hand-written query rather than a typegen'd one;
+ * malformed documents are skipped per-entry in `getCmsRoutes` rather than
+ * failing the whole fetch.
+ */
+const routableDocumentSchema = z.object({
+  _type: z.enum(['page', 'article']),
+  title: z.string().nullable(),
+  slug: z.object({ current: z.string() }).nullable(),
+  _updatedAt: z.string(),
+})
+
+const routableContentQuery = defineQuery(`
+  *[_type in ["page", "article"] && defined(slug.current)] {
+    _type,
+    title,
+    slug,
+    _updatedAt
+  }
+`)
+
+/**
+ * Every published `page`/`article` document, resolved to the same URL
+ * `urlForReference` (`@/integrations/sanity/utils/link`) uses for internal
+ * links elsewhere in the app — so the sitemap and `/llms.txt` can never
+ * disagree with on-page navigation about where a document lives.
+ *
+ * Returns `[]` when Sanity isn't configured (a fresh clone's default
+ * state): no fetch runs, and callers degrade to `STATIC_ROUTES` only.
+ *
+ * `'use cache'` is required: `sanityFetch` calls `cacheTag()` internally,
+ * which Cache Components (`cacheComponents: true`) only allows inside a
+ * `'use cache'` boundary — see `app/(site)/(examples)/sanity/page.tsx` for
+ * the same constraint applied to a page-level fetch. `perspective`/`stega`
+ * are hardcoded to the published, non-stega variant: crawlers never see
+ * draft content, so there's no request-level (draft mode) state to branch
+ * on here, unlike a rendered page.
+ */
+export async function getCmsRoutes(): Promise<ContentRoute[]> {
+  'use cache'
+
+  if (!isConfigured('sanity')) return []
+
+  // A schema-valid env (`isConfigured`) doesn't guarantee the project/dataset
+  // it points to actually exists or is reachable — `sanityFetch` throws on a
+  // Sanity API error (wrong project ID, deleted dataset, network failure).
+  // Crawlers depend on `sitemap.xml`/`llms.txt` always responding, even for
+  // the static routes, so a broken CMS connection degrades to no CMS routes
+  // instead of taking the whole response down.
+  let data: unknown
+  try {
+    ;({ data } = await sanityFetch({
+      query: routableContentQuery,
+      perspective: 'published',
+      stega: false,
+    }))
+  } catch (error) {
+    console.error(
+      '[seo/routes] Sanity fetch failed, omitting CMS routes:',
+      error
+    )
+    return []
+  }
+
+  const parsed = z.array(routableDocumentSchema).safeParse(data)
+  if (!parsed.success) return []
+
+  const routes = new Map<string, ContentRoute>()
+
+  for (const doc of parsed.data) {
+    if (!doc.slug) continue
+
+    const lastModified = new Date(doc._updatedAt)
+    if (Number.isNaN(lastModified.getTime())) continue
+
+    const path = urlForReference({
+      linkType: 'internal',
+      internalLink: { _type: doc._type, slug: doc.slug },
+    })
+
+    // `path === '#'` is unresolvable; a `staticPaths` hit means the document
+    // stands in for an already-listed static route (e.g. a `page` with slug
+    // `home`, which `urlForReference` resolves to `/`).
+    if (path === '#' || staticPaths.has(path)) continue
+
+    routes.set(path, {
+      path,
+      label: doc.title ?? doc.slug.current,
+      lastModified,
+    })
+  }
+
+  return [...routes.values()]
+}

lib/seo/site.ts

@@ -42,13 +42,18 @@ export interface SiteFacts {
 }
 
 /**
- * `NEXT_PUBLIC_BASE_URL` is validated with `z.url()`, which permits a trailing
- * slash. Everything below concatenates onto this, so an unnormalized value
- * would emit `//icon.png` and `//#organization` — a broken logo and a JSON-LD
- * `@id` that no longer matches the one other nodes reference. Strip it once,
- * here, rather than in every consumer.
+ * Canonical, normalized base URL — the single place every absolute URL in
+ * the app (sitemap, robots, `/llms.txt`, JSON-LD) derives from.
+ *
+ * `APP_BASE_URL` (`lib/env.ts`) already carries the fallback chain
+ * (`NEXT_PUBLIC_BASE_URL` ?? `https://localhost:3000`); this just normalizes
+ * it once. `NEXT_PUBLIC_BASE_URL` is validated with `z.url()`, which permits
+ * a trailing slash. Everything below concatenates onto this, so an
+ * unnormalized value would emit `//icon.png` and `//#organization` — a
+ * broken logo and a JSON-LD `@id` that no longer matches the one other nodes
+ * reference. Strip it once, here, rather than in every consumer.
  */
-const BASE_URL = APP_BASE_URL.replace(/\/+$/, '')
+export const BASE_URL = APP_BASE_URL.replace(/\/+$/, '')
 
 export const SITE: SiteFacts = {
   name: 'Satūs',