From 3167961bd63057051ad0b7a1ac113c12d8f1fbbf Mon Sep 17 00:00:00 2001 From: Angelo Date: Thu, 30 Jul 2026 15:30:15 -0400 Subject: [PATCH] fix(search): discover pages from the sitemap and let deploys settle before scraping The scraper discovered pages only by following links from /, so a scrape racing a deploy crawled the stale cached /guides index and missed most of a 38-guide batch (verified against the prod index: llm-gateway present, code-execution-sandboxes/webhooks-at-scale/ uptime-kuma absent, run 30519545306). Discovery now also reads sitemap-0.xml, which regenerates with every build, and deployment-triggered scrapes wait 3 minutes for the rollout. Co-Authored-By: Claude Fable 5 --- .github/workflows/build-search-index.yml | 7 + meilisearch-docs-scraper.config.json | 560 ++++++++++++++++++----- 2 files changed, 465 insertions(+), 102 deletions(-) diff --git a/.github/workflows/build-search-index.yml b/.github/workflows/build-search-index.yml index 21b149104..e5f369644 100644 --- a/.github/workflows/build-search-index.yml +++ b/.github/workflows/build-search-index.yml @@ -17,6 +17,13 @@ jobs: contains(github.event.deployment.environment, 'production')) steps: - uses: actions/checkout@v7 + # The deploy-success event can precede edge caches serving the new + # build; scraping immediately indexes stale pages (this dropped most + # of a 38-guide batch from the index on 2026-07-30). Let the rollout + # settle before crawling. + - name: Wait for rollout to settle + if: github.event_name == 'deployment_status' + run: sleep 180 - name: Scrape and index docs env: HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} diff --git a/meilisearch-docs-scraper.config.json b/meilisearch-docs-scraper.config.json index 9b71c710c..a599ca91d 100644 --- a/meilisearch-docs-scraper.config.json +++ b/meilisearch-docs-scraper.config.json @@ -58,109 +58,465 @@ "disableOnAttributes": [] }, "stopWords": [ - "the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", - "of", "with", "by", "from", "as", "is", "was", "are", "were", "been", - "be", "have", "has", "had", "do", "does", "did", "will", "would", - "could", "should", "may", "might", "must", "shall", "can", "need", - "this", "that", "these", "those", "it", "its", "i", "you", "we", - "they", "my", "your", "our", "their", "what", "which", "who", "how" + "the", + "a", + "an", + "and", + "or", + "but", + "in", + "on", + "at", + "to", + "for", + "of", + "with", + "by", + "from", + "as", + "is", + "was", + "are", + "were", + "been", + "be", + "have", + "has", + "had", + "do", + "does", + "did", + "will", + "would", + "could", + "should", + "may", + "might", + "must", + "shall", + "can", + "need", + "this", + "that", + "these", + "those", + "it", + "its", + "i", + "you", + "we", + "they", + "my", + "your", + "our", + "their", + "what", + "which", + "who", + "how" ], "synonyms": { - "postgres": ["postgresql", "pg"], - "postgresql": ["postgres", "pg"], - "mysql": ["mariadb", "maria"], - "mariadb": ["mysql", "maria"], - "redis": ["cache", "in-memory", "key-value"], - "mongodb": ["mongo"], - "mongo": ["mongodb"], - - "preview environment": ["pr environment", "pull request environment", "preview deploy"], - "pr environment": ["preview environment", "pull request environment", "preview deploy"], - "pull request environment": ["preview environment", "pr environment", "preview deploy"], - "staging": ["stage", "test environment", "testing"], - "production": ["prod", "live", "production environment"], - "deploy": ["deployment", "ship", "push", "release"], - "deployment": ["deploy", "release", "rollout"], - - "domain": ["custom domain", "hostname", "url", "fqdn"], - "custom domain": ["domain", "hostname", "own domain"], - "ssl": ["tls", "https", "certificate", "cert"], - "tls": ["ssl", "https", "certificate", "cert"], - "https": ["ssl", "tls", "secure"], - - "private network": ["internal network", "vpn", "private networking"], - "private networking": ["private network", "internal network", "vpn"], - "public network": ["external network", "public networking", "internet"], - "tcp proxy": ["tcp", "proxy", "port forwarding"], - "static ip": ["fixed ip", "static outbound ip", "dedicated ip"], - - "service": ["app", "application", "container", "workload"], - "container": ["docker", "service", "app"], - "dockerfile": ["docker", "container", "containerfile"], - "docker": ["container", "dockerfile", "containerize"], - - "railpack": ["buildpack", "builder"], - "build": ["compile", "bundle", "package"], - - "environment variable": ["env var", "env", "config", "envvar", "variable"], - "env var": ["environment variable", "envvar", "config", "variable"], - "config": ["configuration", "settings", "setup"], - "secret": ["api key", "credential", "token", "password"], - - "cli": ["command line", "terminal", "railway cli", "shell"], - "command line": ["cli", "terminal", "shell"], - - "api": ["graphql", "rest", "endpoint", "public api"], - "graphql": ["gql", "api", "query"], - "gql": ["graphql", "api"], - "rest": ["rest api", "http api", "api"], - - "logs": ["logging", "log", "output", "console"], - "logging": ["logs", "log", "observability"], - "metrics": ["monitoring", "stats", "statistics", "observability"], - "monitoring": ["metrics", "observability", "tracking"], - - "volume": ["persistent storage", "disk", "storage", "mount"], - "storage": ["volume", "disk", "persist", "bucket"], - "bucket": ["s3", "object storage", "blob storage", "storage bucket"], - "s3": ["bucket", "object storage", "storage bucket"], - - "scale": ["scaling", "autoscale", "horizontal scaling", "resize"], - "autoscale": ["auto-scale", "automatic scaling", "scale", "horizontal scaling"], - "horizontal scaling": ["scale out", "autoscale", "replicas"], - "vertical scaling": ["scale up", "resize", "upgrade"], - - "template": ["starter", "boilerplate", "example", "blueprint"], - "starter": ["template", "boilerplate", "quickstart"], - - "team": ["workspace", "organization", "org"], - "workspace": ["team", "organization", "org"], - "member": ["user", "collaborator", "teammate"], - - "pricing": ["cost", "price", "billing", "payment"], - "billing": ["pricing", "invoice", "payment", "cost"], - "credits": ["credit", "balance", "payment"], - "trial": ["free trial", "free tier", "starter"], - - "error": ["bug", "issue", "problem", "fail", "failure"], - "crash": ["error", "failure", "fail", "down"], - - "cron": ["scheduled task", "cron job", "scheduler", "scheduled"], - "cron job": ["cron", "scheduled task", "periodic task", "timer"], - "scheduled": ["cron", "periodic", "recurring", "timed"], - - "healthcheck": ["health check", "health", "liveness", "readiness"], - "health check": ["healthcheck", "liveness probe", "readiness probe"], - - "next.js": ["nextjs", "next", "vercel"], - "nextjs": ["next.js", "next"], - "react": ["reactjs", "react.js"], - "vue": ["vuejs", "vue.js"], - "express": ["expressjs", "express.js", "node"], - "fastapi": ["fast api", "python api"], - "django": ["python", "python web"], - "rails": ["ruby on rails", "ror", "ruby"], - "laravel": ["php", "php framework"] + "postgres": [ + "postgresql", + "pg" + ], + "postgresql": [ + "postgres", + "pg" + ], + "mysql": [ + "mariadb", + "maria" + ], + "mariadb": [ + "mysql", + "maria" + ], + "redis": [ + "cache", + "in-memory", + "key-value" + ], + "mongodb": [ + "mongo" + ], + "mongo": [ + "mongodb" + ], + "preview environment": [ + "pr environment", + "pull request environment", + "preview deploy" + ], + "pr environment": [ + "preview environment", + "pull request environment", + "preview deploy" + ], + "pull request environment": [ + "preview environment", + "pr environment", + "preview deploy" + ], + "staging": [ + "stage", + "test environment", + "testing" + ], + "production": [ + "prod", + "live", + "production environment" + ], + "deploy": [ + "deployment", + "ship", + "push", + "release" + ], + "deployment": [ + "deploy", + "release", + "rollout" + ], + "domain": [ + "custom domain", + "hostname", + "url", + "fqdn" + ], + "custom domain": [ + "domain", + "hostname", + "own domain" + ], + "ssl": [ + "tls", + "https", + "certificate", + "cert" + ], + "tls": [ + "ssl", + "https", + "certificate", + "cert" + ], + "https": [ + "ssl", + "tls", + "secure" + ], + "private network": [ + "internal network", + "vpn", + "private networking" + ], + "private networking": [ + "private network", + "internal network", + "vpn" + ], + "public network": [ + "external network", + "public networking", + "internet" + ], + "tcp proxy": [ + "tcp", + "proxy", + "port forwarding" + ], + "static ip": [ + "fixed ip", + "static outbound ip", + "dedicated ip" + ], + "service": [ + "app", + "application", + "container", + "workload" + ], + "container": [ + "docker", + "service", + "app" + ], + "dockerfile": [ + "docker", + "container", + "containerfile" + ], + "docker": [ + "container", + "dockerfile", + "containerize" + ], + "railpack": [ + "buildpack", + "builder" + ], + "build": [ + "compile", + "bundle", + "package" + ], + "environment variable": [ + "env var", + "env", + "config", + "envvar", + "variable" + ], + "env var": [ + "environment variable", + "envvar", + "config", + "variable" + ], + "config": [ + "configuration", + "settings", + "setup" + ], + "secret": [ + "api key", + "credential", + "token", + "password" + ], + "cli": [ + "command line", + "terminal", + "railway cli", + "shell" + ], + "command line": [ + "cli", + "terminal", + "shell" + ], + "api": [ + "graphql", + "rest", + "endpoint", + "public api" + ], + "graphql": [ + "gql", + "api", + "query" + ], + "gql": [ + "graphql", + "api" + ], + "rest": [ + "rest api", + "http api", + "api" + ], + "logs": [ + "logging", + "log", + "output", + "console" + ], + "logging": [ + "logs", + "log", + "observability" + ], + "metrics": [ + "monitoring", + "stats", + "statistics", + "observability" + ], + "monitoring": [ + "metrics", + "observability", + "tracking" + ], + "volume": [ + "persistent storage", + "disk", + "storage", + "mount" + ], + "storage": [ + "volume", + "disk", + "persist", + "bucket" + ], + "bucket": [ + "s3", + "object storage", + "blob storage", + "storage bucket" + ], + "s3": [ + "bucket", + "object storage", + "storage bucket" + ], + "scale": [ + "scaling", + "autoscale", + "horizontal scaling", + "resize" + ], + "autoscale": [ + "auto-scale", + "automatic scaling", + "scale", + "horizontal scaling" + ], + "horizontal scaling": [ + "scale out", + "autoscale", + "replicas" + ], + "vertical scaling": [ + "scale up", + "resize", + "upgrade" + ], + "template": [ + "starter", + "boilerplate", + "example", + "blueprint" + ], + "starter": [ + "template", + "boilerplate", + "quickstart" + ], + "team": [ + "workspace", + "organization", + "org" + ], + "workspace": [ + "team", + "organization", + "org" + ], + "member": [ + "user", + "collaborator", + "teammate" + ], + "pricing": [ + "cost", + "price", + "billing", + "payment" + ], + "billing": [ + "pricing", + "invoice", + "payment", + "cost" + ], + "credits": [ + "credit", + "balance", + "payment" + ], + "trial": [ + "free trial", + "free tier", + "starter" + ], + "error": [ + "bug", + "issue", + "problem", + "fail", + "failure" + ], + "crash": [ + "error", + "failure", + "fail", + "down" + ], + "cron": [ + "scheduled task", + "cron job", + "scheduler", + "scheduled" + ], + "cron job": [ + "cron", + "scheduled task", + "periodic task", + "timer" + ], + "scheduled": [ + "cron", + "periodic", + "recurring", + "timed" + ], + "healthcheck": [ + "health check", + "health", + "liveness", + "readiness" + ], + "health check": [ + "healthcheck", + "liveness probe", + "readiness probe" + ], + "next.js": [ + "nextjs", + "next", + "vercel" + ], + "nextjs": [ + "next.js", + "next" + ], + "react": [ + "reactjs", + "react.js" + ], + "vue": [ + "vuejs", + "vue.js" + ], + "express": [ + "expressjs", + "express.js", + "node" + ], + "fastapi": [ + "fast api", + "python api" + ], + "django": [ + "python", + "python web" + ], + "rails": [ + "ruby on rails", + "ror", + "ruby" + ], + "laravel": [ + "php", + "php framework" + ] } - } + }, + "sitemap_urls": [ + "https://docs.railway.com/sitemap-0.xml" + ], + "sitemap_alternate_links": false }