diff --git a/.env.example b/.env.example index 761bee7..9d1ed7e 100644 --- a/.env.example +++ b/.env.example @@ -14,7 +14,7 @@ CORPUSKIT_API_BIND_HOST=127.0.0.1 # Prometheus metrics are served at /internal/metrics. Staging and production # require a distinct 32-512 character opaque bearer token. CORPUSKIT_METRICS_ENABLED=true -CORPUSKIT_METRICS_BEARER_TOKEN= +# CORPUSKIT_METRICS_BEARER_TOKEN=<32-512-character-opaque-token> # Authenticated, tenant-scoped fixed-window limits are shared through PostgreSQL. # Staging/production API processes require this to remain enabled. CORPUSKIT_API_RATE_LIMIT_ENABLED=false @@ -64,7 +64,7 @@ CORPUSKIT_TEMPORAL_NAMESPACE=default CORPUSKIT_TEMPORAL_TASK_QUEUE=batch-cpu # Staging/production require TLS. Set an API key only with TLS enabled. CORPUSKIT_TEMPORAL_TLS=false -CORPUSKIT_TEMPORAL_API_KEY= +# CORPUSKIT_TEMPORAL_API_KEY= CORPUSKIT_TEMPORAL_CONNECT_TIMEOUT_SECONDS=10 CORPUSKIT_TEMPORAL_ACTIVITY_HEARTBEAT_SECONDS=5 CORPUSKIT_TEMPORAL_MAX_CONCURRENT_ACTIVITIES=8 @@ -93,13 +93,13 @@ CORPUSKIT_WORKER_MODEL_CACHE_MOUNT_READ_ONLY=false # CORPUSKIT_WORKER_DATG_INDEX_PUBLISH_ROOT=/datg-index-publish CORPUSKIT_WORKER_DATG_CACHE_MOUNT_READ_ONLY=false CORPUSKIT_WORKER_PHON_RL_CACHE_ROOTS={} -CORPUSKIT_DISPATCHER_ID= +# CORPUSKIT_DISPATCHER_ID=corpuskit-local-dispatcher CORPUSKIT_DISPATCHER_BATCH_SIZE=20 CORPUSKIT_DISPATCHER_POLL_SECONDS=1 # Production OIDC. Required when CORPUSKIT_AUTH_MODE=oidc. -CORPUSKIT_OIDC_ISSUER= -CORPUSKIT_OIDC_AUDIENCE= +# CORPUSKIT_OIDC_ISSUER=https://identity.example.com/ +# CORPUSKIT_OIDC_AUDIENCE=corpuskit-api CORPUSKIT_OIDC_ALGORITHMS=["RS256"] CORPUSKIT_OIDC_JWKS_CACHE_SECONDS=300 CORPUSKIT_OIDC_REFRESH_COOLDOWN_SECONDS=10 @@ -109,9 +109,9 @@ CORPUSKIT_OIDC_ORGANIZATION_CLAIM=org_id CORPUSKIT_OIDC_ROLE_CLAIM=role # Optional server-managed provider credentials. Never expose these to the web app. -CORPUSKIT_PROVIDER_API_KEY= -OPENAI_API_KEY= -ANTHROPIC_API_KEY= +# CORPUSKIT_PROVIDER_API_KEY= +# OPENAI_API_KEY= +# ANTHROPIC_API_KEY= # Frontend server/BFF. This value is never exposed to browser JavaScript. CORPUSKIT_API_INTERNAL_URL=http://127.0.0.1:8000/api/v1 @@ -130,8 +130,8 @@ CORPUSKIT_WEB_LOGIN_TRANSACTION_SECONDS=600 CORPUSKIT_WEB_OIDC_TIMEOUT_SECONDS=5 CORPUSKIT_WEB_REDIS_COMMAND_TIMEOUT_MS=2000 CORPUSKIT_WEB_REDIS_PREFIX=corpuskit:web -CORPUSKIT_WEB_SESSION_REDIS_URL= -CORPUSKIT_WEB_OIDC_CLIENT_ID= -CORPUSKIT_WEB_OIDC_CLIENT_SECRET= -CORPUSKIT_WEB_OIDC_REDIRECT_URI= +# CORPUSKIT_WEB_SESSION_REDIS_URL=rediss://sessions.example.com:6379/0 +# CORPUSKIT_WEB_OIDC_CLIENT_ID=corpuskit-web +# CORPUSKIT_WEB_OIDC_CLIENT_SECRET= +# CORPUSKIT_WEB_OIDC_REDIRECT_URI=https://app.example.com/auth/callback CORPUSKIT_WEB_OIDC_SCOPES=openid profile offline_access diff --git a/CHANGELOG.md b/CHANGELOG.md index 53d4ee5..d48aa72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ All notable changes to CorpusKit will be documented here. The project follows ### Added +- A comprehensive Docker-first getting-started guide, copyable direct-development environment + examples, first-use workflow, lifecycle and troubleshooting commands, and an enforced local + startup documentation contract. - Append-only corpus version creation from manual sentences or bounded UTF-8 TXT, CSV, and JSON imports, with parent lineage, atomic quota accounting, audit evidence, API contracts, and an accessible project-workbench flow. diff --git a/README.md b/README.md index ef4de38..ccdc5df 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,29 @@ reproducible user workflows. > The repository contains a hardened release-candidate pipeline, but no registry publication or > production promotion is claimed until its external acceptance gates are recorded. +## Quick start + +For an isolated local website, install Git and a running Docker Engine/Desktop with Docker +Compose v2, then run: + +```text +git clone https://github.com/jemsbhai/corpuskit.git +cd corpuskit +docker compose --profile web up --build --detach --wait +``` + +Open . No host Python, Node.js, npm, eSpeak NG, or `.env` file is +needed for this Docker path. The first run builds the locked API and web images, migrates +PostgreSQL, initializes private MinIO storage, and provisions the pinned PHOIBLE snapshot. + +> **Do not publish this demo:** every browser receives the same local owner identity and the +> checked-in development secrets are deterministic. Keep it on loopback. Shared deployments +> require OIDC and the production topology. + +The comprehensive [getting-started guide](docs/getting-started.md) covers verification, first +corpus creation, ports, logs, updates, persistence and reset, common failures, durable local jobs, +Windows/PowerShell commands, direct source development, and the production boundary. + ## Product scope CorpusKit is designed for TTS/ASR dataset builders, speech researchers, clinical @@ -160,16 +183,19 @@ the [release](https://github.com/jemsbhai/corpuskit/blob/main/docs/operations/re ## Development prerequisites -- Python 3.12 +The [getting-started guide](docs/getting-started.md) is the authoritative first-run path. Use the +Docker quick start unless you are changing application source. + +- Python >=3.12,<3.13 - [`uv`](https://docs.astral.sh/uv/) -- Node.js 24 LTS and npm 11 +- Node.js >=24.18.1,<25 and npm 11.16.0 - eSpeak NG for real G2P workflows - Docker with Compose for the full local stack -Install locked dependencies after lockfiles have been generated: +The lockfiles are committed. Install them without re-resolving dependencies: ```bash -uv sync --all-groups +uv sync --frozen --all-groups npm ci ``` @@ -182,17 +208,24 @@ uv run corpuskit-phoible provision --json uv run corpuskit-phoible status --json ``` -Run the API and web application: +For direct source development, copy the separate API and Next.js demo environment examples and +prepare the local database: ```bash mkdir -p data -export CORPUSKIT_DATABASE_URL=sqlite+aiosqlite:///./data/corpuskit.db -uv run corpuskit-phoible provision +cp .env.example .env +cp apps/web/.env.example apps/web/.env.local +uv run corpuskit-phoible provision --json +uv run corpuskit-phoible status --json +CORPUSKIT_DATABASE_URL=sqlite+aiosqlite:///./data/corpuskit.db \ uv run corpuskit-db upgrade -uv run corpuskit-api -npm run dev ``` +Then run `uv run corpuskit-api` and `npm run dev` in **two separate terminals** from the +repository root. PowerShell equivalents and eSpeak installation notes are in the +[getting-started guide](docs/getting-started.md). The root `.env` is loaded by Python; Next.js +loads `apps/web/.env.local` and does not consume the root file. + Every Python service also requires an exact process posture. `corpuskit-api` uses the default `CORPUSKIT_RUNTIME_ROLE=api`; direct dispatcher, worker, and maintenance processes must set `dispatcher`, `worker`, and `maintenance`, respectively. A mismatched entry point fails before @@ -201,13 +234,12 @@ the complete role/command map. Direct API installs bind only to `127.0.0.1` by d set `CORPUSKIT_API_BIND_HOST=0.0.0.0` explicitly behind the Compose loopback publication or the Helm Service, ingress, and NetworkPolicy boundary. -PowerShell uses `$env:CORPUSKIT_DATABASE_URL = -"sqlite+aiosqlite:///./data/corpuskit.db"` after `New-Item -ItemType Directory -Force data`. -The migration CLI deliberately requires the database URL in the process environment; it -does not infer a production target. Run migrations explicitly before every direct API -startup after pulling schema changes. Docker Compose performs the same development-only -ordering through its one-shot `migrate` and `provision-phoible` services. The latter owns -write access to the shared PHOIBLE volume; API and worker consumers mount it read-only. See +The migration CLI deliberately requires an explicit database URL in the process environment; it +does not load `.env` or infer a production target. Run migrations before direct API startup after +pulling schema changes. The API itself loads the root `.env`. Docker Compose performs the same +development-only ordering through its one-shot `migrate` and `provision-phoible` services. The +latter owns write access to the shared PHOIBLE volume; API and worker consumers mount it +read-only. See the [PHOIBLE provisioning runbook](https://github.com/jemsbhai/corpuskit/blob/main/docs/operations/phoible-provisioning.md) for air-gapped installation, verification, recovery, and release-job guidance. diff --git a/apps/web/.env.example b/apps/web/.env.example new file mode 100644 index 0000000..570da23 --- /dev/null +++ b/apps/web/.env.example @@ -0,0 +1,9 @@ +# Development-only settings for `npm run dev` from the CorpusKit repository root. +# Copy this file to apps/web/.env.local. Never reuse these deterministic values in +# a shared, staging, or production deployment. +CORPUSKIT_ENVIRONMENT=development +CORPUSKIT_API_INTERNAL_URL=http://127.0.0.1:8000/api/v1 +CORPUSKIT_WEB_AUTH_MODE=demo +CORPUSKIT_WEB_STATE_SECRET=AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQE +CORPUSKIT_WEB_SESSION_ENCRYPTION_KEYS=["AgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgI"] +CORPUSKIT_WEB_ALLOWED_RETURN_PATHS=["/","/projects","/evaluate","/analysis","/capabilities","/g2p","/inventory","/coverage","/selection","/generation","/advanced","/jobs","/artifacts"] diff --git a/compose.yaml b/compose.yaml index 41fd3e4..666de57 100644 --- a/compose.yaml +++ b/compose.yaml @@ -56,9 +56,19 @@ services: entrypoint: ["/bin/sh", "-ec"] command: - >- - mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}"; - mc mb --ignore-existing "local/$${MINIO_BUCKET}"; - mc anonymous set none "local/$${MINIO_BUCKET}" + attempt=0; + until + mc alias set local http://minio:9000 "$${MINIO_ROOT_USER}" "$${MINIO_ROOT_PASSWORD}" && + mc mb --ignore-existing "local/$${MINIO_BUCKET}" && + mc anonymous set none "local/$${MINIO_BUCKET}"; + do + attempt=$$((attempt + 1)); + if [ "$${attempt}" -ge 20 ]; then + echo "MinIO initialization failed after $${attempt} attempts" >&2; + exit 1; + fi; + sleep 1; + done environment: MC_CONFIG_DIR: /tmp/.mc MINIO_ROOT_USER: ${CORPUSKIT_MINIO_ROOT_USER:-corpuskit-local} @@ -280,6 +290,7 @@ services: - "127.0.0.1:${CORPUSKIT_WEB_PORT:-3000}:3000" tmpfs: - /tmp:rw,noexec,nosuid,size=64m + - /app/apps/web/.next/cache:rw,noexec,nosuid,size=64m,uid=1000,gid=1000,mode=0700 cap_drop: - ALL security_opt: diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..548b8ec --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,350 @@ +# Getting started with CorpusKit + +This guide takes a new contributor from a clean checkout to a working local CorpusKit website. +The recommended path uses Docker Compose and does not require Python, Node.js, npm, or eSpeak NG +on the host. A separate source-development path is included for people changing the application. + +> **Local use only:** the Compose quick start uses deterministic development secrets and signs +> every browser in as the same demo owner. All published ports bind to `127.0.0.1`, but that is +> not authentication. Do not expose, forward, or tunnel this demo profile to other users. Use the +> production OIDC and deployment runbooks for any shared environment. + +## Recommended: run the website with Docker + +### 1. Install the prerequisites + +You need: + +- Git; +- Docker Desktop on Windows or macOS, or Docker Engine on Linux; +- Docker Compose v2 (`docker compose`, not the legacy `docker-compose` command); +- Linux containers and an internet connection for the first image build and PHOIBLE download. + +The basic stack is qualified on Linux/AMD64. Allocate at least 2 CPUs, 4 GB of memory, and 10 GB +of free Docker disk space. Apple Silicon and other ARM64 hosts may work because the pinned base +images publish ARM64 manifests, but the complete application is not yet an ARM64 release gate. + +Confirm that Docker is running: + +```text +docker version +docker compose version +``` + +`docker version` must show both **Client** and **Server** sections; `docker compose version` only +needs to print a Compose v2 version. On Windows, switch Docker Desktop to Linux containers if it +is using Windows containers. + +### 2. Clone and start CorpusKit + +```text +git clone https://github.com/jemsbhai/corpuskit.git +cd corpuskit +docker compose --profile web up --build --detach --wait +``` + +No `.env` file is required for this isolated demo. On the first run, Docker downloads pinned base +images, builds the API and web images from the committed lockfiles, creates PostgreSQL and MinIO +volumes, applies database migrations, and downloads and verifies CorpusGen's pinned PHOIBLE +snapshot. A cold start can take several minutes. + +Three one-shot services should finish with exit code `0`: `migrate`, `minio-init`, and +`provision-phoible`. Their exited state is expected. PostgreSQL, MinIO, the API, and the web app +should remain healthy and running. + +### 3. Verify the startup + +POSIX shell: + +```bash +curl --fail http://127.0.0.1:8000/api/v1/health/ready +docker compose --profile web ps --all +``` + +PowerShell: + +```powershell +Invoke-RestMethod http://127.0.0.1:8000/api/v1/health/ready +docker compose --profile web ps --all +``` + +The readiness response must contain `"ready": true`. Then open: + +- website: +- first live workspace: +- development API documentation: + +There is no sign-up or password prompt in local demo mode. CorpusKit creates a `Demo user`, a +local organization, and an empty `Demo project` automatically. + +### 4. Create your first corpus + +1. Open and select **Demo project**. +2. Under **Create an immutable corpus**, enter a corpus name and keep `en-us` as the language. +3. Either enter one sentence per line or choose **File import** and select + [`apps/web/e2e/fixtures/demo-corpus.txt`](../apps/web/e2e/fixtures/demo-corpus.txt). +4. Select the new immutable version and inspect its ordered sentences and JSON, TXT, or CSV + export. +5. Try the same sentences in **Evaluate**, **Selection**, or **G2P**. Use **Jobs** for the local + inline job flow. + +The landing page's Riverbend values are illustrative. Results produced in the workbenches above +come from the running API. + +## What the basic Compose profile runs + +| Component | Purpose | Host address | +| ------------------- | ----------------------------------------- | ----------------------- | +| `web` | Next.js website and authenticated BFF | | +| `api` | FastAPI and CorpusGen adapter | | +| `postgres` | Persistent local application database | `127.0.0.1:5432` | +| `minio` | Persistent S3-compatible artifact storage | `127.0.0.1:9000` | +| MinIO console | Local storage inspection | | +| `migrate` | One-shot Alembic database upgrade | exits `0` | +| `minio-init` | One-shot private bucket initialization | exits `0` | +| `provision-phoible` | One-shot pinned PHOIBLE provisioning | exits `0` | + +The basic profile uses the in-process job backend. It deliberately omits Temporal, separate +workers, hosted-provider access, local-model caches, and GPU execution. + +## Stop, restart, update, and reset + +Stop the containers while preserving all local data: + +```text +docker compose --profile web down +``` + +Start the same data again: + +```text +docker compose --profile web up --detach --wait +``` + +After pulling application changes, rebuild and run the migrations again: + +```text +git pull --ff-only +docker compose --profile web up --build --detach --wait +``` + +PostgreSQL data, MinIO objects, artifacts, and the PHOIBLE snapshot live in named Docker volumes. +`docker compose down` preserves them. + +To perform a completely clean reset, first make sure that the Compose project shown by +`docker compose ls` is the local CorpusKit demo. Then run: + +```text +docker compose --profile web down --volumes --remove-orphans +``` + +> **Destructive reset:** `--volumes` permanently removes the local CorpusKit database, corpora, +> artifacts, and downloaded PHOIBLE snapshot for this Compose project. The next startup downloads +> and provisions everything again. + +## Troubleshooting + +### Docker cannot connect to the daemon + +Start Docker Desktop or the Docker Engine service, then rerun `docker version`. Seeing only the +client section is not enough; a server section must be present. + +### A host port is already in use + +The application containers communicate over their private Compose network, so changing a host +port does not change internal service configuration. Set only the conflicting mappings before +starting. + +POSIX example: + +```bash +CORPUSKIT_POSTGRES_PORT=55432 \ +CORPUSKIT_MINIO_PORT=19000 \ +CORPUSKIT_MINIO_CONSOLE_PORT=19001 \ +CORPUSKIT_API_PORT=18000 \ +CORPUSKIT_WEB_PORT=13000 \ +docker compose --profile web up --build --detach --wait +``` + +PowerShell example: + +```powershell +$env:CORPUSKIT_POSTGRES_PORT = "55432" +$env:CORPUSKIT_MINIO_PORT = "19000" +$env:CORPUSKIT_MINIO_CONSOLE_PORT = "19001" +$env:CORPUSKIT_API_PORT = "18000" +$env:CORPUSKIT_WEB_PORT = "13000" +docker compose --profile web up --build --detach --wait +``` + +With those example values, use for the website and + for readiness. Environment changes apply only to the +shell in which they were set. Remove them or open a new shell to restore the defaults. + +### Startup does not become healthy + +Inspect the resolved services and recent logs: + +```text +docker compose --profile web config --services +docker compose --profile web ps --all +docker compose --profile web logs --since=15m postgres migrate minio minio-init provision-phoible api web +``` + +Common causes are an older Compose version without `--wait` support, insufficient Docker disk or +memory, blocked registry/PyPI/npm access during the image build, or blocked GitHub access during +PHOIBLE provisioning. Fix the cause and rerun the same `up --build --detach --wait` command; the +one-shot setup operations are idempotent. + +### PHOIBLE is unavailable + +The API never downloads PHOIBLE during a request. Check the one-shot provisioner explicitly: + +```text +docker compose logs provision-phoible +docker compose run --rm --no-deps provision-phoible status --json +``` + +See the [PHOIBLE provisioning runbook](operations/phoible-provisioning.md) for checksum failures, +air-gapped installation, and recovery. + +### ARM64 build or startup fails + +The current release path is qualified on Linux/AMD64. Docker Desktop users on ARM64 can try its +AMD64 emulation by setting `DOCKER_DEFAULT_PLATFORM=linux/amd64` before building. Treat a native +ARM64 run as development-only until the repository adds an ARM64 acceptance gate. + +## Optional: use durable local jobs + +The basic demo is enough for first use. To add the local Temporal server, dispatcher, and CPU +worker, stop the basic stack and start both profiles with the Temporal backend. + +POSIX shell: + +```bash +docker compose --profile web down +CORPUSKIT_JOB_BACKEND=temporal \ +docker compose --profile web --profile durable up --build --detach --wait +``` + +PowerShell: + +```powershell +docker compose --profile web down +$env:CORPUSKIT_JOB_BACKEND = "temporal" +docker compose --profile web --profile durable up --build --detach --wait +``` + +The Temporal UI is then available at . Read the +[durable-jobs runbook](operations/durable-jobs.md) before enabling hosted, local-model, or GPU +profiles; those profiles require explicit allowlists, credentials, caches, or qualified hardware. + +## Develop directly from source + +Use this path only when changing Python or web code. The Docker quick start above remains the +simplest way to evaluate the application. + +### Host requirements + +- Python `>=3.12,<3.13`; +- [`uv`](https://docs.astral.sh/uv/); +- Node.js `>=24.18.1,<25` (the repository includes [`.nvmrc`](../.nvmrc)); +- npm `11.16.0`; +- eSpeak NG available on `PATH` or installed through the platform's supported mechanism. + +Install eSpeak NG with the OS package manager on Linux/macOS or a current Windows build from the +[eSpeak NG releases](https://github.com/espeak-ng/espeak-ng/releases). Verify it with +`espeak-ng --version`. + +### One-time setup + +POSIX shell: + +```bash +uv sync --frozen --all-groups +npm ci +cp .env.example .env +cp apps/web/.env.example apps/web/.env.local +mkdir -p data artifacts +uv run corpuskit-phoible provision --json +uv run corpuskit-phoible status --json +CORPUSKIT_DATABASE_URL=sqlite+aiosqlite:///./data/corpuskit.db \ +uv run corpuskit-db upgrade +``` + +PowerShell: + +```powershell +uv sync --frozen --all-groups +npm ci +Copy-Item .env.example .env +Copy-Item apps/web/.env.example apps/web/.env.local +New-Item -ItemType Directory -Force data, artifacts +uv run corpuskit-phoible provision --json +uv run corpuskit-phoible status --json +$env:CORPUSKIT_DATABASE_URL = "sqlite+aiosqlite:///./data/corpuskit.db" +uv run corpuskit-db upgrade +``` + +The copied files contain deterministic demo-only settings and are ignored by Git. The root +`.env` configures the Python API. Next.js loads `apps/web/.env.local`; it does not load the root +file when the workspace script runs. The migration CLI deliberately reads only the process +environment, which is why the commands above set `CORPUSKIT_DATABASE_URL` explicitly. + +### Start two development terminals + +Terminal 1, from the repository root: + +```text +uv run corpuskit-api +``` + +Terminal 2, also from the repository root: + +```text +npm run dev +``` + +Keep both processes running. Verify the API at +, then open . + +The direct-development defaults use SQLite, filesystem artifacts, in-process sessions, demo +identity, and inline jobs. They do not provide PostgreSQL RLS, shared sessions, or durable worker +recovery and are not a production deployment. + +## Run the live acceptance walkthrough + +Starting the website does not require host Node.js or Playwright. If you want to execute the +fixed-input browser acceptance suite, install the exact Node/npm versions above and follow the +[15-minute live demo](product/15-minute-demo.md). That guide adds `npm ci`, a Chromium install, +and the live Playwright command to an already understood local stack. + +## Requirements and container inventory + +CorpusKit intentionally does not use a hand-maintained `requirements.txt`: + +- [`pyproject.toml`](../pyproject.toml) declares Python packages and optional worker groups; +- [`uv.lock`](../uv.lock) freezes the complete Python dependency graph; +- [`package.json`](../package.json), [`apps/web/package.json`](../apps/web/package.json), and + [`package-lock.json`](../package-lock.json) define and lock Node dependencies; +- [`docker/api.Dockerfile`](../docker/api.Dockerfile) builds the API, migration, provisioning, + dispatcher, and maintenance runtime; +- [`docker/web.Dockerfile`](../docker/web.Dockerfile) builds the Next.js server; +- [`docker/worker.Dockerfile`](../docker/worker.Dockerfile) contains the batch, external-provider, + GPU-inference, and GPU-training worker targets; +- [`docker/mutation.Dockerfile`](../docker/mutation.Dockerfile) is test-only; +- [`compose.yaml`](../compose.yaml) wires the local profiles, health checks, one-shot setup, and + persistent volumes together. + +Do not generate a second requirements file from the lock. Use `uv sync --frozen` or the +Dockerfiles so local and CI dependency resolution stays identical. + +## Shared and production deployments + +Docker Compose is an isolated development/demo topology, not a public hosting recipe. A shared +deployment requires external OIDC, encrypted Redis/Valkey browser sessions, PostgreSQL roles and +RLS, private S3 storage, TLS Temporal, separate dispatcher/worker identities, secret management, +maintenance scheduling, ingress, and monitoring. Start with the +[Kubernetes production runbook](operations/kubernetes-production.md) and +[OIDC authentication runbook](operations/oidc-authentication.md). The repository is still an +alpha and does not claim completed production promotion evidence. diff --git a/docs/product/15-minute-demo.md b/docs/product/15-minute-demo.md index f2ea5a4..2353f6b 100644 --- a/docs/product/15-minute-demo.md +++ b/docs/product/15-minute-demo.md @@ -5,6 +5,10 @@ selection, scoring, job-control, quota, and audit surfaces. The automated accept Playwright request interception and fails on any API 5xx response. Hosted providers, GPU models, DATG, and Phon-RL remain deployment-gated and are not required for this CPU demo. +If this is your first local run, begin with the [getting-started guide](../getting-started.md). +That path needs only Git and Docker. The Node.js, npm, Chromium, and fixed-fixture steps below are +for executable acceptance evidence, not for simply starting the website. + ## Before the clock starts Use Docker with Compose, Node.js 24 LTS, npm 11, and a clean checkout. Cold image builds and the diff --git a/tests/architecture/test_documentation_contract.py b/tests/architecture/test_documentation_contract.py index de8efda..b7d4218 100644 --- a/tests/architecture/test_documentation_contract.py +++ b/tests/architecture/test_documentation_contract.py @@ -69,3 +69,65 @@ def test_public_execution_routing_claims_match_the_registered_boundaries() -> No assert "advanced-capabilities.v1" not in overview + runtimes assert "execution_routes_exposed" not in overview + runtimes assert "Hugging Face repository imports share the" in compose + + +def test_getting_started_contract_tracks_the_runnable_local_stack() -> None: + """Keep the beginner copy/paste path aligned with deployable files and Compose.""" + + guide = (ROOT / "docs" / "getting-started.md").read_text(encoding="utf-8") + compose = (ROOT / "compose.yaml").read_text(encoding="utf-8") + web_environment = (ROOT / "apps" / "web" / ".env.example").read_text(encoding="utf-8") + + required_files = ( + "compose.yaml", + "docker/api.Dockerfile", + "docker/web.Dockerfile", + "docker/worker.Dockerfile", + "docker/mutation.Dockerfile", + "pyproject.toml", + "uv.lock", + "package.json", + "apps/web/package.json", + "package-lock.json", + ".env.example", + "apps/web/.env.example", + ) + assert not [path for path in required_files if not (ROOT / path).is_file()] + + required_guide_fragments = ( + "docker compose --profile web up --build --detach --wait", + '"ready": true', + "http://127.0.0.1:3000/projects", + "http://127.0.0.1:8000/api/v1/health/ready", + "docker compose --profile web down", + "docker compose --profile web down --volumes --remove-orphans", + "uv sync --frozen --all-groups", + "CORPUSKIT_DATABASE_URL=sqlite+aiosqlite:///./data/corpuskit.db", + '$env:CORPUSKIT_DATABASE_URL = "sqlite+aiosqlite:///./data/corpuskit.db"', + "apps/web/.env.local", + "The migration CLI deliberately reads only the process", + "same demo owner", + ) + assert not [fragment for fragment in required_guide_fragments if fragment not in guide] + + for service in ( + "postgres", + "minio", + "minio-init", + "api", + "migrate", + "provision-phoible", + "web", + ): + assert re.search(rf"^ {re.escape(service)}:$", compose, flags=re.MULTILINE) + assert "profiles: [web]" in compose + + for variable in ( + "CORPUSKIT_ENVIRONMENT", + "CORPUSKIT_API_INTERNAL_URL", + "CORPUSKIT_WEB_AUTH_MODE", + "CORPUSKIT_WEB_STATE_SECRET", + "CORPUSKIT_WEB_SESSION_ENCRYPTION_KEYS", + "CORPUSKIT_WEB_ALLOWED_RETURN_PATHS", + ): + assert re.search(rf"^{variable}=.+$", web_environment, flags=re.MULTILINE) diff --git a/tests/deployment/test_compose_contract.py b/tests/deployment/test_compose_contract.py index 8fbb36e..bf80910 100644 --- a/tests/deployment/test_compose_contract.py +++ b/tests/deployment/test_compose_contract.py @@ -20,6 +20,7 @@ ESPEAK_TMPDIR = "/run/corpuskit-espeak" GENERAL_TMPDIR = "/tmp" # noqa: S108 - asserted container path, not a host temp file XDG_CONFIG_HOME = "/tmp/corpuskit-xdg" # noqa: S108 - asserted container path +WEB_CACHE = "/app/apps/web/.next/cache" def _compose_config() -> dict[str, Any]: @@ -74,3 +75,28 @@ def test_non_execution_services_do_not_receive_executable_temp_storage() -> None assert all(not item.startswith(f"{ESPEAK_TMPDIR}:") for item in service.get("tmpfs", [])), ( name ) + + +def test_read_only_web_runtime_has_only_bounded_writable_storage() -> None: + web = _compose_config()["services"]["web"] + + assert web["read_only"] is True + assert set(web["tmpfs"]) == { + "/tmp:rw,noexec,nosuid,size=64m", # noqa: S108 - asserted container path + f"{WEB_CACHE}:rw,noexec,nosuid,size=64m,uid=1000,gid=1000,mode=0700", + } + + +def test_minio_initialization_retries_transient_startup_failures() -> None: + initializer = _compose_config()["services"]["minio-init"] + command = initializer["command"] + assert isinstance(command, list) + assert len(command) == 1 + script = command[0] + + assert initializer["depends_on"]["minio"]["condition"] == "service_healthy" + assert "until" in script + assert "attempt=$$((attempt + 1))" in script + assert 'if [ "$${attempt}" -ge 20 ]' in script + assert "mc mb --ignore-existing" in script + assert "mc anonymous set none" in script diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index a123efc..7dd917b 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -3,12 +3,16 @@ from __future__ import annotations import json +import os +from pathlib import Path import pytest from pydantic import ValidationError from corpuskit.config import RuntimeRole, Settings +ROOT = Path(__file__).parents[2] + def _production_settings(**overrides: object) -> Settings: values: dict[str, object] = { @@ -46,6 +50,20 @@ def test_development_defaults_are_bounded() -> None: settings.log_level = "DEBUG" +def test_committed_development_env_example_is_loadable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + for variable in tuple(os.environ): + if variable.startswith(("CORPUSKIT_", "OPENAI_", "ANTHROPIC_")): + monkeypatch.delenv(variable) + + settings = Settings(_env_file=ROOT / ".env.example") + + assert settings.environment == "development" + assert settings.database_url == "sqlite+aiosqlite:///./data/corpuskit.db" + assert settings.job_backend == "inline" + + def test_database_credentials_are_excluded_from_settings_repr() -> None: settings = Settings( environment="test",