feat: hosted deployment stack — production compose, /healthz, image publish, ops runbook

.github/workflows/publish.yml

@@ -0,0 +1,45 @@
+# Build and publish the coven-github container image to GHCR.
+# Pushes to main publish `latest` + the commit SHA; version tags publish the
+# semver tag. deploy/hosted/compose.yaml pins deployments to these images.
+name: publish
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/opencoven/coven-github
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=sha,format=long
+            type=semver,pattern={{version}}
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Dockerfile

@@ -23,8 +23,9 @@ RUN cargo build --release --locked -p coven-github
 FROM debian:bookworm-slim AS runtime
 
 # ca-certificates: TLS to api.github.com. git: clone target repos.
+# curl: container healthchecks against /healthz (deploy/hosted/compose.yaml).
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates git \
+    && apt-get install -y --no-install-recommends ca-certificates git curl \
     && rm -rf /var/lib/apt/lists/*
 
 # Run as an unprivileged user. Secrets are mounted read-only at runtime, never

HOSTED.md

@@ -88,6 +88,11 @@ Hosted beta should wait for:
 5. Usage metering by installation, repo, familiar, and task.
 6. Cave oversight dashboard for task history and human intervention.
 
+All six code gates are shipped. The operational side — provisioning, TLS
+ingress, continuous store backup, monitoring, restore drills, upgrades — is
+the [hosted deployment runbook](docs/hosted-deploy.md) with its stack in
+[`deploy/hosted/`](deploy/hosted/).
+
 ## Landing Page Copy
 
 Headline:

crates/server/src/main.rs

@@ -12,7 +12,8 @@ use tracing_subscriber::EnvFilter;
 
 use coven_github_config::Config;
 use coven_github_webhook::routes::{
-    audit, handle_webhook, list_memory, list_tasks, revoke_memory, routing, usage, AppState,
+    audit, handle_webhook, healthz, list_memory, list_tasks, revoke_memory, routing, usage,
+    AppState,
 };
 use coven_github_worker as worker;
 
@@ -172,6 +173,7 @@ async fn main() -> Result<()> {
             };
 
             let app = Router::new()
+                .route("/healthz", get(healthz))
                 .route("/webhook", post(handle_webhook))
                 .route("/api/github/tasks", get(list_tasks))
                 .route("/api/github/memory", get(list_memory))

crates/webhook/src/routes.rs

@@ -43,6 +43,23 @@ fn familiar_names(config: &Config) -> std::collections::HashMap<String, String>
         .collect()
 }
 
+/// GET /healthz — liveness/readiness for ingress and uptime checks.
+/// Unauthenticated by design, so it must never expose tenant data: the body
+/// is only `ok` plus whether the durable store answers a trivial read.
+pub async fn healthz(State(state): State<AppState>) -> impl IntoResponse {
+    match state.store.delivery_routing("__healthz__").await {
+        Ok(_) => (StatusCode::OK, Json(json!({"ok": true}))).into_response(),
+        Err(e) => {
+            error!("healthz store probe failed: {e:#}");
+            (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"ok": false, "store": "unavailable"})),
+            )
+                .into_response()
+        }
+    }
+}
+
 /// GET /api/github/tasks — task state for CovenCave polling, behind the
 /// tenant boundary (issue #3). `token` mode fails closed; a tenant token sees
 /// only its own installation (optionally narrowed to repositories); the
@@ -2996,6 +3013,25 @@ mod metering_route_tests {
     }
 }
 
+#[cfg(test)]
+mod healthz_tests {
+    use super::tests::app_state;
+    use super::*;
+    use axum::extract::State;
+
+    #[tokio::test]
+    async fn healthz_answers_ok_and_leaks_nothing() {
+        let state = app_state();
+        let response = healthz(State(state)).await.into_response();
+        assert_eq!(response.status(), StatusCode::OK);
+        let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("body");
+        let json: serde_json::Value = serde_json::from_slice(&bytes).expect("json");
+        assert_eq!(json, serde_json::json!({"ok": true}));
+    }
+}
+
 #[cfg(test)]
 mod billing_route_tests {
     //! Marketplace plan intake: entitlement recording, plan-derived limits,

deploy/hosted/.env.example

@@ -0,0 +1,20 @@
+# Secrets for deploy/hosted/compose.yaml. Copy to .env (mode 0600) and fill
+# in. NEVER commit the filled-in file.
+
+# Public hostname the GitHub App webhook points at.
+WEBHOOK_DOMAIN=gh.opencoven.ai
+
+# Image tag to run (a released ghcr.io/opencoven/coven-github tag).
+COVEN_GITHUB_TAG=latest
+
+# Cloudflare R2 (S3-compatible) for Litestream SQLite replication.
+# Create an R2 API token scoped to one bucket with object read+write.
+R2_ACCOUNT_ID=
+R2_BUCKET=coven-github-store
+R2_ACCESS_KEY_ID=
+R2_SECRET_ACCESS_KEY=
+
+# Host docker group id, so the unprivileged adapter user can drive the
+# daemon for the container worker backend. provision.sh fills this in
+# (getent group docker | cut -d: -f3).
+DOCKER_GID=

deploy/hosted/Caddyfile

@@ -0,0 +1,13 @@
+# TLS ingress for hosted coven-github. Caddy provisions and renews
+# Let's Encrypt certificates automatically for $WEBHOOK_DOMAIN.
+{$WEBHOOK_DOMAIN} {
+	encode gzip
+
+	# GitHub webhook deliveries and the tenant-scoped API.
+	reverse_proxy coven-github:3000
+
+	log {
+		output stdout
+		format console
+	}
+}

deploy/hosted/compose.yaml

@@ -0,0 +1,77 @@
+# Production Docker Compose for hosted coven-github: adapter + TLS ingress
+# (Caddy) + continuous SQLite replication (Litestream → Cloudflare R2).
+#
+# Runbook: docs/hosted-deploy.md. Secrets come from ./.env (never committed);
+# copy .env.example and fill it in.
+#
+# Layout on the host (created by provision.sh):
+#   /opt/coven-github/
+#     compose.yaml            this file
+#     .env                    secrets (mode 0600)
+#     Caddyfile               TLS ingress
+#     litestream.yml          replication config
+#     config/production.toml  adapter config
+#     keys/app.pem            GitHub App private key (mode 0600)
+#     data/                   SQLite store (replicated)
+
+services:
+  coven-github:
+    image: ghcr.io/opencoven/coven-github:${COVEN_GITHUB_TAG:-latest}
+    command: ["serve", "--config", "/config/production.toml"]
+    restart: unless-stopped
+    expose:
+      - "3000"
+    volumes:
+      - ./config:/config:ro
+      - ./keys:/keys:ro
+      - ./data:/data
+      - ./workspaces:/tmp/coven-github-tasks
+      # Container worker backend: the adapter launches per-task hardened
+      # containers through the host Docker daemon (docs/container-isolation.md).
+      - /var/run/docker.sock:/var/run/docker.sock
+    # The image runs unprivileged (uid 10001); grant the host docker group so
+    # the worker can drive the daemon. provision.sh fills DOCKER_GID in .env.
+    group_add:
+      - "${DOCKER_GID:?set in .env (getent group docker | cut -d: -f3)}"
+    environment:
+      RUST_LOG: "coven_github=info"
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://localhost:3000/healthz"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
+  caddy:
+    image: caddy:2
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - caddy-data:/data
+      - caddy-config:/config
+    environment:
+      WEBHOOK_DOMAIN: ${WEBHOOK_DOMAIN:?set in .env, e.g. gh.opencoven.ai}
+    depends_on:
+      - coven-github
+
+  litestream:
+    image: litestream/litestream:0.3
+    restart: unless-stopped
+    command: ["replicate", "-config", "/etc/litestream.yml"]
+    volumes:
+      - ./litestream.yml:/etc/litestream.yml:ro
+      - ./data:/data
+    environment:
+      LITESTREAM_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:?set in .env}
+      LITESTREAM_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:?set in .env}
+      R2_BUCKET: ${R2_BUCKET:?set in .env}
+      R2_ACCOUNT_ID: ${R2_ACCOUNT_ID:?set in .env}
+    depends_on:
+      - coven-github
+
+volumes:
+  caddy-data:
+  caddy-config:

deploy/hosted/litestream.yml

@@ -0,0 +1,14 @@
+# Litestream: continuous SQLite replication to Cloudflare R2 (S3-compatible).
+# Credentials come from the environment (compose.yaml). Restore drill:
+#   litestream restore -config /etc/litestream.yml /data/coven-github.db
+dbs:
+  - path: /data/coven-github.db
+    replicas:
+      - type: s3
+        bucket: ${R2_BUCKET}
+        path: coven-github
+        endpoint: https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com
+        # R2 ignores regions but the S3 client requires one.
+        region: auto
+        retention: 720h # 30 days of point-in-time history
+        sync-interval: 10s

deploy/hosted/provision.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Provision a fresh Ubuntu 24.04 host (e.g. Hetzner CPX41) to run hosted
+# coven-github. Idempotent; run as root ON THE SERVER:
+#
+#   curl -fsSL https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted/provision.sh | bash
+#
+# Then follow docs/hosted-deploy.md to place config, keys, and .env.
+set -euo pipefail
+
+DEPLOY_DIR=/opt/coven-github
+RAW=https://raw.githubusercontent.com/OpenCoven/coven-github/main/deploy/hosted
+
+echo "==> Installing Docker Engine + compose plugin"
+if ! command -v docker >/dev/null 2>&1; then
+  curl -fsSL https://get.docker.com | sh
+fi
+
+echo "==> Hardening: unattended upgrades + firewall (22/80/443 only)"
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq unattended-upgrades ufw curl
+ufw default deny incoming
+ufw default allow outgoing
+ufw allow 22/tcp
+ufw allow 80/tcp
+ufw allow 443/tcp
+ufw --force enable
+
+echo "==> Laying out ${DEPLOY_DIR}"
+mkdir -p "${DEPLOY_DIR}"/{config,keys,data,workspaces}
+chmod 700 "${DEPLOY_DIR}/keys"
+# The image runs as uid 10001 (user `coven`); it owns its state dirs.
+chown 10001 "${DEPLOY_DIR}/data" "${DEPLOY_DIR}/workspaces"
+cd "${DEPLOY_DIR}"
+for f in compose.yaml Caddyfile litestream.yml .env.example; do
+  curl -fsSL "${RAW}/${f}" -o "${f}"
+done
+[ -f .env ] || { cp .env.example .env && chmod 600 .env; }
+
+echo "==> Recording the docker group id for the compose worker"
+DOCKER_GID="$(getent group docker | cut -d: -f3)"
+grep -q '^DOCKER_GID=' .env \
+  && sed -i "s/^DOCKER_GID=.*/DOCKER_GID=${DOCKER_GID}/" .env \
+  || echo "DOCKER_GID=${DOCKER_GID}" >> .env
+
+cat <<'NEXT'
+
+Provisioned. Next (docs/hosted-deploy.md):
+  1. Fill /opt/coven-github/.env            (domain, image tag, R2 credentials)
+  2. Place /opt/coven-github/config/production.toml and keys/app.pem (0600)
+  3. Point DNS A/AAAA for your webhook domain at this host
+  4. docker compose run --rm coven-github doctor --config /config/production.toml
+  5. docker compose up -d
+  6. curl -fsS https://<domain>/healthz
+NEXT